def test_remove_project_node(self): """ Test removing of a node from a project. """ inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user) project_to_link = model.Project("Link", self.test_user.id, "descript") project_to_link = dao.store_entity(project_to_link) exact_data = dao.get_datatype_by_gid(gid) dao.store_entity(model.Links(exact_data.id, project_to_link.id)) assert dao.get_datatype_by_gid(gid) is not None, "Initialization problem!" operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id)) assert os.path.exists(op_folder) sub_files = os.listdir(op_folder) assert 2 == len(sub_files) ### Validate that no more files are created than needed. if(dao.get_system_user() is None): dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None)) self.project_service._remove_project_node_files(inserted_project.id, gid) sub_files = os.listdir(op_folder) assert 1 == len(sub_files) ### operation.xml file should still be there op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1)) sub_files = os.listdir(op_folder) assert 2 == len(sub_files) assert dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links" self.project_service._remove_project_node_files(project_to_link.id, gid) assert dao.get_datatype_by_gid(gid) is None sub_files = os.listdir(op_folder) assert 1 == len(sub_files)
def update(): """ Update TVB code to SVN revision version 4455. This update was done for release 1.0.2. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: user = dao.get_system_user() adapter = ObjSurfaceImporter() OperationService().fire_operation(adapter, user, project.id, visible=False, surface_type=EEG_CAP, data_file=DATA_FILE_EEG_CAP) adapter = ObjSurfaceImporter() OperationService().fire_operation(adapter, user, project.id, visible=False, surface_type=FACE, data_file=DATA_FILE_FACE) except Exception as excep: LOGGER.exception(excep) TvbProfile.current.manager.add_entries_to_config_file( {stored.KEY_MATLAB_EXECUTABLE: get_matlab_executable()})
def test_remove_project_node(self): """ Test removing of a node from a project. """ inserted_project, gid, op = TestFactory.create_value_wrapper(self.test_user) project_to_link = model_project.Project("Link", self.test_user.id, "descript") project_to_link = dao.store_entity(project_to_link) exact_data = dao.get_datatype_by_gid(gid) assert exact_data is not None, "Initialization problem!" link = dao.store_entity(model_datatype.Links(exact_data.id, project_to_link.id)) vw_h5_path = h5.path_for_stored_index(exact_data) assert os.path.exists(vw_h5_path) if dao.get_system_user() is None: dao.store_entity(model_operation.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None)) self.project_service._remove_project_node_files(inserted_project.id, gid, [link]) assert not os.path.exists(vw_h5_path) exact_data = dao.get_datatype_by_gid(gid) assert exact_data is not None, "Data should still be in DB, because of links" vw_h5_path_new = h5.path_for_stored_index(exact_data) assert os.path.exists(vw_h5_path_new) assert vw_h5_path_new != vw_h5_path self.project_service._remove_project_node_files(project_to_link.id, gid, []) assert dao.get_datatype_by_gid(gid) is None
def update(): """ Update TVB code to SVN revision version 4455. This update was done for release 1.0.2. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: user = dao.get_system_user() adapter = ObjSurfaceImporter() FlowService().fire_operation(adapter, user, project.id, visible=False, surface_type=EEG_CAP, data_file=DATA_FILE_EEG_CAP) adapter = ObjSurfaceImporter() FlowService().fire_operation(adapter, user, project.id, visible=False, surface_type=FACE, data_file=DATA_FILE_FACE) except Exception as excep: LOGGER.exception(excep) TvbProfile.current.manager.add_entries_to_config_file({stored.KEY_MATLAB_EXECUTABLE: get_matlab_executable()})
def update(): """ Update TVB code to SVN revision version 4455. This update was done for release 1.0.2. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: user = dao.get_system_user() adapter = ObjSurfaceImporter() FlowService().fire_operation(adapter, user, project.id, visible=False, surface_type=EEG_CAP, data_file=DATA_FILE_EEG_CAP) adapter = ObjSurfaceImporter() FlowService().fire_operation(adapter, user, project.id, visible=False, surface_type=FACE, data_file=DATA_FILE_FACE) except Exception, excep: LOGGER.exception(excep)
def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(Links, link.id) was_link = True if not was_link: # Create a clone of the operation new_op = Operation( dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.parameters, datatype.parent_operation.meta_data, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project).name new_op_loaded = dao.get_operation_by_id(new_op.id) self.structure_helper.write_operation_metadata( new_op_loaded) full_path = h5.path_for_stored_index(datatype) self.structure_helper.move_datatype( datatype, to_project, str(new_op.id), full_path) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) h5_path = h5.path_for_stored_index(datatype) self.structure_helper.remove_datatype_file(h5_path) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException( "Remove operation failed for unknown reasons.Please contact system administrator." )
def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) if links: was_link = False for link in links: #This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(model.Links, link.id) was_link = True if not was_link: # Create a clone of the operation new_op = model.Operation(dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.parameters, datatype.parent_operation.meta_data, datatype.parent_operation.method_name, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project).name new_op_loaded = dao.get_operation_by_id(new_op.id) self.structure_helper.write_operation_metadata(new_op_loaded) self.structure_helper.move_datatype(datatype, to_project, str(new_op.id)) datatype.set_operation_id(new_op.id) datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(model.Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) self.structure_helper.remove_datatype(datatype) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def test_initialize_startup(self): """ Test "reset_database" and "initialize_startup" calls. """ reset_database() # Table USERS should not exist: self.assertRaises(Exception, dao.get_all_users) initialize_startup() # Table exists, but no rows self.assertEqual(0, len(dao.get_all_users())) self.assertEqual(None, dao.get_system_user()) # DB revisions folder should exist: self.assertTrue(os.path.exists(TvbProfile.current.db.DB_VERSIONING_REPO))
def update(): """ Update TVB code to SVN revision version 4770. This update was done for release 1.0.5 """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: handle_event(EVENT_FILE_IDENTIFIER, dao.get_system_user(), project) except Exception, excep: LOGGER.exception(excep)
def test_initialize_startup(self): """ Test "reset_database" and "initialize_startup" calls. """ reset_database() # Table USERS should not exist: with pytest.raises(Exception): dao.get_all_users() initialize_startup() # Table exists, but no rows assert 0 == len(dao.get_all_users()) assert None == dao.get_system_user() # DB revisions folder should exist: assert os.path.exists(TvbProfile.current.db.DB_VERSIONING_REPO)
def update(): """ Update TVB code to SVN revision version 6093. This update was done for release 1.1.5. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: handle_event(EVENT_FILE_IDENTIFIER, dao.get_system_user(), project) except Exception: LOGGER.exception("could not migrate project id: %s, name %s" % (project.id, project.name))
def update(): """ Update TVB code to SVN revision version 4770. This update was done for release 1.0.5 """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: adapter = SensorsImporter() FlowService().fire_operation(adapter, dao.get_system_user(), project.id, visible=False, sensors_file=DATA_FILE, sensors_type=SensorsImporter.INTERNAL_SENSORS) except Exception as excep: LOGGER.exception(excep)
def update(): """ Update TVB code to SVN revision version 4770. This update was done for release 1.0.5 """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: adapter = Sensors_Importer() FlowService().fire_operation(adapter, dao.get_system_user(), project.id, visible=False, sensors_file=DATA_FILE, sensors_type=Sensors_Importer.INTERNAL_SENSORS) except Exception, excep: LOGGER.exception(excep)
def update(): """ Update TVB code to SVN revision version 6093. This update was done for release 1.1.5. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: user = dao.get_system_user() adapter = ObjSurfaceImporter() FlowService().fire_operation(adapter, user, project.id, visible=False, surface_type=FACE, data_file=DATA_FILE_FACE) except Exception: LOGGER.exception("could not migrate project id: %s, name %s" % (project.id, project.name))
def update(): """ Update TVB code to SVN revision version 6093. This update was done for release 1.1.5. """ projects_count = dao.get_all_projects(is_count=True) for page_start in range(0, projects_count, PAGE_SIZE): projects_page = dao.get_all_projects(page_start=page_start, page_size=PAGE_SIZE) for project in projects_page: try: user = dao.get_system_user() adapter = ObjSurfaceImporter() OperationService().fire_operation(adapter, user, project.id, visible=False, surface_type=SurfaceTypesEnum.FACE_SURFACE.value, data_file=DATA_FILE_FACE) except Exception: LOGGER.exception("could not migrate project id: %s, name %s" % (project.id, project.name))
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project): new_op = Operation(op.view_model_gid, dao.get_system_user().id, fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op.visible = datatype.parent_operation.visible new_op = dao.store_entity(new_op) to_project = self.find_project(fk_to_project) to_project_path = self.storage_interface.get_project_folder(to_project.name) full_path = h5.path_for_stored_index(datatype) old_folder = self.storage_interface.get_project_folder(project.name, str(op.id)) file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0] file_paths.append(full_path) # The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation # folder containing the file if datatype.is_ts and datatype.fk_parent_burst is not None: bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst, project.name) if os.path.exists(bc_path): file_paths.append(bc_path) bc = dao.get_burst_for_operation_id(op.id) bc.fk_simulation = new_op.id dao.store_entity(bc) # Move all files to the new operation folder self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) return new_op
def _remove_project_node_files(self, project_id, gid, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) links = dao.get_links_for_datatype(datatype.id) op = dao.get_operation_by_id(datatype.fk_from_operation) adapter = ABCAdapter.build_adapter(op.algorithm) if links: was_link = False for link in links: # This means it's only a link and we need to remove it if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id: dao.remove_entity(Links, link.id) was_link = True if not was_link: # Create a clone of the operation # There is no view_model so the view_model_gid is None new_op = Operation(op.view_model_gid, dao.get_system_user().id, links[0].fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op = dao.store_entity(new_op) to_project = self.find_project(links[0].fk_to_project) to_project_path = self.structure_helper.get_project_folder(to_project) encryption_handler.set_project_active(to_project) encryption_handler.sync_folders(to_project_path) to_project_name = to_project.name full_path = h5.path_for_stored_index(datatype) self.structure_helper.move_datatype(datatype, to_project_name, str(new_op.id), full_path) # Move also the ViewModel H5 old_folder = self.structure_helper.get_project_folder(project, str(op.id)) view_model = adapter.load_view_model(op) vm_full_path = h5.determine_filepath(op.view_model_gid, old_folder) self.structure_helper.move_datatype(view_model, to_project_name, str(new_op.id), vm_full_path) encryption_handler.sync_folders(to_project_path) encryption_handler.set_project_inactive(to_project) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) dao.remove_entity(Links, links[0].id) else: specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) h5_path = h5.path_for_stored_index(datatype) self.structure_helper.remove_datatype_file(h5_path) encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder_from_h5(h5_path)) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")