def __create_operation(self): """ Create a operation entity. Return the operation, algo_id and the storage path. """ meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"} algorithm = FlowService().get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS) operation = model.Operation(self.user.id, self.project.id, algorithm.id, json.dumps(''), meta=json.dumps(meta), status=model.STATUS_STARTED) operation = dao.store_entity(operation) storage_path = FilesHelper().get_project_folder(self.project, str(operation.id)) return operation, algorithm.id, storage_path
def test_zip_import(self): """ This method tests import of TVB data in zip format (which imply multiple data types in the same zip file - exported from a group) """ self._import(self.zip_file_path) count = FlowService().get_available_datatypes( self.test_project.id, self.datatype.module + "." + self.datatype.type)[1] self.assertEqual(9, count, "9 datatypes should have been imported from group.")
def import_conn_zip(project_id, zip_path): project = dao.get_project_by_id(project_id) group = dao.get_algorithm_by_module( 'tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') importer = ABCAdapter.build_adapter(group) ### Launch Operation FlowService().fire_operation(importer, project.administrator, project_id, uploaded=zip_path)
def transactional_setup_method(self): """ Prepare some entities to work with during tests:""" self.flow_service = FlowService() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(admin=self.test_user) category = dao.get_uploader_categories()[0] self.algorithm = dao.store_entity( model_operation.Algorithm(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, category.id))
def import_conn_zip(project_id, zip_path): project = dao.get_project_by_id(project_id) importer = ABCAdapter.build_adapter_from_class(ZIPConnectivityImporter) params = {"_uploaded": zip_path} form = ZIPConnectivityImporterForm() form.uploaded.data = zip_path importer.submit_form(form) FlowService().fire_operation(importer, project.administrator, project_id, **params)
def get_entity(project, expected_data, filters=None): """ Return the first entity with class given by `expected_data` :param expected_data: specifies the class whose entity is returned """ data_types = FlowService().get_available_datatypes( project.id, expected_data.module + "." + expected_data.type, filters)[0] entity = ABCAdapter.load_entity_by_gid(data_types[0][2]) return entity
def __init__(self): self.logger = get_logger(self.__class__.__module__) self.user_service = UserService() self.flow_service = FlowService() self.analyze_category_link = '/flow/step_analyzers' self.analyze_adapters = None self.connectivity_tab_link = '/flow/step_connectivity' view_category = self.flow_service.get_visualisers_category() conn_id = self.flow_service.get_algorithm_by_module_and_class( CONNECTIVITY_MODULE, CONNECTIVITY_CLASS).id connectivity_link = self.get_url_adapter(view_category.id, conn_id) self.connectivity_submenu = [ dict(title="Large Scale Connectivity", link=connectivity_link, subsection=WebStructure.SUB_SECTION_CONNECTIVITY, description= "View Connectivity Regions. Perform Connectivity lesions"), dict(title="Local Connectivity", link='/spatial/localconnectivity/step_1/1', subsection=WebStructure.SUB_SECTION_LOCAL_CONNECTIVITY, description= "Create or view existent Local Connectivity entities.") ] allen_algo = self.flow_service.get_algorithm_by_module_and_class( ALLEN_CREATOR_MODULE, ALLEN_CREATOR_CLASS) if allen_algo and not allen_algo.removed: # Only add the Allen Creator if AllenSDK is installed allen_link = self.get_url_adapter(allen_algo.fk_category, allen_algo.id) self.connectivity_submenu.append( dict( title="Allen Connectome Builder", link=allen_link, subsection=WebStructure.SUB_SECTION_ALLEN, description= "Download data from Allen dataset and create a mouse connectome" )) self.burst_submenu = [ dict(link='/burst', subsection=WebStructure.SUB_SECTION_BURST, title='Simulation Cockpit', description='Manage simulations'), dict(link='/burst/dynamic', subsection='dynamic', title='Phase plane', description='Configure model dynamics') ]
def _import(self, import_file_name): ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.connectivity_measure_importer', 'ConnectivityMeasureImporter') path = os.path.join(os.path.dirname(test_data.__file__), import_file_name) args = {'data_file': path, 'connectivity': self.connectivity.gid, DataTypeMetaData.KEY_SUBJECT: "John"} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _run_cff_importer(self, cff_path): ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.cff_importer', 'CFF_Importer') args = { 'cff': cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def import_projection_matrix(user, project, file_path, sensors_gid, surface_gid): importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixSurfaceEEGImporter') form = ProjectionMatrixImporterForm() form.fill_from_post({ 'projection_file': Part(file_path, HeaderMap({}), ''), 'dataset_name': 'ProjectionMatrix', 'sensors': sensors_gid, 'surface': surface_gid, 'Data_Subject': 'John Doe' }) form.projection_file.data = file_path view_model = form.get_view_model()() form.fill_trait(view_model) importer.submit_form(form) FlowService().fire_operation(importer, user, project.id, view_model=view_model) data_types = FlowService().get_available_datatypes( project.id, ProjectionMatrixIndex)[0] assert 1 == len( data_types ), "Project should contain only one data type = Projection Matrix." projection_matrix = ABCAdapter.load_entity_by_gid(data_types[0][2]) assert projection_matrix is not None, "Projection Matrix instance should not be none" return projection_matrix
def import_surface_gifti(user, project, path): """ This method is used for importing data in GIFIT format :param path: absolute path of the file to be imported """ # Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter') form = GIFTISurfaceImporterForm() form.fill_from_post({ 'file_type': form.get_view_model().KEY_OPTION_READ_METADATA, 'data_file': Part(path, HeaderMap({}), ''), 'data_file_part2': Part('', HeaderMap({}), ''), 'should_center': 'False', 'Data_Subject': 'John Doe', }) form.data_file.data = path view_model = form.get_view_model()() form.fill_trait(view_model) importer.submit_form(form) # Launch import Operation FlowService().fire_operation(importer, user, project.id, view_model=view_model) surface = CorticalSurface data_types = FlowService().get_available_datatypes( project.id, surface.__module__ + "." + surface.__name__)[0] assert 1, len( data_types) == "Project should contain only one data type." surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) assert surface is not None == "TimeSeries should not be none" return surface
def import_surface_zip(user, project, zip_path, surface_type, zero_based): ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter') args = { 'uploaded': zip_path, 'surface_type': surface_type, 'zero_based_triangles': zero_based } ### Launch Operation FlowService().fire_operation(importer, user, project.id, **args)
def _run_cff_importer(self, cff_path): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.cff_importer', 'CFF_Importer') importer = ABCAdapter.build_adapter(group) args = { 'cff': cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _import(self, import_file_path=None): """ This method is used for importing data in TVB format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.tvb_importer', 'TVBImporter') args = {'data_file': import_file_path} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def import_surface_zip(user, project, zip_path, surface_type, zero_based): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter') importer = ABCAdapter.build_adapter(group) args = { 'uploaded': zip_path, 'surface_type': surface_type, 'zero_based_triangles': zero_based } ### Launch Operation FlowService().fire_operation(importer, user, project.id, **args)
def setUp(self): """ Sets up the testing environment; saves config file; creates a test user, a test project; creates burst, operation, flow and workflow services """ self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user) self.workflow_service = WorkflowService() self.burst_service = BurstService() self.operation_service = OperationService() self.flow_service = FlowService()
def _launch_test_algo_on_cluster(self, **data): module = "tvb.tests.framework.adapters.testadapter1" class_name = "TestAdapter1" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) algo_group = adapter.algorithm_group algo_category = dao.get_category_by_id(algo_group.fk_category) algo = dao.get_algorithm_by_group(algo_group.id) operations, _ = self.operation_service.prepare_operations( self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) return operations
def setUp(self): """ Prepare before each test. """ self.project_service = ProjectService() self.flow_service = FlowService() self.structure_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, "ProjectStructure") self.relevant_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.RELEVANT_VIEW) self.full_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.FULL_VIEW)
def test_h5_import(self, transactional_setup_fixture): """ This method tests import of TVB data in h5 format. Single data type / import """ self._import(self.h5_file_path) data_types = FlowService().get_available_datatypes(self.test_project.id, self.datatype.module + "." + self.datatype.type)[0] assert 1, len(data_types) == "Project should contain only one data type." data_type_entity = ABCAdapter.load_entity_by_gid(data_types[0][2]) assert data_type_entity is not None, "Datatype should not be none" assert self.datatype.gid, data_type_entity.gid == "Imported datatype should have the same gid"
def _import(self, import_file_path, sensors_type, expected_data): """ This method is used for importing sensors :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer') args = {'sensors_file': import_file_path, 'sensors_type': sensors_type} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes(self.test_project.id, expected_data.module + "." + expected_data.type)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type = Sensors.") time_series = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(time_series is not None, "Sensors instance should not be none") return time_series
def import_surface_zip(user, project, zip_path, surface_type, zero_based='True'): # Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter') form = ZIPSurfaceImporterForm() form.fill_from_post({ 'uploaded': Part(zip_path, HeaderMap({}), ''), 'zero_based_triangles': zero_based, 'should_center': 'True', 'surface_type': surface_type, 'Data_Subject': 'John Doe' }) form.uploaded.data = zip_path view_model = form.get_view_model()() form.fill_trait(view_model) importer.submit_form(form) # Launch import Operation FlowService().fire_operation(importer, user, project.id, view_model=view_model) data_types = FlowService().get_available_datatypes( project.id, SurfaceIndex)[0] assert 1, len( data_types) == "Project should contain only one data type." surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) surface.user_tag_3 = '' assert surface is not None, "Surface should not be None" return surface
def import_surface_obj(user, project, obj_path, surface_type): # Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter') form = ObjSurfaceImporterForm() form.fill_from_post({'_data_file': Part(obj_path, HeaderMap({}), ''), '_surface_type': surface_type, '_Data_Subject': 'John Doe' }) form.data_file.data = obj_path view_model = form.get_view_model()() form.fill_trait(view_model) importer.submit_form(form) # Launch import Operation FlowService().fire_operation(importer, user, project.id, view_model=view_model) data_types = FlowService().get_available_datatypes(project.id, SurfaceIndex)[0] assert 1, len(data_types) == "Project should contain only one data type." surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) assert surface is not None, "Surface should not be None" return surface
def _importSurface(self, import_file_path=None): """ This method is used for importing data in GIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter') args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: ""} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) surface = CorticalSurface() data_types = FlowService().get_available_datatypes(self.test_project.id, surface.module + "." + surface.type)[0] assert 1, len(data_types) == "Project should contain only one data type." surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) assert surface is not None == "TimeSeries should not be none" return surface
def fire_simulation(project_id, simulator): project = dao.get_project_by_id(project_id) assert isinstance(simulator, Simulator) # Load the SimulatorAdapter algorithm from DB cached_simulator_algorithm = FlowService().get_algorithm_by_module_and_class(IntrospectionRegistry.SIMULATOR_MODULE, IntrospectionRegistry.SIMULATOR_CLASS) # Instantiate a SimulatorService and launch the configured simulation simulator_service = SimulatorService() launched_operation = simulator_service.async_launch_and_prepare_simulation(None, project.administrator, project, cached_simulator_algorithm, simulator, None) LOG.info("Operation launched ....") return launched_operation
def __init__(self, overwrites=None, settings_file=None): """ Parameters can be overwritten either from a settigns file or from a dictionary. """ if overwrites is not None: self.overwrites.update(overwrites) if settings_file is not None: settings = open(sys.argv[1]).read() for line in settings.split('\n'): key, value = line.split('=') self.overwrites[key.strip()] = value.strip() if KEY_PROJECT not in self.overwrites: raise Exception("Settings file should contain the id of the project: %s=1" % KEY_PROJECT) self.project = dao.get_project_by_id(self.overwrites[KEY_PROJECT]) self.flow_service = FlowService() self.operation_service = OperationService()
def _import_connectivity(self): importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, uploaded=self.connectivity_path, Data_Subject='QL') self.connectivity = TestFactory.get_entity(self.test_project, Connectivity())
def _importSurface(self, import_file_path=None): ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter') args = { 'data_file': import_file_path, "surface_type": FACE, DataTypeMetaData.KEY_SUBJECT: "John" } ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes( self.test_project.id, FaceSurface)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "Surface should not be None") return surface
def _compute_connectivity_global_params(self, connectivity): """ Returns a dictionary which contains the data needed for drawing a connectivity. :param connectivity: the `Connectivity(HasTraits)` object """ conn_gid = connectivity.gid.hex path_weights = SurfaceURLGenerator.paths2url(conn_gid, 'ordered_weights') path_pos = SurfaceURLGenerator.paths2url(conn_gid, 'ordered_centres') path_tracts = SurfaceURLGenerator.paths2url(conn_gid, 'ordered_tracts') path_labels = SurfaceURLGenerator.paths2url(conn_gid, 'ordered_labels') path_hemisphere_order_indices = SurfaceURLGenerator.paths2url( conn_gid, 'hemisphere_order_indices') algo = FlowService().get_algorithm_by_module_and_class( CONNECTIVITY_CREATOR_MODULE, CONNECTIVITY_CREATOR_CLASS) submit_url = '/{}/{}/{}'.format(SurfaceURLGenerator.FLOW, algo.fk_category, algo.id) global_pages = dict(controlPage="connectivity/top_right_controls") minimum, maximum, minimum_non_zero = self._compute_matrix_extrema( connectivity.ordered_weights) minimum_t, maximum_t, minimum_non_zero_t = self._compute_matrix_extrema( connectivity.ordered_tracts) global_params = dict( urlWeights=path_weights, urlPositions=path_pos, urlTracts=path_tracts, urlLabels=path_labels, originalConnectivity=conn_gid, title="Connectivity Control", submitURL=submit_url, positions=connectivity.ordered_centres, tractsMin=minimum_t, tractsMax=maximum_t, weightsMin=minimum, weightsMax=maximum, tractsNonZeroMin=minimum_non_zero_t, weightsNonZeroMin=minimum_non_zero, pointsLabels=connectivity.ordered_labels, conductionSpeed=connectivity.speed or 1, connectivity_entity=connectivity, base_selection=connectivity.saved_selection_labels, hemisphereOrderUrl=path_hemisphere_order_indices) global_params.update( self.build_params_for_selectable_connectivity(connectivity)) return global_params, global_pages
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) self.assertTrue( len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def __init__(self): BaseController.__init__(self) self.flow_service = FlowService() self.logger = get_logger(__name__) editable_entities = [ dict(link='/spatial/stimulus/region/step_1_submit/1/1', title='Region Stimulus', subsection='regionstim', description='Create a new Stimulus on Region level'), dict(link='/spatial/stimulus/surface/step_1_submit/1/1', title='Surface Stimulus', subsection='surfacestim', description='Create a new Stimulus on Surface level') ] self.submenu_list = editable_entities