def test_initiate_operation(self): """ Test the actual operation flow by executing a test adapter. """ module = "tvb.tests.framework.adapters.testadapter1" class_name = "TestAdapter1" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) output = adapter.get_output() output_type = output[0].__name__ data = {"test1_val1": 5, "test1_val2": 5} tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") res = self.operation_service.initiate_operation( self.test_user, self.test_project.id, adapter, tmp_folder, **data ) self.assertTrue(res.index("has finished.") > 10, "Operation didn't finish") group = dao.find_group(module, class_name) self.assertEqual(group.module, "tvb.tests.framework.adapters.testadapter1", "Wrong data stored.") self.assertEqual(group.classname, "TestAdapter1", "Wrong data stored.") dts, count = dao.get_values_of_datatype(self.test_project.id, Datatype1) self.assertEqual(count, 1) self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_initiate_operation(self): """ Test the actual operation flow by executing a test adapter. """ module = "tvb_test.adapters.testadapter1" class_name = "TestAdapter1" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) output = adapter.get_output() output_type = output[0].__name__ data = {"test1_val1": 5, "test1_val2": 5} tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") res = self.operation_service.initiate_operation( self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) self.assertTrue( res.index("has finished.") > 10, "Operation didn't finish") group = dao.find_group(module, class_name) self.assertEqual(group.module, 'tvb_test.adapters.testadapter1', "Wrong data stored.") self.assertEqual(group.classname, 'TestAdapter1', "Wrong data stored.") dts = dao.get_values_of_datatype(self.test_project.id, Datatype1) self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def build_adapter_from_declaration(cls, adapter_declaration): """ Build and adapter from the declaration in the portlets xml. """ adapter_import_path = adapter_declaration[ABCAdapter.KEY_TYPE] class_name = adapter_import_path.split(".")[-1] module = adapter_import_path.replace("." + class_name, "") if "initparam" in adapter_declaration: algo_group = dao.find_group(module, class_name, adapter_declaration["initparam"]) else: algo_group = dao.find_group(module, class_name) if algo_group is not None: return ABCAdapter.build_adapter(algo_group), algo_group else: return None, None
def build_adapter_from_declaration(cls, adapter_declaration): """ Build and adapter from the declaration in the portlets xml. """ adapter_import_path = adapter_declaration[ABCAdapter.KEY_TYPE] class_name = adapter_import_path.split('.')[-1] module = adapter_import_path.replace('.' + class_name, '') if 'initparam' in adapter_declaration: algo_group = dao.find_group(module, class_name, adapter_declaration['initparam']) else: algo_group = dao.find_group(module, class_name) if algo_group is not None: return ABCAdapter.build_adapter(algo_group), algo_group else: return None, None
def _import(self, import_file_path, sensors_type, expected_data): """ This method is used for importing sensors :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer') importer = ABCAdapter.build_adapter(group) importer.meta_data = {DataTypeMetaData.KEY_SUBJECT: "", DataTypeMetaData.KEY_STATE: "RAW"} args = {'sensors_file': import_file_path, 'sensors_type': sensors_type} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes(self.test_project.id, expected_data.module + "." + expected_data.type) self.assertEqual(1, len(data_types), "Project should contain only one data type = Sensors.") time_series = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(time_series is not None, "Sensors instance should not be none") return time_series
def test_launch_two_ops_HDD_with_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ module = "tvb_test.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) output = adapter.get_output() output_type = output[0].__name__ data = {"test": 100} TVBSettings.MAX_DISK_SPACE = 2 * float(adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2) self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.") #Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TVBSettings.MAX_DISK_SPACE = float(datatype.disk_size) + float(adapter.get_required_disk_size(**data)) self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2) self.assertEqual(len(dts), 2) datatype = dao.get_datatype_by_id(dts[1][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_launch_two_ops_HDD_full_space(self): """ Launch two operations and give available space for user so that the first should finish, but after the update to the user hdd size the second should not. """ module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) output = adapter.get_output() output_type = output[0].__name__ data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data))) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0] self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.") #Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \ float(adapter.get_required_disk_size(**data) - 1) self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0] self.assertEqual(len(dts), 1)
def _create_mapped_arrays(self, project_id): """ :param project_id: the project in which the arrays are created :return: a list of dummy `MappedArray` """ count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1] self.assertEqual(count, 0) algo_group = dao.find_group('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter') group, _ = self.flow_service.prepare_adapter(project_id, algo_group) adapter_instance = self.flow_service.build_adapter_instance(group) data = {'param_1': 'some value'} #create 3 data types self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1] self.assertEqual(count, 1) self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1] self.assertEqual(count, 2) self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) array_wrappers, count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray") self.assertEqual(count, 3) return array_wrappers
def _import_csv_test_connectivity(self, reference_connectivity_gid, subject): ### First prepare input data: data_dir = path.abspath(path.dirname(tvb_data.__file__)) torronto_dir = path.join(data_dir, 'dti_pipeline', 'Output_Toronto') weights = path.join(torronto_dir, 'output_ConnectionCapacityMatrix.csv') tracts = path.join(torronto_dir, 'output_ConnectionDistanceMatrix.csv') weights_tmp = weights + '.tmp' tracts_tmp = tracts + '.tmp' self.helper.copy_file(weights, weights_tmp) self.helper.copy_file(tracts, tracts_tmp) ### Find importer and Launch Operation group = dao.find_group( 'tvb.adapters.uploaders.csv_connectivity_importer', 'CSVConnectivityImporter') importer = ABCAdapter.build_adapter(group) FlowService().fire_operation(importer, self.test_user, self.test_project.id, weights=weights_tmp, tracts=tracts_tmp, Data_Subject=subject, input_data=reference_connectivity_gid)
def test_wrong_shape(self): """ Verifies that importing a different shape throws exception """ group = dao.find_group( 'tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixSurfaceEEGImporter') importer = ABCAdapter.build_adapter(group) file_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), 'surface_reg_13_eeg_62.mat') args = { 'projection_file': file_path, 'dataset_name': 'ProjectionMatrix', 'sensors': self.sensors.gid, 'surface': self.surface.gid, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } try: FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) self.fail( "This was expected not to run! 62 rows in proj matrix, but 65 sensors" ) except OperationException: pass
def _import(self, import_file_path, surface_gid, connectivity_gid): """ This method is used for importing region mappings :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group( 'tvb.adapters.uploaders.region_mapping_importer', 'RegionMapping_Importer') importer = ABCAdapter.build_adapter(group) args = { 'mapping_file': import_file_path, 'surface': surface_gid, 'connectivity': connectivity_gid, DataTypeMetaData.KEY_SUBJECT: "test" } now = datetime.datetime.now() ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) # During setup we import a CFF which creates an additional RegionMapping # So, here we have to find our mapping (just imported) data_filter = FilterChain( fields=[FilterChain.datatype + ".create_date"], operations=[">"], values=[now]) region_mapping = self._get_entity(RegionMapping(), data_filter) return region_mapping
def create_operation(algorithm=None, test_user=None, test_project=None, operation_status=model.STATUS_FINISHED, parameters="test params"): """ Create persisted operation. :param algorithm: When not None, introspect TVB and TVB_TEST for adapters. :return: Operation entity after persistence. """ if algorithm is None: algo_group = dao.find_group('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter') algorithm = dao.get_algorithm_by_group(algo_group.id) if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"} operation = model.Operation(test_user.id, test_project.id, algorithm.id, parameters, meta=json.dumps(meta), status=operation_status, method_name=ABCAdapter.LAUNCH_METHOD) dao.store_entity(operation) ### Make sure lazy attributes are correctly loaded. return dao.get_operation_by_id(operation.id)
def _import(self, import_file_path=None, expected_result_class=StructuralMRI, connectivity=None): """ This method is used for importing data in NIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter') importer = ABCAdapter.build_adapter(group) args = { 'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "bla bla", 'apply_corrections': True, 'connectivity': connectivity } ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dts, count = dao.get_values_of_datatype(self.test_project.id, expected_result_class, None) self.assertEqual(1, count, "Project should contain only one data type.") result = ABCAdapter.load_entity_by_gid(dts[0][2]) self.assertTrue(result is not None, "Result should not be none") return result
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3") group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def test_build_adapter_instance(self): """ Test standard flow for building an adapter instance. """ algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS) adapter = ABCAdapter.build_adapter(algo_group) self.assertTrue(isinstance(adapter, ABCSynchronous), "Something went wrong with valid data!")
def test_launch_operation_HDD_full_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) started_operation = model.Operation( self.test_user.id, self.test_project.id, group.id, "", status=model.STATUS_STARTED, estimated_disk_size=space_taken_by_started, ) dao.store_entity(started_operation) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) + space_taken_by_started - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises( NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data ) self._assert_no_dt2()
def test_launch_two_ops_HDD_full_space(self): """ Launch two operations and give available space for user so that the first should finish, but after the update to the user hdd size the second should not. """ module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = 1 + float(adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() # Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB) # plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + float( adapter.get_required_disk_size(**data) - 1 ) self.assertRaises( NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data ) self._assert_stored_dt2()
def import_sensors(user, project, zip_path, sensors_type): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer') importer = ABCAdapter.build_adapter(group) args = {'sensors_file': zip_path, 'sensors_type': sensors_type} ### Launch Operation FlowService().fire_operation(importer, user, project.id, **args)
def create_group(test_user=None, test_project=None, subject="John Doe"): """ Create a group of 2 operations, each with at least one resultant DataType. """ if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) ### Retrieve Adapter instance algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') algo_category = dao.get_category_by_id(algo_group.fk_category) algo = dao.get_algorithm_by_group(algo_group.id) adapter_inst = TestFactory.create_adapter(algo_group=algo_group, test_project=test_project) adapter_inst.meta_data = {DataTypeMetaData.KEY_SUBJECT: subject} args = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ### Prepare Operations group. Execute them synchronously service = OperationService() operations = service.prepare_operations(test_user.id, test_project.id, algo, algo_category, {}, **args)[0] service.launch_operation(operations[0].id, False, adapter_inst) service.launch_operation(operations[1].id, False, adapter_inst) resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group) return resulted_dts, operations[0].fk_operation_group
def test_happy_flow_region_import(self): """ Verifies the happy flow for importing a region. """ dt_count_before = TestFactory.get_entity_count(self.test_project, ProjectionRegionEEG()) group = dao.find_group( 'tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixRegionEEGImporter') importer = ABCAdapter.build_adapter(group) zip_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), 'region_conn_74_eeg_1020_62.mat') args = { 'projection_file': zip_path, 'dataset_name': 'ProjectionMatrix', 'connectivity': self.connectivity.gid, 'sensors': self.sensors.gid, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dt_count_after = TestFactory.get_entity_count(self.test_project, ProjectionRegionEEG()) self.assertEqual(dt_count_before + 1, dt_count_after)
def test_launch_operation_HDD_full_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 module = "tvb_test.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) started_operation = model.Operation( self.test_user.id, self.test_project.id, group.id, "", status=model.STATUS_STARTED, result_disk_size=space_taken_by_started) dao.store_entity(started_operation) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TVBSettings.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data) + space_taken_by_started - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2) self.assertEqual(len(dts), 0)
def create_operation(algorithm=None, test_user=None, test_project=None, operation_status=model.STATUS_FINISHED, parameters="test params"): """ Create persisted operation. :param algorithm: When not None, introspect TVB and TVB_TEST for adapters. :return: Operation entity after persistence. """ if algorithm is None: algo_group = dao.find_group('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter') algorithm = dao.get_algorithm_by_group(algo_group.id) if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe", DataTypeMetaData.KEY_STATE: "RAW_DATA"} operation = model.Operation(test_user.id, test_project.id, algorithm.id, parameters, meta=json.dumps(meta), status=operation_status) dao.store_entity(operation) ### Make sure lazy attributes are correctly loaded. return dao.get_operation_by_id(operation.id)
def test_launch_operation_HDD_with_space(self): """ Test the actual operation flow by executing a test adapter. """ module = "tvb_test.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) output = adapter.get_output() output_type = output[0].__name__ data = {"test": 100} TVBSettings.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation( self.test_user, self.test_project.id, adapter, tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data) dts = dao.get_values_of_datatype(self.test_project.id, Datatype2) self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_launch_operation_HDD_full_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) started_operation = model.Operation( self.test_user.id, self.test_project.id, group.id, "", status=model.STATUS_STARTED, estimated_disk_size=space_taken_by_started) dao.store_entity(started_operation) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data) + space_taken_by_started - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_no_dt2()
def test_launch_two_ops_HDD_full_space(self): """ Launch two operations and give available space for user so that the first should finish, but after the update to the user hdd size the second should not. """ module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = ( 1 + float(adapter.get_required_disk_size(**data))) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \ float(adapter.get_required_disk_size(**data) - 1) self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def test_launch_two_ops_HDD_with_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = 2 * float( adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float( adapter.get_required_disk_size(**data)) self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2(2)
def parse_event_node(self): """ Parse the stored event node to get required data and arguments. """ kw_parameters = {} for one_arg in self.event_node.childNodes: if one_arg.nodeType != Node.ELEMENT_NODE: continue if one_arg.nodeName == ELEM_ADAPTER: #TODO: so far there is no need for it, but we should maybe #handle cases where same module/class but different init parameter group = dao.find_group(one_arg.getAttribute(ATT_MODULE), one_arg.getAttribute(ATT_CLASS)) adapter = ABCAdapter.build_adapter(group) result_uid = one_arg.getAttribute(ATT_UID) if result_uid: kw_parameters[ATT_UID] = result_uid LOGGER.debug("Adapter used is %s", str(adapter.__class__)) self.callable_object = adapter continue if one_arg.nodeName == ELEM_METHOD: self.call_method = one_arg.getAttribute(ATT_NAME) if one_arg.getAttribute(ATT_OPERATION_HIDDEN): self.operation_visible = False continue if one_arg.nodeName == ELEM_ARGS: kw_parameters.update(_parse_arguments(one_arg)) continue LOGGER.info("Ignored undefined node %s", str(one_arg.nodeName)) self.arguments.update(kw_parameters)
def test_happy_flow_import(self): """ Test that importing a CFF generates at least one DataType in DB. """ dt_count_before = TestFactory.get_entity_count(self.test_project, Connectivity()) group = dao.find_group( 'tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') importer = ABCAdapter.build_adapter(group) importer.meta_data = { DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT, DataTypeMetaData.KEY_STATE: "RAW" } zip_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), 'connectivity_regions_96.zip') args = {'uploaded': zip_path} ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dt_count_after = TestFactory.get_entity_count(self.test_project, Connectivity()) self.assertTrue(dt_count_after == dt_count_before + 1)
def test_happy_flow_surface_import(self): """ Verifies the happy flow for importing a surface. """ dt_count_before = TestFactory.get_entity_count(self.test_project, ProjectionSurfaceEEG()) group = dao.find_group( "tvb.adapters.uploaders.projection_matrix_importer", "ProjectionMatrixSurfaceEEGImporter" ) importer = ABCAdapter.build_adapter(group) file_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), "projection_eeg_65_surface_16k.npy" ) args = { "projection_file": file_path, "dataset_name": "ProjectionMatrix", "sensors": self.sensors.gid, "surface": self.surface.gid, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT, } FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dt_count_after = TestFactory.get_entity_count(self.test_project, ProjectionSurfaceEEG()) self.assertEqual(dt_count_before + 1, dt_count_after)
def prepare_adapter(self, project_id, algo_group): """ Having a given Adapter, specified by Module and ClassName, create a instance of it and return the instance. Actually return a Tuple: Adapter Instance, Dictionary for Adapter Interface specification. """ adapter_module = algo_group.module.replace('-', '.') adapter_name = algo_group.classname try: # Prepare Adapter Interface, by populating with existent data, # in case of a parameter of type DataType. group = dao.find_group(adapter_module, adapter_name, algo_group.init_parameter) adapter_instance = self.build_adapter_instance(group) interface = adapter_instance.get_input_tree() interface = self.prepare_parameters(interface, project_id, group.fk_category) interface = ABCAdapter.prepare_param_names(interface) return group, interface except Exception, excep: self.logger.exception(excep) self.logger.error('Not found:' + adapter_name + ' in:' + adapter_module) raise OperationException("Could not prepare " + adapter_name)
def _import(self, import_file_path, surface_gid, connectivity_gid): """ This method is used for importing region mappings :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group("tvb.adapters.uploaders.region_mapping_importer", "RegionMapping_Importer") importer = ABCAdapter.build_adapter(group) args = { "mapping_file": import_file_path, "surface": surface_gid, "connectivity": connectivity_gid, DataTypeMetaData.KEY_SUBJECT: "test", } now = datetime.datetime.now() ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) # During setup we import a CFF which creates an additional RegionMapping # So, here we have to find our mapping (just imported) data_filter = FilterChain(fields=[FilterChain.datatype + ".create_date"], operations=[">"], values=[now]) region_mapping = self._get_entity(RegionMapping(), data_filter) return region_mapping
def _create_mapped_arrays(self, project_id): array_wrappers = self.flow_service.get_available_datatypes( project_id, "tvb.datatypes.arrays.MappedArray") self.assertEqual(len(array_wrappers), 0) algo_group = dao.find_group('tvb_test.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter') group, _ = self.flow_service.prepare_adapter(project_id, algo_group) adapter_instance = self.flow_service.build_adapter_instance(group) data = {'param_1': 'some value'} #create 3 data types self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) array_wrappers = self.flow_service.get_available_datatypes( project_id, "tvb.datatypes.arrays.MappedArray") self.assertEqual(len(array_wrappers), 1) self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) array_wrappers = self.flow_service.get_available_datatypes( project_id, "tvb.datatypes.arrays.MappedArray") self.assertEqual(len(array_wrappers), 2) self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data) array_wrappers = self.flow_service.get_available_datatypes( project_id, "tvb.datatypes.arrays.MappedArray") self.assertEqual(len(array_wrappers), 3) return array_wrappers
def test_happy_flow_surface_import(self): dt_count_before = TestFactory.get_entity_count(self.test_project, ProjectionSurfaceEEG()) group = dao.find_group( 'tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixSurfaceEEGImporter') importer = ABCAdapter.build_adapter(group) importer.meta_data = { DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT, DataTypeMetaData.KEY_STATE: "RAW" } zip_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), 'region_conn_74_eeg_1020_62.mat') args = { 'projection_file': zip_path, 'dataset_name': 'ProjectionMatrix', 'connectivity': self.connectivity.gid, 'sensors': self.sensors.gid, 'surface': self.surface.gid } FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dt_count_after = TestFactory.get_entity_count(self.test_project, ProjectionRegionEEG()) self.assertTrue(dt_count_after == dt_count_before + 1)
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def test_wrong_shape(self): """ Verifies that importing a different shape throws exception """ group = dao.find_group( "tvb.adapters.uploaders.projection_matrix_importer", "ProjectionMatrixSurfaceEEGImporter" ) importer = ABCAdapter.build_adapter(group) file_path = os.path.join( os.path.abspath(os.path.dirname(dataset.__file__)), "projection_eeg_62_surface_16k.mat" ) args = { "projection_file": file_path, "dataset_name": "ProjectionMatrix", "sensors": self.sensors.gid, "surface": self.surface.gid, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT, } try: FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) self.fail("This was expected not to run! 62 rows in proj matrix, but 65 sensors") except OperationException: pass
def import_cff(cff_path=None, test_user=None, test_project=None): """ This method is used for importing a CFF data-set (load CFF_Importer, launch it). :param cff_path: absolute path where CFF file exists. When None, a default CFF will be used. :param test_user: optional persisted User instance, to use as Operation->launcher :param test_project: optional persisted Project instance, to use for launching Operation in it. """ ### Prepare Data if cff_path is None: cff_path = os.path.join(os.path.dirname(cff_dataset.__file__), 'connectivities.cff') if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.cff_importer', 'CFF_Importer') importer = ABCAdapter.build_adapter(group) args = { 'cff': cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } ### Launch Operation FlowService().fire_operation(importer, test_user, test_project.id, **args)
def _export_linked_datatypes(self, project, zip_file): files_helper = FilesHelper() linked_paths = self._get_linked_datatypes_storage_path(project) if not linked_paths: # do not export an empty operation return # Make a import operation which will contain links to other projects alg_group = dao.find_group(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS) algo = dao.get_algorithm_by_group(alg_group.id) op = model.Operation(None, project.id, algo.id, '') op.project = project op.algorithm = algo op.id = 'links-to-external-projects' op.start_now() op.mark_complete(model.STATUS_FINISHED) # write operation.xml to disk files_helper.write_operation_metadata(op) op_folder = files_helper.get_operation_folder(op.project.name, op.id) operation_xml = files_helper.get_operation_meta_file_path(op.project.name, op.id) op_folder_name = os.path.basename(op_folder) # add operation.xml zip_file.write(operation_xml, op_folder_name + '/' + os.path.basename(operation_xml)) # add linked datatypes to archive in the import operation for pth in linked_paths: zip_pth = op_folder_name + '/' + os.path.basename(pth) zip_file.write(pth, zip_pth) # remove these files, since we only want them in export archive files_helper.remove_folder(op_folder)
def test_delete_dt_free_HDD_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ module = "tvb.tests.framework.adapters.testadapter3" class_name = "TestAdapterHDDRequired" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() # Now free some space and relaunch ProjectService().remove_datatype(self.test_project.id, datatype.gid) self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def _import(self, import_file_path=None): """ This method is used for importing data in NIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter') importer = ABCAdapter.build_adapter(group) importer.meta_data = { DataTypeMetaData.KEY_SUBJECT: "", DataTypeMetaData.KEY_STATE: "RAW" } args = {'data_file': import_file_path} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) time_series = TimeSeries() data_types = FlowService().get_available_datatypes( self.test_project.id, time_series.module + "." + time_series.type) self.assertEqual(1, len(data_types), "Project should contain only one data type.") time_series = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(time_series is not None, "TimeSeries should not be none") return time_series
def _importSurface(self, import_file_path=None): """ This method is used for importing data in GIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter') importer = ABCAdapter.build_adapter(group) args = { 'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "" } ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) surface = CorticalSurface() data_types = FlowService().get_available_datatypes( self.test_project.id, surface.module + "." + surface.type)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "TimeSeries should not be none") return surface
def remove_visualizer_references(): """ As we removed an algorithm, remove left-overs. """ LOGGER.info("Starting to remove references towards old viewer ....") pearson_group = dao.find_group( 'tvb.adapters.visualizers.cross_correlation', 'PearsonCorrelationCoefficientVisualizer') pearson_algorithm = dao.get_algorithm_by_group(pearson_group.id) pearson_operations = dao.get_generic_entity(model.Operation, pearson_algorithm.id, "fk_from_algo") for op in pearson_operations: dao.remove_entity(model.Operation, op.id) pearson_workflows = dao.get_generic_entity(model.WorkflowStepView, pearson_algorithm.id, "fk_algorithm") for ws in pearson_workflows: dao.remove_entity(model.WorkflowStepView, ws.id) LOGGER.info("References removed.")
def test_adapter_memory(self): """ Test that a method not implemented exception is raised in case the get_required_memory_size method is not implemented. """ algo_group = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") adapter = FlowService().build_adapter_instance(algo_group) self.assertEqual(42, adapter.get_required_memory_size())
def get_algorithm_by_module_and_class(module, classname): """ Get the db entry from the algorithm table for the given module and class. """ group = dao.find_group(module, classname) algo = dao.get_algorithm_by_group(group.id) return algo, group
def _run_cff_importer(self, cff_path): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.cff_importer', 'CFF_Importer') importer = ABCAdapter.build_adapter(group) args = {'cff': cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT} ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _import_connectivity(self): group = dao.find_group('tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') importer = ABCAdapter.build_adapter(group) ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, uploaded=self.connectivity_path, Data_Subject='QL') self.connectivity = TestFactory.get_entity(self.test_project, Connectivity())
def import_surface_obj(user, project, obj_path, surface_type): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter') importer = ABCAdapter.build_adapter(group) args = {'data_file': obj_path, 'surface_type': surface_type} ### Launch Operation FlowService().fire_operation(importer, user, project.id, **args)
def test_fire_operation(self): """ Test preparation of an adapter and launch mechanism. """ algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS) adapter = self.flow_service.build_adapter_instance(algo_group) data = {"test": 5} result = self.flow_service.fire_operation(adapter, self.test_user, self.test_project.id, **data) self.assertTrue(result.endswith("has finished."), "Operation fail")
def import_surface_zip(user, project, zip_path, surface_type, zero_based): ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter') importer = ABCAdapter.build_adapter(group) args = {'uploaded': zip_path, 'surface_type': surface_type, 'zero_based_triangles': zero_based} ### Launch Operation FlowService().fire_operation(importer, user, project.id, **args)
def upgrade(_migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ db_group = dao.find_group('tvb.adapters.uploaders.csv_connectivity_importer', 'ZIPConnectivityImporter') if db_group is not None: db_group.module = 'tvb.adapters.uploaders.csv_connectivity_importer' db_group.classname = 'CSVConnectivityImporter' dao.store_entity(db_group)
def downgrade(_migrate_engine): """Operations to reverse the above upgrade go here.""" simulator_db_group = dao.find_group('tvb.adapters.simulator.simulator_adapter', 'SimulatorAdapter') if simulator_db_group is not None: simulator_db_group.module = 'tvb.adapters.simulator.simulatorAdapter' simulator_db_group.classname = 'SimulatorAdapter' dao.store_entity(simulator_db_group)
def upgrade(_migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ simulator_db_group = dao.find_group('tvb.adapters.simulator.simulatorAdapter', 'SimulatorAdapter') if simulator_db_group is not None: simulator_db_group.module = 'tvb.adapters.simulator.simulator_adapter' simulator_db_group.classname = 'SimulatorAdapter' dao.store_entity(simulator_db_group)
def downgrade(_migrate_engine): """Operations to reverse the above upgrade go here.""" db_group = dao.find_group('tvb.adapters.uploaders.csv_connectivity_importer', 'CSVConnectivityImporter') if db_group is not None: db_group.module = 'tvb.adapters.uploaders.csv_connectivity_importer' db_group.classname = 'ZIPConnectivityImporter' dao.store_entity(db_group)
def import_test_connectivity96(test_user, test_project, subject=DataTypeMetaData.DEFAULT_SUBJECT): """ Import a connectivity with 96 regions from tvb_data. """ group = dao.find_group('tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') importer = ABCAdapter.build_adapter(group) data_dir = path.abspath(path.dirname(tvb_data.__file__)) zip_path = path.join(data_dir, 'connectivity', 'connectivity_96.zip') ### Launch Operation FlowService().fire_operation(importer, test_user, test_project.id, uploaded=zip_path, Data_Subject=subject)
def _import(self, import_file_path=None): """ This method is used for importing data in TVB format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance group = dao.find_group('tvb.adapters.uploaders.tvb_importer', 'TVBImporter') importer = ABCAdapter.build_adapter(group) args = {'data_file': import_file_path} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _launch_test_algo_on_cluster(self, **data): module = "tvb.tests.framework.adapters.testadapter1" class_name = "TestAdapter1" group = dao.find_group(module, class_name) adapter = FlowService().build_adapter_instance(group) algo_group = adapter.algorithm_group algo_category = dao.get_category_by_id(algo_group.fk_category) algo = dao.get_algorithm_by_group(algo_group.id) operations, _ = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id, algo, algo_category, {}, ABCAdapter.LAUNCH_METHOD, **data) self.operation_service._send_to_cluster(operations, adapter) return operations