def _run_cff_importer(self, cff_path): ### Retrieve Adapter instance importer = TestFactory.create_adapter("tvb.adapters.uploaders.cff_importer", "CFF_Importer") args = {"cff": cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT} ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def test_delete_dt_free_HDD_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() # Now free some space and relaunch ProjectService().remove_datatype(self.test_project.id, datatype.gid) self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def _import(self, import_file_path, surface_gid, connectivity_gid): """ This method is used for importing region mappings :param import_file_path: absolute path of the file to be imported """ # Retrieve Adapter instance test_subject = "test" importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.region_mapping_importer', 'RegionMapping_Importer') args = { 'mapping_file': import_file_path, 'surface': surface_gid, 'connectivity': connectivity_gid, DataTypeMetaData.KEY_SUBJECT: test_subject } # Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) # During setup we import a CFF which creates an additional RegionMapping # So, here we have to find our mapping (just imported) data_filter = FilterChain(fields=[FilterChain.datatype + ".subject"], operations=["=="], values=[test_subject]) region_mapping = self._get_entity(RegionMapping, data_filter) return region_mapping
def test_adapter_memory(self): """ Test that a method not implemented exception is raised in case the get_required_memory_size method is not implemented. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") self.assertEqual(42, adapter.get_required_memory_size())
def _import_csv_test_connectivity(self, reference_connectivity_gid, subject): ### First prepare input data: data_dir = path.abspath(path.dirname(tvb_data.__file__)) torronto_dir = path.join(data_dir, 'dti_pipeline', 'Output_Toronto') weights = path.join(torronto_dir, 'output_ConnectionCapacityMatrix.csv') tracts = path.join(torronto_dir, 'output_ConnectionDistanceMatrix.csv') weights_tmp = weights + '.tmp' tracts_tmp = tracts + '.tmp' self.helper.copy_file(weights, weights_tmp) self.helper.copy_file(tracts, tracts_tmp) ### Find importer and Launch Operation importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.csv_connectivity_importer', 'CSVConnectivityImporter') FlowService().fire_operation(importer, self.test_user, self.test_project.id, weights=weights_tmp, tracts=tracts_tmp, Data_Subject=subject, input_data=reference_connectivity_gid)
def _importSurface(self, import_file_path=None): """ This method is used for importing data in GIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter') args = { 'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "" } ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) surface = CorticalSurface() data_types = FlowService().get_available_datatypes( self.test_project.id, surface.module + "." + surface.type)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "TimeSeries should not be none") return surface
def test_launch_two_ops_HDD_full_space(self): """ Launch two operations and give available space for user so that the first should finish, but after the update to the user hdd size the second should not. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = ( 1 + float(adapter.get_required_disk_size(**data))) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \ float(adapter.get_required_disk_size(**data) - 1) self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def test_import_bold(self): ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.mat_timeseries_importer', 'MatTimeSeriesImporter') args = dict( data_file=self.bold_path, dataset_name='QL_20120824_DK_BOLD_timecourse', structure_path='', transpose=False, slice=None, sampling_rate=1000, start_time=0, tstype='region', tstype_parameters_option_region_connectivity=self.connectivity.gid, Data_Subject="QL") ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) tsr = TestFactory.get_entity(self.test_project, TimeSeriesRegion()) self.assertEqual((661, 1, 68, 1), tsr.read_data_shape())
def test_launch_two_ops_HDD_with_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = 2 * float( adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float( adapter.get_required_disk_size(**data)) self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2(2)
def test_launch_operation_HDD_full_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") started_operation = model.Operation( self.test_user.id, self.test_project.id, adapter.stored_adapter.id, "", status=model.STATUS_STARTED, estimated_disk_size=space_taken_by_started) dao.store_entity(started_operation) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data) + space_taken_by_started - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_no_dt2()
def setUp(self): """ Reset the database before each test. """ self.test_user = TestFactory.create_user("UserPM") self.test_project = TestFactory.create_project(self.test_user) zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'eeg_brainstorm_65.txt') TestFactory.import_sensors(self.test_user, self.test_project, zip_path, Sensors_Importer.EEG_SENSORS) zip_path = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, CORTICAL, True) self.surface = TestFactory.get_entity(self.test_project, CorticalSurface()) self.assertTrue(self.surface is not None) self.sensors = TestFactory.get_entity(self.test_project, SensorsEEG()) self.assertTrue(self.sensors is not None) self.importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixSurfaceEEGImporter')
def _import(self, import_file_path=None, expected_result_class=StructuralMRI, connectivity=None): """ This method is used for importing data in NIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter') args = { 'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "bla bla", 'apply_corrections': True, 'connectivity': connectivity } ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dts, count = dao.get_values_of_datatype(self.test_project.id, expected_result_class, None) self.assertEqual(1, count, "Project should contain only one data type.") result = ABCAdapter.load_entity_by_gid(dts[0][2]) self.assertTrue(result is not None, "Result should not be none") return result
def setUp(self): """ Sets up the environment for running the tests; creates a test user, a test project, a connectivity and a list of BCT adapters; imports a CFF data-set """ self.test_user = TestFactory.create_user("BCT_User") self.test_project = TestFactory.create_project(self.test_user, "BCT-Project") ### Make sure Connectivity is in DB TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project) self.connectivity = dao.get_generic_entity(Connectivity, 'John Doe', 'subject')[0] # make weights matrix symmetric, or else some BCT algorithms will run infinitely: w = self.connectivity.weights self.connectivity.weights = w + w.T - numpy.diag(w.diagonal()) self.algo_groups = dao.get_generic_entity(model.AlgorithmGroup, 'MatlabAdapter', 'classname') self.assertTrue(self.algo_groups is not None) self.assertEquals(6, len(self.algo_groups)) self.bct_adapters = [] for group in self.algo_groups: self.bct_adapters.append( TestFactory.create_adapter(group, self.test_project))
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def test_initiate_operation(self): """ Test the actual operation flow by executing a test adapter. """ module = "tvb.tests.framework.adapters.testadapter1" class_name = "TestAdapter1" adapter = TestFactory.create_adapter(module, class_name) output = adapter.get_output() output_type = output[0].__name__ data = {"test1_val1": 5, "test1_val2": 5} tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") res = self.operation_service.initiate_operation( self.test_user, self.test_project.id, adapter, tmp_folder, **data) self.assertTrue( res.index("has finished.") > 10, "Operation didn't finish") group = dao.get_algorithm_by_module(module, class_name) self.assertEqual(group.module, 'tvb.tests.framework.adapters.testadapter1', "Wrong data stored.") self.assertEqual(group.classname, 'TestAdapter1', "Wrong data stored.") dts, count = dao.get_values_of_datatype(self.test_project.id, Datatype1) self.assertEqual(count, 1) self.assertEqual(len(dts), 1) datatype = dao.get_datatype_by_id(dts[0][0]) self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.") self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_build_adapter_instance(self): """ Test standard flow for building an adapter instance. """ adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS) self.assertTrue(isinstance(adapter, ABCSynchronous), "Something went wrong with valid data!")
def _launch_test_algo_on_cluster(self, **data): adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter1", "TestAdapter1") algo = adapter.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations, _ = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) return operations
def test_fire_operation(self): """ Test preparation of an adapter and launch mechanism. """ adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS) data = {"test": 5} result = self.flow_service.fire_operation(adapter, self.test_user, self.test_project.id, **data) self.assertTrue(result.endswith("has finished."), "Operation fail")
def _import_connectivity(self): importer = TestFactory.create_adapter('tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, uploaded=self.connectivity_path, Data_Subject='QL') self.connectivity = TestFactory.get_entity(self.test_project, Connectivity())
def import_test_connectivity96(test_user, test_project, subject=DataTypeMetaData.DEFAULT_SUBJECT): """ Import a connectivity with 96 regions from tvb_data. """ importer = TestFactory.create_adapter('tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') data_dir = path.abspath(path.dirname(tvb_data.__file__)) zip_path = path.join(data_dir, 'connectivity', 'connectivity_96.zip') ### Launch Operation FlowService().fire_operation(importer, test_user, test_project.id, uploaded=zip_path, Data_Subject=subject)
def test_launch_operation_HDD_full_space(self): """ Test the actual operation flow by executing a test adapter. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_no_dt2()
def _import(self, import_file_path=None): """ This method is used for importing data in TVB format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.tvb_importer', 'TVBImporter') args = {'data_file': import_file_path} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _run_cff_importer(self, cff_path): ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.cff_importer', 'CFF_Importer') args = { 'cff': cff_path, DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT } ### Launch Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def _import(self, import_file_name): ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.connectivity_measure_importer', 'ConnectivityMeasureImporter') path = os.path.join(os.path.dirname(test_data.__file__), import_file_name) args = {'data_file': path, 'connectivity': self.connectivity.gid, DataTypeMetaData.KEY_SUBJECT: "John"} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def test_adapter_huge_memory_requirement(self): """ Test that an MemoryException is raised in case adapter cant launch due to lack of memory. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHugeMemoryRequired") data = {"test": 5} operation = model.Operation(self.test_user.id, self.test_project.id, adapter.stored_adapter.id, json.dumps(data), json.dumps({}), status=model.STATUS_STARTED) operation = dao.store_entity(operation) self.assertRaises(NoMemoryAvailableException, OperationService().initiate_prelaunch, operation, adapter, {})
def _import(self, import_file_path=None): """ This method is used for importing data in TVB format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.tvb_importer', 'TVBImporter') args = {'data_file': import_file_path} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
def test_stop_operation(self): """ Test that an operation is successfully stopped. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter2", "TestAdapter2") data = {"test": 5} algo = adapter.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations, _ = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) self.operation_service.stop_operation(operations[0].id) operation = dao.get_operation_by_id(operations[0].id) self.assertEqual(operation.status, model.STATUS_CANCELED, "Operation should have been canceled!")
def setUp(self): """ Reset the database before each test. """ self.import_service = ImportService() self.flow_service = FlowService() self.project_service = ProjectService() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, name="GeneratedProject", description="test_desc") self.operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) self.adapter_instance = TestFactory.create_adapter(test_project=self.test_project) TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project) self.zip_path = None
def test_launch_operation_HDD_with_space_started_ops(self): """ Test the actual operation flow by executing a test adapter. """ space_taken_by_started = 100 adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") started_operation = model.Operation(self.test_user.id, self.test_project.id, adapter.stored_adapter.id, "", status=model.STATUS_STARTED, estimated_disk_size=space_taken_by_started) dao.store_entity(started_operation) data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) + space_taken_by_started) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def _importSurface(self, import_file_path=None): ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter') args = {'uploaded': import_file_path, 'surface_type': OUTER_SKULL, 'zero_based_triangles': True, DataTypeMetaData.KEY_SUBJECT: "John"} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes(self.test_project.id, SkullSkin)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "Surface should not be None") return surface
def test_import_bold(self): ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.mat_timeseries_importer', 'MatTimeSeriesImporter') args = dict(data_file=self.bold_path, dataset_name='QL_20120824_DK_BOLD_timecourse', structure_path='', transpose=False, slice=None, sampling_rate=1000, start_time=0, tstype='region', tstype_parameters_option_region_connectivity=self.connectivity.gid, Data_Subject="QL") ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) tsr = TestFactory.get_entity(self.test_project, TimeSeriesRegion()) self.assertEqual((661, 1, 68, 1), tsr.read_data_shape())
def test_stop_operation(self): """ Test that an operation is successfully stopped. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter2", "TestAdapter2") data = {"test": 5} algo = adapter.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations, _ = self.operation_service.prepare_operations( self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) self.operation_service.stop_operation(operations[0].id) operation = dao.get_operation_by_id(operations[0].id) self.assertEqual(operation.status, model.STATUS_CANCELED, "Operation should have been canceled!")
def _importSurface(self, import_file_path=None): ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter') args = {'data_file': import_file_path, "surface_type": FACE, DataTypeMetaData.KEY_SUBJECT: "John"} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes(self.test_project.id, FaceSurface)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "Surface should not be None") return surface
def test_launch_operation_HDD_full_space(self): """ Test the actual operation flow by executing a test adapter. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float( adapter.get_required_disk_size(**data) - 1) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_no_dt2()
def test_stop_operation_finished(self): """ Test that an operation that is already finished is not changed by the stop operation. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter1", "TestAdapter1") data = {"test1_val1": 5, 'test1_val2': 5} algo = adapter.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations, _ = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) operation = dao.get_operation_by_id(operations[0].id) operation.status = model.STATUS_FINISHED dao.store_entity(operation) self.operation_service.stop_operation(operations[0].id) operation = dao.get_operation_by_id(operations[0].id) self.assertEqual(operation.status, model.STATUS_FINISHED, "Operation shouldn't have been canceled!")
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") adapter_instance = TestFactory.create_adapter( 'tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) self.assertTrue( len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def test_delete_dt_free_HDD_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() # Now free some space and relaunch ProjectService().remove_datatype(self.test_project.id, datatype.gid) self._assert_no_dt2() self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def test_launch_two_ops_HDD_with_space(self): """ Launch two operations and give enough available space for user so that both should finish. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = 2 * float(adapter.get_required_disk_size(**data)) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float(adapter.get_required_disk_size(**data)) self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2(2)
def setUp(self): """ Prepare the database before each test. """ self.import_service = ImportService() self.flow_service = FlowService() self.project_service = ProjectService() self.test_user = TestFactory.create_user() self.delete_project_folders() result = self.count_all_entities(DataType) self.assertEqual(0, result, "There should be no data type in DB") result = self.count_all_entities(Project) self.assertEqual(0, result) self.test_project = TestFactory.import_default_project(self.test_user) self.operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) self.adapter_instance = TestFactory.create_adapter(test_project=self.test_project)
def import_test_connectivity96(test_user, test_project, subject=DataTypeMetaData.DEFAULT_SUBJECT): """ Import a connectivity with 96 regions from tvb_data. """ importer = TestFactory.create_adapter( 'tvb.adapters.uploaders.zip_connectivity_importer', 'ZIPConnectivityImporter') data_dir = path.abspath(path.dirname(tvb_data.__file__)) zip_path = path.join(data_dir, 'connectivity', 'connectivity_96.zip') ### Launch Operation FlowService().fire_operation(importer, test_user, test_project.id, uploaded=zip_path, Data_Subject=subject)
def setUp(self): """ Reset the database before each test. """ self.import_service = ImportService() self.flow_service = FlowService() self.project_service = ProjectService() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, name="GeneratedProject", description="test_desc") self.operation = TestFactory.create_operation( test_user=self.test_user, test_project=self.test_project) self.adapter_instance = TestFactory.create_adapter() TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project) self.zip_path = None
def _import_csv_test_connectivity(self, reference_connectivity_gid, subject): ### First prepare input data: data_dir = path.abspath(path.dirname(tvb_data.__file__)) torronto_dir = path.join(data_dir, 'dti_pipeline', 'Output_Toronto') weights = path.join(torronto_dir, 'output_ConnectionCapacityMatrix.csv') tracts = path.join(torronto_dir, 'output_ConnectionDistanceMatrix.csv') weights_tmp = weights + '.tmp' tracts_tmp = tracts + '.tmp' self.helper.copy_file(weights, weights_tmp) self.helper.copy_file(tracts, tracts_tmp) ### Find importer and Launch Operation importer = TestFactory.create_adapter('tvb.adapters.uploaders.csv_connectivity_importer', 'CSVConnectivityImporter') FlowService().fire_operation(importer, self.test_user, self.test_project.id, weights=weights_tmp, tracts=tracts_tmp, Data_Subject=subject, input_data=reference_connectivity_gid)
def test_import(self): count_before = self.count_all_entities(Connectivity) self.assertEqual(0, count_before) ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.networkx_importer', 'NetworkxConnectivityImporter') args = {'data_file': self.upload_file, DataTypeMetaData.KEY_SUBJECT: "John"} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) count_after = self.count_all_entities(Connectivity) self.assertEqual(1, count_after) conn = self.get_all_entities(Connectivity)[0] self.assertEqual(83, conn.number_of_regions)
def _import(self, import_file_path=None, expected_result_class=StructuralMRI, connectivity=None): """ This method is used for importing data in NIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter') args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "bla bla", 'apply_corrections': True, 'connectivity': connectivity} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) dts, count = dao.get_values_of_datatype(self.test_project.id, expected_result_class, None) self.assertEqual(1, count, "Project should contain only one data type.") result = ABCAdapter.load_entity_by_gid(dts[0][2]) self.assertTrue(result is not None, "Result should not be none") return result
def test_stop_operation_finished(self): """ Test that an operation that is already finished is not changed by the stop operation. """ adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter1", "TestAdapter1") data = {"test1_val1": 5, 'test1_val2': 5} algo = adapter.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) operations, _ = self.operation_service.prepare_operations( self.test_user.id, self.test_project.id, algo, algo_category, {}, **data) self.operation_service._send_to_cluster(operations, adapter) operation = dao.get_operation_by_id(operations[0].id) operation.status = model.STATUS_FINISHED dao.store_entity(operation) self.operation_service.stop_operation(operations[0].id) operation = dao.get_operation_by_id(operations[0].id) self.assertEqual(operation.status, model.STATUS_FINISHED, "Operation shouldn't have been canceled!")
def setUp(self): """ Reset the database before each test. """ self.test_user = TestFactory.create_user("UserPM") self.test_project = TestFactory.create_project(self.test_user) zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'eeg_brainstorm_65.txt') TestFactory.import_sensors(self.test_user, self.test_project, zip_path, Sensors_Importer.EEG_SENSORS) zip_path = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, CORTICAL, True) self.surface = TestFactory.get_entity(self.test_project, CorticalSurface()) self.assertTrue(self.surface is not None) self.sensors = TestFactory.get_entity(self.test_project, SensorsEEG()) self.assertTrue(self.sensors is not None) self.importer = TestFactory.create_adapter('tvb.adapters.uploaders.projection_matrix_importer', 'ProjectionMatrixSurfaceEEGImporter')
def test_launch_two_ops_HDD_full_space(self): """ Launch two operations and give available space for user so that the first should finish, but after the update to the user hdd size the second should not. """ adapter = TestFactory.create_adapter("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired") data = {"test": 100} TvbProfile.current.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data))) tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP") self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data) datatype = self._assert_stored_dt2() #Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB) #plus what is estimated to be required from the next one (transform from B to MB) TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \ float(adapter.get_required_disk_size(**data) - 1) self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user, self.test_project.id, adapter, tmp_folder, **data) self._assert_stored_dt2()
def _import(self, import_file_path, sensors_type, expected_data): """ This method is used for importing sensors :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer') args = {'sensors_file': import_file_path, 'sensors_type': sensors_type} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) data_types = FlowService().get_available_datatypes(self.test_project.id, expected_data.module + "." + expected_data.type)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type = Sensors.") time_series = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(time_series is not None, "Sensors instance should not be none") return time_series
def setUp(self): """ Sets up the environment for running the tests; creates a test user, a test project, a connectivity and a list of BCT adapters; imports a CFF data-set """ self.test_user = TestFactory.create_user("BCT_User") self.test_project = TestFactory.create_project(self.test_user, "BCT-Project") ### Make sure Connectivity is in DB TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project) self.connectivity = dao.get_generic_entity(Connectivity, 'John Doe', 'subject')[0] # make weights matrix symmetric, or else some BCT algorithms will run infinitely: w = self.connectivity.weights self.connectivity.weights = w + w.T - numpy.diag(w.diagonal()) self.algo_groups = dao.get_generic_entity(model.AlgorithmGroup, 'MatlabAdapter', 'classname') self.assertTrue(self.algo_groups is not None) self.assertEquals(6, len(self.algo_groups)) self.bct_adapters = [] for group in self.algo_groups: self.bct_adapters.append(TestFactory.create_adapter(group, self.test_project))
def _import(self, import_file_path, surface_gid, connectivity_gid): """ This method is used for importing region mappings :param import_file_path: absolute path of the file to be imported """ # Retrieve Adapter instance test_subject = "test" importer = TestFactory.create_adapter('tvb.adapters.uploaders.region_mapping_importer', 'RegionMapping_Importer') args = {'mapping_file': import_file_path, 'surface': surface_gid, 'connectivity': connectivity_gid, DataTypeMetaData.KEY_SUBJECT: test_subject} # Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) # During setup we import a CFF which creates an additional RegionMapping # So, here we have to find our mapping (just imported) data_filter = FilterChain(fields=[FilterChain.datatype + ".subject"], operations=["=="], values=[test_subject]) region_mapping = self._get_entity(RegionMapping, data_filter) return region_mapping
def _importSurface(self, import_file_path=None): """ This method is used for importing data in GIFIT format :param import_file_path: absolute path of the file to be imported """ ### Retrieve Adapter instance importer = TestFactory.create_adapter('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter') args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: ""} ### Launch import Operation FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args) surface = CorticalSurface() data_types = FlowService().get_available_datatypes(self.test_project.id, surface.module + "." + surface.type)[0] self.assertEqual(1, len(data_types), "Project should contain only one data type.") surface = ABCAdapter.load_entity_by_gid(data_types[0][2]) self.assertTrue(surface is not None, "TimeSeries should not be none") return surface