def launch(self, weights, tracts, input_data): """ Execute import operations: process the weights and tracts csv files, then use the reference connectivity passed as input_data for the rest of the attributes. """ dti_service = dtipipelineservice.DTIPipelineService() dti_service._process_csv_file(weights, dti_service.WEIGHTS_FILE) dti_service._process_csv_file(tracts, dti_service.TRACT_FILE) weights_matrix = read_list_data(os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE)) tract_matrix = read_list_data(os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)) FilesHelper.remove_files([os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE), os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)]) if weights_matrix.shape[0] != input_data.orientations.shape[0]: raise LaunchException("The csv files define %s nodes but the connectivity you selected as reference has only %s nodes."%( weights_matrix.shape[0], input_data.orientations.shape[0])) result = Connectivity() result.storage_path = self.storage_path result.nose_correction = input_data.nose_correction result.centres = input_data.centres result.region_labels = input_data.region_labels result.weights = weights_matrix result.tract_lengths = tract_matrix result.orientations = input_data.orientations result.areas = input_data.areas result.cortical = input_data.cortical result.hemispheres = input_data.hemispheres return result
class FilesHelperTest(TransactionalTestCase): """ This class contains tests for the tvb.core.entities.file.fileshelper module. """ PROJECT_NAME = "test_proj" def setUp(self): """ Set up the context needed by the tests. """ # self.clean_database() self.files_helper = FilesHelper() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME) def tearDown(self): """ Remove generated project during tests. """ self.delete_project_folders() def test_check_created(self): """ Test standard flows for check created. """ self.files_helper.check_created() self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.files_helper.check_created(os.path.join(root_storage, "test")) self.assertTrue(os.path.exists(root_storage), "Storage not created!") self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!") def test_get_project_folder(self): """ Test the get_project_folder method which should create a folder in case it doesn't already exist. """ project_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") folder_path = self.files_helper.get_project_folder(self.test_project, "43") self.assertTrue(os.path.exists(project_path), "Folder doesn't exist") self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist") def test_rename_project_structure(self): """ Try to rename the folder structure of a project. Standard flow. """ self.files_helper.get_project_folder(self.test_project) path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name") self.assertNotEqual(path, name, "Rename didn't take effect.") def test_rename_structure_same_name(self): """ Try to rename the folder structure of a project. Same name. """ self.files_helper.get_project_folder(self.test_project) self.assertRaises(FileStructureException, self.files_helper.rename_project_structure, self.test_project.name, self.PROJECT_NAME) def test_remove_project_structure(self): """ Check that remove project structure deletes the corresponding folder. Standard flow. """ full_path = self.files_helper.get_project_folder(self.test_project) self.assertTrue(os.path.exists(full_path), "Folder was not created.") self.files_helper.remove_project_structure(self.test_project.name) self.assertFalse(os.path.exists(full_path), "Project folder not deleted.") def test_write_project_metadata(self): """ Write XML for test-project. """ self.files_helper.write_project_metadata(self.test_project) expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME) self.assertTrue(os.path.exists(expected_file)) project_meta = XMLReader(expected_file).read_metadata() loaded_project = model.Project(None, None) loaded_project.from_dict(project_meta, self.test_user.id) self.assertEqual(self.test_project.name, loaded_project.name) self.assertEqual(self.test_project.description, loaded_project.description) self.assertEqual(self.test_project.gid, loaded_project.gid) expected_dict = self.test_project.to_dict()[1] del expected_dict['last_updated'] found_dict = loaded_project.to_dict()[1] del found_dict['last_updated'] self.assertEqual(expected_dict, found_dict) def test_write_operation_metadata(self): """ Test that a correct XML is created for an operation. """ operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id) self.assertFalse(os.path.exists(expected_file)) self.files_helper.write_operation_metadata(operation) self.assertTrue(os.path.exists(expected_file)) operation_meta = XMLReader(expected_file).read_metadata() loaded_operation = model.Operation(None, None, None, None) loaded_operation.from_dict(operation_meta, dao) expected_dict = operation.to_dict()[1] found_dict = loaded_operation.to_dict()[1] for key, value in expected_dict.iteritems(): self.assertEqual(str(value), str(found_dict[key])) # Now validate that operation metaData can be also updated self.assertNotEqual("new_group_name", found_dict['user_group']) self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id) found_dict = XMLReader(expected_file).read_metadata() self.assertEqual("new_group_name", found_dict['user_group']) def test_remove_dt_happy_flow(self): """ Happy flow for removing a file related to a DataType. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.remove_datatype(datatype) self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!") def test_remove_dt_non_existent(self): """ Try to call remove on a dataType with no H5 file. Should throw an exception. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path self.assertFalse(os.path.exists(datatype.get_storage_file_path())) self.assertRaises(FileStructureException, self.files_helper.remove_datatype, datatype) def test_move_datatype(self): """ Make sure associated H5 file is moved to a correct new location. """ folder_path = self.files_helper.get_project_folder(self.test_project, "42") datatype = MappedType() datatype.storage_path = folder_path open(datatype.get_storage_file_path(), 'w') self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43") self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!") datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43") self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!") def test_find_relative_path(self): rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here") self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.") def test_remove_files_valid(self): """ Pass a valid list of files and check they are all removed. """ file_list = ["test1", "test2", "test3"] for file_n in file_list: fp = open(file_n, 'w') fp.write('test') fp.close() for file_n in file_list: self.assertTrue(os.path.isfile(file_n)) self.files_helper.remove_files(file_list) for file_n in file_list: self.assertFalse(os.path.isfile(file_n)) def test_remove_folder(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" os.mkdir(folder_name) self.assertTrue(os.path.isdir(folder_name), "Folder should be created.") self.files_helper.remove_folder(folder_name) self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.") def test_remove_folder_non_existing_ignore_exc(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.files_helper.remove_folder(folder_name, ignore_errors=True) def test_remove_folder_non_existing(self): """ Pass an open file pointer, but ignore exceptions. """ folder_name = "test_folder" self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.") self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
def launch(self, uploaded, surface_type, zero_based_triangles = False): """ Execute import operations: unpack ZIP and build Surface object as result. """ if uploaded is None: raise LaunchException ("Please select ZIP file which contains data to import") self.logger.debug("Start to import surface: '%s' from file: %s"%(surface_type, uploaded)) try: files = FilesHelper().unpack_zip(uploaded, self.storage_path) except IOError: exception_str = "Did not find the specified ZIP at %s" % uploaded raise LaunchException(exception_str) vertices = [] normals = [] triangles = [] for file_name in files: if file_name.lower().find(self.VERTICES_TOKEN) >= 0: vertices.append(file_name) continue if file_name.lower().find(self.NORMALS_TOKEN) >= 0: normals.append(file_name) continue if file_name.lower().find(self.TRIANGLES_TOKEN) >= 0: triangles.append(file_name) # Now detect and instantiate correct surface type self.logger.debug("Create surface instance") if surface_type == CORTICAL: surface = CorticalSurface() elif surface_type == INNER_SKULL: surface = BrainSkull() elif surface_type == OUTER_SKULL: surface = SkullSkin() elif surface_type == OUTER_SKIN: surface = SkinAir() elif surface_type == EEG_CAP: surface = EEGCap() elif surface_type == FACE: surface = FaceSurface() else: exception_str = "Could not determine surface type (selected option %s)" % surface_type raise LaunchException(exception_str) surface.storage_path = self.storage_path all_vertices, all_normals, all_triangles = self._process_files(vertices, normals, triangles) FilesHelper.remove_files(files, True) surface.zero_based_triangles = zero_based_triangles surface.vertices = all_vertices surface.vertex_normals = all_normals if zero_based_triangles: surface.triangles = all_triangles else : surface.triangles = all_triangles - 1 surface.triangle_normals = None # Now check if the triangles of the surface are valid triangles_min_vertex = numpy.amin(surface.triangles) if triangles_min_vertex < 0: if triangles_min_vertex == -1 and not zero_based_triangles: raise RuntimeError("Triangles contain a negative vertex index. Maybe you have a ZERO based surface.") else: raise RuntimeError("Your triangles contain a negative vertex index: %d" %triangles_min_vertex) no_of_vertices = len(surface.vertices) triangles_max_vertex = numpy.amax(surface.triangles) if triangles_max_vertex >= no_of_vertices: if triangles_max_vertex == no_of_vertices and zero_based_triangles: raise RuntimeError("Your triangles contain an invalid vertex index: %d. \ Maybe your surface is NOT ZERO Based."%triangles_max_vertex) else: raise RuntimeError("Your triangles contain an invalid vertex index: %d." %triangles_max_vertex) self.logger.debug("Surface ready to be stored") return surface
def launch(self, uploaded, rotate_x=0, rotate_y=0, rotate_z=0): """ Execute import operations: unpack ZIP and build Connectivity object as result. """ if uploaded is None: raise LaunchException( "Please select ZIP file which contains data to import") files = FilesHelper().unpack_zip(uploaded, self.storage_path) weights_matrix = None centres = None labels_vector = None tract_matrix = None orientation = None areas = None cortical_vector = None hemisphere_vector = None for file_name in files: if file_name.lower().find(self.WEIGHT_TOKEN) >= 0: weights_matrix = read_list_data(file_name) continue if file_name.lower().find(self.POSITION_TOKEN) >= 0: centres = read_list_data(file_name, skiprows=1, usecols=[1, 2, 3]) labels_vector = read_list_data(file_name, dtype=numpy.str, skiprows=1, usecols=[0]) continue if file_name.lower().find(self.TRACT_TOKEN) >= 0: tract_matrix = read_list_data(file_name) continue if file_name.lower().find(self.ORIENTATION_TOKEN) >= 0: orientation = read_list_data(file_name) continue if file_name.lower().find(self.AREA_TOKEN) >= 0: areas = read_list_data(file_name) continue if file_name.lower().find(self.CORTICAL_INFO) >= 0: cortical_vector = read_list_data(file_name, dtype=numpy.bool) continue if file_name.lower().find(self.HEMISPHERE_INFO) >= 0: hemisphere_vector = read_list_data(file_name, dtype=numpy.bool) continue ### Clean remaining text-files. FilesHelper.remove_files(files, True) result = Connectivity() result.storage_path = self.storage_path result.nose_correction = [rotate_x, rotate_y, rotate_z] ### Fill positions if centres is None: raise Exception( "Positions for Connectivity Regions are required! " "We expect a file *position* inside the uploaded ZIP.") expected_number_of_nodes = len(centres) if expected_number_of_nodes < 2: raise Exception("A connectivity with at least 2 nodes is expected") result.centres = centres if labels_vector is not None: result.region_labels = labels_vector ### Fill and check weights if weights_matrix is not None: if numpy.any([x < 0 for x in weights_matrix.flatten()]): raise Exception( "Negative values are not accepted in weights matrix! " "Please check your file, and use values >= 0") if weights_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception( "Unexpected shape for weights matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.weights = weights_matrix ### Fill and check tracts if tract_matrix is not None: if numpy.any([x < 0 for x in tract_matrix.flatten()]): raise Exception( "Negative values are not accepted in tracts matrix! " "Please check your file, and use values >= 0") if tract_matrix.shape != (expected_number_of_nodes, expected_number_of_nodes): raise Exception( "Unexpected shape for tracts matrix! " "Should be %d x %d " % (expected_number_of_nodes, expected_number_of_nodes)) result.tract_lengths = tract_matrix if orientation is not None: if len(orientation) != expected_number_of_nodes: raise Exception( "Invalid size for vector orientation. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.orientations = orientation if areas is not None: if len(areas) != expected_number_of_nodes: raise Exception( "Invalid size for vector areas. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.areas = areas if cortical_vector is not None: if len(cortical_vector) != expected_number_of_nodes: raise Exception( "Invalid size for vector cortical. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.cortical = cortical_vector if hemisphere_vector is not None: if len(hemisphere_vector) != expected_number_of_nodes: raise Exception( "Invalid size for vector hemispheres. " "Expected the same as region-centers number %d" % expected_number_of_nodes) result.hemispheres = hemisphere_vector return result