def launch(self, weights, tracts, input_data):
"""
Execute import operations: process the weights and tracts csv files, then use
the reference connectivity passed as input_data for the rest of the attributes.
"""
dti_service = dtipipelineservice.DTIPipelineService()
dti_service._process_csv_file(weights, dti_service.WEIGHTS_FILE)
dti_service._process_csv_file(tracts, dti_service.TRACT_FILE)
weights_matrix = read_list_data(os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE))
tract_matrix = read_list_data(os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE))
FilesHelper.remove_files([os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE),
os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)])
if weights_matrix.shape[0] != input_data.orientations.shape[0]:
raise LaunchException("The csv files define %s nodes but the connectivity you selected as reference has only %s nodes."%(
weights_matrix.shape[0], input_data.orientations.shape[0]))
result = Connectivity()
result.storage_path = self.storage_path
result.nose_correction = input_data.nose_correction
result.centres = input_data.centres
result.region_labels = input_data.region_labels
result.weights = weights_matrix
result.tract_lengths = tract_matrix
result.orientations = input_data.orientations
result.areas = input_data.areas
result.cortical = input_data.cortical
result.hemispheres = input_data.hemispheres
return result
class FilesHelperTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.entities.file.fileshelper module.
"""
PROJECT_NAME = "test_proj"
def setUp(self):
"""
Set up the context needed by the tests.
"""
# self.clean_database()
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def tearDown(self):
""" Remove generated project during tests. """
self.delete_project_folders()
def test_check_created(self):
""" Test standard flows for check created. """
self.files_helper.check_created()
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.files_helper.check_created(os.path.join(root_storage, "test"))
self.assertTrue(os.path.exists(root_storage), "Storage not created!")
self.assertTrue(os.path.exists(os.path.join(root_storage, "test")), "Test directory not created!")
def test_get_project_folder(self):
"""
Test the get_project_folder method which should create a folder in case
it doesn't already exist.
"""
project_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
folder_path = self.files_helper.get_project_folder(self.test_project, "43")
self.assertTrue(os.path.exists(project_path), "Folder doesn't exist")
self.assertTrue(os.path.exists(folder_path), "Folder doesn't exist")
def test_rename_project_structure(self):
""" Try to rename the folder structure of a project. Standard flow. """
self.files_helper.get_project_folder(self.test_project)
path, name = self.files_helper.rename_project_structure(self.test_project.name, "new_name")
self.assertNotEqual(path, name, "Rename didn't take effect.")
def test_rename_structure_same_name(self):
""" Try to rename the folder structure of a project. Same name. """
self.files_helper.get_project_folder(self.test_project)
self.assertRaises(FileStructureException, self.files_helper.rename_project_structure,
self.test_project.name, self.PROJECT_NAME)
def test_remove_project_structure(self):
""" Check that remove project structure deletes the corresponding folder. Standard flow. """
full_path = self.files_helper.get_project_folder(self.test_project)
self.assertTrue(os.path.exists(full_path), "Folder was not created.")
self.files_helper.remove_project_structure(self.test_project.name)
self.assertFalse(os.path.exists(full_path), "Project folder not deleted.")
def test_write_project_metadata(self):
""" Write XML for test-project. """
self.files_helper.write_project_metadata(self.test_project)
expected_file = self.files_helper.get_project_meta_file_path(self.PROJECT_NAME)
self.assertTrue(os.path.exists(expected_file))
project_meta = XMLReader(expected_file).read_metadata()
loaded_project = model.Project(None, None)
loaded_project.from_dict(project_meta, self.test_user.id)
self.assertEqual(self.test_project.name, loaded_project.name)
self.assertEqual(self.test_project.description, loaded_project.description)
self.assertEqual(self.test_project.gid, loaded_project.gid)
expected_dict = self.test_project.to_dict()[1]
del expected_dict['last_updated']
found_dict = loaded_project.to_dict()[1]
del found_dict['last_updated']
self.assertEqual(expected_dict, found_dict)
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
self.assertFalse(os.path.exists(expected_file))
self.files_helper.write_operation_metadata(operation)
self.assertTrue(os.path.exists(expected_file))
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.iteritems():
self.assertEqual(str(value), str(found_dict[key]))
# Now validate that operation metaData can be also updated
self.assertNotEqual("new_group_name", found_dict['user_group'])
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
self.assertEqual("new_group_name", found_dict['user_group'])
def test_remove_dt_happy_flow(self):
"""
Happy flow for removing a file related to a DataType.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.remove_datatype(datatype)
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not deleted!")
def test_remove_dt_non_existent(self):
"""
Try to call remove on a dataType with no H5 file.
Should throw an exception.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
self.assertFalse(os.path.exists(datatype.get_storage_file_path()))
self.assertRaises(FileStructureException, self.files_helper.remove_datatype, datatype)
def test_move_datatype(self):
"""
Make sure associated H5 file is moved to a correct new location.
"""
folder_path = self.files_helper.get_project_folder(self.test_project, "42")
datatype = MappedType()
datatype.storage_path = folder_path
open(datatype.get_storage_file_path(), 'w')
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
self.files_helper.move_datatype(datatype, self.PROJECT_NAME + '11', "43")
self.assertFalse(os.path.exists(datatype.get_storage_file_path()), "Test file was not moved!")
datatype.storage_path = self.files_helper.get_project_folder(self.PROJECT_NAME + '11', "43")
self.assertTrue(os.path.exists(datatype.get_storage_file_path()), "Test file was not created!")
def test_find_relative_path(self):
rel_path = self.files_helper.find_relative_path("/root/up/to/here/test/it/now", "/root/up/to/here")
self.assertEqual(rel_path, os.sep.join(["test", "it", "now"]), "Did not extract relative path as expected.")
def test_remove_files_valid(self):
"""
Pass a valid list of files and check they are all removed.
"""
file_list = ["test1", "test2", "test3"]
for file_n in file_list:
fp = open(file_n, 'w')
fp.write('test')
fp.close()
for file_n in file_list:
self.assertTrue(os.path.isfile(file_n))
self.files_helper.remove_files(file_list)
for file_n in file_list:
self.assertFalse(os.path.isfile(file_n))
def test_remove_folder(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
os.mkdir(folder_name)
self.assertTrue(os.path.isdir(folder_name), "Folder should be created.")
self.files_helper.remove_folder(folder_name)
self.assertFalse(os.path.isdir(folder_name), "Folder should be deleted.")
def test_remove_folder_non_existing_ignore_exc(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.files_helper.remove_folder(folder_name, ignore_errors=True)
def test_remove_folder_non_existing(self):
"""
Pass an open file pointer, but ignore exceptions.
"""
folder_name = "test_folder"
self.assertFalse(os.path.isdir(folder_name), "Folder should not exist before call.")
self.assertRaises(FileStructureException, self.files_helper.remove_folder, folder_name, False)
def launch(self, uploaded, surface_type, zero_based_triangles = False):
"""
Execute import operations: unpack ZIP and build Surface object as result.
"""
if uploaded is None:
raise LaunchException ("Please select ZIP file which contains data to import")
self.logger.debug("Start to import surface: '%s' from file: %s"%(surface_type, uploaded))
try:
files = FilesHelper().unpack_zip(uploaded, self.storage_path)
except IOError:
exception_str = "Did not find the specified ZIP at %s" % uploaded
raise LaunchException(exception_str)
vertices = []
normals = []
triangles = []
for file_name in files:
if file_name.lower().find(self.VERTICES_TOKEN) >= 0:
vertices.append(file_name)
continue
if file_name.lower().find(self.NORMALS_TOKEN) >= 0:
normals.append(file_name)
continue
if file_name.lower().find(self.TRIANGLES_TOKEN) >= 0:
triangles.append(file_name)
# Now detect and instantiate correct surface type
self.logger.debug("Create surface instance")
if surface_type == CORTICAL:
surface = CorticalSurface()
elif surface_type == INNER_SKULL:
surface = BrainSkull()
elif surface_type == OUTER_SKULL:
surface = SkullSkin()
elif surface_type == OUTER_SKIN:
surface = SkinAir()
elif surface_type == EEG_CAP:
surface = EEGCap()
elif surface_type == FACE:
surface = FaceSurface()
else:
exception_str = "Could not determine surface type (selected option %s)" % surface_type
raise LaunchException(exception_str)
surface.storage_path = self.storage_path
all_vertices, all_normals, all_triangles = self._process_files(vertices, normals, triangles)
FilesHelper.remove_files(files, True)
surface.zero_based_triangles = zero_based_triangles
surface.vertices = all_vertices
surface.vertex_normals = all_normals
if zero_based_triangles:
surface.triangles = all_triangles
else :
surface.triangles = all_triangles - 1
surface.triangle_normals = None
# Now check if the triangles of the surface are valid
triangles_min_vertex = numpy.amin(surface.triangles)
if triangles_min_vertex < 0:
if triangles_min_vertex == -1 and not zero_based_triangles:
raise RuntimeError("Triangles contain a negative vertex index. Maybe you have a ZERO based surface.")
else:
raise RuntimeError("Your triangles contain a negative vertex index: %d" %triangles_min_vertex)
no_of_vertices = len(surface.vertices)
triangles_max_vertex = numpy.amax(surface.triangles)
if triangles_max_vertex >= no_of_vertices:
if triangles_max_vertex == no_of_vertices and zero_based_triangles:
raise RuntimeError("Your triangles contain an invalid vertex index: %d. \
Maybe your surface is NOT ZERO Based."%triangles_max_vertex)
else:
raise RuntimeError("Your triangles contain an invalid vertex index: %d." %triangles_max_vertex)
self.logger.debug("Surface ready to be stored")
return surface
def launch(self, uploaded, rotate_x=0, rotate_y=0, rotate_z=0):
"""
Execute import operations: unpack ZIP and build Connectivity object as result.
"""
if uploaded is None:
raise LaunchException(
"Please select ZIP file which contains data to import")
files = FilesHelper().unpack_zip(uploaded, self.storage_path)
weights_matrix = None
centres = None
labels_vector = None
tract_matrix = None
orientation = None
areas = None
cortical_vector = None
hemisphere_vector = None
for file_name in files:
if file_name.lower().find(self.WEIGHT_TOKEN) >= 0:
weights_matrix = read_list_data(file_name)
continue
if file_name.lower().find(self.POSITION_TOKEN) >= 0:
centres = read_list_data(file_name,
skiprows=1,
usecols=[1, 2, 3])
labels_vector = read_list_data(file_name,
dtype=numpy.str,
skiprows=1,
usecols=[0])
continue
if file_name.lower().find(self.TRACT_TOKEN) >= 0:
tract_matrix = read_list_data(file_name)
continue
if file_name.lower().find(self.ORIENTATION_TOKEN) >= 0:
orientation = read_list_data(file_name)
continue
if file_name.lower().find(self.AREA_TOKEN) >= 0:
areas = read_list_data(file_name)
continue
if file_name.lower().find(self.CORTICAL_INFO) >= 0:
cortical_vector = read_list_data(file_name, dtype=numpy.bool)
continue
if file_name.lower().find(self.HEMISPHERE_INFO) >= 0:
hemisphere_vector = read_list_data(file_name, dtype=numpy.bool)
continue
### Clean remaining text-files.
FilesHelper.remove_files(files, True)
result = Connectivity()
result.storage_path = self.storage_path
result.nose_correction = [rotate_x, rotate_y, rotate_z]
### Fill positions
if centres is None:
raise Exception(
"Positions for Connectivity Regions are required! "
"We expect a file *position* inside the uploaded ZIP.")
expected_number_of_nodes = len(centres)
if expected_number_of_nodes < 2:
raise Exception("A connectivity with at least 2 nodes is expected")
result.centres = centres
if labels_vector is not None:
result.region_labels = labels_vector
### Fill and check weights
if weights_matrix is not None:
if numpy.any([x < 0 for x in weights_matrix.flatten()]):
raise Exception(
"Negative values are not accepted in weights matrix! "
"Please check your file, and use values >= 0")
if weights_matrix.shape != (expected_number_of_nodes,
expected_number_of_nodes):
raise Exception(
"Unexpected shape for weights matrix! "
"Should be %d x %d " %
(expected_number_of_nodes, expected_number_of_nodes))
result.weights = weights_matrix
### Fill and check tracts
if tract_matrix is not None:
if numpy.any([x < 0 for x in tract_matrix.flatten()]):
raise Exception(
"Negative values are not accepted in tracts matrix! "
"Please check your file, and use values >= 0")
if tract_matrix.shape != (expected_number_of_nodes,
expected_number_of_nodes):
raise Exception(
"Unexpected shape for tracts matrix! "
"Should be %d x %d " %
(expected_number_of_nodes, expected_number_of_nodes))
result.tract_lengths = tract_matrix
if orientation is not None:
if len(orientation) != expected_number_of_nodes:
raise Exception(
"Invalid size for vector orientation. "
"Expected the same as region-centers number %d" %
expected_number_of_nodes)
result.orientations = orientation
if areas is not None:
if len(areas) != expected_number_of_nodes:
raise Exception(
"Invalid size for vector areas. "
"Expected the same as region-centers number %d" %
expected_number_of_nodes)
result.areas = areas
if cortical_vector is not None:
if len(cortical_vector) != expected_number_of_nodes:
raise Exception(
"Invalid size for vector cortical. "
"Expected the same as region-centers number %d" %
expected_number_of_nodes)
result.cortical = cortical_vector
if hemisphere_vector is not None:
if len(hemisphere_vector) != expected_number_of_nodes:
raise Exception(
"Invalid size for vector hemispheres. "
"Expected the same as region-centers number %d" %
expected_number_of_nodes)
result.hemispheres = hemisphere_vector
return result
