def _import_csv_test_connectivity(self, reference_connectivity_gid,
subject):
### First prepare input data:
data_dir = path.abspath(path.dirname(tvb_data.__file__))
toronto_dir = path.join(data_dir, 'dti_pipeline_toronto')
weights = path.join(toronto_dir, 'output_ConnectionCapacityMatrix.csv')
tracts = path.join(toronto_dir, 'output_ConnectionDistanceMatrix.csv')
weights_tmp = weights + '.tmp'
tracts_tmp = tracts + '.tmp'
self.helper.copy_file(weights, weights_tmp)
self.helper.copy_file(tracts, tracts_tmp)
### Find importer and Launch Operation
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.csv_connectivity_importer',
'CSVConnectivityImporter')
form = CSVConnectivityImporterForm()
form.fill_from_post({
'_weights': Part(weights_tmp, HeaderMap({}), ''),
'_tracts': Part(tracts_tmp, HeaderMap({}), ''),
'_weights_delimiter': 'comma',
'_tracts_delimiter': 'comma',
'_Data_Subject': subject,
'_input_data': reference_connectivity_gid
})
form.weights.data = weights_tmp
form.tracts.data = tracts_tmp
importer.submit_form(form)
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **form.get_dict())
def import_surface_gifti(user, project, path):
"""
This method is used for importing data in GIFIT format
:param import_file_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter')
form = GIFTISurfaceImporterForm()
form.fill_from_post({'_file_type': form.get_view_model().KEY_OPTION_READ_METADATA,
'_data_file': Part(path, HeaderMap({}), ''),
'_data_file_part2': Part('', HeaderMap({}), ''),
'_should_center': 'False',
'_Data_Subject': 'John Doe',
})
form.data_file.data = path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer, user, project.id, view_model=view_model)
surface = CorticalSurface
data_types = FlowService().get_available_datatypes(project.id,
surface.__module__ + "." + surface.__name__)[0]
assert 1, len(data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None == "TimeSeries should not be none"
return surface
def import_surface_zip(user,
project,
zip_path,
surface_type,
zero_based='True'):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_surface_importer',
'ZIPSurfaceImporter')
form = ZIPSurfaceImporterForm()
form.fill_from_post({
'_uploaded': Part(zip_path, HeaderMap({}), ''),
'_zero_based_triangles': zero_based,
'_should_center': 'True',
'_surface_type': surface_type,
'_Data_Subject': 'John Doe'
})
form.uploaded.data = zip_path
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer, user, project.id,
**form.get_form_values())
data_types = FlowService().get_available_datatypes(
project.id, SurfaceIndex)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
surface.user_tag_3 = ''
assert surface is not None, "Surface should not be None"
return surface
def _import(self, import_file_path, surface_gid, connectivity_gid):
"""
This method is used for importing region mappings
:param import_file_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
test_subject = "test"
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.region_mapping_importer',
'RegionMappingImporter')
form = RegionMappingImporterForm()
form.fill_from_post({
'_mapping_file':
Part(import_file_path, HeaderMap({}), ''),
'_surface':
surface_gid,
'_connectivity':
connectivity_gid,
'_Data_Subject':
'John Doe'
})
form.mapping_file.data = import_file_path
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **form.get_dict())
region_mapping = TestFactory.get_entity(self.test_project,
RegionMappingIndex)
return region_mapping
def import_zip_connectivity(user,
project,
zip_path=None,
subject=DataTypeMetaData.DEFAULT_SUBJECT):
if zip_path is None:
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_76.zip')
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_connectivity_importer',
'ZIPConnectivityImporter')
form = ZIPConnectivityImporterForm()
form.fill_from_post({
'uploaded': Part(zip_path, HeaderMap({}), ''),
'normalization': None,
'project_id': {1},
'Data_Subject': subject
})
form.uploaded.data = zip_path
view_model = form.get_view_model()()
view_model.data_subject = subject
form.fill_trait(view_model)
importer.submit_form(form)
# Launch Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
def _import(self, import_file_name):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.connectivity_measure_importer',
'ConnectivityMeasureImporter')
path = os.path.join(os.path.dirname(test_data.__file__),
import_file_name)
form = ConnectivityMeasureImporterForm()
form.fill_from_post({
'data_file': Part(path, HeaderMap({}), ''),
'dataset_name': 'M',
'connectivity': self.connectivity.gid,
'Data_Subject': 'John Doe'
})
form.data_file.data = path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer,
self.test_user,
self.test_project.id,
view_model=view_model)
def import_sensors(user, project, zip_path, sensors_type):
"""
This method is used for importing sensors
:param zip_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.sensors_importer', 'SensorsImporter')
form = SensorsImporterForm()
form.fill_from_post({
'sensors_file': Part(zip_path, HeaderMap({}), ''),
'sensors_type': sensors_type,
'Data_Subject': 'John Doe'
})
form.sensors_file.data = zip_path
form.sensors_type.data = sensors_type
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, SensorsIndex)[0]
assert 1 == len(
data_types), "Project should contain only one data type = Sensors."
sensors = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert sensors is not None, "Sensors instance should not be none"
return sensors
def import_surface_obj(user, project, obj_path, surface_type):
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter')
form = ObjSurfaceImporterForm()
form.fill_from_post({
'data_file': Part(obj_path, HeaderMap({}), ''),
'surface_type': surface_type,
'Data_Subject': 'John Doe'
})
form.data_file.data = obj_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, SurfaceIndex)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None, "Surface should not be None"
return surface
def _import(self,
import_file_path=None,
expected_result_class=StructuralMRIIndex,
connectivity_gid=None):
"""
This method is used for importing data in NIFIT format
:param import_file_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter')
form = NIFTIImporterForm()
form.fill_from_post({
'data_file':
Part(import_file_path, HeaderMap({}), ''),
'apply_corrections':
'True',
'connectivity':
connectivity_gid,
'mappings_file':
Part(self.TXT_FILE, HeaderMap({}), ''),
'Data_Subject':
'bla bla'
})
form.data_file.data = import_file_path
form.mappings_file.data = self.TXT_FILE
view_model = form.get_view_model()()
view_model.data_subject = 'bla bla'
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
self.test_user,
self.test_project.id,
view_model=view_model)
dts, count = dao.get_values_of_datatype(self.test_project.id,
expected_result_class, None)
assert 1, count == "Project should contain only one data type."
result = ABCAdapter.load_entity_by_gid(dts[0][2])
assert result is not None, "Result should not be none"
return result
def _import(self, import_file_path=None):
"""
This method is used for importing data in TVB format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.tvb_importer', 'TVBImporter')
form = TVBImporterForm()
form.fill_from_post({ '_data_file': Part(import_file_path, HeaderMap({}), ''),
'_Data_Subject': 'John Doe'
})
form.data_file.data = import_file_path
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **form.get_form_values())
def test_import(self):
count_before = self.count_all_entities(ConnectivityIndex)
assert 0 == count_before
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.networkx_importer',
'NetworkxConnectivityImporter')
form = NetworkxConnectivityImporterForm()
form.fill_from_post({
'_data_file':
Part(self.upload_file, HeaderMap({}), ''),
'_key_edge_weight':
NetworkxParser.KEY_EDGE_WEIGHT[0],
'_key_edge_tract':
NetworkxParser.KEY_EDGE_TRACT[0],
'_key_node_coordinates':
NetworkxParser.KEY_NODE_COORDINATES[0],
'_key_node_label':
NetworkxParser.KEY_NODE_LABEL[0],
'_key_node_region':
NetworkxParser.KEY_NODE_REGION[0],
'_key_node_hemisphere':
NetworkxParser.KEY_NODE_HEMISPHERE[0],
'_Data_Subject':
'John Doe'
})
view_model = form.get_view_model()()
view_model.data_subject = 'John Doe'
form.data_file.data = self.upload_file
form.fill_trait(view_model)
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer,
self.test_user,
self.test_project.id,
view_model=view_model)
count_after = self.count_all_entities(ConnectivityIndex)
assert 1 == count_after
conn = self.get_all_entities(ConnectivityIndex)[0]
assert 83 == conn.number_of_regions
def import_zip_connectivity(user,
project,
zip_path,
subject=DataTypeMetaData.DEFAULT_SUBJECT):
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_connectivity_importer',
'ZIPConnectivityImporter')
form = ZIPConnectivityImporterForm()
form.fill_from_post({
'_uploaded': Part(zip_path, HeaderMap({}), ''),
'_project_id': {1},
'_Data_Subject': subject
})
form.uploaded.data = zip_path
importer.submit_form(form)
### Launch Operation
FlowService().fire_operation(importer, user, project.id,
**form.get_form_values())
def import_projection_matrix(user, project, file_path, sensors_gid,
surface_gid):
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.projection_matrix_importer',
'ProjectionMatrixSurfaceEEGImporter')
form = ProjectionMatrixImporterForm()
form.fill_from_post({
'projection_file':
Part(file_path, HeaderMap({}), ''),
'dataset_name':
'ProjectionMatrix',
'sensors':
sensors_gid,
'surface':
surface_gid,
'Data_Subject':
'John Doe'
})
form.projection_file.data = file_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, ProjectionMatrixIndex)[0]
assert 1 == len(
data_types
), "Project should contain only one data type = Projection Matrix."
projection_matrix = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert projection_matrix is not None, "Projection Matrix instance should not be none"
return projection_matrix
def test_wrong_shape(self):
"""
Verifies that importing a different shape throws exception
"""
file_path = os.path.join(os.path.abspath(os.path.dirname(dataset.__file__)),
'projection_eeg_62_surface_16k.mat')
form = ProjectionMatrixImporterForm()
form.fill_from_post({'_projection_file': Part(file_path, HeaderMap({}), ''),
'_dataset_name': 'ProjectionMatrix',
'_sensors': self.sensors.gid,
'_surface': self.surface.gid,
'_Data_Subject': 'John Doe'
})
form.projection_file.data = file_path
self.importer.submit_form(form)
try:
FlowService().fire_operation(self.importer, self.test_user, self.test_project.id, **form.get_dict())
raise AssertionError("This was expected not to run! 62 rows in proj matrix, but 65 sensors")
except OperationException:
pass
def test_happy_flow_surface_import(self):
"""
Verifies the happy flow for importing a surface.
"""
dt_count_before = TestFactory.get_entity_count(self.test_project, ProjectionMatrixIndex())
file_path = os.path.join(os.path.abspath(os.path.dirname(dataset.__file__)),
'projection_eeg_65_surface_16k.npy')
form = ProjectionMatrixImporterForm()
form.fill_from_post({'_projection_file': Part(file_path, HeaderMap({}), ''),
'_dataset_name': 'ProjectionMatrix',
'_sensors': self.sensors.gid,
'_surface': self.surface.gid,
'_Data_Subject': 'John Doe'
})
form.projection_file.data = file_path
self.importer.submit_form(form)
FlowService().fire_operation(self.importer, self.test_user, self.test_project.id, **form.get_dict())
dt_count_after = TestFactory.get_entity_count(self.test_project, ProjectionMatrixIndex())
assert dt_count_before + 1 == dt_count_after
def test_import_bold(self):
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.mat_timeseries_importer',
'RegionTimeSeriesImporter')
form = RegionMatTimeSeriesImporterForm()
form.fill_from_post({
'data_file':
Part(self.bold_path, HeaderMap({}), ''),
'dataset_name':
'QL_20120824_DK_BOLD_timecourse',
'structure_path':
'',
'slice':
'',
'start_time':
'0',
'Data_Subject':
'QL',
'datatype':
self.connectivity
})
form.data_file.data = self.bold_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
self.test_user,
self.test_project.id,
view_model=view_model)
tsr = TestFactory.get_entity(self.test_project, TimeSeriesRegionIndex)
assert (661, 1, 68, 1) == (tsr.data_length_1d, tsr.data_length_2d,
tsr.data_length_3d, tsr.data_length_4d)
def test_import_bold(self):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.mat_timeseries_importer',
'MatTimeSeriesImporter')
form = RegionMatTimeSeriesImporterForm()
form.fill_from_post({
'_data_file':
Part(self.bold_path, HeaderMap({}), ''),
'_dataset_name':
'QL_20120824_DK_BOLD_timecourse',
'_structure_path':
'',
'_slice':
'',
'_sampling_rate':
"100",
'_start_time':
'0',
'_Data_Subject':
'QL',
'_tstype_parameters':
self.connectivity.gid
})
form.data_file.data = self.bold_path
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **form.get_dict())
tsr = TestFactory.get_entity(self.test_project, TimeSeriesRegionIndex)
assert (661, 1, 68, 1) == (tsr.data_length_1d, tsr.data_length_2d,
tsr.data_length_3d, tsr.data_length_4d)
def import_region_mapping(user, project, import_file_path, surface_gid,
connectivity_gid):
"""
This method is used for importing region mappings
:param import_file_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.region_mapping_importer',
'RegionMappingImporter')
form = RegionMappingImporterForm()
form.fill_from_post({
'mapping_file':
Part(import_file_path, HeaderMap({}), ''),
'surface':
surface_gid,
'connectivity':
connectivity_gid,
'Data_Subject':
'John Doe'
})
form.mapping_file.data = import_file_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
region_mapping = TestFactory.get_entity(project, RegionMappingIndex)
return region_mapping