def read_datatype_attribute(self, entity_gid, dataset_name, flatten=False, datatype_kwargs='null', **kwargs):
"""
Retrieve from a given DataType a property or a method result.
:returns: JSON representation of the attribute.
:param entity_gid: GID for DataType entity
:param dataset_name: name of the dataType property /method
:param flatten: result should be flatten before return (use with WebGL data mainly e.g vertices/triangles)
Ignored if the attribute is not an ndarray
:param datatype_kwargs: if passed, will contain a dictionary of type {'name' : 'gid'}, and for each such
pair, a load_entity will be performed and kwargs will be updated to contain the result
:param kwargs: extra parameters to be passed when dataset_name is method.
"""
self.logger.debug("Starting to read HDF5: " + entity_gid + "/" + dataset_name + "/" + str(kwargs))
entity = ABCAdapter.load_entity_by_gid(entity_gid)
datatype_kwargs = json.loads(datatype_kwargs)
if datatype_kwargs:
for key, value in datatype_kwargs.iteritems():
kwargs[key] = ABCAdapter.load_entity_by_gid(value)
dataset = getattr(entity, dataset_name)
if not kwargs:
# why the deep copy?
result = copy.deepcopy(dataset)
else:
result = dataset(**kwargs)
if isinstance(result, numpy.ndarray):
# for ndarrays honor the flatten kwarg and convert to lists as ndarrs are not json-able
if flatten is True or flatten == "True":
result = result.flatten()
return result.tolist()
else:
return result
def read_datatype_attribute(self, entity_gid, dataset_name, flatten=False, datatype_kwargs='null', **kwargs):
"""
Retrieve from a given DataType a property or a method result.
:returns: JSON with a NumPy array
:param entity_gid: GID for DataType entity
:param dataset_name: name of the dataType property /method
:param flatten: result should be flatten before return (use with WebGL data mainly e.g vertices/triangles)
:param datatype_kwargs: if passed, will contain a dictionary of type {'name' : 'gid'}, and for each such
pair, a load_entity will be performed and kwargs will be updated to contain the result
:param kwargs: extra parameters to be passed when dataset_name is method.
"""
try:
self.logger.debug("Starting to read HDF5: " + entity_gid + "/" + dataset_name + "/" + str(kwargs))
entity = ABCAdapter.load_entity_by_gid(entity_gid)
if kwargs is None:
kwargs = {}
datatype_kwargs = json.loads(datatype_kwargs)
if datatype_kwargs is not None:
for key in datatype_kwargs:
kwargs[key] = ABCAdapter.load_entity_by_gid(datatype_kwargs[key])
if len(kwargs) < 1:
numpy_array = copy.deepcopy(getattr(entity, dataset_name))
else:
numpy_array = eval("entity." + dataset_name + "(**kwargs)")
if (flatten is True) or (flatten == "True"):
numpy_array = numpy_array.flatten()
return numpy_array.tolist()
except Exception, excep:
self.logger.error("Could not retrieve complex entity field:" + str(entity_gid) + "/" + str(dataset_name))
self.logger.exception(excep)
def cdata2local_connectivity(local_connectivity_data, meta, storage_path, expected_length=0):
"""
From a CData entry in CFF, create LocalConnectivity entity.
"""
##### expected_length = cortex.region_mapping.shape[0]
tmpdir = os.path.join(gettempdir(), local_connectivity_data.parent_cfile.get_unique_cff_name())
LOG.debug("Using temporary folder for Local Connectivity import: " + tmpdir)
_zipfile = ZipFile(local_connectivity_data.parent_cfile.src, 'r', ZIP_DEFLATED)
local_connectivity_path = _zipfile.extract(local_connectivity_data.src, tmpdir)
gid = dao.get_last_data_with_uid(meta[constants.KEY_SURFACE_UID], surfaces.CorticalSurface)
surface_data = ABCAdapter.load_entity_by_gid(gid)
local_connectivity = surfaces.LocalConnectivity()
local_connectivity.storage_path = storage_path
local_connectivity_data = read_matlab_data(local_connectivity_path, constants.DATA_NAME_LOCAL_CONN)
if local_connectivity_data.shape[0] < expected_length:
padding = sparse.csc_matrix((local_connectivity_data.shape[0],
expected_length - local_connectivity_data.shape[0]))
local_connectivity_data = sparse.hstack([local_connectivity_data, padding])
padding = sparse.csc_matrix((expected_length - local_connectivity_data.shape[0],
local_connectivity_data.shape[1]))
local_connectivity_data = sparse.vstack([local_connectivity_data, padding])
local_connectivity.equation = None
local_connectivity.matrix = local_connectivity_data
local_connectivity.surface = surface_data
uid = meta[constants.KEY_UID] if constants.KEY_UID in meta else None
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
return local_connectivity, uid
def step_2(self, **kwargs):
"""
Generate the html for the second step of the local connectivity page.
:param kwargs: not actually used, but parameters are still submitted from UI since we just\
use the same js function for this. TODO: do this in a smarter way
"""
context = common.get_from_session(KEY_LCONN_CONTEXT)
left_side_interface = self.get_select_existent_entities('Load Local Connectivity:', LocalConnectivity,
context.selected_entity)
template_specification = dict(title="Surface - Local Connectivity")
template_specification['mainContent'] = 'spatial/local_connectivity_step2_main'
template_specification['existentEntitiesInputList'] = left_side_interface
template_specification['loadExistentEntityUrl'] = LOAD_EXISTING_URL
template_specification['resetToDefaultUrl'] = RELOAD_DEFAULT_PAGE_URL
template_specification['next_step_url'] = '/spatial/localconnectivity/step_1'
msg, _ = common.get_message_from_session()
template_specification['displayedMessage'] = msg
context = common.get_from_session(KEY_LCONN_CONTEXT)
if context.selected_entity is not None:
selected_local_conn = ABCAdapter.load_entity_by_gid(context.selected_entity)
template_specification.update(self.display_surface(selected_local_conn.surface.gid))
template_specification['no_local_connectivity'] = False
min_value, max_value = selected_local_conn.get_min_max_values()
template_specification['minValue'] = min_value
template_specification['maxValue'] = max_value
else:
template_specification['no_local_connectivity'] = True
template_specification[common.KEY_PARAMETERS_CONFIG] = False
return self.fill_default_attributes(template_specification)
def load_region_stimulus(self, region_stimulus_gid, from_step=None):
"""
Loads the interface for the selected region stimulus.
"""
selected_region_stimulus = ABCAdapter.load_entity_by_gid(region_stimulus_gid)
temporal_eq = selected_region_stimulus.temporal
spatial_eq = selected_region_stimulus.spatial
connectivity = selected_region_stimulus.connectivity
weights = selected_region_stimulus.weight
temporal_eq_type = temporal_eq.__class__.__name__
spatial_eq_type = spatial_eq.__class__.__name__
default_dict = {'temporal': temporal_eq_type, 'spatial': spatial_eq_type,
'connectivity': connectivity.gid, 'weight': json.dumps(weights)}
for param in temporal_eq.parameters:
prepared_name = 'temporal_parameters_option_' + str(temporal_eq_type)
prepared_name = prepared_name + '_parameters_parameters_' + str(param)
default_dict[prepared_name] = str(temporal_eq.parameters[param])
for param in spatial_eq.parameters:
prepared_name = 'spatial_parameters_option_' + str(spatial_eq_type) + '_parameters_parameters_' + str(param)
default_dict[prepared_name] = str(spatial_eq.parameters[param])
input_list = self.get_creator_and_interface(REGION_STIMULUS_CREATOR_MODULE,
REGION_STIMULUS_CREATOR_CLASS, StimuliRegion())[1]
input_list = InputTreeManager.fill_defaults(input_list, default_dict)
context = common.get_from_session(KEY_REGION_CONTEXT)
context.reset()
context.update_from_interface(input_list)
context.equation_kwargs[DataTypeMetaData.KEY_TAG_1] = selected_region_stimulus.user_tag_1
context.set_active_stimulus(region_stimulus_gid)
return self.do_step(from_step)
def _read_datatype_attribute(self, entity_gid, dataset_name, datatype_kwargs='null', **kwargs):
self.logger.debug("Starting to read HDF5: " + entity_gid + "/" + dataset_name + "/" + str(kwargs))
entity = ABCAdapter.load_entity_by_gid(entity_gid)
datatype_kwargs = json.loads(datatype_kwargs)
if datatype_kwargs:
for key, value in datatype_kwargs.iteritems():
kwargs[key] = ABCAdapter.load_entity_by_gid(value)
result = getattr(entity, dataset_name)
if callable(result):
if kwargs:
result = result(**kwargs)
else:
result = result()
return result
def _create_value_wrapper(self):
"""Persist ValueWrapper"""
value_ = ValueWrapper(data_value=5.0, data_name="my_value")
self._store_entity(value_, "ValueWrapper", "tvb.datatypes.mapped_values")
valuew = self.get_all_entities(ValueWrapper)
self.assertEqual(1, len(valuew), "Should be one value wrapper")
return ABCAdapter.load_entity_by_gid(valuew[0].gid)
def cdata2eeg_mapping(eeg_mapping_data, meta, storage_path, expected_shape=0):
"""
Currently not used
"""
tmpdir = os.path.join(gettempdir(), eeg_mapping_data.parent_cfile.get_unique_cff_name())
LOG.debug("Using temporary folder for EEG_Mapping import: " + tmpdir)
_zipfile = ZipFile(eeg_mapping_data.parent_cfile.src, 'r', ZIP_DEFLATED)
eeg_projection_path = _zipfile.extract(eeg_mapping_data.src, tmpdir)
eeg_projection_data = read_matlab_data(eeg_projection_path, constants.DATA_NAME_PROJECTION)
if eeg_projection_data.shape[1] < expected_shape:
padding = numpy.zeros((eeg_projection_data.shape[0], expected_shape - eeg_projection_data.shape[1]))
eeg_projection_data = numpy.hstack((eeg_projection_data, padding))
gid = dao.get_last_data_with_uid(meta[constants.KEY_SURFACE_UID], surfaces.CorticalSurface)
surface_data = ABCAdapter.load_entity_by_gid(gid)
projection_matrix = projections.ProjectionSurfaceEEG(storage_path = storage_path)
projection_matrix.projection_data = eeg_projection_data
projection_matrix.sources = surface_data
projection_matrix.sensors = None
### TODO if we decide to use this method, we will need to find a manner to fill the sensors.
return projection_matrix
def _create_value_wrapper(self):
"""Persist ValueWrapper"""
value_ = ValueWrapper(data_value=5.0, data_name="my_value")
self._store_entity(value_, "ValueWrapper", "tvb.datatypes.mapped_values")
valuew = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
self.assertEqual(len(valuew), 1, "Should be only one value wrapper")
return ABCAdapter.load_entity_by_gid(valuew[0][2])
def test_import_export(self):
"""
Test the import/export mechanism for a project structure.
The project contains the following data types: Connectivity, Surface, MappedArray and ValueWrapper.
"""
result = self.get_all_datatypes()
expected_results = {}
for one_data in result:
expected_results[one_data.gid] = (one_data.module, one_data.type)
#create an array mapped in DB
data = {'param_1': 'some value'}
OperationService().initiate_prelaunch(self.operation, self.adapter_instance, {}, **data)
inserted = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(1, inserted, "Problems when inserting data")
#create a value wrapper
value_wrapper = self._create_value_wrapper()
count_operations = dao.get_filtered_operations(self.test_project.id, None, is_count=True)
self.assertEqual(2, count_operations, "Invalid ops number before export!")
# Export project as ZIP
self.zip_path = ExportManager().export_project(self.test_project)
self.assertTrue(self.zip_path is not None, "Exported file is none")
# Remove the original project
self.project_service.remove_project(self.test_project.id)
result, lng_ = self.project_service.retrieve_projects_for_user(self.test_user.id)
self.assertEqual(0, len(result), "Project Not removed!")
self.assertEqual(0, lng_, "Project Not removed!")
# Now try to import again project
self.import_service.import_project_structure(self.zip_path, self.test_user.id)
result = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertEqual(len(result), 1, "There should be only one project.")
self.assertEqual(result[0].name, "GeneratedProject", "The project name is not correct.")
self.assertEqual(result[0].description, "test_desc", "The project description is not correct.")
self.test_project = result[0]
count_operations = dao.get_filtered_operations(self.test_project.id, None, is_count=True)
#1 op. - import cff; 2 op. - save the array wrapper;
self.assertEqual(2, count_operations, "Invalid ops number after export and import !")
for gid in expected_results:
datatype = dao.get_datatype_by_gid(gid)
self.assertEqual(datatype.module, expected_results[gid][0], 'DataTypes not imported correctly')
self.assertEqual(datatype.type, expected_results[gid][1], 'DataTypes not imported correctly')
#check the value wrapper
new_val = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
self.assertEqual(1, len(new_val), "One !=" + str(len(new_val)))
new_val = ABCAdapter.load_entity_by_gid(new_val[0][2])
self.assertEqual(value_wrapper.data_value, new_val.data_value, "Data value incorrect")
self.assertEqual(value_wrapper.data_type, new_val.data_type, "Data type incorrect")
self.assertEqual(value_wrapper.data_name, new_val.data_name, "Data name incorrect")
def get_entity(project, expected_data, filters=None):
"""
Return the first entity with class given by `expected_data`
:param expected_data: specifies the class whose entity is returned
"""
data_types = FlowService().get_available_datatypes(project.id,
expected_data.module + "." + expected_data.type, filters)[0]
entity = ABCAdapter.load_entity_by_gid(data_types[0][2])
return entity
def display_connectivity(connectivity_gid):
"""
Generates the html for displaying the connectivity matrix.
"""
connectivity = ABCAdapter.load_entity_by_gid(connectivity_gid)
connectivity_viewer_params = ConnectivityViewer.get_connectivity_parameters(connectivity)
template_specification = dict()
template_specification['isSingleMode'] = True
template_specification.update(connectivity_viewer_params)
return template_specification
def _is_compatible(self, algorithm, datatype_group_gid):
"""
Check if PSE view filters are compatible with current DataType.
:param algorithm: Algorithm instance to get filters from it.
:param datatype_group_gid: Current DataTypeGroup to validate against.
:returns: True when DataTypeGroup can be displayed with current algorithm, False when incompatible.
"""
datatype_group = ABCAdapter.load_entity_by_gid(datatype_group_gid)
filter_chain = FilterChain.from_json(algorithm.datatype_filter)
if datatype_group and (not filter_chain or filter_chain.get_python_filter_equivalent(datatype_group)):
return True
return False
def test_h5_import(self):
"""
This method tests import of TVB data in h5 format. Single data type / import
"""
self._import(self.h5_file_path)
data_types = FlowService().get_available_datatypes(self.test_project.id,
self.datatype.module + "." + self.datatype.type)[0]
self.assertEqual(1, len(data_types), "Project should contain only one data type.")
data_type_entity = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(data_type_entity is not None, "Datatype should not be none")
self.assertEqual(self.datatype.gid, data_type_entity.gid, "Imported datatype should have the same gid")
def downloaddata(self, data_gid, export_module):
""" Export the data to a default path of TVB_STORAGE/PROJECTS/project_name """
current_prj = common.get_current_project()
# Load data by GID
entity = ABCAdapter.load_entity_by_gid(data_gid)
# Do real export
export_mng = ExportManager()
file_name, file_path, delete_file = export_mng.export_data(entity, export_module, current_prj)
if delete_file:
# We force parent folder deletion because export process generated it.
self.mark_file_for_delete(file_path, True)
self.logger.debug("Data exported in file: " + str(file_path))
return serve_file(file_path, "application/x-download", "attachment", file_name)
def cdata2region_mapping(region_mapping_data, meta, storage_path):
"""
From a CData entry in CFF, create RegionMapping entity.
"""
tmpdir = os.path.join(gettempdir(), region_mapping_data.parent_cfile.get_unique_cff_name())
LOG.debug("Using temporary folder for Region_Mapping import: " + tmpdir)
_zipfile = ZipFile(region_mapping_data.parent_cfile.src, 'r', ZIP_DEFLATED)
region_mapping_path = _zipfile.extract(region_mapping_data.src, tmpdir)
gid = dao.get_last_data_with_uid(meta[constants.KEY_SURFACE_UID], surfaces.CorticalSurface)
surface_data = ABCAdapter.load_entity_by_gid(gid)
gid = dao.get_last_data_with_uid(meta[constants.KEY_CONNECTIVITY_UID], Connectivity)
connectivity = ABCAdapter.load_entity_by_gid(gid)
region_mapping = surfaces.RegionMapping(storage_path = storage_path)
region_mapping.array_data = read_list_data(region_mapping_path, dtype=numpy.int32)
region_mapping.connectivity = connectivity
region_mapping.surface = surface_data
uid = meta[constants.KEY_UID] if constants.KEY_UID in meta else None
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
return region_mapping, uid
def _get_entity(self, expected_data, filters=None):
"""
Checks there is exactly one datatype with required specifications and returns it
:param expected_data: a class whose entity is to be returned
:param filters: optional, the returned entity will also have the required filters
:return: an object of class `expected_data`
"""
dt_full_name = expected_data.__module__ + "." + expected_data.__name__
data_types = FlowService().get_available_datatypes(self.test_project.id,dt_full_name, filters)[0]
self.assertEqual(1, len(data_types), "Project should contain only one data type:" + str(expected_data.type))
entity = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(entity is not None, "Instance should not be none")
return entity
def _importSurface(self, import_file_path=None):
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.zip_surface_importer', 'ZIPSurfaceImporter')
args = {'uploaded': import_file_path, 'surface_type': OUTER_SKULL,
'zero_based_triangles': True,
DataTypeMetaData.KEY_SUBJECT: "John"}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(self.test_project.id, SkullSkin)[0]
assert 1, len(data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None, "Surface should not be None"
return surface
def _importSurface(self, import_file_path=None):
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter')
args = {'data_file': import_file_path,
"surface_type": FACE,
DataTypeMetaData.KEY_SUBJECT: "John"}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(self.test_project.id, FaceSurface)[0]
self.assertEqual(1, len(data_types), "Project should contain only one data type.")
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(surface is not None, "Surface should not be None")
return surface
def display_surface(surface_gid):
"""
Generates the HTML for displaying the surface with the given ID.
"""
surface = ABCAdapter.load_entity_by_gid(surface_gid)
common.add2session(PARAM_SURFACE, surface_gid)
url_vertices_pick, url_normals_pick, url_triangles_pick = surface.get_urls_for_pick_rendering()
url_vertices, url_normals, _, url_triangles = surface.get_urls_for_rendering()
return {
'urlVerticesPick': json.dumps(url_vertices_pick),
'urlTrianglesPick': json.dumps(url_triangles_pick),
'urlNormalsPick': json.dumps(url_normals_pick),
'urlVertices': json.dumps(url_vertices),
'urlTriangles': json.dumps(url_triangles),
'urlNormals': json.dumps(url_normals),
'brainCenter': json.dumps(surface.center())
}
def _get_effective_data_type(self, data):
"""
This method returns the data type for the provided data.
- If current data is a simple data type is returned.
- If it is an data type group, we return the first element. Only one element is
necessary since all group elements are the same type.
"""
# first check if current data is a DataTypeGroup
if self.is_data_a_group(data):
data_types = ProjectService.get_datatypes_from_datatype_group(data.id)
if data_types is not None and len(data_types) > 0:
# Since all objects in a group are the same type it's enough
return ABCAdapter.load_entity_by_gid(data_types[0].gid)
else:
return None
else:
return data
def test_remove_used_surface(self):
"""
Tries to remove an used surface
"""
mapping, mapping_count = self.flow_service.get_available_datatypes(self.test_project.id, RegionMapping)
self.assertEquals(1, mapping_count, "There should be one Mapping.")
mapping_gid = mapping[0][2]
mapping = ABCAdapter.load_entity_by_gid(mapping_gid)
surface = dao.get_datatype_by_gid(mapping.surface.gid)
self.assertEqual(surface.gid, mapping.surface.gid, "The surfaces should have the same GID")
try:
self.project_service.remove_datatype(self.test_project.id, surface.gid)
self.fail("The surface should still be used by a RegionMapping " + str(surface.gid))
except RemoveDataTypeException:
#OK, do nothing
pass
res = dao.get_datatype_by_gid(surface.gid)
self.assertEqual(surface.id, res.id, "A used surface was deleted")
def step_2(self):
"""
Generate the required template dictionary for the second step.
"""
context = common.get_from_session(KEY_REGION_CONTEXT)
selected_stimulus_gid = context.selected_stimulus
left_side_interface = self.get_select_existent_entities('Load Region Stimulus:',
StimuliRegion, selected_stimulus_gid)
template_specification = dict(title="Spatio temporal - Region stimulus")
template_specification['mainContent'] = 'spatial/stimulus_region_step2_main'
template_specification['next_step_url'] = '/spatial/stimulus/region/step_2_submit'
template_specification['existentEntitiesInputList'] = left_side_interface
default_weights = context.get_weights()
if len(default_weights) == 0:
selected_connectivity = ABCAdapter.load_entity_by_gid(context.get_session_connectivity())
default_weights = StimuliRegion.get_default_weights(selected_connectivity.number_of_regions)
template_specification['node_weights'] = json.dumps(default_weights)
template_specification[common.KEY_PARAMETERS_CONFIG] = False
template_specification.update(self.display_connectivity(context.get_session_connectivity()))
return self.fill_default_attributes(template_specification)
def _import(self, import_file_path=None, expected_result_class=StructuralMRI, connectivity=None):
"""
This method is used for importing data in NIFIT format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter')
args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "bla bla",
'apply_corrections': True, 'connectivity': connectivity}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
dts, count = dao.get_values_of_datatype(self.test_project.id, expected_result_class, None)
self.assertEqual(1, count, "Project should contain only one data type.")
result = ABCAdapter.load_entity_by_gid(dts[0][2])
self.assertTrue(result is not None, "Result should not be none")
return result
def _get_all_data_types_arr(self, data):
"""
This method builds an array with all data types to be processed later.
- If current data is a simple data type is added to an array.
- If it is an data type group all its children are loaded and added to array.
"""
# first check if current data is a DataTypeGroup
if self.is_data_a_group(data):
data_types = ProjectService.get_datatypes_from_datatype_group(data.id)
result = []
if data_types is not None and len(data_types) > 0:
for data_type in data_types:
entity = ABCAdapter.load_entity_by_gid(data_type.gid)
result.append(entity)
return result
else:
return [data]
def display_surface(surface_gid):
"""
Generates the HTML for displaying the surface with the given ID.
"""
surface = ABCAdapter.load_entity_by_gid(surface_gid)
common.add2session(PARAM_SURFACE, surface_gid)
url_vertices_pick, url_normals_pick, url_triangles_pick = surface.get_urls_for_pick_rendering(
)
url_vertices, url_normals, _, url_triangles = surface.get_urls_for_rendering(
)
return {
'urlVerticesPick': json.dumps(url_vertices_pick),
'urlTrianglesPick': json.dumps(url_triangles_pick),
'urlNormalsPick': json.dumps(url_normals_pick),
'urlVertices': json.dumps(url_vertices),
'urlTriangles': json.dumps(url_triangles),
'urlNormals': json.dumps(url_normals),
'brainCenter': json.dumps(surface.center())
}
def display_surface(surface_gid):
"""
Generates the HTML for displaying the surface with the given ID.
"""
surface = ABCAdapter.load_entity_by_gid(surface_gid)
base.add2session(PARAM_SURFACE, surface_gid)
url_vertices_pick, url_normals_pick, url_triangles_pick = surface.get_urls_for_pick_rendering()
url_vertices, url_normals, url_triangles, alphas, alphas_indices = surface.get_urls_for_rendering(True, None)
template_specification = dict()
template_specification['urlVerticesPick'] = json.dumps(url_vertices_pick)
template_specification['urlTrianglesPick'] = json.dumps(url_triangles_pick)
template_specification['urlNormalsPick'] = json.dumps(url_normals_pick)
template_specification['urlVertices'] = json.dumps(url_vertices)
template_specification['urlTriangles'] = json.dumps(url_triangles)
template_specification['urlNormals'] = json.dumps(url_normals)
template_specification['alphas'] = json.dumps(alphas)
template_specification['alphas_indices'] = json.dumps(alphas_indices)
template_specification['brainCenter'] = json.dumps(surface.center())
return template_specification
def _import(self,
import_file_path=None,
expected_result_class=StructuralMRIIndex,
connectivity=None):
"""
This method is used for importing data in NIFIT format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter')
form = NIFTIImporterForm()
form.fill_from_post({
'_data_file':
Part(import_file_path, HeaderMap({}), ''),
'_apply_corrections':
'True',
'_connectivity':
connectivity,
'_mappings_file':
Part(self.TXT_FILE, HeaderMap({}), ''),
'_Data_Subject':
'bla bla'
})
form.data_file.data = import_file_path
form.mappings_file.data = self.TXT_FILE
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **form.get_dict())
dts, count = dao.get_values_of_datatype(self.test_project.id,
expected_result_class, None)
assert 1, count == "Project should contain only one data type."
result = ABCAdapter.load_entity_by_gid(dts[0][2])
assert result is not None, "Result should not be none"
return result
def setUp(self):
"""
Reset the database before each test;
creates a test user, a test project, a connectivity;
sets context model parameters and a Generic2dOscillator as a default model
"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
TestFactory.import_cff(test_user=self.test_user,
test_project=self.test_project)
self.default_model = models_module.Generic2dOscillator()
all_connectivities = self.flow_service.get_available_datatypes(
self.test_project.id, Connectivity)
self.connectivity = ABCAdapter.load_entity_by_gid(
all_connectivities[0][2])
self.connectivity.number_of_regions = 74
self.context_model_param = ContextModelParameters(
self.connectivity, self.default_model)
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
parameters = json.loads(operation.parameters)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return adapter.review_operation_inputs(parameters)
except Exception:
self.logger.exception("Could not load details for operation %s" % operation_gid)
inputs_datatypes = []
changed_parameters = dict(Warning="Algorithm changed dramatically. We can not offer more details")
for submit_param in parameters.values():
self.logger.debug("Searching DT by GID %s" % submit_param)
datatype = ABCAdapter.load_entity_by_gid(str(submit_param))
if datatype is not None:
inputs_datatypes.append(datatype)
return inputs_datatypes, changed_parameters
def _get_all_data_types_arr(self, data):
"""
This method builds an array with all data types to be processed later.
- If current data is a simple data type is added to an array.
- If it is an data type group all its children are loaded and added to array.
"""
# first check if current data is a DataTypeGroup
if self.is_data_a_group(data):
data_types = ProjectService.get_datatypes_from_datatype_group(
data.id)
result = []
if data_types is not None and len(data_types) > 0:
for data_type in data_types:
entity = ABCAdapter.load_entity_by_gid(data_type.gid)
result.append(entity)
return result
else:
return [data]
def _review_operation_inputs(self, operation_gid):
"""
:returns: A list of DataTypes that are used as input parameters for the specified operation.
And a dictionary will all operation parameters different then the default ones.
"""
operation = dao.get_operation_by_gid(operation_gid)
parameters = json.loads(operation.parameters)
try:
adapter = ABCAdapter.build_adapter(operation.algorithm)
return adapter.review_operation_inputs(parameters)
except IntrospectionException:
self.logger.warning("Could not find adapter class for operation %s" % operation_gid)
inputs_datatypes = []
changed_parameters = dict(Warning="Algorithm was Removed. We can not offer more details")
for submit_param in parameters.values():
self.logger.debug("Searching DT by GID %s" % submit_param)
datatype = ABCAdapter.load_entity_by_gid(str(submit_param))
if datatype is not None:
inputs_datatypes.append(datatype)
return inputs_datatypes, changed_parameters
def import_projection_matrix(user, project, file_path, sensors_gid,
surface_gid):
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.projection_matrix_importer',
'ProjectionMatrixSurfaceEEGImporter')
form = ProjectionMatrixImporterForm()
form.fill_from_post({
'projection_file':
Part(file_path, HeaderMap({}), ''),
'dataset_name':
'ProjectionMatrix',
'sensors':
sensors_gid,
'surface':
surface_gid,
'Data_Subject':
'John Doe'
})
form.projection_file.data = file_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, ProjectionMatrixIndex)[0]
assert 1 == len(
data_types
), "Project should contain only one data type = Projection Matrix."
projection_matrix = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert projection_matrix is not None, "Projection Matrix instance should not be none"
return projection_matrix
def view_stimulus(self, focal_points):
"""
Just create the stimulus to view the actual data, don't store to db.
Hold the entity in session without the surface, so the next time you need
data just get from that one.
"""
try:
context = base.get_from_session(KEY_SURFACE_CONTEXT)
context.set_focal_points(focal_points)
kwargs = copy.deepcopy(context.equation_kwargs)
surface_stimulus_creator = self.get_creator_and_interface(SURFACE_STIMULUS_CREATOR_MODULE,
SURFACE_STIMULUS_CREATOR_CLASS,
StimuliSurface())[0]
min_time = float(kwargs.get('min_tmp_x', 0))
max_time = float(kwargs.get('max_tmp_x', 100))
kwargs = surface_stimulus_creator.prepare_ui_inputs(kwargs)
stimulus = surface_stimulus_creator.launch(**kwargs)
surface_gid = base.get_from_session(PARAM_SURFACE)
surface = ABCAdapter.load_entity_by_gid(surface_gid)
stimulus.surface = surface
stimulus.configure_space()
time = numpy.arange(min_time, max_time, 1)
time = time[numpy.newaxis, :]
stimulus.configure_time(time)
data = []
max_value = numpy.max(stimulus())
min_value = numpy.min(stimulus())
for i in range(min(CHUNK_SIZE, stimulus.temporal_pattern.shape[1])):
step_data = stimulus(i).tolist()
data.append(step_data)
stimulus.surface = surface.gid
base.add2session(KEY_STIMULUS, stimulus)
result = {'status': 'ok', 'max': max_value, 'min': min_value,
'data': data, "time_min": min_time, "time_max": max_time, "chunk_size": CHUNK_SIZE}
return result
except (NameError, ValueError, SyntaxError):
return {'status': 'error',
'errorMsg': "Could not generate stimulus data. Some of the parameters hold invalid characters."}
except Exception, ex:
return {'allSeries': 'error', 'errorMsg': ex.message}
def import_surface_gifti(user, project, path):
"""
This method is used for importing data in GIFIT format
:param path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.gifti_surface_importer',
'GIFTISurfaceImporter')
form = GIFTISurfaceImporterForm()
form.fill_from_post({
'file_type': form.get_view_model().KEY_OPTION_READ_METADATA,
'data_file': Part(path, HeaderMap({}), ''),
'data_file_part2': Part('', HeaderMap({}), ''),
'should_center': 'False',
'Data_Subject': 'John Doe',
})
form.data_file.data = path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
surface = CorticalSurface
data_types = FlowService().get_available_datatypes(
project.id, surface.__module__ + "." + surface.__name__)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None == "TimeSeries should not be none"
return surface
def view_stimulus(self, focal_points):
"""
Just create the stimulus to view the actual data, don't store to db.
Hold the entity in session without the surface, so the next time you need
data just get from that one.
"""
try:
context = common.get_from_session(KEY_SURFACE_CONTEXT)
context.set_focal_points(focal_points)
kwargs = copy.deepcopy(context.equation_kwargs)
surface_stimulus_creator = self.get_creator_and_interface(SURFACE_STIMULUS_CREATOR_MODULE,
SURFACE_STIMULUS_CREATOR_CLASS,
StimuliSurface())[0]
min_time = float(kwargs.get('min_tmp_x', 0))
max_time = float(kwargs.get('max_tmp_x', 100))
kwargs = surface_stimulus_creator.prepare_ui_inputs(kwargs)
stimulus = surface_stimulus_creator.launch(**kwargs)
surface_gid = common.get_from_session(PARAM_SURFACE)
surface = ABCAdapter.load_entity_by_gid(surface_gid)
stimulus.surface = surface
stimulus.configure_space()
time = numpy.arange(min_time, max_time, 1)
time = time[numpy.newaxis, :]
stimulus.configure_time(time)
data = []
max_value = numpy.max(stimulus())
min_value = numpy.min(stimulus())
for i in range(min(CHUNK_SIZE, stimulus.temporal_pattern.shape[1])):
step_data = stimulus(i).tolist()
data.append(step_data)
stimulus.surface = surface.gid
common.add2session(KEY_STIMULUS, stimulus)
result = {'status': 'ok', 'max': max_value, 'min': min_value,
'data': data, "time_min": min_time, "time_max": max_time, "chunk_size": CHUNK_SIZE}
return result
except (NameError, ValueError, SyntaxError):
return {'status': 'error',
'errorMsg': "Could not generate stimulus data. Some of the parameters hold invalid characters."}
except Exception, ex:
return {'allSeries': 'error', 'errorMsg': ex.message}
def get_template_from_context(self):
"""
Return the parameters for the local connectivity in case one is stored in context. Load the entity
and use it to populate the defaults from the interface accordingly.
"""
context = common.get_from_session(KEY_LCONN_CONTEXT)
selected_local_conn = ABCAdapter.load_entity_by_gid(context.selected_entity)
cutoff = selected_local_conn.cutoff
equation = selected_local_conn.equation
surface = selected_local_conn.surface
default_dict = {"surface": surface.gid, "cutoff": cutoff}
if equation is not None:
equation_type = equation.__class__.__name__
default_dict["equation"] = equation_type
for param in equation.parameters:
prepared_name = "equation_parameters_option_" + str(equation_type)
prepared_name = prepared_name + "_parameters_parameters_" + str(param)
default_dict[prepared_name] = equation.parameters[param]
else:
msg = "There is no equation specified for this local connectivity. "
msg += "The default equation is displayed into the spatial field."
self.logger.warning(msg)
common.set_info_message(msg)
default_dict[DataTypeMetaData.KEY_TAG_1] = selected_local_conn.user_tag_1
input_list = self.get_creator_and_interface(
LOCAL_CONN_CREATOR_MODULE, LOCAL_CONN_CREATOR_CLASS, LocalConnectivity(), lock_midpoint_for_eq=[1]
)[1]
input_list = self._add_extra_fields_to_interface(input_list)
input_list = ABCAdapter.fill_defaults(input_list, default_dict)
template_specification = {
"inputList": input_list,
common.KEY_PARAMETERS_CONFIG: False,
"equationViewerUrl": "/spatial/localconnectivity/get_equation_chart",
"equationsPrefixes": json.dumps(self.plotted_equations_prefixes),
}
return template_specification
def _import(self, import_file_path, sensors_type, expected_data):
"""
This method is used for importing sensors
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer')
args = {'sensors_file': import_file_path, 'sensors_type': sensors_type}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(self.test_project.id,
expected_data.module + "." + expected_data.type)[0]
self.assertEqual(1, len(data_types), "Project should contain only one data type = Sensors.")
time_series = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(time_series is not None, "Sensors instance should not be none")
return time_series
def _import(self, import_file_path, sensors_type, expected_data):
"""
This method is used for importing sensors
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer')
args = {'sensors_file': import_file_path, 'sensors_type': sensors_type}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(self.test_project.id,
expected_data.module + "." + expected_data.type)[0]
assert 1 == len(data_types), "Project should contain only one data type = Sensors."
time_series = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert time_series is not None, "Sensors instance should not be none"
return time_series
def display_surface(surface_gid, region_mapping_gid=None):
"""
Generates the HTML for displaying the surface with the given ID.
"""
surface = ABCAdapter.load_entity_by_gid(surface_gid)
common.add2session(PARAM_SURFACE, surface_gid)
surface_h5 = h5.h5_file_for_index(surface)
url_vertices_pick, url_normals_pick, url_triangles_pick = SurfaceURLGenerator.get_urls_for_pick_rendering(
surface_h5)
url_vertices, url_normals, _, url_triangles, _ = SurfaceURLGenerator.get_urls_for_rendering(
surface_h5, region_mapping_gid)
surface_h5.close()
return {
'urlVerticesPick': json.dumps(url_vertices_pick),
'urlTrianglesPick': json.dumps(url_triangles_pick),
'urlNormalsPick': json.dumps(url_normals_pick),
'urlVertices': json.dumps(url_vertices),
'urlTriangles': json.dumps(url_triangles),
'urlNormals': json.dumps(url_normals),
'brainCenter': json.dumps(surface_h5.center())
}
def _create_value_wrapper(test_user, test_project=None):
"""
Creates a ValueWrapper dataType, and the associated parent Operation.
This is also used in ProjectStructureTest.
"""
if test_project is None:
test_project = TestFactory.create_project(test_user, 'test_proj')
operation = TestFactory.create_operation(test_user=test_user, test_project=test_project)
value_wrapper = ValueWrapper(data_value=5.0, data_name="my_value")
value_wrapper.type = "ValueWrapper"
value_wrapper.module = "tvb.datatypes.mapped_values"
value_wrapper.subject = "John Doe"
value_wrapper.state = "RAW_STATE"
value_wrapper.set_operation_id(operation.id)
adapter_instance = StoreAdapter([value_wrapper])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
all_value_wrappers = FlowService().get_available_datatypes(test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
if len(all_value_wrappers) != 1:
raise Exception("Should be only one value wrapper.")
result_vw = ABCAdapter.load_entity_by_gid(all_value_wrappers[0][2])
return test_project, result_vw.gid, operation.gid
def _create_value_wrapper(test_user, test_project=None):
"""
Creates a ValueWrapper dataType, and the associated parent Operation.
This is also used in ProjectStructureTest.
"""
if test_project is None:
test_project = TestFactory.create_project(test_user, 'test_proj')
operation = TestFactory.create_operation(test_user=test_user, test_project=test_project)
value_wrapper = ValueWrapper(data_value=5.0, data_name="my_value")
value_wrapper.type = "ValueWrapper"
value_wrapper.module = "tvb.datatypes.mapped_values"
value_wrapper.subject = "John Doe"
value_wrapper.state = "RAW_STATE"
value_wrapper.set_operation_id(operation.id)
adapter_instance = StoreAdapter([value_wrapper])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
all_value_wrappers = FlowService().get_available_datatypes(test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
if len(all_value_wrappers) != 1:
raise Exception("Should be only one value wrapper.")
result_vw = ABCAdapter.load_entity_by_gid(all_value_wrappers[0][2])
return test_project, result_vw.gid, operation.gid
def set_connectivity(self, **data):
session_stored_simulator = common.get_from_session(
common.KEY_SIMULATOR_CONFIG)
is_simulator_copy = common.get_from_session(
common.KEY_IS_SIMULATOR_COPY)
is_simulator_load = common.get_from_session(
common.KEY_IS_SIMULATOR_LOAD)
form = SimulatorAdapterForm()
if cherrypy.request.method == 'POST':
is_simulator_copy = False
form.fill_from_post(data)
connectivity_index_gid = form.connectivity.value
conduction_speed = form.conduction_speed.value
coupling = form.coupling.value
connectivity_index = ABCAdapter.load_entity_by_gid(
connectivity_index_gid)
connectivity = h5.load_from_index(connectivity_index)
# TODO: handle this cases in a better manner
session_stored_simulator.connectivity = connectivity
session_stored_simulator.conduction_speed = conduction_speed
session_stored_simulator.coupling = coupling()
next_form = get_form_for_coupling(
type(session_stored_simulator.coupling))()
self.range_parameters.coupling_parameters = next_form.get_range_parameters(
)
next_form.fill_from_trait(session_stored_simulator.coupling)
dict_to_render = copy.deepcopy(self.dict_to_render)
dict_to_render[self.FORM_KEY] = next_form
dict_to_render[self.ACTION_KEY] = '/burst/set_coupling_params'
dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_connectivity'
dict_to_render[self.IS_COPY] = is_simulator_copy
dict_to_render[self.IS_LOAD] = is_simulator_load
return dict_to_render
def step_2(self):
"""
Generate the required template dictionary for the second step.
"""
current_region_stimulus = common.get_from_session(KEY_REGION_STIMULUS)
region_stim_selector_form = StimulusRegionSelectorForm(
common.get_current_project().id)
region_stim_selector_form.region_stimulus.data = current_region_stimulus.gid.hex
region_stim_selector_form.display_name.data = common.get_from_session(
KEY_REGION_STIMULUS_NAME)
template_specification = dict(
title="Spatio temporal - Region stimulus")
template_specification[
'mainContent'] = 'spatial/stimulus_region_step2_main'
template_specification[
'next_step_url'] = '/spatial/stimulus/region/step_2_submit'
template_specification[
'regionStimSelectorForm'] = self.render_adapter_form(
region_stim_selector_form)
default_weights = current_region_stimulus.weight
if len(default_weights) == 0:
selected_connectivity = ABCAdapter.load_entity_by_gid(
current_region_stimulus.connectivity.hex)
if selected_connectivity is None:
common.set_error_message(self.MSG_MISSING_CONNECTIVITY)
default_weights = numpy.array([])
else:
default_weights = StimuliRegion.get_default_weights(
selected_connectivity.number_of_regions)
template_specification['node_weights'] = json.dumps(
default_weights.tolist())
template_specification[common.KEY_PARAMETERS_CONFIG] = False
template_specification.update(
self.display_connectivity(
current_region_stimulus.connectivity.hex))
return self.fill_default_attributes(template_specification)
def import_surface_zip(user,
project,
zip_path,
surface_type,
zero_based='True'):
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_surface_importer',
'ZIPSurfaceImporter')
form = ZIPSurfaceImporterForm()
form.fill_from_post({
'uploaded': Part(zip_path, HeaderMap({}), ''),
'zero_based_triangles': zero_based,
'should_center': 'True',
'surface_type': surface_type,
'Data_Subject': 'John Doe'
})
form.uploaded.data = zip_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, SurfaceIndex)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
surface.user_tag_3 = ''
assert surface is not None, "Surface should not be None"
return surface
def _importSurface(self, import_file_path=None):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter')
args = {
'data_file': import_file_path,
"surface_type": FACE,
DataTypeMetaData.KEY_SUBJECT: "John"
}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(
self.test_project.id, FaceSurface)[0]
self.assertEqual(1, len(data_types),
"Project should contain only one data type.")
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(surface is not None, "Surface should not be None")
return surface
def compute_data_for_gradient_view(self, local_connectivity_gid, selected_triangle):
"""
When the user loads an existent local connectivity and he picks a vertex from the used surface, this
method computes the data needed for drawing a gradient view corresponding to that vertex.
Returns a json which contains the data needed for drawing a gradient view for the selected vertex.
"""
selected_local_conn = ABCAdapter.load_entity_by_gid(local_connectivity_gid)
surface = selected_local_conn.surface
triangle_index = int(selected_triangle)
vertex_index = int(surface.triangles[triangle_index][0])
picked_data = list(selected_local_conn.matrix[vertex_index].toarray().squeeze())
result = []
if surface.number_of_split_slices <= 1:
result.append(picked_data)
else:
for slice_number in range(surface.number_of_split_slices):
start_idx, end_idx = surface.get_slice_vertex_boundaries(slice_number)
result.append(picked_data[start_idx:end_idx])
result = {'data': json.dumps(result)}
return result
def import_surface_obj(user, project, obj_path, surface_type):
# Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter')
form = ObjSurfaceImporterForm()
form.fill_from_post({'_data_file': Part(obj_path, HeaderMap({}), ''),
'_surface_type': surface_type,
'_Data_Subject': 'John Doe'
})
form.data_file.data = obj_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer, user, project.id, view_model=view_model)
data_types = FlowService().get_available_datatypes(project.id, SurfaceIndex)[0]
assert 1, len(data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None, "Surface should not be None"
return surface
def test_remove_used_surface(self):
"""
Tries to remove an used surface
"""
mapping, mapping_count = self.flow_service.get_available_datatypes(
self.test_project.id, RegionMapping)
assert 1 == mapping_count, "There should be one Mapping."
mapping_gid = mapping[0][2]
mapping = ABCAdapter.load_entity_by_gid(mapping_gid)
surface = dao.get_datatype_by_gid(mapping.surface.gid)
assert surface.gid == mapping.surface.gid, "The surfaces should have the same GID"
try:
self.project_service.remove_datatype(self.test_project.id,
surface.gid)
raise AssertionError(
"The surface should still be used by a RegionMapping " +
str(surface.gid))
except RemoveDataTypeException:
#OK, do nothing
pass
res = dao.get_datatype_by_gid(surface.gid)
assert surface.id == res.id, "A used surface was deleted"
def _importSurface(self, import_file_path=None):
"""
This method is used for importing data in GIFIT format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter')
args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: ""}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
surface = CorticalSurface()
data_types = FlowService().get_available_datatypes(self.test_project.id,
surface.module + "." + surface.type)[0]
assert 1, len(data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None == "TimeSeries should not be none"
return surface
def get_template_from_context(self):
"""
Return the parameters for the local connectivity in case one is stored in context. Load the entity
and use it to populate the defaults from the interface accordingly.
"""
context = common.get_from_session(KEY_LCONN_CONTEXT)
selected_local_conn = ABCAdapter.load_entity_by_gid(context.selected_entity)
cutoff = selected_local_conn.cutoff
equation = selected_local_conn.equation
surface = selected_local_conn.surface
default_dict = {'surface': surface.gid, 'cutoff': cutoff}
if equation is not None:
equation_type = equation.__class__.__name__
default_dict['equation'] = equation_type
for param in equation.parameters:
prepared_name = 'equation_parameters_option_' + str(equation_type)
prepared_name = prepared_name + '_parameters_parameters_' + str(param)
default_dict[prepared_name] = equation.parameters[param]
else:
msg = "There is no equation specified for this local connectivity. "
msg += "The default equation is displayed into the spatial field."
self.logger.warning(msg)
common.set_info_message(msg)
default_dict[DataTypeMetaData.KEY_TAG_1] = selected_local_conn.user_tag_1
input_list = self.get_creator_and_interface(LOCAL_CONN_CREATOR_MODULE,
LOCAL_CONN_CREATOR_CLASS, LocalConnectivity(),
lock_midpoint_for_eq=[1])[1]
input_list = self._add_extra_fields_to_interface(input_list)
input_list = InputTreeManager.fill_defaults(input_list, default_dict)
template_specification = {'inputList': input_list, common.KEY_PARAMETERS_CONFIG: False,
'equationViewerUrl': '/spatial/localconnectivity/get_equation_chart',
'equationsPrefixes': json.dumps(self.plotted_equations_prefixes)}
return template_specification
def _review_operation_inputs_for_adapter_model(form_fields, form_model,
view_model):
changed_attr = {}
inputs_datatypes = []
for field in form_fields:
if not hasattr(view_model, field.name):
continue
attr_vm = getattr(view_model, field.name)
if type(field) == TraitUploadField:
attr_vm = os.path.basename(attr_vm)
if isinstance(field, TraitDataTypeSelectField):
data_type = None
if attr_vm:
data_type = ABCAdapter.load_entity_by_gid(attr_vm)
changed_attr[
field.
label] = data_type.display_name if data_type else "None"
inputs_datatypes.append(data_type)
else:
attr_default = None
if hasattr(form_model, field.name):
attr_default = getattr(form_model, field.name)
if attr_vm != attr_default:
if isinstance(attr_vm, float) or isinstance(
attr_vm, int) or isinstance(attr_vm, str):
changed_attr[field.label] = attr_vm
elif isinstance(attr_vm, tuple) or isinstance(attr_vm, list):
changed_attr[field.label] = ', '.join(
[str(sub_attr) for sub_attr in attr_vm])
else:
# All HasTraits instances will show as being different than default, even if the same!! Is this ok?
changed_attr[field.label] = str(attr_vm)
return inputs_datatypes, changed_attr
def set_stimulus(self, **data):
session_stored_simulator = common.get_from_session(
common.KEY_SIMULATOR_CONFIG)
is_simulator_copy = common.get_from_session(
common.KEY_IS_SIMULATOR_COPY)
is_simulator_load = common.get_from_session(
common.KEY_IS_SIMULATOR_LOAD)
if cherrypy.request.method == 'POST':
is_simulator_copy = False
stimuli_fragment = SimulatorStimulusFragment(
'',
common.get_current_project().id,
session_stored_simulator.is_surface_simulation)
stimuli_fragment.fill_from_post(data)
stimulus_gid = stimuli_fragment.stimulus.value
if stimulus_gid != None:
stimulus_index = ABCAdapter.load_entity_by_gid(stimulus_gid)
stimulus = h5.load_from_index(stimulus_index)
session_stored_simulator.stimulus = stimulus
model_fragment = SimulatorModelFragment(
'',
common.get_current_project().id)
model_fragment.fill_from_trait(session_stored_simulator)
dict_to_render = copy.deepcopy(self.dict_to_render)
dict_to_render[self.FORM_KEY] = model_fragment
dict_to_render[self.ACTION_KEY] = '/burst/set_model'
dict_to_render[self.PREVIOUS_ACTION_KEY] = '/burst/set_stimulus'
dict_to_render[self.IS_MODEL_FRAGMENT_KEY] = True
dict_to_render[self.IS_COPY] = is_simulator_copy
dict_to_render[self.IS_LOAD] = is_simulator_load
dict_to_render[
self.
IS_SURFACE_SIMULATION_KEY] = session_stored_simulator.is_surface_simulation
return dict_to_render
def _import(self, import_file_path=None):
"""
This method is used for importing data in NIFIT format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
group = dao.find_group('tvb.adapters.uploaders.nifti_importer', 'NIFTIImporter')
importer = ABCAdapter.build_adapter(group)
args = {'data_file': import_file_path, DataTypeMetaData.KEY_SUBJECT: "bla bla",
'apply_corrections': False, 'connectivity': None}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user, self.test_project.id, **args)
time_series = TimeSeries()
data_types = FlowService().get_available_datatypes(self.test_project.id,
time_series.module + "." + time_series.type)[0]
self.assertEqual(1, len(data_types), "Project should contain only one data type.")
time_series = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(time_series is not None, "TimeSeries should not be none")
return time_series
def test_import_nii_without_time_dimension(self):
"""
This method tests import of a NIFTI file.
"""
structural_mri_index = self._import(self.NII_FILE)
assert "T1" == structural_mri_index.weighting
structural_mri = h5.load_from_index(structural_mri_index)
data_shape = structural_mri.array_data.shape
assert 3 == len(data_shape)
assert 64 == data_shape[0]
assert 64 == data_shape[1]
assert 10 == data_shape[2]
volume_index = ABCAdapter.load_entity_by_gid(
structural_mri_index.volume_gid)
assert volume_index is not None
volume = h5.load_from_index(volume_index)
assert numpy.equal(self.DEFAULT_ORIGIN, volume.origin).all()
assert numpy.equal([3.0, 3.0, 3.0], volume.voxel_size).all()
assert self.UNKNOWN_STR == volume.voxel_unit
def step_2(self, **kwargs):
"""
Generate the html for the second step of the local connectivity page.
:param kwargs: not actually used, but parameters are still submitted from UI since we just\
use the same js function for this.
"""
current_lconn = common.get_from_session(KEY_LCONN)
left_side_form = LocalConnectivitySelectorForm(
project_id=common.get_current_project().id)
left_side_form.existentEntitiesSelect.data = current_lconn.gid.hex
template_specification = dict(title="Surface - Local Connectivity")
template_specification[
'mainContent'] = 'spatial/local_connectivity_step2_main'
template_specification[
'existentEntitiesInputList'] = self.render_adapter_form(
left_side_form)
template_specification['loadExistentEntityUrl'] = LOAD_EXISTING_URL
template_specification['resetToDefaultUrl'] = RELOAD_DEFAULT_PAGE_URL
template_specification[
'next_step_url'] = '/spatial/localconnectivity/step_1'
msg, _ = common.get_message_from_session()
template_specification['displayedMessage'] = msg
if current_lconn is not None:
selected_local_conn = ABCAdapter.load_entity_by_gid(
current_lconn.gid.hex)
template_specification.update(
self.display_surface(selected_local_conn.fk_surface_gid))
template_specification['no_local_connectivity'] = False
template_specification[
'minValue'] = selected_local_conn.matrix_non_zero_min
template_specification[
'maxValue'] = selected_local_conn.matrix_non_zero_max
else:
template_specification['no_local_connectivity'] = True
template_specification[common.KEY_PARAMETERS_CONFIG] = False
return self.fill_default_attributes(template_specification)
def _importSurface(self, import_file_path=None):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_surface_importer',
'ZIPSurfaceImporter')
args = {
'uploaded': import_file_path,
'surface_type': OUTER_SKULL,
'zero_based_triangles': True,
DataTypeMetaData.KEY_SUBJECT: "John"
}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(
self.test_project.id, SkullSkin)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None, "Surface should not be None"
return surface
def load_surface_stimulus(self, surface_stimulus_gid, from_step):
"""
Loads the interface for the selected surface stimulus.
"""
context = base.get_from_session(KEY_SURFACE_CONTEXT)
selected_surface_stimulus = ABCAdapter.load_entity_by_gid(surface_stimulus_gid)
temporal_eq = selected_surface_stimulus.temporal
spatial_eq = selected_surface_stimulus.spatial
surface = selected_surface_stimulus.surface
focal_points_surface = selected_surface_stimulus.focal_points_surface
focal_points_triangles = selected_surface_stimulus.focal_points_triangles
temporal_eq_type = temporal_eq.__class__.__name__
spatial_eq_type = spatial_eq.__class__.__name__
default_dict = {'temporal': temporal_eq_type, 'spatial': spatial_eq_type,
'surface': surface.gid, 'focal_points_surface': json.dumps(focal_points_surface),
'focal_points_triangles': json.dumps(focal_points_triangles)}
for param in temporal_eq.parameters:
prepared_name = 'temporal_parameters_option_' + str(temporal_eq_type)
prepared_name = prepared_name + '_parameters_parameters_' + str(param)
default_dict[prepared_name] = str(temporal_eq.parameters[param])
for param in spatial_eq.parameters:
prepared_name = 'spatial_parameters_option_' + str(spatial_eq_type) + '_parameters_parameters_' + str(param)
default_dict[prepared_name] = str(spatial_eq.parameters[param])
default_dict[DataTypeMetaData.KEY_TAG_1] = selected_surface_stimulus.user_tag_1
input_list = self.get_creator_and_interface(SURFACE_STIMULUS_CREATOR_MODULE,
SURFACE_STIMULUS_CREATOR_CLASS, StimuliSurface(),
lock_midpoint_for_eq=[1])[1]
input_list = ABCAdapter.fill_defaults(input_list, default_dict)
context.reset()
context.update_from_interface(input_list)
context.equation_kwargs[DataTypeMetaData.KEY_TAG_1] = selected_surface_stimulus.user_tag_1
context.set_active_stimulus(surface_stimulus_gid)
return self.do_step(from_step)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
"""
Return a Tuple with Project entity and Operations for current Project.
:param project_id: Current Project Identifier
:param applied_filters: Filters to apply on Operations
:param current_page: Number for current page in operations
"""
selected_project = self.find_project(project_id)
total_filtered = self.count_filtered_operations(project_id, applied_filters)
pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
total_ops_nr = self.count_filtered_operations(project_id)
start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
if current_ops is None:
return selected_project, 0, [], 0
operations = []
view_categ_id = dao.get_visualisers_categories()[0].id
for one_op in current_ops:
try:
result = {}
if one_op[0] != one_op[1]:
result["id"] = str(one_op[0]) + "-" + str(one_op[1])
else:
result["id"] = str(one_op[0])
burst = dao.get_burst_for_operation_id(one_op[0])
result["burst_name"] = burst.name if burst else '-'
result["count"] = one_op[2]
result["gid"] = one_op[13]
if one_op[3] is not None and one_op[3]:
try:
operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0]
result["group"] = operation_group.name
result["group"] = result["group"].replace("_", " ")
result["operation_group_id"] = operation_group.id
datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
result["datatype_group_gid"] = datatype_group.gid
result["gid"] = operation_group.gid
## Filter only viewers for current DataTypeGroup entity:
result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
except Exception:
self.logger.exception("We will ignore group on entity:" + str(one_op))
result["datatype_group_gid"] = None
else:
result['group'] = None
result['datatype_group_gid'] = None
result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
result["user"] = dao.get_user_by_id(one_op[5])
if type(one_op[6]) is str:
result["create"] = string2date(str(one_op[6]))
else:
result["create"] = one_op[6]
if type(one_op[7]) is str:
result["start"] = string2date(str(one_op[7]))
else:
result["start"] = one_op[7]
if type(one_op[8]) is str:
result["complete"] = string2date(str(one_op[8]))
else:
result["complete"] = one_op[8]
if result["complete"] is not None and result["start"] is not None:
result["duration"] = format_timedelta(result["complete"] - result["start"])
result["status"] = one_op[9]
result["additional"] = one_op[10]
result["visible"] = True if one_op[11] > 0 else False
result['operation_tag'] = one_op[12]
result['figures'] = None
if not result['group']:
datatype_results = dao.get_results_for_operation(result['id'])
result['results'] = []
for dt in datatype_results:
dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
if dt_loaded:
result['results'].append(dt_loaded)
else:
self.logger.warning("Could not retrieve datatype %s" % str(dt))
operation_figures = dao.get_figures_for_operation(result['id'])
# Compute the full path to the figure / image on disk
for figure in operation_figures:
figures_folder = self.structure_helper.get_images_folder(figure.project.name)
figure_full_path = os.path.join(figures_folder, figure.file_path)
# Compute the path available from browser
figure.figure_path = utils.path2url_part(figure_full_path)
result['figures'] = operation_figures
else:
result['results'] = None
operations.append(result)
except Exception:
## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
self.logger.exception("Could not prepare operation for display:" + str(one_op))
return selected_project, total_ops_nr, operations, pages_no
def test_import_export(self):
"""
Test the import/export mechanism for a project structure.
The project contains the following data types: Connectivity, Surface, MappedArray and ValueWrapper.
"""
result = self.get_all_datatypes()
expected_results = {}
for one_data in result:
expected_results[one_data.gid] = (one_data.module, one_data.type)
#create an array mapped in DB
data = {'param_1': 'some value'}
OperationService().initiate_prelaunch(self.operation,
self.adapter_instance, {},
**data)
inserted = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
assert 1 == inserted, "Problems when inserting data"
#create a value wrapper
value_wrapper = self._create_value_wrapper()
count_operations = dao.get_filtered_operations(self.test_project.id,
None,
is_count=True)
assert 2 == count_operations, "Invalid ops number before export!"
# Export project as ZIP
self.zip_path = ExportManager().export_project(self.test_project)
assert self.zip_path is not None, "Exported file is none"
# Remove the original project
self.project_service.remove_project(self.test_project.id)
result, lng_ = self.project_service.retrieve_projects_for_user(
self.test_user.id)
assert 0 == len(result), "Project Not removed!"
assert 0 == lng_, "Project Not removed!"
# Now try to import again project
self.import_service.import_project_structure(self.zip_path,
self.test_user.id)
result = self.project_service.retrieve_projects_for_user(
self.test_user.id)[0]
assert len(result) == 1, "There should be only one project."
assert result[
0].name == "GeneratedProject", "The project name is not correct."
assert result[
0].description == "test_desc", "The project description is not correct."
self.test_project = result[0]
count_operations = dao.get_filtered_operations(self.test_project.id,
None,
is_count=True)
#1 op. - import cff; 2 op. - save the array wrapper;
assert 2 == count_operations, "Invalid ops number after export and import !"
for gid in expected_results:
datatype = dao.get_datatype_by_gid(gid)
assert datatype.module == expected_results[gid][
0], 'DataTypes not imported correctly'
assert datatype.type == expected_results[gid][
1], 'DataTypes not imported correctly'
#check the value wrapper
new_val = self.flow_service.get_available_datatypes(
self.test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
assert 1 == len(new_val), "One !=" + str(len(new_val))
new_val = ABCAdapter.load_entity_by_gid(new_val[0][2])
assert value_wrapper.data_value == new_val.data_value, "Data value incorrect"
assert value_wrapper.data_type == new_val.data_type, "Data type incorrect"
assert value_wrapper.data_name == new_val.data_name, "Data name incorrect"
