def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user)
project_to_link = model.Project("Link", self.test_user.id, "descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
dao.store_entity(model.Links(exact_data.id, project_to_link.id))
self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Initialization problem!")
operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id
op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id))
self.assertTrue(os.path.exists(op_folder))
sub_files = os.listdir(op_folder)
self.assertEqual(2, len(sub_files))
### Validate that no more files are created than needed.
self.project_service._remove_project_node_files(inserted_project.id, gid)
sub_files = os.listdir(op_folder)
self.assertEqual(1, len(sub_files))
### operation.xml file should still be there
op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1))
sub_files = os.listdir(op_folder)
self.assertEqual(2, len(sub_files))
self.assertTrue(dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links")
self.project_service._remove_project_node_files(project_to_link.id, gid)
self.assertTrue(dao.get_datatype_by_gid(gid) is None)
sub_files = os.listdir(op_folder)
self.assertEqual(1, len(sub_files))
def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, gid_op = self._create_value_wrapper(self.test_user)
project_to_link = model.Project("Link", self.test_user.id, "descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
dao.store_entity(model.Links(exact_data.id, project_to_link.id))
assert dao.get_datatype_by_gid(gid) is not None, "Initialization problem!"
operation_id = dao.get_generic_entity(model.Operation, gid_op, 'gid')[0].id
op_folder = self.structure_helper.get_project_folder("test_proj", str(operation_id))
assert os.path.exists(op_folder)
sub_files = os.listdir(op_folder)
assert 2 == len(sub_files)
### Validate that no more files are created than needed.
if(dao.get_system_user() is None):
dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None))
self.project_service._remove_project_node_files(inserted_project.id, gid)
sub_files = os.listdir(op_folder)
assert 1 == len(sub_files)
### operation.xml file should still be there
op_folder = self.structure_helper.get_project_folder("Link", str(operation_id + 1))
sub_files = os.listdir(op_folder)
assert 2 == len(sub_files)
assert dao.get_datatype_by_gid(gid) is not None, "Data should still be in DB, because of links"
self.project_service._remove_project_node_files(project_to_link.id, gid)
assert dao.get_datatype_by_gid(gid) is None
sub_files = os.listdir(op_folder)
assert 1 == len(sub_files)
def test_remove_used_surface(self):
"""
Tries to remove an used surface
"""
mapping = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.surfaces.RegionMapping")
self.assertEquals(len(mapping), 1, "There should be one Mapping.")
mapping_gid = mapping[0][2]
mapping = ABCAdapter.load_entity_by_gid(mapping_gid)
#delete surface
surfaces = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.surfaces.CorticalSurface")
self.assertTrue(len(surfaces) > 0, "At least one Cortex expected")
surface = dao.get_datatype_by_gid(mapping.surface.gid)
self.assertEqual(surface.gid, mapping.surface.gid,
"The surfaces should have the same GID")
try:
self.project_service.remove_datatype(self.test_project.id,
surface.gid)
self.fail(
"The surface is still used by a RegionMapping. It should not be possible to remove it."
)
except RemoveDataTypeException:
#OK, do nothing
pass
res = dao.get_datatype_by_gid(surface.gid)
self.assertEqual(surface.id, res.id, "A used surface was deleted")
def populate_surface_fields(self, time_series_index):
"""
To be overwritten for populating fields: one_to_one_map/connectivity/region_map/surface fields
"""
self.one_to_one_map = isinstance(time_series_index, TimeSeriesSurfaceIndex)
if self.one_to_one_map:
self.PAGE_SIZE /= 10
surface_gid = time_series_index.fk_surface_gid
surface_index = dao.get_datatype_by_gid(surface_gid)
region_map_indexes = dao.get_generic_entity(RegionMappingIndex, surface_gid, 'fk_surface_gid')
if len(region_map_indexes) < 1:
region_map_index = None
connectivity_index = None
else:
region_map_index = region_map_indexes[0]
connectivity_index = dao.get_datatype_by_gid(region_map_index.fk_connectivity_gid)
else:
connectivity_index = dao.get_datatype_by_gid(time_series_index.fk_connectivity_gid)
if time_series_index.fk_region_mapping_gid:
region_map_index = dao.get_datatype_by_gid(time_series_index.fk_region_mapping_gid)
else:
region_map_indexes = dao.get_generic_entity(RegionMappingIndex, connectivity_index.gid,
'fk_connectivity_gid')
region_map_index = region_map_indexes[0]
surface_index = dao.get_datatype_by_gid(region_map_index.fk_surface_gid)
self.connectivity_index = connectivity_index
self.region_map_gid = None if region_map_index is None else region_map_index.gid
self.surface_gid = None if surface_index is None else surface_index.gid
self.surface_h5 = None if surface_index is None else h5.h5_file_for_index(surface_index)
def test_remove_project_node(self):
"""
Test removing of a node from a project.
"""
inserted_project, gid, op = TestFactory.create_value_wrapper(self.test_user)
project_to_link = model_project.Project("Link", self.test_user.id, "descript")
project_to_link = dao.store_entity(project_to_link)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Initialization problem!"
link = dao.store_entity(model_datatype.Links(exact_data.id, project_to_link.id))
vw_h5_path = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path)
if dao.get_system_user() is None:
dao.store_entity(model_operation.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME,
TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True,
None))
self.project_service._remove_project_node_files(inserted_project.id, gid, [link])
assert not os.path.exists(vw_h5_path)
exact_data = dao.get_datatype_by_gid(gid)
assert exact_data is not None, "Data should still be in DB, because of links"
vw_h5_path_new = h5.path_for_stored_index(exact_data)
assert os.path.exists(vw_h5_path_new)
assert vw_h5_path_new != vw_h5_path
self.project_service._remove_project_node_files(project_to_link.id, gid, [])
assert dao.get_datatype_by_gid(gid) is None
def launch(self, view_model):
# type: (BaseSurfaceViewerModel) -> dict
connectivity_measure_index = self.load_entity_by_gid(
view_model.connectivity_measure)
cm_connectivity_gid = connectivity_measure_index.fk_connectivity_gid
cm_connectivity_index = dao.get_datatype_by_gid(cm_connectivity_gid)
region_map_index = None
rm_connectivity_index = None
if view_model.region_map:
region_map_index = self.load_entity_by_gid(view_model.region_map)
rm_connectivity_gid = region_map_index.fk_connectivity_gid
rm_connectivity_index = dao.get_datatype_by_gid(
rm_connectivity_gid)
if not region_map_index or rm_connectivity_index.number_of_regions != cm_connectivity_index.number_of_regions:
region_maps = dao.get_generic_entity(RegionMappingIndex,
cm_connectivity_gid,
'fk_connectivity_gid')
if region_maps:
region_map_index = region_maps[0]
surface_gid = region_map_index.fk_surface_gid
surface_viewer_model = SurfaceViewerModel(
surface=surface_gid,
region_map=region_map_index.gid,
connectivity_measure=view_model.connectivity_measure,
shell_surface=view_model.shell_surface)
surface_viewer_model.title = self._ui_name
return SurfaceViewer.launch(self, surface_viewer_model)
def _store_imported_datatypes_in_db(self, project, all_datatypes):
# type: (Project, dict) -> int
sorted_dts = sorted(
all_datatypes.items(),
key=lambda dt_item: dt_item[1].create_date or datetime.now())
count = 0
for dt_path, datatype in sorted_dts:
datatype_already_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_already_in_tvb:
self.store_datatype(datatype, dt_path)
count += 1
else:
AlgorithmService.create_link([datatype_already_in_tvb.id],
project.id)
file_path = h5.h5_file_for_index(datatype).path
h5_class = H5File.h5_class_from_file(file_path)
reference_list = h5_class(file_path).gather_references()
for _, reference_gid in reference_list:
if not reference_gid:
continue
ref_index = dao.get_datatype_by_gid(reference_gid.hex)
if ref_index is None:
os.remove(file_path)
dao.remove_entity(datatype.__class__, datatype.id)
raise MissingReferenceException(
'Imported file depends on datatypes that do not exist. Please upload '
'those first!')
return count
def test_set_visibility_datatype(self):
"""
Set datatype visibility to true and false and check results are updated.
"""
datatype = DatatypesFactory().create_datatype_with_storage()
self.assertTrue(datatype.visible)
self.project_c.set_visibility('datatype', datatype.gid, 'False')
datatype = dao.get_datatype_by_gid(datatype.gid)
self.assertFalse(datatype.visible)
self.project_c.set_visibility('datatype', datatype.gid, 'True')
datatype = dao.get_datatype_by_gid(datatype.gid)
self.assertTrue(datatype.visible)
def test_set_visibility_datatype(self, dummy_datatype_index_factory):
"""
Set datatype visibility to true and false and check results are updated.
"""
datatype = dummy_datatype_index_factory()
assert datatype.visible
self.project_c.set_visibility('datatype', datatype.gid, 'False')
datatype = dao.get_datatype_by_gid(datatype.gid)
assert not datatype.visible
self.project_c.set_visibility('datatype', datatype.gid, 'True')
datatype = dao.get_datatype_by_gid(datatype.gid)
assert datatype.visible
def test_set_visibility_datatype(self):
"""
Set datatype visibility to true and false and check results are updated.
"""
datatype = DatatypesFactory().create_datatype_with_storage()
self.assertTrue(datatype.visible)
self.project_c.set_visibility('datatype', datatype.gid, 'False')
datatype = dao.get_datatype_by_gid(datatype.gid)
self.assertFalse(datatype.visible)
self.project_c.set_visibility('datatype', datatype.gid, 'True')
datatype = dao.get_datatype_by_gid(datatype.gid)
self.assertTrue(datatype.visible)
def deserialize_simulator(simulator_gid, storage_path):
simulator_in_path = h5.path_for(storage_path, SimulatorH5,
simulator_gid)
simulator_in = Simulator()
with SimulatorH5(simulator_in_path) as simulator_in_h5:
simulator_in_h5.load_into(simulator_in)
connectivity_gid = simulator_in_h5.connectivity.load()
stimulus_gid = simulator_in_h5.stimulus.load()
simulation_state_gid = simulator_in_h5.simulation_state.load()
conn_index = dao.get_datatype_by_gid(connectivity_gid.hex)
conn = h5.load_from_index(conn_index)
simulator_in.connectivity = conn
if simulator_in.surface:
cortex_path = h5.path_for(storage_path, CortexH5,
simulator_in.surface.gid)
with CortexH5(cortex_path) as cortex_h5:
local_conn_gid = cortex_h5.local_connectivity.load()
region_mapping_gid = cortex_h5.region_mapping_data.load()
region_mapping_index = dao.get_datatype_by_gid(
region_mapping_gid.hex)
region_mapping_path = h5.path_for_stored_index(
region_mapping_index)
region_mapping = RegionMapping()
with RegionMappingH5(region_mapping_path) as region_mapping_h5:
region_mapping_h5.load_into(region_mapping)
region_mapping.gid = region_mapping_h5.gid.load()
surf_gid = region_mapping_h5.surface.load()
surf_index = dao.get_datatype_by_gid(surf_gid.hex)
surf_h5 = h5.h5_file_for_index(surf_index)
surf = CorticalSurface()
surf_h5.load_into(surf)
surf_h5.close()
region_mapping.surface = surf
simulator_in.surface.region_mapping_data = region_mapping
if local_conn_gid:
local_conn_index = dao.get_datatype_by_gid(local_conn_gid.hex)
local_conn = h5.load_from_index(local_conn_index)
simulator_in.surface.local_connectivity = local_conn
if stimulus_gid:
stimulus_index = dao.get_datatype_by_gid(stimulus_gid.hex)
stimulus = h5.load_from_index(stimulus_index)
simulator_in.stimulus = stimulus
return simulator_in, simulation_state_gid
def test_remove_value_wrapper(self):
"""
Test the deletion of a value wrapper dataType
"""
count_vals = self.count_all_entities(ValueWrapperIndex)
assert 0 == count_vals, "There should be no value wrapper"
value_wrapper_gid = TestFactory.create_value_wrapper(self.test_user, self.test_project)[1]
res = dao.get_datatype_by_gid(value_wrapper_gid)
assert res is not None, "The value wrapper was not created."
self.project_service.remove_datatype(self.test_project.id, value_wrapper_gid)
res = dao.get_datatype_by_gid(value_wrapper_gid)
assert res is None, "The value wrapper was not deleted."
def update_dt(dt_id, new_create_date):
dt = dao.get_datatype_by_id(dt_id)
dt.create_date = new_create_date
dao.store_entity(dt)
# Update MetaData in H5 as well.
dt = dao.get_datatype_by_gid(dt.gid)
dt.persist_full_metadata()
def _get_launchable_algorithms(self, datatype_gid, categories):
datatype_instance = dao.get_datatype_by_gid(datatype_gid)
data_class = datatype_instance.__class__
all_compatible_classes = [data_class.__name__]
for one_class in getmro(data_class):
if issubclass(
one_class, MappedType
) and one_class.__name__ not in all_compatible_classes:
all_compatible_classes.append(one_class.__name__)
self.logger.debug("Searching in categories: " + str(categories) +
" for classes " + str(all_compatible_classes))
categories_ids = [categ.id for categ in categories]
launchable_adapters = dao.get_applicable_adapters(
all_compatible_classes, categories_ids)
filtered_adapters = []
for stored_adapter in launchable_adapters:
filter_chain = FilterChain.from_json(
stored_adapter.datatype_filter)
if not filter_chain or filter_chain.get_python_filter_equivalent(
datatype_instance):
filtered_adapters.append(stored_adapter)
return datatype_instance, filtered_adapters
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
dt.persist_full_metadata()
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
# update the datatype or datatype group.
set_visibility(datatype)
def test_remove_used_surface(self):
"""
Tries to remove an used surface
"""
filter = FilterChain(fields=[FilterChain.datatype + '.surface_type'],
operations=["=="],
values=[CORTICAL])
mapping = try_get_last_datatype(self.test_project.id,
RegionMappingIndex)
surface = try_get_last_datatype(self.test_project.id, SurfaceIndex,
filter)
assert mapping is not None, "There should be one Mapping."
assert surface is not None, "There should be one Costical Surface."
assert surface.gid == mapping.fk_surface_gid, "The surfaces should have the same GID"
try:
self.project_service.remove_datatype(self.test_project.id,
surface.gid)
raise AssertionError(
"The surface should still be used by a RegionMapping " +
str(surface.gid))
except RemoveDataTypeException:
# OK, do nothing
pass
res = dao.get_datatype_by_gid(surface.gid)
assert surface.id == res.id, "A used surface was deleted"
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def create_timeseries(self, connectivity, ts_type=None, sensors=None):
"""
Create a stored TimeSeries entity.
"""
operation, _, storage_path = self.__create_operation()
if ts_type == "EEG":
time_series = TimeSeriesEEG(storage_path=storage_path,
sensors=sensors)
else:
rm = dao.get_generic_entity(RegionMapping, connectivity.gid,
'_connectivity')
if len(rm) < 1:
rm = None
else:
rm = rm[0]
time_series = TimeSeriesRegion(storage_path=storage_path,
connectivity=connectivity,
region_mapping=rm)
data = numpy.random.random((10, 10, 10, 10))
time_series.write_data_slice(data)
time_series.write_time_slice(numpy.arange(10))
adapter_instance = StoreAdapter([time_series])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
time_series = dao.get_datatype_by_gid(time_series.gid)
return time_series
def configure(self, simulator_gid):
"""
Make preparations for the adapter launch.
"""
self.log.debug("%s: Instantiating requested simulator..." % str(self))
simulator_service = SimulatorService()
self.algorithm, connectivity_gid, simulation_state_gid = simulator_service.deserialize_simulator(
simulator_gid, self.storage_path)
self.branch_simulation_state_gid = simulation_state_gid
# for monitor in self.algorithm.monitors:
# if issubclass(monitor, Projection):
# # TODO: add a service that loads a RM with Surface and Connectivity
# pass
connectivity_index = dao.get_datatype_by_gid(connectivity_gid.hex)
connectivity = h5.load_from_index(connectivity_index)
connectivity.gid = connectivity_gid
self.algorithm.connectivity = connectivity
self.simulation_length = self.algorithm.simulation_length
self.log.debug("%s: Initializing storage..." % str(self))
try:
self.algorithm.preconfigure()
except ValueError as err:
raise LaunchException(
"Failed to configure simulator due to invalid Input Values. It could be because "
"of an incompatibility between different version of TVB code.",
err)
def index(self):
des = SerializationManager(common.get_from_session(common.KEY_SIMULATOR_CONFIG))
connectivity = des.conf.connectivity
conn_idx = dao.get_datatype_by_gid(connectivity.hex)
model = des.conf.model
integrator = des.conf.integrator
state_vars = model.state_variables
noise_values = self.init_noise_config_values(model, integrator, conn_idx)
initial_noise = self.group_noise_array_by_state_var(noise_values, state_vars, conn_idx.number_of_regions)
current_project = common.get_current_project()
file_handler = FilesHelper()
conn_path = file_handler.get_project_folder(current_project, str(conn_idx.fk_from_operation))
params = ConnectivityViewer.get_connectivity_parameters(conn_idx, conn_path)
params.update({
'title': 'Noise configuration',
'mainContent': 'burst/noise',
'isSingleMode': True,
'submit_parameters_url': '/burst/noise/submit',
'stateVars': state_vars,
'stateVarsJson' : json.dumps(state_vars),
'noiseInputValues' : initial_noise[0],
'initialNoiseValues': json.dumps(initial_noise)
})
return self.fill_default_attributes(params, 'regionmodel')
def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
dt_entity = dao.get_datatype_by_gid(dt.gid)
if dt_entity is None:
self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt))
continue
# Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.display_type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username
data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name)
def _compute_measure_params(self, region_mapping_volume, measure,
data_slice):
# prepare the url that will project the measure onto the region volume map
measure_h5_class, measure_h5_path = self._load_h5_of_gid(measure.gid)
measure_h5 = measure_h5_class(measure_h5_path)
min_value, max_value = measure_h5.get_min_max_values()
measure_shape = measure_h5.array_data.shape
if not data_slice:
conn_index = dao.get_datatype_by_gid(
region_mapping_volume.connectivity.load().hex)
data_slice = self.get_default_slice(measure_shape,
conn_index.number_of_regions)
data_slice = slice_str(data_slice)
url_volume_data = URLGenerator.build_url(
self.stored_adapter.id,
'get_mapped_array_volume_view',
region_mapping_volume.gid.load(),
parameter='')
url_volume_data += 'mapped_array_gid=' + measure.gid + ';mapped_array_slice=' + data_slice + ';'
return dict(minValue=min_value,
maxValue=max_value,
urlVolumeData=url_volume_data,
measureShape=slice_str(measure_shape),
measureSlice=data_slice)
def update_dt(dt_id, new_create_date):
dt = dao.get_datatype_by_id(dt_id)
dt.create_date = new_create_date
dao.store_entity(dt)
# Update MetaData in H5 as well.
dt = dao.get_datatype_by_gid(dt.gid)
dt.persist_full_metadata()
def _store_imported_datatypes_in_db(self, project, all_datatypes, dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id], project.id)
def __upgrade_datatype_list(self, datatypes):
"""
Upgrade a list of DataTypes to the current version.
:param datatypes: The list of DataTypes that should be upgraded.
:returns: (nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault) a two-tuple of integers representing
the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which
some kind of fault occurred
"""
nr_of_dts_upgraded_fine = 0
nr_of_dts_upgraded_fault = 0
for datatype in datatypes:
specific_datatype = dao.get_datatype_by_gid(datatype.gid,
load_lazy=False)
if isinstance(specific_datatype, MappedType):
try:
self.upgrade_file(
specific_datatype.get_storage_file_path(),
specific_datatype)
nr_of_dts_upgraded_fine += 1
except (MissingDataFileException,
FileVersioningException) as ex:
# The file is missing for some reason. Just mark the DataType as invalid.
datatype.invalid = True
dao.store_entity(datatype)
nr_of_dts_upgraded_fault += 1
self.log.exception(ex)
return nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault
def _store_imported_datatypes_in_db(self, project, all_datatypes,
dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(
datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id],
project.id)
def get_project_structure(self, project, visibility_filter, first_level, second_level, filter_value):
"""
Find all DataTypes (including the linked ones and the groups) relevant for the current project.
In case of a problem, will return an empty list.
"""
metadata_list = []
dt_list = dao.get_data_in_project(project.id, visibility_filter, filter_value)
for dt in dt_list:
# Prepare the DT results from DB, for usage in controller, by converting into DataTypeMetaData objects
data = {}
is_group = False
group_op = None
dt_entity = dao.get_datatype_by_gid(dt.gid)
if dt_entity is None:
self.logger.warning("Ignored entity (possibly removed DT class)" + str(dt))
continue
## Filter by dt.type, otherwise Links to individual DT inside a group will be mistaken
if dt.type == "DataTypeGroup" and dt.parent_operation.operation_group is not None:
is_group = True
group_op = dt.parent_operation.operation_group
# All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_DATATYPE_ID] = dt.id
data[DataTypeMetaData.KEY_GID] = dt.gid
data[DataTypeMetaData.KEY_NODE_TYPE] = dt.type
data[DataTypeMetaData.KEY_STATE] = dt.state
data[DataTypeMetaData.KEY_SUBJECT] = str(dt.subject)
data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name
data[DataTypeMetaData.KEY_RELEVANCY] = dt.visible
data[DataTypeMetaData.KEY_LINK] = dt.parent_operation.fk_launched_in != project.id
data[DataTypeMetaData.KEY_TAG_1] = dt.user_tag_1 if dt.user_tag_1 else ''
data[DataTypeMetaData.KEY_TAG_2] = dt.user_tag_2 if dt.user_tag_2 else ''
data[DataTypeMetaData.KEY_TAG_3] = dt.user_tag_3 if dt.user_tag_3 else ''
data[DataTypeMetaData.KEY_TAG_4] = dt.user_tag_4 if dt.user_tag_4 else ''
data[DataTypeMetaData.KEY_TAG_5] = dt.user_tag_5 if dt.user_tag_5 else ''
# Operation related fields:
operation_name = CommonDetails.compute_operation_name(
dt.parent_operation.algorithm.algorithm_category.displayname,
dt.parent_operation.algorithm.displayname)
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = dt.parent_operation.algorithm.displayname
data[DataTypeMetaData.KEY_AUTHOR] = dt.parent_operation.user.username
data[DataTypeMetaData.KEY_OPERATION_TAG] = group_op.name if is_group else dt.parent_operation.user_group
data[DataTypeMetaData.KEY_OP_GROUP_ID] = group_op.id if is_group else None
completion_date = dt.parent_operation.completion_date
string_year = completion_date.strftime(MONTH_YEAR_FORMAT) if completion_date is not None else ""
string_month = completion_date.strftime(DAY_MONTH_YEAR_FORMAT) if completion_date is not None else ""
data[DataTypeMetaData.KEY_DATE] = date2string(completion_date) if (completion_date is not None) else ''
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = string_year
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = string_month
data[DataTypeMetaData.KEY_BURST] = dt._parent_burst.name if dt._parent_burst is not None else '-None-'
metadata_list.append(DataTypeMetaData(data, dt.invalid))
return StructureNode.metadata2tree(metadata_list, first_level, second_level, project.id, project.name)
def index(self):
current_user_id = common.get_logged_user().id
# In case the number of dynamics gets big we should add a filter in the ui.
dynamics = dao.get_dynamics_for_user(current_user_id)
if not dynamics:
return self.no_dynamics_page()
sim_config = common.get_from_session(common.KEY_SIMULATOR_CONFIG)
connectivity = sim_config.connectivity
if connectivity is None:
msg = 'You have to select a connectivity before setting up the region Model. '
common.set_error_message(msg)
raise ValueError(msg)
current_project = common.get_current_project()
file_handler = FilesHelper()
conn_idx = dao.get_datatype_by_gid(connectivity.hex)
conn_path = file_handler.get_project_folder(current_project, str(conn_idx.fk_from_operation))
params = ConnectivityViewer.get_connectivity_parameters(conn_idx, conn_path)
burst_config = common.get_from_session(common.KEY_BURST_CONFIG)
params.update({
'title': 'Model parameters',
'mainContent': 'burst/model_param_region',
'isSingleMode': True,
'submit_parameters_url': '/burst/modelparameters/regions/submit_model_parameters',
'dynamics': dynamics,
'dynamics_json': self._dynamics_json(dynamics),
'initial_dynamic_ids': burst_config.dynamic_ids
})
return self.fill_default_attributes(params, 'regionmodel')
def prepare_range_labels(operation_group, range_json):
"""
Prepare Range labels for display in UI.
When the current range_json is empty, returns None, [RANGE_MISSING_STRING], [RANGE_MISSING_STRING]
:param operation_group: model.OperationGroup instance
:param range_json: Stored JSON for for a given range
:return: String with current range label, Array of ranged numbers, Array of labels for current range
"""
contains_numbers, range_name, range_values = operation_group.load_range_numbers(
range_json)
if contains_numbers is None:
return None, range_values, range_values, True
if contains_numbers:
range_labels = range_values
else:
# when datatypes are in range, get the display name for those and use as labels.
range_labels = []
for data_gid in range_values:
range_labels.append(
dao.get_datatype_by_gid(data_gid).display_name)
return range_name, range_values, range_labels, contains_numbers
def build():
"""
Project src will have 3 datatypes, and a link to the VW from the dest project.
Project dest will have the derived VW and links
"""
# add a connectivity to src project and link it to dest project
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_96.zip')
conn = TestFactory.import_zip_connectivity(self.dst_user,
self.src_project,
zip_path, "John")
self.flow_service.create_link([conn.id], self.dest_project.id)
# in dest derive a ValueWrapper from the linked conn
vw_gid = TestFactory.create_value_wrapper(self.dst_user,
self.dest_project)[1]
vw = dao.get_datatype_by_gid(vw_gid)
# then link the time series in the src project
self.flow_service.create_link([vw.id], self.src_project.id)
assert 3 == len(dao.get_datatypes_in_project(self.src_project.id))
assert 1 == len(
dao.get_linked_datatypes_in_project(self.src_project.id))
assert 1 == len(dao.get_datatypes_in_project(self.dest_project.id))
assert 3 == len(
dao.get_linked_datatypes_in_project(self.dest_project.id))
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(model.DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception, excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
dt.persist_full_metadata()
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
# update the datatype or datatype group.
set_visibility(datatype)
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
def set_visibility(dt):
""" set visibility flag, persist in db and h5"""
dt.visible = is_visible
dt = dao.store_entity(dt)
h5_path = h5.path_for_stored_index(dt)
with H5File.from_file(h5_path) as f:
f.visible.store(is_visible)
def set_group_descendants_visibility(datatype_group_id):
datatypes_in_group = dao.get_datatypes_from_datatype_group(datatype_group_id)
for group_dt in datatypes_in_group:
set_visibility(group_dt)
datatype = dao.get_datatype_by_gid(datatype_gid)
if isinstance(datatype, DataTypeGroup): # datatype is a group
set_group_descendants_visibility(datatype.id)
datatype.visible = is_visible
dao.store_entity(datatype)
elif datatype.fk_datatype_group is not None: # datatype is member of a group
set_group_descendants_visibility(datatype.fk_datatype_group)
# the datatype to be updated is the parent datatype group
parent = dao.get_datatype_by_id(datatype.fk_datatype_group)
parent.visible = is_visible
dao.store_entity(parent)
else:
# update the single datatype.
set_visibility(datatype)
def test_full_import(self):
"""
Test that importing a CFF generates at least one DataType in DB.
"""
all_dt = self.get_all_datatypes()
self.assertEqual(0, len(all_dt))
TestFactory.import_cff(cff_path=self.VALID_CFF,
test_user=self.test_user,
test_project=self.test_project)
flow_service = FlowService()
### Check that at one Connectivity was persisted
gid_list = flow_service.get_available_datatypes(
self.test_project.id, 'tvb.datatypes.connectivity.Connectivity')
self.assertEquals(len(gid_list), 1)
### Check that at one RegionMapping was persisted
gid_list = flow_service.get_available_datatypes(
self.test_project.id, 'tvb.datatypes.surfaces.RegionMapping')
self.assertEquals(len(gid_list), 1)
### Check that at one LocalConnectivity was persisted
gids = flow_service.get_available_datatypes(
self.test_project.id, 'tvb.datatypes.surfaces.LocalConnectivity')
self.assertEquals(len(gids), 1)
connectivity = dao.get_datatype_by_gid(gids[0][2])
metadata = connectivity.get_metadata()
self.assertEqual(metadata['Cutoff'], '40.0')
self.assertEqual(metadata['Equation'], 'null')
self.assertFalse(metadata['Invalid'])
self.assertFalse(metadata['Is_nan'])
self.assertEqual(metadata['Type'], 'LocalConnectivity')
### Check that at 2 Surfaces were persisted
gid_list = flow_service.get_available_datatypes(
self.test_project.id, 'tvb.datatypes.surfaces_data.SurfaceData')
self.assertEquals(len(gid_list), 2)
def _store_imported_datatypes_in_db(self, project, all_datatypes,
dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
self.store_datatype(datatype)
else:
AlgorithmService.create_link([datatype_allready_in_tvb.id],
project.id)
def __upgrade_datatype_list(self, datatypes):
"""
Upgrade a list of DataTypes to the current version.
:param datatypes: The list of DataTypes that should be upgraded.
:returns: (nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault) a two-tuple of integers representing
the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which
some kind of fault occurred
"""
nr_of_dts_upgraded_fine = 0
nr_of_dts_upgraded_fault = 0
for datatype in datatypes:
specific_datatype = dao.get_datatype_by_gid(datatype.gid)
if isinstance(specific_datatype, MappedType):
try:
self.upgrade_file(specific_datatype.get_storage_file_path())
nr_of_dts_upgraded_fine += 1
except (MissingDataFileException, FileVersioningException) as ex:
# The file is missing for some reason. Just mark the DataType as invalid.
datatype.invalid = True
dao.store_entity(datatype)
nr_of_dts_upgraded_fault += 1
self.log.exception(ex)
return nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault
def _remove_project_node_files(self,
project_id,
gid,
skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
new_op = Operation(
dao.get_system_user().id, links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.parameters,
datatype.parent_operation.meta_data,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project).name
new_op_loaded = dao.get_operation_by_id(new_op.id)
self.structure_helper.write_operation_metadata(
new_op_loaded)
full_path = h5.path_for_stored_index(datatype)
self.structure_helper.move_datatype(
datatype, to_project, str(new_op.id), full_path)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
h5_path = h5.path_for_stored_index(datatype)
self.structure_helper.remove_datatype_file(h5_path)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException(
"Remove operation failed for unknown reasons.Please contact system administrator."
)
def test_import_export(self, user_factory, project_factory,
value_wrapper_factory):
"""
Test the import/export mechanism for a project structure.
The project contains the following data types: Connectivity, Surface, MappedArray and ValueWrapper.
"""
test_user = user_factory()
test_project = project_factory(test_user, "TestImportExport",
"test_desc")
zip_path = os.path.join(os.path.dirname(tvb_data.__file__),
'connectivity', 'connectivity_66.zip')
TestFactory.import_zip_connectivity(test_user, test_project, zip_path)
value_wrapper = value_wrapper_factory(test_user, test_project)
result = self.get_all_datatypes()
expected_results = {}
for one_data in result:
expected_results[one_data.gid] = (one_data.module, one_data.type)
# Export project as ZIP
self.zip_path = ExportManager().export_project(test_project)
assert self.zip_path is not None, "Exported file is none"
# Remove the original project
self.project_service.remove_project(test_project.id)
result, lng_ = self.project_service.retrieve_projects_for_user(
test_user.id)
assert 0 == len(result), "Project Not removed!"
assert 0 == lng_, "Project Not removed!"
# Now try to import again project
self.import_service.import_project_structure(self.zip_path,
test_user.id)
result = self.project_service.retrieve_projects_for_user(
test_user.id)[0]
assert len(result) == 1, "There should be only one project."
assert result[
0].name == "TestImportExport", "The project name is not correct."
assert result[
0].description == "test_desc", "The project description is not correct."
test_project = result[0]
count_operations = dao.get_filtered_operations(test_project.id,
None,
is_count=True)
# 1 op. - import conn; 2 op. - BCT Analyzer
assert 2 == count_operations, "Invalid ops number after export and import !"
for gid in expected_results:
datatype = dao.get_datatype_by_gid(gid)
assert datatype.module == expected_results[gid][
0], 'DataTypes not imported correctly'
assert datatype.type == expected_results[gid][
1], 'DataTypes not imported correctly'
# check the value wrapper
new_val = try_get_last_datatype(test_project.id, ValueWrapperIndex)
assert value_wrapper.data_value == new_val.data_value, "Data value incorrect"
assert value_wrapper.data_type == new_val.data_type, "Data type incorrect"
assert value_wrapper.data_name == new_val.data_name, "Data name incorrect"
def build_structure_for_datatype(datatype_gid):
datatype = dao.get_datatype_by_gid(datatype_gid)
is_group = dao.is_datatype_group(datatype_gid)
structure = NodeStructure(datatype_gid, datatype.type)
structure.data = NodeData.build_node_for_datatype(datatype.id, datatype.display_name, is_group=is_group)
return structure
def remove_datatype(self, project_id, datatype_gid, skip_validation=False):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid)
return
user = dao.get_user_for_datatype(datatype.id)
freed_space = datatype.disk_size or 0
is_datatype_group = False
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
freed_space = dao.get_datatype_group_disk_size(datatype.id)
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
datatype = dao.get_datatype_by_id(datatype.fk_datatype_group)
freed_space = dao.get_datatype_group_disk_size(datatype.id)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
self.logger.debug("Removing datatype group %s" % datatype)
data_list = dao.get_datatypes_from_datatype_group(datatype.id)
for adata in data_list:
self._remove_project_node_files(project_id, adata.gid, skip_validation)
if adata.fk_from_operation not in operations_set:
operations_set.append(adata.fk_from_operation)
datatype_group = dao.get_datatype_group_by_gid(datatype.gid)
dao.remove_datatype(datatype_gid)
correct = correct and dao.remove_entity(model.OperationGroup, datatype_group.fk_operation_group)
else:
self.logger.debug("Removing datatype %s" % datatype)
self._remove_project_node_files(project_id, datatype.gid, skip_validation)
## Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(model.DataType, operation_id, "fk_from_operation")
if len(dependent_dt) > 0:
### Do not remove Operation in case DataType still exist referring it.
continue
correct = correct and dao.remove_entity(model.Operation, operation_id)
## Make sure Operation folder is removed
self.structure_helper.remove_operation_data(project.name, datatype.fk_from_operation)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid))
user.used_disk_space = user.used_disk_space - freed_space
dao.store_entity(user)
def test_import_export(self):
"""
Test the import/export mechanism for a project structure.
The project contains the following data types: Connectivity, Surface, MappedArray and ValueWrapper.
"""
result = self.get_all_datatypes()
expected_results = {}
for one_data in result:
expected_results[one_data.gid] = (one_data.module, one_data.type)
#create an array mapped in DB
data = {'param_1': 'some value'}
OperationService().initiate_prelaunch(self.operation, self.adapter_instance, {}, **data)
inserted = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(1, inserted, "Problems when inserting data")
#create a value wrapper
value_wrapper = self._create_value_wrapper()
count_operations = dao.get_filtered_operations(self.test_project.id, None, is_count=True)
self.assertEqual(2, count_operations, "Invalid ops number before export!")
# Export project as ZIP
self.zip_path = ExportManager().export_project(self.test_project)
self.assertTrue(self.zip_path is not None, "Exported file is none")
# Remove the original project
self.project_service.remove_project(self.test_project.id)
result, lng_ = self.project_service.retrieve_projects_for_user(self.test_user.id)
self.assertEqual(0, len(result), "Project Not removed!")
self.assertEqual(0, lng_, "Project Not removed!")
# Now try to import again project
self.import_service.import_project_structure(self.zip_path, self.test_user.id)
result = self.project_service.retrieve_projects_for_user(self.test_user.id)[0]
self.assertEqual(len(result), 1, "There should be only one project.")
self.assertEqual(result[0].name, "GeneratedProject", "The project name is not correct.")
self.assertEqual(result[0].description, "test_desc", "The project description is not correct.")
self.test_project = result[0]
count_operations = dao.get_filtered_operations(self.test_project.id, None, is_count=True)
#1 op. - import cff; 2 op. - save the array wrapper;
self.assertEqual(2, count_operations, "Invalid ops number after export and import !")
for gid in expected_results:
datatype = dao.get_datatype_by_gid(gid)
self.assertEqual(datatype.module, expected_results[gid][0], 'DataTypes not imported correctly')
self.assertEqual(datatype.type, expected_results[gid][1], 'DataTypes not imported correctly')
#check the value wrapper
new_val = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.mapped_values.ValueWrapper")[0]
self.assertEqual(1, len(new_val), "One !=" + str(len(new_val)))
new_val = ABCAdapter.load_entity_by_gid(new_val[0][2])
self.assertEqual(value_wrapper.data_value, new_val.data_value, "Data value incorrect")
self.assertEqual(value_wrapper.data_type, new_val.data_type, "Data type incorrect")
self.assertEqual(value_wrapper.data_name, new_val.data_name, "Data name incorrect")
def test_remove_value_wrapper(self):
"""
Test the deletion of a value wrapper dataType
"""
count_vals = self.count_all_entities(ValueWrapper)
self.assertEqual(0, count_vals, "There should be no value wrapper")
value_wrapper = self._create_value_wrapper()
self.project_service.remove_datatype(self.test_project.id, value_wrapper.gid)
res = dao.get_datatype_by_gid(value_wrapper.gid)
self.assertEqual(None, res, "The value wrapper was not deleted.")
def test_remove_used_surface(self):
"""
Tries to remove an used surface
"""
mapping, mapping_count = self.flow_service.get_available_datatypes(self.test_project.id, RegionMapping)
self.assertEquals(1, mapping_count, "There should be one Mapping.")
mapping_gid = mapping[0][2]
mapping = ABCAdapter.load_entity_by_gid(mapping_gid)
surface = dao.get_datatype_by_gid(mapping.surface.gid)
self.assertEqual(surface.gid, mapping.surface.gid, "The surfaces should have the same GID")
try:
self.project_service.remove_datatype(self.test_project.id, surface.gid)
self.fail("The surface should still be used by a RegionMapping " + str(surface.gid))
except RemoveDataTypeException:
#OK, do nothing
pass
res = dao.get_datatype_by_gid(surface.gid)
self.assertEqual(surface.id, res.id, "A used surface was deleted")
def set_datatype_visibility(datatype_gid, is_visible):
"""
Sets the dataType visibility. If the given dataType is a dataType group or it is part of a
dataType group than this method will set the visibility for each dataType from this group.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype.fk_datatype_group is not None:
datatype_gid = dao.get_datatype_by_id(datatype.fk_datatype_group).gid
dao.set_datatype_visibility(datatype_gid, is_visible)
def update(input_file):
"""
:param input_file: the file that needs to be converted to a newer file storage version.
"""
if not os.path.isfile(input_file):
raise IncompatibleFileManagerException("The input path %s received for upgrading from 3 -> 4 is not a "
"valid file on the disk." % input_file)
folder, file_name = os.path.split(input_file)
storage_manager = HDF5StorageManager(folder, file_name)
root_metadata = storage_manager.get_metadata()
if DataTypeMetaData.KEY_CLASS_NAME not in root_metadata:
raise IncompatibleFileManagerException("File %s received for upgrading 3 -> 4 is not valid, due to missing "
"metadata: %s" % (input_file, DataTypeMetaData.KEY_CLASS_NAME))
class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME]
if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata:
LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder))
projection_type = projections.EEG_POLYMORPHIC_IDENTITY
if "SEEG" in class_name:
projection_type = projections.SEEG_POLYMORPHIC_IDENTITY
elif "MEG" in class_name:
projection_type = projections.MEG_POLYMORPHIC_IDENTITY
root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type)
LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type))
elif "TimeSeries" in class_name:
LOGGER.info("Updating TS %s from %s" % (file_name, folder))
service = ImportService()
operation_id = int(os.path.split(folder)[1])
dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False)
dt_db = dao.get_datatype_by_gid(dt.gid)
if dt_db is not None:
# DT already in DB (update of own storage, by making sure all fields are being correctly populated)
dt_db.configure()
dt_db.persist_full_metadata()
try:
# restore in DB, in case TVB 1.4 had wrongly imported flags
dao.store_entity(dt_db)
except Exception:
LOGGER.exception("Could not update flags in DB, but we continue with the update!")
elif FIELD_SURFACE_MAPPING not in root_metadata:
# Have default values, to avoid the full project not being imported
root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False)
root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False)
root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION
storage_manager.set_metadata(root_metadata)
def _update_datatype_disk_size(self, file_path):
"""
Computes and updates the disk_size attribute of the DataType, for which was created the given file.
"""
file_handler = FilesHelper()
datatype_gid = self._get_manager(file_path).get_gid_attribute()
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is not None:
datatype.disk_size = file_handler.compute_size_on_disk(file_path)
dao.store_entity(datatype)
def _remove_entity(self, data_class, before_number):
"""
Try to remove entity. Fail otherwise.
"""
dts, count = self.flow_service.get_available_datatypes(self.test_project.id, data_class)
self.assertEquals(count, before_number)
for dt in dts:
data_gid = dt[2]
self.project_service.remove_datatype(self.test_project.id, data_gid)
res = dao.get_datatype_by_gid(data_gid)
self.assertEqual(None, res, "The entity was not deleted")
def test_remove_time_series(self):
"""
Tests the happy flow for the deletion of a time series.
"""
count_ts = self.count_all_entities(TimeSeries)
self.assertEqual(0, count_ts, "There should be no time series")
self._create_timeseries()
series = self.get_all_entities(TimeSeries)
self.assertEqual(1, len(series), "There should be only one time series")
self.project_service.remove_datatype(self.test_project.id, series[0].gid)
res = dao.get_datatype_by_gid(series[0].gid)
self.assertEqual(None, res, "The time series was not deleted.")
def __init__(self, datatype_gid):
NodeStructure.__init__(self, datatype_gid, "")
datatype_shape = DATATYPE_SHAPE
if dao.is_datatype_group(datatype_gid):
datatype_shape = DATATYPE_GROUP_SHAPE
datatype = dao.get_datatype_by_gid(datatype_gid)
node_data = NodeData(MAX_SHAPE_SIZE, DATATYPE_SHAPE_COLOR, datatype_shape,
NODE_DATATYPE_TYPE, datatype.id, datatype.display_name)
self.name = str(datatype.type)
self.data = node_data
def _get_linked_datatypes_storage_path(project):
"""
:return: the file paths to the datatypes that are linked in `project`
"""
paths = []
for lnk_dt in dao.get_linked_datatypes_in_project(project.id):
# get datatype as a mapped type
lnk_dt = dao.get_datatype_by_gid(lnk_dt.gid)
if lnk_dt.storage_path is not None:
paths.append(lnk_dt.get_storage_file_path())
else:
LOG.warning("Problem when trying to retrieve path on %s:%s for export!" % (lnk_dt.type, lnk_dt.gid))
return paths
def _remove_project_node_files(self, project_id, gid, skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
if links:
was_link = False
for link in links:
#This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(model.Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
new_op = model.Operation(dao.get_system_user().id,
links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.parameters,
datatype.parent_operation.meta_data,
datatype.parent_operation.method_name,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project).name
new_op_loaded = dao.get_operation_by_id(new_op.id)
self.structure_helper.write_operation_metadata(new_op_loaded)
self.structure_helper.move_datatype(datatype, to_project, str(new_op.id))
datatype.set_operation_id(new_op.id)
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(model.Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
self.structure_helper.remove_datatype(datatype)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def update():
"""
Try to import Default_Project, so that new users created with the latest code can share this project.
"""
try:
all_surfaces = dao.get_generic_entity(model.DataType, "tvb.datatypes.surfaces", "module")
for srf in all_surfaces:
surface = dao.get_datatype_by_gid(srf.gid)
if isinstance(surface, Surface):
surface._find_edge_lengths()
dao.store_entity(surface)
except Exception:
LOGGER.exception("Could update Surface entities!")
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
datatype = dao.get_datatype_by_gid(data_gid)
if isinstance(datatype, MappedType):
datatype_path = datatype.get_storage_file_path()
files_update_manager = FilesUpdateManager()
if not files_update_manager.is_file_up_to_date(datatype_path):
datatype.invalid = True
dao.store_entity(datatype)
raise FileVersioningException("Encountered DataType with an incompatible storage or data version. "
"The DataType was marked as invalid.")
return datatype
def test_remove_array_wrapper(self):
"""
Tests the happy flow for the deletion of an array wrapper.
"""
count_array = self.count_all_entities(MappedArray)
self.assertEqual(1, count_array)
data = {'param_1': 'some value'}
OperationService().initiate_prelaunch(self.operation, self.adapter_instance, {}, **data)
array_wrappers = self.get_all_entities(MappedArray)
self.assertEqual(2, len(array_wrappers))
array_gid = array_wrappers[0].gid
self.project_service.remove_datatype(self.test_project.id, array_gid)
res = dao.get_datatype_by_gid(array_gid)
self.assertEqual(None, res, "The array wrapper was not deleted.")
def update():
"""
Update Surface metadata
"""
try:
all_surfaces = dao.get_generic_entity(model.DataType, "tvb.datatypes.surfaces", "module")
for srf in all_surfaces:
surface = dao.get_datatype_by_gid(srf.gid)
if isinstance(surface, Surface):
surface._find_edge_lengths()
dao.store_entity(surface)
surface.persist_full_metadata()
except Exception:
LOGGER.exception("Could update Surface entities!")
def create_timeseries(self, connectivity, ts_type=None, sensors=None):
"""
Create a stored TimeSeries entity.
"""
operation, _, storage_path = self.__create_operation()
if ts_type == "EEG":
time_series = TimeSeriesEEG(storage_path=storage_path, sensors=sensors)
else:
time_series = TimeSeriesRegion(storage_path=storage_path, connectivity=connectivity)
data = numpy.random.random((10, 10, 10, 10))
time = numpy.arange(10)
time_series.write_data_slice(data)
time_series.write_time_slice(time)
adapter_instance = StoreAdapter([time_series])
OperationService().initiate_prelaunch(operation, adapter_instance, {})
time_series = dao.get_datatype_by_gid(time_series.gid)
return time_series
def __datatype2metastructure(row, dt_ids):
"""
Convert a list of data retrieved from DB and create a DataTypeMetaData object.
"""
data = {}
is_group = False
group = None
if row[7] is not None and row[7] and row[14] in dt_ids:
is_group = True
group = dao.get_generic_entity(model.OperationGroup, row[7])
if group and len(group):
group = group[0]
else:
is_group = False
datatype_group = None
if row[14] is not None and row[14] in dt_ids:
datatype_group = dao.get_datatype_by_id(row[14])
dt_entity = dao.get_datatype_by_gid(row[9])
data[DataTypeMetaData.KEY_TITLE] = dt_entity.display_name
## All these fields are necessary here for dynamic Tree levels.
data[DataTypeMetaData.KEY_NODE_TYPE] = datatype_group.type if datatype_group is not None else row[0]
data[DataTypeMetaData.KEY_STATE] = row[1]
data[DataTypeMetaData.KEY_SUBJECT] = str(row[2])
operation_name = CommonDetails.compute_operation_name(row[3], row[4], row[5])
data[DataTypeMetaData.KEY_OPERATION_TYPE] = operation_name
data[DataTypeMetaData.KEY_AUTHOR] = row[6]
data[DataTypeMetaData.KEY_OPERATION_TAG] = group.name if is_group else row[8]
data[DataTypeMetaData.KEY_OP_GROUP_ID] = group.id if is_group else None
data[DataTypeMetaData.KEY_GID] = datatype_group.gid if datatype_group is not None else row[9]
data[DataTypeMetaData.KEY_DATE] = date2string(row[10]) if (row[10] is not None) else ''
data[DataTypeMetaData.KEY_DATATYPE_ID] = datatype_group.id if datatype_group is not None else row[11]
data[DataTypeMetaData.KEY_LINK] = row[12]
data[DataTypeMetaData.KEY_OPERATION_ALGORITHM] = row[5]
date_string = row[10].strftime(MONTH_YEAR_FORMAT) if row[10] is not None else ""
data[DataTypeMetaData.KEY_CREATE_DATA_MONTH] = date_string
date_string = row[10].strftime(DAY_MONTH_YEAR_FORMAT) if row[10] is not None else ""
data[DataTypeMetaData.KEY_CREATE_DATA_DAY] = date_string
data[DataTypeMetaData.KEY_BURST] = row[15] if row[15] is not None else '-None-'
data[DataTypeMetaData.KEY_TAG_1] = row[16] if row[16] else ''
data[DataTypeMetaData.KEY_TAG_2] = row[17] if row[17] else ''
data[DataTypeMetaData.KEY_TAG_3] = row[18] if row[18] else ''
data[DataTypeMetaData.KEY_TAG_4] = row[19] if row[19] else ''
data[DataTypeMetaData.KEY_TAG_5] = row[20] if row[20] else ''
data[DataTypeMetaData.KEY_RELEVANCY] = True if row[21] > 0 else False
invalid = True if row[13] else False
return DataTypeMetaData(data, invalid)
def test_handle_event(self):
"""
Test a defined handler for the store project method.
"""
path_to_events = os.path.dirname(__file__)
event_handlers.read_events([path_to_events])
data = dict(name="test_project", description="test_description", users=[])
initial_projects = dao.get_projects_for_user(self.test_user.id)
self.assertEqual(len(initial_projects), 0, "Database reset probably failed!")
test_project = self.project_service.store_project(self.test_user, True, None, **data)
# Operations will start asynchronously; Give them time.
time.sleep(1)
gid = dao.get_last_data_with_uid("test_uid")
self.assertTrue(gid is not None, "Nothing was stored in database!")
datatype = dao.get_datatype_by_gid(gid)
self.assertEqual(datatype.type, "Datatype1", "Wrong data stored!")
self.project_service._remove_project_node_files(test_project.id, gid)
def test_update_meta_data_simple(self):
"""
Test the new update metaData for a simple data that is not part of a group.
"""
inserted_project, gid, _ = self._create_value_wrapper(self.test_user)
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "second_state",
DataTypeOverlayDetails.CODE_GID: gid,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'new user group'}
self.project_service.update_metadata(new_meta_data)
new_datatype = dao.get_datatype_by_gid(gid)
self.__check_meta_data(new_meta_data, new_datatype)
op_path = FilesHelper().get_operation_meta_file_path(inserted_project.name, new_datatype.parent_operation.id)
op_meta = XMLReader(op_path).read_metadata()
self.assertEqual(op_meta['user_group'], 'new user group', 'UserGroup not updated!')
