def test_gather_view_model_and_datatype_references(connectivity_index_factory,
operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
only_vm_references, _ = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path, True)
vm_references, dt_references = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path)
assert len(only_vm_references) == 5
assert len(vm_references) + len(dt_references) == 6
def _prepare_input(operation, simulator_gid):
# type: (Operation, str) -> list
storage_path = StorageInterface().get_project_folder(
operation.project.name, str(operation.id))
vm_files, dt_files = h5.gather_references_of_view_model(
simulator_gid, storage_path)
vm_files.extend(dt_files)
return vm_files
def test_gather_view_model_and_datatype_references_multiple_monitors(
connectivity_index_factory, operation_factory, sensors_index_factory,
surface_index_factory, region_mapping_index_factory):
conn = connectivity_index_factory()
_, surface = surface_index_factory(cortical=True)
region_mapping_idx = region_mapping_index_factory(
conn_gid=conn.gid, surface_gid=surface.gid.hex)
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors,
sources=surface,
projection_data=numpy.ones(3))
op = operation_factory()
storage_path = StorageInterface().get_project_folder(
op.project.name, str(op.id))
prj_db = h5.store_complete(proj, storage_path)
prj_db.fk_from_operation = op.id
dao.store_entity(prj_db)
seeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
seeg_monitor.region_mapping = region_mapping_idx.gid
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.monitors = [TemporalAverageViewModel(), seeg_monitor]
op = operation_factory()
storage_path = StorageInterface().get_project_folder(
op.project.name, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
only_vm_references, _ = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path, True)
assert len(only_vm_references) == 7
vm_references, dt_references = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path)
assert len(vm_references + dt_references) == 12
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " + str(burst_id))
op_folder = self.storage_interface.get_project_folder(burst.project.name, str(burst.fk_simulation))
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
zip_filename = ABCExporter.get_export_file_name(burst, self.storage_interface.TVB_ZIP_FILE_EXTENSION)
result_path = self.storage_interface.export_simulator_configuration(burst, all_view_model_paths,
all_datatype_paths, zip_filename)
return result_path
def export_simulator_configuration(self, burst_id):
burst = dao.get_burst_by_id(burst_id)
if burst is None:
raise InvalidExportDataException("Could not find burst with ID " +
str(burst_id))
op_folder = self.files_helper.get_project_folder(
burst.project, str(burst.fk_simulation))
tmp_export_folder = self._build_data_export_folder(burst)
tmp_sim_folder = os.path.join(tmp_export_folder,
self.EXPORTED_SIMULATION_NAME)
if not os.path.exists(tmp_sim_folder):
os.makedirs(tmp_sim_folder)
all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(
burst.simulator_gid, op_folder)
burst_path = h5.determine_filepath(burst.gid, op_folder)
all_view_model_paths.append(burst_path)
for vm_path in all_view_model_paths:
dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path))
self.files_helper.copy_file(vm_path, dest)
for dt_path in all_datatype_paths:
dest = os.path.join(tmp_sim_folder,
self.EXPORTED_SIMULATION_DTS_DIR,
os.path.basename(dt_path))
self.files_helper.copy_file(dt_path, dest)
main_vm_path = h5.determine_filepath(burst.simulator_gid,
tmp_sim_folder)
H5File.remove_metadata_param(main_vm_path, 'history_gid')
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, str(burst_id),
self.ZIP_FILE_EXTENSION)
result_path = os.path.join(tmp_export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
zip_file.write_folder(tmp_sim_folder)
self.files_helper.remove_folder(tmp_sim_folder)
return result_path
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project):
new_op = Operation(op.view_model_gid,
dao.get_system_user().id,
fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op.visible = datatype.parent_operation.visible
new_op = dao.store_entity(new_op)
to_project = self.find_project(fk_to_project)
to_project_path = self.storage_interface.get_project_folder(to_project.name)
full_path = h5.path_for_stored_index(datatype)
old_folder = self.storage_interface.get_project_folder(project.name, str(op.id))
file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0]
file_paths.append(full_path)
# The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation
# folder containing the file
if datatype.is_ts and datatype.fk_parent_burst is not None:
bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst,
project.name)
if os.path.exists(bc_path):
file_paths.append(bc_path)
bc = dao.get_burst_for_operation_id(op.id)
bc.fk_simulation = new_op.id
dao.store_entity(bc)
# Move all files to the new operation folder
self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
return new_op
def prepare_datatypes_for_export(data):
"""
Method used for exporting data type groups. This method returns a list of all datatype indexes needed to be
exported and a dictionary where keys are operation folder names and the values are lists containing the paths
that belong to one particular operation folder.
"""
all_datatypes = ProjectService.get_all_datatypes_from_data(data)
first_datatype = all_datatypes[0]
# We are exporting a group of datatype measures so we need to find the group of time series
if hasattr(first_datatype, 'fk_source_gid'):
ts = h5.load_entity_by_gid(first_datatype.fk_source_gid)
dt_metric_group = dao.get_datatypegroup_by_op_group_id(
ts.parent_operation.fk_operation_group)
datatype_measure_list = ProjectService.get_all_datatypes_from_data(
dt_metric_group)
all_datatypes = datatype_measure_list + all_datatypes
else:
ts_group = dao.get_datatype_measure_group_from_ts_from_pse(
first_datatype.gid, DatatypeMeasureIndex)
time_series_list = ProjectService.get_all_datatypes_from_data(
ts_group)
all_datatypes = all_datatypes + time_series_list
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data!")
op_file_dict = dict()
for dt in all_datatypes:
h5_path = h5.path_for_stored_index(dt)
op_folder = os.path.dirname(h5_path)
op_file_dict[op_folder] = [h5_path]
op = dao.get_operation_by_id(dt.fk_from_operation)
vms = h5.gather_references_of_view_model(op.view_model_gid,
os.path.dirname(h5_path),
only_view_models=True)
op_file_dict[op_folder].extend(vms[0])
return all_datatypes, op_file_dict