def test_gather_view_model_and_datatype_references(connectivity_index_factory, operation_factory): conn = connectivity_index_factory() sim_view_model = SimulatorAdapterModel() sim_view_model.connectivity = conn.gid op = operation_factory() storage_path = FilesHelper().get_project_folder(op.project, str(op.id)) h5.store_view_model(sim_view_model, storage_path) only_vm_references, _ = h5.gather_references_of_view_model( sim_view_model.gid, storage_path, True) vm_references, dt_references = h5.gather_references_of_view_model( sim_view_model.gid, storage_path) assert len(only_vm_references) == 5 assert len(vm_references) + len(dt_references) == 6
def _prepare_input(operation, simulator_gid): # type: (Operation, str) -> list storage_path = StorageInterface().get_project_folder( operation.project.name, str(operation.id)) vm_files, dt_files = h5.gather_references_of_view_model( simulator_gid, storage_path) vm_files.extend(dt_files) return vm_files
def test_gather_view_model_and_datatype_references_multiple_monitors( connectivity_index_factory, operation_factory, sensors_index_factory, surface_index_factory, region_mapping_index_factory): conn = connectivity_index_factory() _, surface = surface_index_factory(cortical=True) region_mapping_idx = region_mapping_index_factory( conn_gid=conn.gid, surface_gid=surface.gid.hex) sensors_idx, sensors = sensors_index_factory() proj = ProjectionSurfaceEEG(sensors=sensors, sources=surface, projection_data=numpy.ones(3)) op = operation_factory() storage_path = StorageInterface().get_project_folder( op.project.name, str(op.id)) prj_db = h5.store_complete(proj, storage_path) prj_db.fk_from_operation = op.id dao.store_entity(prj_db) seeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid) seeg_monitor.region_mapping = region_mapping_idx.gid sim_view_model = SimulatorAdapterModel() sim_view_model.connectivity = conn.gid sim_view_model.monitors = [TemporalAverageViewModel(), seeg_monitor] op = operation_factory() storage_path = StorageInterface().get_project_folder( op.project.name, str(op.id)) h5.store_view_model(sim_view_model, storage_path) only_vm_references, _ = h5.gather_references_of_view_model( sim_view_model.gid, storage_path, True) assert len(only_vm_references) == 7 vm_references, dt_references = h5.gather_references_of_view_model( sim_view_model.gid, storage_path) assert len(vm_references + dt_references) == 12
def export_simulator_configuration(self, burst_id): burst = dao.get_burst_by_id(burst_id) if burst is None: raise InvalidExportDataException("Could not find burst with ID " + str(burst_id)) op_folder = self.storage_interface.get_project_folder(burst.project.name, str(burst.fk_simulation)) all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model(burst.simulator_gid, op_folder) burst_path = h5.determine_filepath(burst.gid, op_folder) all_view_model_paths.append(burst_path) zip_filename = ABCExporter.get_export_file_name(burst, self.storage_interface.TVB_ZIP_FILE_EXTENSION) result_path = self.storage_interface.export_simulator_configuration(burst, all_view_model_paths, all_datatype_paths, zip_filename) return result_path
def export_simulator_configuration(self, burst_id): burst = dao.get_burst_by_id(burst_id) if burst is None: raise InvalidExportDataException("Could not find burst with ID " + str(burst_id)) op_folder = self.files_helper.get_project_folder( burst.project, str(burst.fk_simulation)) tmp_export_folder = self._build_data_export_folder(burst) tmp_sim_folder = os.path.join(tmp_export_folder, self.EXPORTED_SIMULATION_NAME) if not os.path.exists(tmp_sim_folder): os.makedirs(tmp_sim_folder) all_view_model_paths, all_datatype_paths = h5.gather_references_of_view_model( burst.simulator_gid, op_folder) burst_path = h5.determine_filepath(burst.gid, op_folder) all_view_model_paths.append(burst_path) for vm_path in all_view_model_paths: dest = os.path.join(tmp_sim_folder, os.path.basename(vm_path)) self.files_helper.copy_file(vm_path, dest) for dt_path in all_datatype_paths: dest = os.path.join(tmp_sim_folder, self.EXPORTED_SIMULATION_DTS_DIR, os.path.basename(dt_path)) self.files_helper.copy_file(dt_path, dest) main_vm_path = h5.determine_filepath(burst.simulator_gid, tmp_sim_folder) H5File.remove_metadata_param(main_vm_path, 'history_gid') now = datetime.now() date_str = now.strftime("%Y-%m-%d_%H-%M") zip_file_name = "%s_%s.%s" % (date_str, str(burst_id), self.ZIP_FILE_EXTENSION) result_path = os.path.join(tmp_export_folder, zip_file_name) with TvbZip(result_path, "w") as zip_file: zip_file.write_folder(tmp_sim_folder) self.files_helper.remove_folder(tmp_sim_folder) return result_path
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project): new_op = Operation(op.view_model_gid, dao.get_system_user().id, fk_to_project, datatype.parent_operation.fk_from_algo, datatype.parent_operation.status, datatype.parent_operation.start_date, datatype.parent_operation.completion_date, datatype.parent_operation.fk_operation_group, datatype.parent_operation.additional_info, datatype.parent_operation.user_group, datatype.parent_operation.range_values) new_op.visible = datatype.parent_operation.visible new_op = dao.store_entity(new_op) to_project = self.find_project(fk_to_project) to_project_path = self.storage_interface.get_project_folder(to_project.name) full_path = h5.path_for_stored_index(datatype) old_folder = self.storage_interface.get_project_folder(project.name, str(op.id)) file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0] file_paths.append(full_path) # The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation # folder containing the file if datatype.is_ts and datatype.fk_parent_burst is not None: bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst, project.name) if os.path.exists(bc_path): file_paths.append(bc_path) bc = dao.get_burst_for_operation_id(op.id) bc.fk_simulation = new_op.id dao.store_entity(bc) # Move all files to the new operation folder self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths) datatype.fk_from_operation = new_op.id datatype.parent_operation = new_op dao.store_entity(datatype) return new_op
def prepare_datatypes_for_export(data): """ Method used for exporting data type groups. This method returns a list of all datatype indexes needed to be exported and a dictionary where keys are operation folder names and the values are lists containing the paths that belong to one particular operation folder. """ all_datatypes = ProjectService.get_all_datatypes_from_data(data) first_datatype = all_datatypes[0] # We are exporting a group of datatype measures so we need to find the group of time series if hasattr(first_datatype, 'fk_source_gid'): ts = h5.load_entity_by_gid(first_datatype.fk_source_gid) dt_metric_group = dao.get_datatypegroup_by_op_group_id( ts.parent_operation.fk_operation_group) datatype_measure_list = ProjectService.get_all_datatypes_from_data( dt_metric_group) all_datatypes = datatype_measure_list + all_datatypes else: ts_group = dao.get_datatype_measure_group_from_ts_from_pse( first_datatype.gid, DatatypeMeasureIndex) time_series_list = ProjectService.get_all_datatypes_from_data( ts_group) all_datatypes = all_datatypes + time_series_list if all_datatypes is None or len(all_datatypes) == 0: raise ExportException( "Could not export a data type group with no data!") op_file_dict = dict() for dt in all_datatypes: h5_path = h5.path_for_stored_index(dt) op_folder = os.path.dirname(h5_path) op_file_dict[op_folder] = [h5_path] op = dao.get_operation_by_id(dt.fk_from_operation) vms = h5.gather_references_of_view_model(op.view_model_gid, os.path.dirname(h5_path), only_view_models=True) op_file_dict[op_folder].extend(vms[0]) return all_datatypes, op_file_dict