def launch(self, data_file): """ Execute import operations: unpack ZIP, build and store generic DataType objects. :param data_file: an archive (ZIP / HDF5) containing the `DataType` :raises LaunchException: when data_file is None, nonexistent, or invalid \ (e.g. incomplete meta-data, not in ZIP / HDF5 format etc. ) """ if data_file is None: raise LaunchException( "Please select file which contains data to import") if os.path.exists(data_file): if zipfile.is_zipfile(data_file): current_op = dao.get_operation_by_id(self.operation_id) # Creates a new TMP folder where to extract data tmp_folder = os.path.join(self.storage_path, "tmp_import") FilesHelper().unpack_zip(data_file, tmp_folder) operations = ImportService().import_project_operations( current_op.project, self.storage_path) shutil.rmtree(tmp_folder) self.nr_of_datatypes += len(operations) else: # upgrade file if necessary file_update_manager = FilesUpdateManager() file_update_manager.upgrade_file(data_file) folder, h5file = os.path.split(data_file) manager = HDF5StorageManager(folder, h5file) if manager.is_valid_hdf5_file(): datatype = None try: service = ImportService() datatype = service.load_datatype_from_file( folder, h5file, self.operation_id, final_storage=self.storage_path) service.store_datatype(datatype) self.nr_of_datatypes += 1 except Exception as excep: # If import operation failed delete file from disk. if datatype is not None and os.path.exists( datatype.get_storage_file_path()): os.remove(datatype.get_storage_file_path()) self.log.exception(excep) raise LaunchException( "Invalid file received as input. Most probably incomplete " "meta-data ... " + str(excep)) else: raise LaunchException( "Uploaded file: %s is neither in ZIP or HDF5 format" % data_file) else: raise LaunchException("File: %s to import does not exists." % data_file)
def update(input_file, burst_match_dict=None): """ :param input_file: the file that needs to be converted to a newer file storage version. """ if not os.path.isfile(input_file): raise IncompatibleFileManagerException("The input path %s received for upgrading from 3 -> 4 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = HDF5StorageManager(folder, file_name) root_metadata = storage_manager.get_metadata() if DataTypeMetaData.KEY_CLASS_NAME not in root_metadata: raise IncompatibleFileManagerException("File %s received for upgrading 3 -> 4 is not valid, due to missing " "metadata: %s" % (input_file, DataTypeMetaData.KEY_CLASS_NAME)) class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] class_name = str(class_name, 'utf-8') if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata: LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder)) projection_type = ProjectionsType.EEG.value if "SEEG" in class_name: projection_type = ProjectionsType.SEEG.value elif "MEG" in class_name: projection_type = ProjectionsType.MEG.value root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type) LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type)) elif "TimeSeries" in class_name: LOGGER.info("Updating TS %s from %s" % (file_name, folder)) service = ImportService() try: operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(os.path.join(folder, file_name), operation_id) dt_db = dao.get_datatype_by_gid(dt.gid) except ValueError: dt_db = None if dt_db is not None: # DT already in DB (update of own storage, by making sure all fields are being correctly populated) dt_db.configure() dt_db.persist_full_metadata() try: # restore in DB, in case TVB 1.4 had wrongly imported flags dao.store_entity(dt_db) except Exception: LOGGER.exception("Could not update flags in DB, but we continue with the update!") elif FIELD_SURFACE_MAPPING not in root_metadata: # Have default values, to avoid the full project not being imported root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False) root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False) root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)
def _update_localconnectivity_metadata(folder, file_name): service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False) info_dict = SparseMatrix.extract_sparse_matrix_metadata(dt.matrix) dt.set_metadata(info_dict, "", True, SparseMatrix.ROOT_PATH + "matrix")
def launch(self, view_model): # type: (TVBImporterModel) -> [] """ Execute import operations: unpack ZIP, build and store generic DataType objects. :raises LaunchException: when data_file is None, nonexistent, or invalid \ (e.g. incomplete meta-data, not in ZIP / HDF5 format etc. ) """ if view_model.data_file is None: raise LaunchException( "Please select file which contains data to import") service = ImportService() if os.path.exists(view_model.data_file): if zipfile.is_zipfile(view_model.data_file): current_op = dao.get_operation_by_id(self.operation_id) # Creates a new TMP folder where to extract data tmp_folder = os.path.join(self.storage_path, "tmp_import") FilesHelper().unpack_zip(view_model.data_file, tmp_folder) operations = service.import_project_operations( current_op.project, tmp_folder) shutil.rmtree(tmp_folder) self.nr_of_datatypes += len(operations) else: # upgrade file if necessary file_update_manager = FilesUpdateManager() file_update_manager.upgrade_file(view_model.data_file) folder, h5file = os.path.split(view_model.data_file) manager = HDF5StorageManager(folder, h5file) if manager.is_valid_hdf5_file(): datatype = None try: datatype = service.load_datatype_from_file( view_model.data_file, self.operation_id) service.check_import_references( view_model.data_file, datatype) service.store_datatype(datatype, view_model.data_file) self.nr_of_datatypes += 1 except ImportException as excep: self.log.exception(excep) if datatype is not None: target_path = h5.path_for_stored_index(datatype) if os.path.exists(target_path): os.remove(target_path) raise LaunchException( "Invalid file received as input. " + str(excep)) else: raise LaunchException( "Uploaded file: %s is neither in ZIP or HDF5 format" % view_model.data_file) else: raise LaunchException("File: %s to import does not exists." % view_model.data_file)
def update(input_file): """ :param input_file: the file that needs to be converted to a newer file storage version. """ if not os.path.isfile(input_file): raise IncompatibleFileManagerException("The input path %s received for upgrading from 3 -> 4 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = HDF5StorageManager(folder, file_name) root_metadata = storage_manager.get_metadata() if DataTypeMetaData.KEY_CLASS_NAME not in root_metadata: raise IncompatibleFileManagerException("File %s received for upgrading 3 -> 4 is not valid, due to missing " "metadata: %s" % (input_file, DataTypeMetaData.KEY_CLASS_NAME)) class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata: LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder)) projection_type = projections.EEG_POLYMORPHIC_IDENTITY if "SEEG" in class_name: projection_type = projections.SEEG_POLYMORPHIC_IDENTITY elif "MEG" in class_name: projection_type = projections.MEG_POLYMORPHIC_IDENTITY root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type) LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type)) elif "TimeSeries" in class_name: LOGGER.info("Updating TS %s from %s" % (file_name, folder)) service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False) dt_db = dao.get_datatype_by_gid(dt.gid) if dt_db is not None: # DT already in DB (update of own storage, by making sure all fields are being correctly populated) dt_db.configure() dt_db.persist_full_metadata() try: # restore in DB, in case TVB 1.4 had wrongly imported flags dao.store_entity(dt_db) except Exception: LOGGER.exception("Could not update flags in DB, but we continue with the update!") elif FIELD_SURFACE_MAPPING not in root_metadata: # Have default values, to avoid the full project not being imported root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False) root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False) root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)
def _update_localconnectivity_metadata(folder, file_name): service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False) info_dict = SparseMatrix.extract_sparse_matrix_metadata(dt.matrix) dt.set_metadata(info_dict, '', True, SparseMatrix.ROOT_PATH + 'matrix')
def update_localconnectivity_metadata(folder, file_name): service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(os.path.join(folder, file_name), operation_id) info_dict = {"dtype": dt.matrix.dtype.str, "format": dt.matrix.format, "Shape": str(dt.matrix.shape), "Maximum": dt.matrix.data.max(), "Minimum": dt.matrix.data.min(), "Mean": dt.matrix.mean()} dt.set_metadata(info_dict, '', True, '/matrix')
def launch(self, data_file): """ Execute import operations: unpack ZIP, build and store generic DataType objects. :param data_file: an archive (ZIP / HDF5) containing the `DataType` :raises LaunchException: when data_file is None, nonexistent, or invalid \ (e.g. incomplete meta-data, not in ZIP / HDF5 format etc. ) """ if data_file is None: raise LaunchException("Please select file which contains data to import") if os.path.exists(data_file): if zipfile.is_zipfile(data_file): current_op = dao.get_operation_by_id(self.operation_id) # Creates a new TMP folder where to extract data tmp_folder = os.path.join(self.storage_path, "tmp_import") FilesHelper().unpack_zip(data_file, tmp_folder) operations = ImportService().import_project_operations(current_op.project, self.storage_path) shutil.rmtree(tmp_folder) self.nr_of_datatypes += len(operations) else: # upgrade file if necessary file_update_manager = FilesUpdateManager() file_update_manager.upgrade_file(data_file) folder, h5file = os.path.split(data_file) manager = HDF5StorageManager(folder, h5file) if manager.is_valid_hdf5_file(): datatype = None try: service = ImportService() datatype = service.load_datatype_from_file(folder, h5file, self.operation_id) service.store_datatype(datatype) self.nr_of_datatypes += 1 except Exception as excep: # If import operation failed delete file from disk. if datatype is not None and os.path.exists(datatype.get_storage_file_path()): os.remove(datatype.get_storage_file_path()) self.log.exception(excep) raise LaunchException("Invalid file received as input. Most probably incomplete " "meta-data ... " + str(excep)) else: raise LaunchException("Uploaded file: %s is neither in ZIP or HDF5 format" % data_file) else: raise LaunchException("File: %s to import does not exists." % data_file)
def update(input_file): """ :param input_file: the file that needs to be converted to a newer file storage version. """ if not os.path.isfile(input_file): raise FileVersioningException("The input path %s received for upgrading from 3 -> 4 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = HDF5StorageManager(folder, file_name) root_metadata = storage_manager.get_metadata() class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata: LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder)) projection_type = projections_data.EEG_POLYMORPHIC_IDENTITY if "SEEG" in class_name: projection_type = projections_data.SEEG_POLYMORPHIC_IDENTITY elif "MEG" in class_name: projection_type = projections_data.MEG_POLYMORPHIC_IDENTITY root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type) LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type)) elif "TimeSeries" in class_name: LOGGER.info("Updating TS %s from %s" % (file_name, folder)) service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False) dt_db = dao.get_datatype_by_gid(dt.gid) if dt_db is not None: # DT already in DB (update of own storage, by making sure all fields are being correctly populated) dt_db.configure() dt_db.persist_full_metadata() # restore in DB, in case TVB 1.4 had wrongly imported flags dao.store_entity(dt_db) elif FIELD_SURFACE_MAPPING not in root_metadata: # Have default values, to avoid the full project not being imported root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False) root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False) root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)
def read_h5(full_paths): # We need to load the DataType in the context of an operation, and a project all_projects = dao.get_all_projects() all_operations = dao.get_generic_entity(model.Operation, all_projects[0].id, "fk_launched_in") service = ImportService() results = [] for full_path in full_paths: folder, h5file = os.path.split(full_path) # The actual read of H5: datatype = service.load_datatype_from_file(folder, h5file, all_operations[0].id, move=False) print "We've build DataType: [%s]" % datatype.__class__.__name__, datatype results.append(datatype) return results
def read_h5(full_paths): # We need to load the DataType in the context of an operation, and a project all_projects = dao.get_all_projects() all_operations = dao.get_generic_entity(Operation, all_projects[0].id, "fk_launched_in") service = ImportService() results = [] for full_path in full_paths: # The actual read of H5: datatype = service.load_datatype_from_file(full_path, all_operations[0].id) print("We've build DataType: [%s]" % datatype.__class__.__name__, datatype) results.append(datatype) return results
def launch(self, view_model): # type: (TVBImporterModel) -> [] """ Execute import operations: unpack ZIP, build and store generic DataType objects. :raises LaunchException: when data_file is None, nonexistent, or invalid \ (e.g. incomplete meta-data, not in ZIP / HDF5 format etc. ) """ if view_model.data_file is None: raise LaunchException( "Please select file which contains data to import") service = ImportService() if os.path.exists(view_model.data_file): current_op = dao.get_operation_by_id(self.operation_id) if zipfile.is_zipfile(view_model.data_file): # Creates a new TMP folder where to extract data tmp_folder = os.path.join(self.storage_path, "tmp_import") self.storage_interface.unpack_zip(view_model.data_file, tmp_folder) is_group = False current_op_id = current_op.id for file in os.listdir(tmp_folder): # In case we import a DatatypeGroup, we want the default import flow if os.path.isdir(os.path.join(tmp_folder, file)): current_op_id = None is_group = True break try: operations, all_dts, stored_dts_count = service.import_project_operations( current_op.project, tmp_folder, is_group, current_op_id) self.nr_of_datatypes += stored_dts_count if stored_dts_count == 0: current_op.additional_info = 'All chosen datatypes already exist!' dao.store_entity(current_op) elif stored_dts_count < all_dts: current_op.additional_info = 'Part of the chosen datatypes already exist!' dao.store_entity(current_op) except ImportException as excep: self.log.exception(excep) current_op.additional_info = excep.message current_op.status = STATUS_ERROR raise LaunchException("Invalid file received as input. " + str(excep)) finally: shutil.rmtree(tmp_folder) else: # upgrade file if necessary file_update_manager = FilesUpdateManager() file_update_manager.upgrade_file(view_model.data_file) if self.storage_interface.get_storage_manager( view_model.data_file).is_valid_tvb_file(): datatype = None try: datatype = service.load_datatype_from_file( view_model.data_file, self.operation_id) stored_new_dt = service.store_or_link_datatype( datatype, view_model.data_file, current_op.project.id) if stored_new_dt == 0: current_op.additional_info = 'The chosen datatype already exists!' dao.store_entity(current_op) self.nr_of_datatypes += stored_new_dt except ImportException as excep: self.log.exception(excep) if datatype is not None: target_path = h5.path_for_stored_index(datatype) if os.path.exists(target_path): os.remove(target_path) raise LaunchException( "Invalid file received as input. " + str(excep)) else: raise LaunchException( "Uploaded file: %s is neither in ZIP or HDF5 format" % view_model.data_file) else: raise LaunchException("File: %s to import does not exists." % view_model.data_file)