def update(input_file): """ In order to avoid segmentation faults when updating a batch of files just start every conversion on a different Python process. :param input_file: the file that needs to be converted to a newer file storage version. This should be a file that still uses TVB 1.0 storage (pyTables) """ # Just to avoid any problems about renaming open files, do a rename from the start # and if case of a fault in the os.system call just rename back and remove the output file. if not os.path.isfile(input_file): raise FileVersioningException( "The input path %s received for upgrading from 1 -> 2 is not a " "valid file on the disk." % input_file) # Use a file-path with no spaces both for the temporary file and the input file # that is passed to the os.system call and just rename to original file at the # end of the processing to avoid any problems with parameters passed to os.system. input_file_no_spaces = input_file.replace(' ', '') path_to, file_name = os.path.split(input_file_no_spaces) tmp_convert_file = os.path.join(path_to, 'tmp_' + file_name) os.rename(input_file, tmp_convert_file) ok_status = os.system(PYTHON_EXE_PATH + ' -m %s %s %s' % (__name__, tmp_convert_file, input_file_no_spaces)) if ok_status == 0: # Call finished successfully os.remove(tmp_convert_file) os.rename(input_file_no_spaces, input_file) else: # Call failed for some reason, just rename back the input file os.rename(tmp_convert_file, input_file) raise FileVersioningException( "Something went wrong during the upgrade to file %s." % input_file)
def update(input_file): """ In order to avoid segmentation faults when updating a batch of files just start every conversion on a different Python process. :param input_file: the file that needs to be converted to a newer file storage version. This should be a file that still uses TVB 2.0 storage """ if not os.path.isfile(input_file): raise FileVersioningException( "The input path %s received for upgrading from 2 -> 3 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = HDF5StorageManager(folder, file_name) root_metadata = storage_manager.get_metadata() class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] if class_name == "LocalConnectivity": root_metadata[ DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.local_connectivity" storage_manager.set_metadata(root_metadata) _update_localconnectivity_metadata(folder, file_name) elif class_name == "RegionMapping": root_metadata[ DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.region_mapping" root_metadata[ TvbProfile.current.version. DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)
def update(input_file): """ :param input_file: the file that needs to be converted to a newer file storage version. """ if not os.path.isfile(input_file): raise FileVersioningException("The input path %s received for upgrading from 3 -> 4 is not a " "valid file on the disk." % input_file) folder, file_name = os.path.split(input_file) storage_manager = HDF5StorageManager(folder, file_name) root_metadata = storage_manager.get_metadata() class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME] if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata: LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder)) projection_type = projections_data.EEG_POLYMORPHIC_IDENTITY if "SEEG" in class_name: projection_type = projections_data.SEEG_POLYMORPHIC_IDENTITY elif "MEG" in class_name: projection_type = projections_data.MEG_POLYMORPHIC_IDENTITY root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type) LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type)) elif "TimeSeries" in class_name: LOGGER.info("Updating TS %s from %s" % (file_name, folder)) service = ImportService() operation_id = int(os.path.split(folder)[1]) dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False) dt_db = dao.get_datatype_by_gid(dt.gid) if dt_db is not None: # DT already in DB (update of own storage, by making sure all fields are being correctly populated) dt_db.configure() dt_db.persist_full_metadata() # restore in DB, in case TVB 1.4 had wrongly imported flags dao.store_entity(dt_db) elif FIELD_SURFACE_MAPPING not in root_metadata: # Have default values, to avoid the full project not being imported root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False) root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False) root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION storage_manager.set_metadata(root_metadata)
def load_entity_by_gid(data_gid): """ Load a generic DataType, specified by GID. """ datatype = dao.get_datatype_by_gid(data_gid) if isinstance(datatype, MappedType): datatype_path = datatype.get_storage_file_path() files_update_manager = FilesUpdateManager() if not files_update_manager.is_file_up_to_date(datatype_path): datatype.invalid = True dao.store_entity(datatype) raise FileVersioningException( "Encountered DataType with an incompatible storage or data version. " "The DataType was marked as invalid.") return datatype
# Use a file-path with no spaces both for the temporary file and the input file # that is passed to the os.system call and just rename to original file at the # end of the processing to avoid any problems with parameters passed to os.system. input_file_no_spaces = input_file.replace(' ', '') path_to, file_name = os.path.split(input_file_no_spaces) tmp_convert_file = os.path.join(path_to, 'tmp_' + file_name) os.rename(input_file, tmp_convert_file) ok_status = os.system(PYTHON_EXE_PATH + ' -m %s %s %s' % (__name__, tmp_convert_file, input_file_no_spaces)) if ok_status == 0: # Call finished successfully os.remove(tmp_convert_file) os.rename(input_file_no_spaces, input_file) else: # Call failed for some reason, just rename back the input file os.rename(tmp_convert_file, input_file) raise FileVersioningException( "Something went wrong during the upgrade to file %s." % input_file) ### This main is important, and used by the update() method from above. ### Do not drop this __main__ if __name__ == '__main__': if len(sys.argv) != 3: raise FileVersioningException( "Usage is `python -m tvb.core.entities.file.file_update_scripts.002_update_files" " input_file_name output_file_name`.") input_file = sys.argv[1] output_file = sys.argv[2] __upgrade_file(input_file, output_file)