def update(input_file):
"""
In order to avoid segmentation faults when updating a batch of files just
start every conversion on a different Python process.
:param input_file: the file that needs to be converted to a newer file storage version.
This should be a file that still uses TVB 1.0 storage (pyTables)
"""
# Just to avoid any problems about renaming open files, do a rename from the start
# and if case of a fault in the os.system call just rename back and remove the output file.
if not os.path.isfile(input_file):
raise FileVersioningException(
"The input path %s received for upgrading from 1 -> 2 is not a "
"valid file on the disk." % input_file)
# Use a file-path with no spaces both for the temporary file and the input file
# that is passed to the os.system call and just rename to original file at the
# end of the processing to avoid any problems with parameters passed to os.system.
input_file_no_spaces = input_file.replace(' ', '')
path_to, file_name = os.path.split(input_file_no_spaces)
tmp_convert_file = os.path.join(path_to, 'tmp_' + file_name)
os.rename(input_file, tmp_convert_file)
ok_status = os.system(PYTHON_EXE_PATH + ' -m %s %s %s' %
(__name__, tmp_convert_file, input_file_no_spaces))
if ok_status == 0:
# Call finished successfully
os.remove(tmp_convert_file)
os.rename(input_file_no_spaces, input_file)
else:
# Call failed for some reason, just rename back the input file
os.rename(tmp_convert_file, input_file)
raise FileVersioningException(
"Something went wrong during the upgrade to file %s." % input_file)
def update(input_file):
"""
In order to avoid segmentation faults when updating a batch of files just
start every conversion on a different Python process.
:param input_file: the file that needs to be converted to a newer file storage version.
This should be a file that still uses TVB 2.0 storage
"""
if not os.path.isfile(input_file):
raise FileVersioningException(
"The input path %s received for upgrading from 2 -> 3 is not a "
"valid file on the disk." % input_file)
folder, file_name = os.path.split(input_file)
storage_manager = HDF5StorageManager(folder, file_name)
root_metadata = storage_manager.get_metadata()
class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME]
if class_name == "LocalConnectivity":
root_metadata[
DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.local_connectivity"
storage_manager.set_metadata(root_metadata)
_update_localconnectivity_metadata(folder, file_name)
elif class_name == "RegionMapping":
root_metadata[
DataTypeMetaData.KEY_MODULE] = "tvb.datatypes.region_mapping"
root_metadata[
TvbProfile.current.version.
DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION
storage_manager.set_metadata(root_metadata)
def update(input_file):
"""
:param input_file: the file that needs to be converted to a newer file storage version.
"""
if not os.path.isfile(input_file):
raise FileVersioningException("The input path %s received for upgrading from 3 -> 4 is not a "
"valid file on the disk." % input_file)
folder, file_name = os.path.split(input_file)
storage_manager = HDF5StorageManager(folder, file_name)
root_metadata = storage_manager.get_metadata()
class_name = root_metadata[DataTypeMetaData.KEY_CLASS_NAME]
if "ProjectionSurface" in class_name and FIELD_PROJECTION_TYPE not in root_metadata:
LOGGER.info("Updating ProjectionSurface %s from %s" % (file_name, folder))
projection_type = projections_data.EEG_POLYMORPHIC_IDENTITY
if "SEEG" in class_name:
projection_type = projections_data.SEEG_POLYMORPHIC_IDENTITY
elif "MEG" in class_name:
projection_type = projections_data.MEG_POLYMORPHIC_IDENTITY
root_metadata[FIELD_PROJECTION_TYPE] = json.dumps(projection_type)
LOGGER.debug("Setting %s = %s" % (FIELD_PROJECTION_TYPE, projection_type))
elif "TimeSeries" in class_name:
LOGGER.info("Updating TS %s from %s" % (file_name, folder))
service = ImportService()
operation_id = int(os.path.split(folder)[1])
dt = service.load_datatype_from_file(folder, file_name, operation_id, move=False)
dt_db = dao.get_datatype_by_gid(dt.gid)
if dt_db is not None:
# DT already in DB (update of own storage, by making sure all fields are being correctly populated)
dt_db.configure()
dt_db.persist_full_metadata()
# restore in DB, in case TVB 1.4 had wrongly imported flags
dao.store_entity(dt_db)
elif FIELD_SURFACE_MAPPING not in root_metadata:
# Have default values, to avoid the full project not being imported
root_metadata[FIELD_SURFACE_MAPPING] = json.dumps(False)
root_metadata[FIELD_VOLUME_MAPPING] = json.dumps(False)
root_metadata[TvbProfile.current.version.DATA_VERSION_ATTRIBUTE] = TvbProfile.current.version.DATA_VERSION
storage_manager.set_metadata(root_metadata)
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
datatype = dao.get_datatype_by_gid(data_gid)
if isinstance(datatype, MappedType):
datatype_path = datatype.get_storage_file_path()
files_update_manager = FilesUpdateManager()
if not files_update_manager.is_file_up_to_date(datatype_path):
datatype.invalid = True
dao.store_entity(datatype)
raise FileVersioningException(
"Encountered DataType with an incompatible storage or data version. "
"The DataType was marked as invalid.")
return datatype
# Use a file-path with no spaces both for the temporary file and the input file
# that is passed to the os.system call and just rename to original file at the
# end of the processing to avoid any problems with parameters passed to os.system.
input_file_no_spaces = input_file.replace(' ', '')
path_to, file_name = os.path.split(input_file_no_spaces)
tmp_convert_file = os.path.join(path_to, 'tmp_' + file_name)
os.rename(input_file, tmp_convert_file)
ok_status = os.system(PYTHON_EXE_PATH + ' -m %s %s %s' %
(__name__, tmp_convert_file, input_file_no_spaces))
if ok_status == 0:
# Call finished successfully
os.remove(tmp_convert_file)
os.rename(input_file_no_spaces, input_file)
else:
# Call failed for some reason, just rename back the input file
os.rename(tmp_convert_file, input_file)
raise FileVersioningException(
"Something went wrong during the upgrade to file %s." % input_file)
### This main is important, and used by the update() method from above.
### Do not drop this __main__
if __name__ == '__main__':
if len(sys.argv) != 3:
raise FileVersioningException(
"Usage is `python -m tvb.core.entities.file.file_update_scripts.002_update_files"
" input_file_name output_file_name`.")
input_file = sys.argv[1]
output_file = sys.argv[2]
__upgrade_file(input_file, output_file)