def _load_datatypes_from_operation_folder(self, op_path, operation_entity,
datatype_group):
"""
Loads datatypes from operation folder
:returns: Datatypes ordered by creation date (to solve any dependencies)
"""
all_datatypes = []
for file_name in os.listdir(op_path):
if file_name.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(op_path, file_name)
try:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
datatype = self.load_datatype_from_file(
op_path, file_name, operation_entity.id,
datatype_group)
all_datatypes.append(datatype)
except IncompatibleFileManagerException:
os.remove(h5_file)
self.logger.warning(
"Incompatible H5 file will be ignored: %s" % h5_file)
self.logger.exception("Incompatibility details ...")
all_datatypes.sort(key=lambda dt_date: dt_date.create_date)
for dt in all_datatypes:
self.logger.debug("Import order %s: %s" % (dt.type, dt.gid))
return all_datatypes
def _load_datatypes_from_operation_folder(self, src_op_path,
operation_entity,
datatype_group):
"""
Loads datatypes from operation folder
:returns: Datatype entities as dict {original_path: Dt instance}
"""
all_datatypes = {}
for file_name in os.listdir(src_op_path):
if file_name.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(src_op_path, file_name)
try:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
datatype = self.load_datatype_from_file(
h5_file, operation_entity.id, datatype_group,
operation_entity.fk_launched_in)
all_datatypes[h5_file] = datatype
except IncompatibleFileManagerException:
os.remove(h5_file)
self.logger.warning(
"Incompatible H5 file will be ignored: %s" % h5_file)
self.logger.exception("Incompatibility details ...")
return all_datatypes
def launch(self, data_file):
"""
Execute import operations: unpack ZIP, build and store generic DataType objects.
:param data_file: an archive (ZIP / HDF5) containing the `DataType`
:raises LaunchException: when data_file is None, nonexistent, or invalid \
(e.g. incomplete meta-data, not in ZIP / HDF5 format etc. )
"""
if data_file is None:
raise LaunchException(
"Please select file which contains data to import")
if os.path.exists(data_file):
if zipfile.is_zipfile(data_file):
current_op = dao.get_operation_by_id(self.operation_id)
# Creates a new TMP folder where to extract data
tmp_folder = os.path.join(self.storage_path, "tmp_import")
FilesHelper().unpack_zip(data_file, tmp_folder)
operations = ImportService().import_project_operations(
current_op.project, self.storage_path)
shutil.rmtree(tmp_folder)
self.nr_of_datatypes += len(operations)
else:
# upgrade file if necessary
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(data_file)
folder, h5file = os.path.split(data_file)
manager = HDF5StorageManager(folder, h5file)
if manager.is_valid_hdf5_file():
datatype = None
try:
service = ImportService()
datatype = service.load_datatype_from_file(
folder,
h5file,
self.operation_id,
final_storage=self.storage_path)
service.store_datatype(datatype)
self.nr_of_datatypes += 1
except Exception as excep:
# If import operation failed delete file from disk.
if datatype is not None and os.path.exists(
datatype.get_storage_file_path()):
os.remove(datatype.get_storage_file_path())
self.log.exception(excep)
raise LaunchException(
"Invalid file received as input. Most probably incomplete "
"meta-data ... " + str(excep))
else:
raise LaunchException(
"Uploaded file: %s is neither in ZIP or HDF5 format" %
data_file)
else:
raise LaunchException("File: %s to import does not exists." %
data_file)
def launch(self, view_model):
# type: (TVBImporterModel) -> []
"""
Execute import operations: unpack ZIP, build and store generic DataType objects.
:raises LaunchException: when data_file is None, nonexistent, or invalid \
(e.g. incomplete meta-data, not in ZIP / HDF5 format etc. )
"""
if view_model.data_file is None:
raise LaunchException(
"Please select file which contains data to import")
service = ImportService()
if os.path.exists(view_model.data_file):
if zipfile.is_zipfile(view_model.data_file):
current_op = dao.get_operation_by_id(self.operation_id)
# Creates a new TMP folder where to extract data
tmp_folder = os.path.join(self.storage_path, "tmp_import")
FilesHelper().unpack_zip(view_model.data_file, tmp_folder)
operations = service.import_project_operations(
current_op.project, tmp_folder)
shutil.rmtree(tmp_folder)
self.nr_of_datatypes += len(operations)
else:
# upgrade file if necessary
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(view_model.data_file)
folder, h5file = os.path.split(view_model.data_file)
manager = HDF5StorageManager(folder, h5file)
if manager.is_valid_hdf5_file():
datatype = None
try:
datatype = service.load_datatype_from_file(
view_model.data_file, self.operation_id)
service.check_import_references(
view_model.data_file, datatype)
service.store_datatype(datatype, view_model.data_file)
self.nr_of_datatypes += 1
except ImportException as excep:
self.log.exception(excep)
if datatype is not None:
target_path = h5.path_for_stored_index(datatype)
if os.path.exists(target_path):
os.remove(target_path)
raise LaunchException(
"Invalid file received as input. " + str(excep))
else:
raise LaunchException(
"Uploaded file: %s is neither in ZIP or HDF5 format" %
view_model.data_file)
else:
raise LaunchException("File: %s to import does not exists." %
view_model.data_file)
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
datatype = dao.get_datatype_by_gid(data_gid)
if isinstance(datatype, MappedType):
datatype_path = datatype.get_storage_file_path()
files_update_manager = FilesUpdateManager()
if not files_update_manager.is_file_up_to_date(datatype_path):
datatype.invalid = True
dao.store_entity(datatype)
raise FileVersioningException("Encountered DataType with an incompatible storage or data version. "
"The DataType was marked as invalid.")
return datatype
def launch(self, data_file):
"""
Execute import operations: unpack ZIP, build and store generic DataType objects.
:param data_file: an archive (ZIP / HDF5) containing the `DataType`
:raises LaunchException: when data_file is None, nonexistent, or invalid \
(e.g. incomplete meta-data, not in ZIP / HDF5 format etc. )
"""
if data_file is None:
raise LaunchException("Please select file which contains data to import")
if os.path.exists(data_file):
if zipfile.is_zipfile(data_file):
current_op = dao.get_operation_by_id(self.operation_id)
# Creates a new TMP folder where to extract data
tmp_folder = os.path.join(self.storage_path, "tmp_import")
FilesHelper().unpack_zip(data_file, tmp_folder)
operations = ImportService().import_project_operations(current_op.project, self.storage_path)
shutil.rmtree(tmp_folder)
self.nr_of_datatypes += len(operations)
else:
# upgrade file if necessary
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(data_file)
folder, h5file = os.path.split(data_file)
manager = HDF5StorageManager(folder, h5file)
if manager.is_valid_hdf5_file():
datatype = None
try:
service = ImportService()
datatype = service.load_datatype_from_file(folder, h5file, self.operation_id)
service.store_datatype(datatype)
self.nr_of_datatypes += 1
except Exception as excep:
# If import operation failed delete file from disk.
if datatype is not None and os.path.exists(datatype.get_storage_file_path()):
os.remove(datatype.get_storage_file_path())
self.log.exception(excep)
raise LaunchException("Invalid file received as input. Most probably incomplete "
"meta-data ... " + str(excep))
else:
raise LaunchException("Uploaded file: %s is neither in ZIP or HDF5 format" % data_file)
else:
raise LaunchException("File: %s to import does not exists." % data_file)
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
datatype = dao.get_datatype_by_gid(data_gid)
if isinstance(datatype, MappedType):
datatype_path = datatype.get_storage_file_path()
files_update_manager = FilesUpdateManager()
if not files_update_manager.is_file_up_to_date(datatype_path):
datatype.invalid = True
dao.store_entity(datatype)
raise FileVersioningException(
"Encountered DataType with an incompatible storage or data version. "
"The DataType was marked as invalid.")
return datatype
def initialize(introspected_modules, load_xml_events=True):
"""
Initialize when Application is starting.
Check for new algorithms or new DataTypes.
"""
SettingsService().check_db_url(TvbProfile.current.db.DB_URL)
## Initialize DB
is_db_empty = initialize_startup()
## Create Projects storage root in case it does not exist.
initialize_storage()
## Populate DB algorithms, by introspection
event_folders = []
start_introspection_time = datetime.datetime.now()
for module in introspected_modules:
introspector = Introspector(module)
# Introspection is always done, even if DB was not empty.
introspector.introspect(True)
event_path = introspector.get_events_path()
if event_path:
event_folders.append(event_path)
# Now remove or mark as removed any unverified Algo-Group, Algo-Category or Portlet
to_invalidate, to_remove = dao.get_non_validated_entities(
start_introspection_time)
for entity in to_invalidate:
entity.removed = True
dao.store_entities(to_invalidate)
for entity in to_remove:
dao.remove_entity(entity.__class__, entity.id)
## Populate events
if load_xml_events:
read_events(event_folders)
if not TvbProfile.is_first_run():
## Create default users.
if is_db_empty:
dao.store_entity(
model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None,
None, True, None))
UserService().create_user(
username=TvbProfile.current.web.admin.ADMINISTRATOR_NAME,
password=TvbProfile.current.web.admin.ADMINISTRATOR_PASSWORD,
email=TvbProfile.current.web.admin.ADMINISTRATOR_EMAIL,
role=model.ROLE_ADMINISTRATOR)
## In case actions related to latest code-changes are needed, make sure they are executed.
CodeUpdateManager().run_all_updates()
## In case the H5 version changed, run updates on all DataTypes
if TvbProfile.current.version.DATA_CHECKED_TO_VERSION < TvbProfile.current.version.DATA_VERSION:
thread = threading.Thread(
target=FilesUpdateManager().run_all_updates)
thread.start()
## Clean tvb-first-time-run temporary folder, as we are no longer at the first run:
shutil.rmtree(TvbProfile.current.FIRST_RUN_STORAGE, True)
def _load_datatypes_from_operation_folder(self, op_path, operation_entity, datatype_group):
"""
Loads datatypes from operation folder
:returns: Datatypes ordered by creation date (to solve any dependencies)
"""
all_datatypes = []
for file_name in os.listdir(op_path):
if file_name.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(op_path, file_name)
try:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
datatype = self.load_datatype_from_file(op_path, file_name, operation_entity.id, datatype_group)
all_datatypes.append(datatype)
except IncompatibleFileManagerException:
os.remove(h5_file)
self.logger.warning("Incompatible H5 file will be ignored: %s" % h5_file)
all_datatypes.sort(key=lambda dt_date: dt_date.create_date)
for dt in all_datatypes:
self.logger.debug("Import order %s: %s" % (dt.type, dt.gid))
return all_datatypes
def initialize(skip_import=False, skip_updates=False):
"""
Initialize when Application is starting.
Check for new algorithms or new DataTypes.
"""
SettingsService().check_db_url(TvbProfile.current.db.DB_URL)
# Initialize DB
is_db_empty = initialize_startup()
# Create Projects storage root in case it does not exist.
initialize_storage()
# Populate DB algorithms, by introspection
start_introspection_time = datetime.now()
# Introspection is always done, even if DB was not empty.
introspector = Introspector()
introspector.introspect()
to_invalidate = dao.get_non_validated_entities(start_introspection_time)
for entity in to_invalidate:
entity.removed = True
dao.store_entities(to_invalidate)
if not TvbProfile.is_first_run() and not skip_updates:
# Create default users.
if is_db_empty:
dao.store_entity(
User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, TvbProfile.current.web.admin.SYSTEM_USER_NAME, None,
None, True, None))
UserService().create_user(username=TvbProfile.current.web.admin.ADMINISTRATOR_NAME,
display_name=TvbProfile.current.web.admin.ADMINISTRATOR_DISPLAY_NAME,
password=TvbProfile.current.web.admin.ADMINISTRATOR_PASSWORD,
email=TvbProfile.current.web.admin.ADMINISTRATOR_EMAIL,
role=ROLE_ADMINISTRATOR, skip_import=skip_import)
# In case actions related to latest code-changes are needed, make sure they are executed.
CodeUpdateManager().run_all_updates()
# In case the H5 version changed, run updates on all DataTypes
thread = None
if TvbProfile.current.version.DATA_CHECKED_TO_VERSION < TvbProfile.current.version.DATA_VERSION:
thread = threading.Thread(target=FilesUpdateManager().run_all_updates)
thread.start()
# Clean tvb-first-time-run temporary folder, as we are no longer at the first run:
shutil.rmtree(TvbProfile.current.FIRST_RUN_STORAGE, True)
return thread
def _retrieve_operations_in_order(self, project, import_path):
# type: (Project, str) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters = self.__build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(view_model
) in VIEW_MODEL2ADAPTER.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = VIEW_MODEL2ADAPTER[type(main_view_model)]
operation = Operation(main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.
generic_attributes.operation_tag,
start_date=datetime.now(),
completion_date=datetime.now())
operation.create_date = main_view_model.create_date
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
def launch(self, view_model):
# type: (TVBImporterModel) -> []
"""
Execute import operations: unpack ZIP, build and store generic DataType objects.
:raises LaunchException: when data_file is None, nonexistent, or invalid \
(e.g. incomplete meta-data, not in ZIP / HDF5 format etc. )
"""
if view_model.data_file is None:
raise LaunchException(
"Please select file which contains data to import")
service = ImportService()
if os.path.exists(view_model.data_file):
current_op = dao.get_operation_by_id(self.operation_id)
if zipfile.is_zipfile(view_model.data_file):
# Creates a new TMP folder where to extract data
tmp_folder = os.path.join(self.storage_path, "tmp_import")
self.storage_interface.unpack_zip(view_model.data_file,
tmp_folder)
is_group = False
current_op_id = current_op.id
for file in os.listdir(tmp_folder):
# In case we import a DatatypeGroup, we want the default import flow
if os.path.isdir(os.path.join(tmp_folder, file)):
current_op_id = None
is_group = True
break
try:
operations, all_dts, stored_dts_count = service.import_project_operations(
current_op.project, tmp_folder, is_group,
current_op_id)
self.nr_of_datatypes += stored_dts_count
if stored_dts_count == 0:
current_op.additional_info = 'All chosen datatypes already exist!'
dao.store_entity(current_op)
elif stored_dts_count < all_dts:
current_op.additional_info = 'Part of the chosen datatypes already exist!'
dao.store_entity(current_op)
except ImportException as excep:
self.log.exception(excep)
current_op.additional_info = excep.message
current_op.status = STATUS_ERROR
raise LaunchException("Invalid file received as input. " +
str(excep))
finally:
shutil.rmtree(tmp_folder)
else:
# upgrade file if necessary
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(view_model.data_file)
if self.storage_interface.get_storage_manager(
view_model.data_file).is_valid_tvb_file():
datatype = None
try:
datatype = service.load_datatype_from_file(
view_model.data_file, self.operation_id)
stored_new_dt = service.store_or_link_datatype(
datatype, view_model.data_file,
current_op.project.id)
if stored_new_dt == 0:
current_op.additional_info = 'The chosen datatype already exists!'
dao.store_entity(current_op)
self.nr_of_datatypes += stored_new_dt
except ImportException as excep:
self.log.exception(excep)
if datatype is not None:
target_path = h5.path_for_stored_index(datatype)
if os.path.exists(target_path):
os.remove(target_path)
raise LaunchException(
"Invalid file received as input. " + str(excep))
else:
raise LaunchException(
"Uploaded file: %s is neither in ZIP or HDF5 format" %
view_model.data_file)
else:
raise LaunchException("File: %s to import does not exists." %
view_model.data_file)
import sys
from tvb.basic.profile import TvbProfile
from tvb.config.init.initializer import initialize
from tvb.core.entities.file.files_update_manager import FilesUpdateManager
if __name__ == '__main__':
"""
Script written for testing the migration from version 1.5.8 to 2.1.0.
"""
# set web profile
TvbProfile.set_profile(TvbProfile.WEB_PROFILE)
# migrate the database and h5 files
h5_migrating_thread = initialize()
# wait for thread to finish before processing
h5_migrating_thread.join()
# copy files in tvb_root folder so Jenkins can find them
shutil.copytree(
TvbProfile.current.TVB_LOG_FOLDER,
os.path.join(TvbProfile.current.EXTERNALS_FOLDER_PARENT, 'logs'))
# test if there are any files which were not migrated
number_of_unmigrated_files = len(FilesUpdateManager.get_all_h5_paths())
if number_of_unmigrated_files != 0:
sys.exit(-1)
sys.exit(0)
def import_project_operations(self, project, import_path, dt_burst_mappings=None, burst_ids_mapping=None):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
# Identify folders containing operations
operations = []
for root, _, files in os.walk(import_path):
if FilesHelper.TVB_OPERARATION_FILE in files:
# Found an operation folder - append TMP to its name
tmp_op_folder = os.path.join(os.path.split(root)[0], os.path.split(root)[1] + 'tmp')
os.rename(root, tmp_op_folder)
operation_file_path = os.path.join(tmp_op_folder, FilesHelper.TVB_OPERARATION_FILE)
operation = self.__build_operation_from_file(project, operation_file_path)
operation.import_file = operation_file_path
operations.append(operation)
imported_operations = []
# Now we sort operations by start date, to be sure data dependency is resolved correct
operations = sorted(operations, key=lambda operation: operation.start_date)
# Here we process each operation found
for operation in operations:
old_operation_folder, _ = os.path.split(operation.import_file)
operation_entity, datatype_group = self.__import_operation(operation)
# Rename operation folder with the ID of the stored operation
new_operation_path = FilesHelper().get_operation_folder(project.name, operation_entity.id)
if old_operation_folder != new_operation_path:
# Delete folder of the new operation, otherwise move will fail
shutil.rmtree(new_operation_path)
shutil.move(old_operation_folder, new_operation_path)
# Now process data types for each operation
all_datatypes = []
for file_name in os.listdir(new_operation_path):
if file_name.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(os.path.join(new_operation_path, file_name))
datatype = self.load_datatype_from_file(new_operation_path, file_name,
operation_entity.id, datatype_group)
all_datatypes.append(datatype)
# Before inserting into DB sort data types by creation date (to solve any dependencies)
all_datatypes = sorted(all_datatypes, key=lambda datatype: datatype.create_date)
# Now store data types into DB
for datatype in all_datatypes:
if datatype.gid in dt_burst_mappings:
old_burst_id = dt_burst_mappings[datatype.gid]
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
self.store_datatype(datatype)
# Now import all images from current operation
images_root = self.files_helper.get_images_folder(project.name, operation_entity.id)
if os.path.exists(images_root):
for root, _, files in os.walk(images_root):
for file_name in files:
if file_name.endswith(FilesHelper.TVB_FILE_EXTENSION):
self.__populate_image(os.path.join(root, file_name), project.id, operation_entity.id)
imported_operations.append(operation_entity)
return imported_operations