def load_datatype_from_file(self, current_file, op_id, datatype_group=None, current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
index_class = h5.REGISTRY.get_index_for_datatype(datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = StorageInterface.compute_size_on_disk(associated_file)
result = datatype_index
return result
class FilesUpdateManager(UpdateManager):
"""
Manager for updating H5 files version, when code gets changed.
"""
UPDATE_SCRIPTS_SUFFIX = "_update_files"
PROJECTS_PAGE_SIZE = 20
DATA_TYPES_PAGE_SIZE = 500
STATUS = True
MESSAGE = "Done"
def __init__(self):
super(FilesUpdateManager, self).__init__(file_update_scripts,
TvbProfile.current.version.DATA_CHECKED_TO_VERSION,
TvbProfile.current.version.DATA_VERSION)
self.storage_interface = StorageInterface()
def get_file_data_version(self, file_path):
"""
Return the data version for the given file.
:param file_path: the path on disk to the file for which you need the TVB data version
:returns: a number representing the data version for which the input file was written
"""
data_version = TvbProfile.current.version.DATA_VERSION_ATTRIBUTE
return self.storage_interface.get_storage_manager(file_path).get_file_data_version(data_version)
def is_file_up_to_date(self, file_path):
"""
Returns True only if the data version of the file is equal with the
data version specified into the TVB configuration file.
"""
try:
file_version = self.get_file_data_version(file_path)
except MissingDataFileException as ex:
self.log.exception(ex)
return False
except FileStructureException as ex:
self.log.exception(ex)
return False
if file_version == TvbProfile.current.version.DATA_VERSION:
return True
return False
def upgrade_file(self, input_file_name, datatype=None, burst_match_dict=None):
"""
Upgrades the given file to the latest data version. The file will be upgraded
sequentially, up until the current version from tvb.basic.config.settings.VersionSettings.DB_STRUCTURE_VERSION
:param input_file_name the path to the file which needs to be upgraded
:return True when update was successful and False when it resulted in an error.
"""
if self.is_file_up_to_date(input_file_name):
# Avoid running the DB update of size, when H5 is not being changed, to speed-up
return True
file_version = self.get_file_data_version(input_file_name)
self.log.info("Updating from version %s , file: %s " % (file_version, input_file_name))
for script_name in self.get_update_scripts(file_version):
temp_file_path = os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
os.path.basename(input_file_name) + '.tmp')
self.storage_interface.copy_file(input_file_name, temp_file_path)
try:
self.run_update_script(script_name, input_file=input_file_name, burst_match_dict=burst_match_dict)
except FileMigrationException as excep:
self.storage_interface.copy_file(temp_file_path, input_file_name)
os.remove(temp_file_path)
self.log.error(excep)
return False
if datatype:
# Compute and update the disk_size attribute of the DataType in DB:
datatype.disk_size = self.storage_interface.compute_size_on_disk(input_file_name)
dao.store_entity(datatype)
return True
def __upgrade_h5_list(self, h5_files):
"""
Upgrade a list of DataTypes to the current version.
:returns: (nr_of_dts_upgraded_fine, nr_of_dts_ignored) a two-tuple of integers representing
the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which
the upgrade has failed.
"""
nr_of_dts_upgraded_fine = 0
nr_of_dts_failed = 0
burst_match_dict = {}
for path in h5_files:
update_result = self.upgrade_file(path, burst_match_dict=burst_match_dict)
if update_result:
nr_of_dts_upgraded_fine += 1
else:
nr_of_dts_failed += 1
return nr_of_dts_upgraded_fine, nr_of_dts_failed
# TO DO: We should migrate the older scripts to Python 3 if we want to support migration for versions < 4
def run_all_updates(self):
"""
Upgrades all the data types from TVB storage to the latest data version.
:returns: a two entry tuple (status, message) where status is a boolean that is True in case
the upgrade was successfully for all DataTypes and False otherwise, and message is a status
update message.
"""
if TvbProfile.current.version.DATA_CHECKED_TO_VERSION < TvbProfile.current.version.DATA_VERSION:
start_time = datetime.now()
file_paths = self.get_all_h5_paths()
total_count = len(file_paths)
no_ok, no_error = self.__upgrade_h5_list(file_paths)
self.log.info("Updated H5 files in total: %d [fine:%d, failed:%d in: %s min]" % (
total_count, no_ok, no_error, int((datetime.now() - start_time).seconds / 60)))
delete_old_burst_table_after_migration()
# Now update the configuration file since update was done
config_file_update_dict = {stored.KEY_LAST_CHECKED_FILE_VERSION: TvbProfile.current.version.DATA_VERSION}
if no_error == 0:
# Everything went fine
config_file_update_dict[stored.KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_VALID
FilesUpdateManager.STATUS = True
FilesUpdateManager.MESSAGE = ("File upgrade finished successfully for all %s entries. "
"Thank you for your patience!" % total_count)
self.log.info(FilesUpdateManager.MESSAGE)
else:
# Keep track of how many DataTypes were properly updated and how many
# were marked as invalid due to missing files or invalid manager.
config_file_update_dict[stored.KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_INVALID
FilesUpdateManager.STATUS = False
FilesUpdateManager.MESSAGE = ("Out of %s stored DataTypes, %s were upgraded successfully, but %s had "
"faults and were marked invalid" % (total_count, no_ok, no_error))
self.log.warning(FilesUpdateManager.MESSAGE)
TvbProfile.current.version.DATA_CHECKED_TO_VERSION = TvbProfile.current.version.DATA_VERSION
TvbProfile.current.manager.add_entries_to_config_file(config_file_update_dict)
@staticmethod
def get_all_h5_paths():
"""
This method returns a list of all h5 files and it is used in the migration from version 4 to 5.
The h5 files inside a certain project are retrieved in numerical order (1, 2, 3 etc.).
"""
h5_files = []
projects_folder = StorageInterface().get_projects_folder()
for project_path in os.listdir(projects_folder):
# Getting operation folders inside the current project
project_full_path = os.path.join(projects_folder, project_path)
try:
project_operations = os.listdir(project_full_path)
except NotADirectoryError:
continue
project_operations_base_names = [os.path.basename(op) for op in project_operations]
for op_folder in project_operations_base_names:
try:
int(op_folder)
op_folder_path = os.path.join(project_full_path, op_folder)
for file in os.listdir(op_folder_path):
if StorageInterface().ends_with_tvb_storage_file_extension(file):
h5_file = os.path.join(op_folder_path, file)
try:
if FilesUpdateManager._is_empty_file(h5_file):
continue
h5_files.append(h5_file)
except FileStructureException:
continue
except ValueError:
pass
# Sort all h5 files based on their creation date stored in the files themselves
sorted_h5_files = sorted(h5_files, key=lambda h5_path: FilesUpdateManager._get_create_date_for_sorting(
h5_path) or datetime.now())
return sorted_h5_files
@staticmethod
def _is_empty_file(h5_file):
return H5File.get_metadata_param(h5_file, 'Create_date') is None
@staticmethod
def _get_create_date_for_sorting(h5_file):
create_date_str = str(H5File.get_metadata_param(h5_file, 'Create_date'), 'utf-8')
create_date = string2date(create_date_str, date_format='datetime:%Y-%m-%d %H:%M:%S.%f')
return create_date
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# model.Algorithm instance that will be set for each adapter class created by in build_adapter method
stored_adapter = None
launch_mode = AdapterLaunchModeEnum.ASYNC_DIFF_MEM
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.storage_interface = StorageInterface()
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
def get_adapter_fragments(self, view_model):
"""
The result will be used for introspecting and checking operation changed input
params from the defaults, to show in web gui.
:return: a list of ABCAdapterForm classes, in case the current Adapter GUI
will be composed of multiple sub-forms.
"""
return {}
def get_view_model_class(self):
return self.get_form_class().get_view_model()
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, view_model):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, view_model):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self, view_model):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Takes a ViewModel with data, dependency direction is: Adapter -> Form -> ViewModel
Any returned DataType will be stored in DB, by the Framework.
:param view_model: the data model corresponding to the current adapter
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def extract_operation_data(self, operation):
operation = dao.get_operation_by_id(operation.id)
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
def _ensure_enough_resources(self, available_disk_space, view_model):
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(view_model)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(view_model)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
return required_disk_space
def _update_operation_entity(self, operation, required_disk_space):
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
@nan_not_allowed()
def _prelaunch(self, operation, view_model, available_disk_space=0):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.extract_operation_data(operation)
self.generic_attributes.fill_from(view_model.generic_attributes)
self.configure(view_model)
required_disk_size = self._ensure_enough_resources(available_disk_space, view_model)
self._update_operation_entity(operation, required_disk_size)
result = self.launch(view_model)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
count_stored = 0
if result is None:
return "", count_stored
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.storage_interface.compute_size_on_disk(associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
def load_entity_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> DataType
"""
Load a generic DataType, specified by GID.
"""
idx = load_entity_by_gid(data_gid)
if idx and self.generic_attributes.parent_burst is None:
# Only in case the BurstConfiguration references hasn't been set already, take it from the current DT
self.generic_attributes.parent_burst = idx.fk_parent_burst
return idx
def load_traited_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
"""
Load a generic HasTraits instance, specified by GID.
"""
index = self.load_entity_by_gid(data_gid)
return h5.load_from_index(index)
def load_with_references(self, dt_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
dt_index = self.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def view_model_to_has_traits(self, view_model):
# type: (ViewModel) -> HasTraits
has_traits_class = view_model.linked_has_traits
has_traits = has_traits_class()
view_model_class = type(view_model)
if not has_traits_class:
raise Exception("There is no linked HasTraits for this ViewModel {}".format(type(view_model)))
for attr_name in has_traits_class.declarative_attrs:
view_model_class_attr = getattr(view_model_class, attr_name)
view_model_attr = getattr(view_model, attr_name)
if isinstance(view_model_class_attr, DataTypeGidAttr) and view_model_attr:
attr_value = self.load_with_references(view_model_attr)
elif isinstance(view_model_class_attr, Attr) and isinstance(view_model_attr, ViewModel):
attr_value = self.view_model_to_has_traits(view_model_attr)
elif isinstance(view_model_class_attr, List) and len(view_model_attr) > 0 and isinstance(view_model_attr[0],
ViewModel):
attr_value = list()
for view_model_elem in view_model_attr:
elem = self.view_model_to_has_traits(view_model_elem)
attr_value.append(elem)
else:
attr_value = view_model_attr
setattr(has_traits, attr_name, attr_value)
return has_traits
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def determine_adapter_class(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Determine the class of an adapter based on module and classname strings from stored_adapter
:param stored_adapter: Algorithm or AlgorithmDTO type
:return: a subclass of ABCAdapter
"""
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
return adapter_class
@staticmethod
def build_adapter(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
adapter_class = ABCAdapter.determine_adapter_class(stored_adapter)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
def load_view_model(self, operation):
storage_path = self.storage_interface.get_project_folder(operation.project.name, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
@staticmethod
def array_size2kb(size):
"""
:param size: size in bytes
:return: size in kB
"""
return size * TvbProfile.current.MAGIC_NUMBER / 8 / 2 ** 10
@staticmethod
def fill_index_from_h5(analyzer_index, analyzer_h5):
"""
Method used only by analyzers that write slices of data.
As they never have the whole array_data in memory, the metadata related to array_data (min, max, etc.) they
store on the index is not correct, so we need to update them.
"""
metadata = analyzer_h5.array_data.get_cached_metadata()
if not metadata.has_complex:
analyzer_index.array_data_max = float(metadata.max)
analyzer_index.array_data_min = float(metadata.min)
analyzer_index.array_data_mean = float(metadata.mean)
analyzer_index.aray_has_complex = metadata.has_complex
analyzer_index.array_is_finite = metadata.is_finite
analyzer_index.shape = json.dumps(analyzer_h5.array_data.shape)
analyzer_index.ndim = len(analyzer_h5.array_data.shape)
def path_for(self, h5_file_class, gid, dt_class=None):
project = dao.get_project_by_id(self.current_project_id)
return h5.path_for(self.operation_id, h5_file_class, gid, project.name, dt_class)
def store_complete(self, datatype, generic_attributes=GenericAttributes()):
project = dao.get_project_by_id(self.current_project_id)
return h5.store_complete(datatype, self.operation_id, project.name, generic_attributes)
def get_storage_path(self):
project = dao.get_project_by_id(self.current_project_id)
return self.storage_interface.get_project_folder(project.name, str(self.operation_id))
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
already_existing_datatype = h5.load_entity_by_gid(datatype.gid)
if datatype_group is not None and already_existing_datatype is not None:
raise DatatypeGroupImportException(
"The datatype group that you are trying to import"
" already exists!")
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
if datatype_group is not None and hasattr(datatype_index, 'fk_source_gid') and \
datatype_index.fk_source_gid is not None:
ts = h5.load_entity_by_gid(datatype_index.fk_source_gid)
if ts is None:
op = dao.get_operations_in_group(
datatype_group.fk_operation_group,
only_first_operation=True)
op.fk_operation_group = None
dao.store_entity(op)
dao.remove_entity(OperationGroup,
datatype_group.fk_operation_group)
dao.remove_entity(DataTypeGroup, datatype_group.id)
raise DatatypeGroupImportException(
"Please import the time series group before importing the"
" datatype measure group!")
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = StorageInterface.compute_size_on_disk(
associated_file)
result = datatype_index
return result
def import_list_of_operations(self,
project,
import_path,
is_group=False,
importer_operation_id=None):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
all_dts_count = 0
all_stored_dts_count = 0
imported_operations = []
ordered_operations = self._retrieve_operations_in_order(
project, import_path, None if is_group else importer_operation_id)
if is_group and len(ordered_operations) > 0:
first_op = dao.get_operation_by_id(importer_operation_id)
vm_path = h5.determine_filepath(first_op.view_model_gid,
os.path.dirname(import_path))
os.remove(vm_path)
ordered_operations[0].operation.id = importer_operation_id
for operation_data in ordered_operations:
if operation_data.is_old_form:
operation_entity, datatype_group = self.import_operation(
operation_data.operation)
new_op_folder = self.storage_interface.get_project_folder(
project.name, str(operation_entity.id))
try:
operation_datatypes = self._load_datatypes_from_operation_folder(
operation_data.operation_folder, operation_entity,
datatype_group)
# Create and store view_model from operation
self.create_view_model(operation_entity, operation_data,
new_op_folder)
self._store_imported_datatypes_in_db(
project, operation_datatypes)
imported_operations.append(operation_entity)
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
elif operation_data.main_view_model is not None:
operation_data.operation.create_date = datetime.now()
operation_data.operation.start_date = datetime.now()
operation_data.operation.completion_date = datetime.now()
do_merge = False
if importer_operation_id:
do_merge = True
operation_entity = dao.store_entity(operation_data.operation,
merge=do_merge)
dt_group = None
op_group = dao.get_operationgroup_by_id(
operation_entity.fk_operation_group)
if op_group:
dt_group = dao.get_datatypegroup_by_op_group_id(
op_group.id)
if not dt_group:
first_op = dao.get_operations_in_group(
op_group.id, only_first_operation=True)
dt_group = DataTypeGroup(
op_group,
operation_id=first_op.id,
state=DEFAULTDATASTATE_INTERMEDIATE)
dt_group = dao.store_entity(dt_group)
# Store the DataTypes in db
dts = {}
all_dts_count += len(operation_data.dt_paths)
for dt_path in operation_data.dt_paths:
dt = self.load_datatype_from_file(dt_path,
operation_entity.id,
dt_group, project.id)
if isinstance(dt, BurstConfiguration):
if op_group:
dt.fk_operation_group = op_group.id
all_stored_dts_count += self._store_or_link_burst_config(
dt, dt_path, project.id)
else:
dts[dt_path] = dt
if op_group:
op_group.fill_operationgroup_name(dt.type)
dao.store_entity(op_group)
try:
stored_dts_count = self._store_imported_datatypes_in_db(
project, dts)
all_stored_dts_count += stored_dts_count
if operation_data.main_view_model.is_metric_operation:
self._update_burst_metric(operation_entity)
imported_operations.append(operation_entity)
new_op_folder = self.storage_interface.get_project_folder(
project.name, str(operation_entity.id))
view_model_disk_size = 0
for h5_file in operation_data.all_view_model_files:
view_model_disk_size += StorageInterface.compute_size_on_disk(
h5_file)
shutil.move(h5_file, new_op_folder)
operation_entity.view_model_disk_size = view_model_disk_size
dao.store_entity(operation_entity)
except MissingReferenceException as excep:
self.storage_interface.remove_operation_data(
project.name, operation_entity.id)
operation_entity.fk_operation_group = None
dao.store_entity(operation_entity)
dao.remove_entity(DataTypeGroup, dt_group.id)
raise excep
else:
self.logger.warning(
"Folder %s will be ignored, as we could not find a serialized "
"operation or DTs inside!" %
operation_data.operation_folder)
# We want importer_operation_id to be kept just for the first operation (the first iteration)
if is_group:
importer_operation_id = None
self._update_dt_groups(project.id)
self._update_burst_configurations(project.id)
return imported_operations, all_dts_count, all_stored_dts_count
def import_project_operations(self, project, import_path, is_group=False, importer_operation_id=None):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
all_dts_count = 0
all_stored_dts_count = 0
imported_operations = []
ordered_operations = self._retrieve_operations_in_order(project, import_path, importer_operation_id)
for operation_data in ordered_operations:
if operation_data.is_old_form:
operation_entity, datatype_group = self.import_operation(operation_data.operation)
new_op_folder = self.storage_interface.get_project_folder(project.name, str(operation_entity.id))
try:
operation_datatypes = self._load_datatypes_from_operation_folder(operation_data.operation_folder,
operation_entity, datatype_group)
# Create and store view_model from operation
self.create_view_model(operation_entity, operation_data, new_op_folder)
self._store_imported_datatypes_in_db(project, operation_datatypes)
imported_operations.append(operation_entity)
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
elif operation_data.main_view_model is not None:
do_merge = False
if importer_operation_id:
do_merge = True
operation_entity = dao.store_entity(operation_data.operation, merge=do_merge)
dt_group = None
op_group = dao.get_operationgroup_by_id(operation_entity.fk_operation_group)
if op_group:
dt_group = dao.get_datatypegroup_by_op_group_id(op_group.id)
if not dt_group:
first_op = dao.get_operations_in_group(op_group.id, only_first_operation=True)
dt_group = DataTypeGroup(op_group, operation_id=first_op.id,
state=DEFAULTDATASTATE_INTERMEDIATE)
dt_group = dao.store_entity(dt_group)
# Store the DataTypes in db
dts = {}
all_dts_count += len(operation_data.dt_paths)
for dt_path in operation_data.dt_paths:
dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id)
if isinstance(dt, BurstConfiguration):
if op_group:
dt.fk_operation_group = op_group.id
all_stored_dts_count += self._store_or_link_burst_config(dt, dt_path, project.id)
else:
dts[dt_path] = dt
if op_group:
op_group.fill_operationgroup_name(dt.type)
dao.store_entity(op_group)
try:
stored_dts_count = self._store_imported_datatypes_in_db(project, dts)
all_stored_dts_count += stored_dts_count
if operation_data.main_view_model.is_metric_operation:
self._update_burst_metric(operation_entity)
#TODO: TVB-2849 to reveiw these flags and simplify condition
if stored_dts_count > 0 or (not operation_data.is_self_generated and not is_group) or importer_operation_id is not None:
imported_operations.append(operation_entity)
new_op_folder = self.storage_interface.get_project_folder(project.name, str(operation_entity.id))
view_model_disk_size = 0
for h5_file in operation_data.all_view_model_files:
view_model_disk_size += StorageInterface.compute_size_on_disk(h5_file)
shutil.move(h5_file, new_op_folder)
operation_entity.view_model_disk_size = view_model_disk_size
dao.store_entity(operation_entity)
else:
# In case all Dts under the current operation were Links and the ViewModel is dummy,
# don't keep the Operation empty in DB
dao.remove_entity(Operation, operation_entity.id)
self.storage_interface.remove_operation_data(project.name, operation_entity.id)
except MissingReferenceException as excep:
dao.remove_entity(Operation, operation_entity.id)
self.storage_interface.remove_operation_data(project.name, operation_entity.id)
raise excep
else:
self.logger.warning("Folder %s will be ignored, as we could not find a serialized "
"operation or DTs inside!" % operation_data.operation_folder)
self._update_dt_groups(project.id)
self._update_burst_configurations(project.id)
return imported_operations, all_dts_count, all_stored_dts_count
