def _store_imported_datatypes_in_db(self, project, all_datatypes,
dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(
datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id],
project.id)
def _store_imported_datatypes_in_db(self, project, all_datatypes, dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id], project.id)
def load_datatype_from_file(self, storage_folder, file_name, op_id, datatype_group=None,
move=True, final_storage=None):
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % file_name)
datatype, generic_attributes = h5.load_with_references(os.path.join(storage_folder, file_name))
index_class = h5.REGISTRY.get_index_for_datatype(datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group is not None:
datatype_index.fk_datatype_group = datatype_group.id
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(associated_file)
# Now move storage file into correct folder if necessary
if move and final_storage is not None:
current_file = os.path.join(storage_folder, file_name)
h5_type = h5.REGISTRY.get_h5file_for_datatype(datatype.__class__)
final_path = h5.path_for(final_storage, h5_type, datatype.gid)
if final_path != current_file and move:
shutil.move(current_file, final_path)
return datatype_index
def _update_datatype_disk_size(self, file_path):
"""
Computes and updates the disk_size attribute of the DataType, for which was created the given file.
"""
file_handler = FilesHelper()
datatype_gid = self._get_manager(file_path).get_gid_attribute()
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is not None:
datatype.disk_size = file_handler.compute_size_on_disk(file_path)
dao.store_entity(datatype)
def load_datatype_from_file(self,
current_file,
op_id,
datatype_group=None,
current_project_id=None):
# type: (str, int, DataTypeGroup, int) -> HasTraitsIndex
"""
Creates an instance of datatype from storage / H5 file
:returns: DatatypeIndex
"""
self.logger.debug("Loading DataType from file: %s" % current_file)
h5_class = H5File.h5_class_from_file(current_file)
if h5_class is BurstConfigurationH5:
if current_project_id is None:
op_entity = dao.get_operationgroup_by_id(op_id)
current_project_id = op_entity.fk_launched_in
h5_file = BurstConfigurationH5(current_file)
burst = BurstConfiguration(current_project_id)
burst.fk_simulation = op_id
h5_file.load_into(burst)
result = burst
else:
datatype, generic_attributes = h5.load_with_links(current_file)
index_class = h5.REGISTRY.get_index_for_datatype(
datatype.__class__)
datatype_index = index_class()
datatype_index.fill_from_has_traits(datatype)
datatype_index.fill_from_generic_attributes(generic_attributes)
# Add all the required attributes
if datatype_group:
datatype_index.fk_datatype_group = datatype_group.id
if len(datatype_group.subject) == 0:
datatype_group.subject = datatype_index.subject
dao.store_entity(datatype_group)
datatype_index.fk_from_operation = op_id
associated_file = h5.path_for_stored_index(datatype_index)
if os.path.exists(associated_file):
datatype_index.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
result = datatype_index
return result
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# todo this constants copy is not nice
TYPE_SELECT = input_tree.TYPE_SELECT
TYPE_MULTIPLE = input_tree.TYPE_MULTIPLE
STATIC_ACCEPTED_TYPES = input_tree.STATIC_ACCEPTED_TYPES
KEY_TYPE = input_tree.KEY_TYPE
KEY_OPTIONS = input_tree.KEY_OPTIONS
KEY_ATTRIBUTES = input_tree.KEY_ATTRIBUTES
KEY_NAME = input_tree.KEY_NAME
KEY_DESCRIPTION = input_tree.KEY_DESCRIPTION
KEY_VALUE = input_tree.KEY_VALUE
KEY_LABEL = input_tree.KEY_LABEL
KEY_DEFAULT = input_tree.KEY_DEFAULT
KEY_DATATYPE = input_tree.KEY_DATATYPE
KEY_DTYPE = input_tree.KEY_DTYPE
KEY_DISABLED = input_tree.KEY_DISABLED
KEY_ALL = input_tree.KEY_ALL
KEY_CONDITION = input_tree.KEY_CONDITION
KEY_FILTERABLE = input_tree.KEY_FILTERABLE
KEY_REQUIRED = input_tree.KEY_REQUIRED
KEY_ID = input_tree.KEY_ID
KEY_UI_HIDE = input_tree.KEY_UI_HIDE
# TODO: move everything related to parameters PRE + POST into parameters_factory
KEYWORD_PARAMS = input_tree.KEYWORD_PARAMS
KEYWORD_SEPARATOR = input_tree.KEYWORD_SEPARATOR
KEYWORD_OPTION = input_tree.KEYWORD_OPTION
INTERFACE_ATTRIBUTES_ONLY = "attributes-only"
INTERFACE_ATTRIBUTES = "attributes"
# model.Algorithm instance that will be set for each adapter created by in build_adapter method
stored_adapter = None
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {
DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT
}
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.tree_manager = InputTreeManager()
self.submitted_form = None
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def get_input_tree(self):
"""
Describes inputs and outputs of the launch method.
"""
return None
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, **kwargs):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, **kwargs):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Any returned DataType will be stored in DB, by the Framework.
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def _prepare_generic_attributes(self, user_tag=None):
self.generic_attributes.subject = str(
self.meta_data.get(DataTypeMetaData.KEY_SUBJECT))
self.generic_attributes.state = self.meta_data.get(
DataTypeMetaData.KEY_STATE)
perpetuated_identifier = self.generic_attributes.user_tag_1
if DataTypeMetaData.KEY_TAG_1 in self.meta_data:
perpetuated_identifier = self.meta_data.get(
DataTypeMetaData.KEY_TAG_1)
if not self.generic_attributes.user_tag_1:
self.generic_attributes.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier
else:
self.generic_attributes.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier
@nan_not_allowed()
def _prelaunch(self,
operation,
uid=None,
available_disk_space=0,
**kwargs):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.meta_data.update(json.loads(operation.meta_data))
self.storage_path = self.file_handler.get_project_folder(
operation.project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
self.configure(**kwargs)
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory(
).free
total_existent_memory = psutil.virtual_memory(
).total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(**kwargs)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(
msg %
(adapter_required_memory / 2**30, memory_reference / 2**30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(**kwargs)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg %
(-available_disk_space / 2**10))
if available_disk_space < required_disk_space:
msg = (
"You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(
msg %
(available_disk_space / 2**20, required_disk_space / 2**20))
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
self._prepare_generic_attributes(uid)
result = self.launch(**kwargs)
if not isinstance(result, (list, tuple)):
result = [
result,
]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
res.subject = self.generic_attributes.subject
res.state = self.generic_attributes.state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
res.user_tag_1 = self.generic_attributes.user_tag_1
res.user_tag_2 = self.generic_attributes.user_tag_2
res.fk_datatype_group = data_type_group_id
# Compute size-on disk, in case file-storage is used
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
res.disk_size = self.file_handler.compute_size_on_disk(
associated_file)
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
dao.store_entity(res)
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
@staticmethod
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
return load_entity_by_gid(data_gid)
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException(
"Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(
adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def build_adapter(stored_adapter):
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
# METHODS for PROCESSING PARAMETERS start here #############################
def review_operation_inputs(self, parameters):
"""
:returns: a list with the inputs from the parameters list that are instances of DataType,\
and a dictionary with all parameters which are different than the declared defauts
"""
flat_interface = self.flaten_input_interface()
return self.tree_manager.review_operation_inputs(
parameters, flat_interface)
def prepare_ui_inputs(self, kwargs, validation_required=True):
"""
Prepare the inputs received from a HTTP Post in a form that will be
used by the Python adapter.
"""
algorithm_inputs = self.get_input_tree()
algorithm_inputs = InputTreeManager.prepare_param_names(
algorithm_inputs)
self.tree_manager.append_required_defaults(kwargs, algorithm_inputs)
return self.convert_ui_inputs(kwargs,
validation_required=validation_required)
def convert_ui_inputs(self, kwargs, validation_required=True):
"""
Convert HTTP POST parameters into Python parameters.
"""
return self.tree_manager.convert_ui_inputs(
self.flaten_input_interface(), kwargs, self.meta_data,
validation_required)
def noise_configurable_parameters(self):
return [
entry[self.KEY_NAME] for entry in self.flaten_input_interface()
if 'configurableNoise' in entry
]
def flaten_input_interface(self):
# TODO: temporary condition to pass introspection on neoforms
form = self.get_form_class()()
if form:
return [
form._get_original_field_name(form_field)
for form_field in form.fields
]
return self.tree_manager.flatten(self.get_input_tree())
def import_project_operations(self,
project,
import_path,
is_group=False,
importer_operation_id=None):
"""
This method scans provided folder and identify all operations that needs to be imported
"""
all_dts_count = 0
all_stored_dts_count = 0
imported_operations = []
ordered_operations = self._retrieve_operations_in_order(
project, import_path, importer_operation_id)
for operation_data in ordered_operations:
if operation_data.is_old_form:
operation_entity, datatype_group = self.import_operation(
operation_data.operation)
new_op_folder = self.files_helper.get_project_folder(
project, str(operation_entity.id))
try:
operation_datatypes = self._load_datatypes_from_operation_folder(
operation_data.operation_folder, operation_entity,
datatype_group)
# Create and store view_model from operation
self.create_view_model(operation_entity, operation_data,
new_op_folder)
self._store_imported_datatypes_in_db(
project, operation_datatypes)
imported_operations.append(operation_entity)
except MissingReferenceException:
operation_entity.status = STATUS_ERROR
dao.store_entity(operation_entity)
elif operation_data.main_view_model is not None:
do_merge = False
if importer_operation_id:
do_merge = True
operation_entity = dao.store_entity(operation_data.operation,
merge=do_merge)
dt_group = None
op_group = dao.get_operationgroup_by_id(
operation_entity.fk_operation_group)
if op_group:
dt_group = dao.get_datatypegroup_by_op_group_id(
op_group.id)
if not dt_group:
first_op = dao.get_operations_in_group(
op_group.id, only_first_operation=True)
dt_group = DataTypeGroup(
op_group,
operation_id=first_op.id,
state=DEFAULTDATASTATE_INTERMEDIATE)
dt_group = dao.store_entity(dt_group)
# Store the DataTypes in db
dts = {}
all_dts_count += len(operation_data.dt_paths)
for dt_path in operation_data.dt_paths:
dt = self.load_datatype_from_file(dt_path,
operation_entity.id,
dt_group, project.id)
if isinstance(dt, BurstConfiguration):
if op_group:
dt.fk_operation_group = op_group.id
all_stored_dts_count += self._store_or_link_burst_config(
dt, dt_path, project.id)
else:
dts[dt_path] = dt
if op_group:
op_group.fill_operationgroup_name(dt.type)
dao.store_entity(op_group)
try:
stored_dts_count = self._store_imported_datatypes_in_db(
project, dts)
all_stored_dts_count += stored_dts_count
if operation_data.main_view_model.is_metric_operation:
self._update_burst_metric(operation_entity)
#TODO: TVB-2849 to reveiw these flags and simplify condition
if stored_dts_count > 0 or (
not operation_data.is_self_generated and
not is_group) or importer_operation_id is not None:
imported_operations.append(operation_entity)
new_op_folder = self.files_helper.get_project_folder(
project, str(operation_entity.id))
view_model_disk_size = 0
for h5_file in operation_data.all_view_model_files:
view_model_disk_size += FilesHelper.compute_size_on_disk(
h5_file)
shutil.move(h5_file, new_op_folder)
operation_entity.view_model_disk_size = view_model_disk_size
dao.store_entity(operation_entity)
else:
# In case all Dts under the current operation were Links and the ViewModel is dummy,
# don't keep the Operation empty in DB
dao.remove_entity(Operation, operation_entity.id)
self.files_helper.remove_operation_data(
project.name, operation_entity.id)
except MissingReferenceException as excep:
dao.remove_entity(Operation, operation_entity.id)
self.files_helper.remove_operation_data(
project.name, operation_entity.id)
raise excep
else:
self.logger.warning(
"Folder %s will be ignored, as we could not find a serialized "
"operation or DTs inside!" %
operation_data.operation_folder)
self._update_dt_groups(project.id)
self._update_burst_configurations(project.id)
return imported_operations, all_dts_count, all_stored_dts_count
class FilesUpdateManager(UpdateManager):
"""
Manager for updating H5 files version, when code gets changed.
"""
UPDATE_SCRIPTS_SUFFIX = "_update_files"
PROJECTS_PAGE_SIZE = 20
DATA_TYPES_PAGE_SIZE = 500
STATUS = True
MESSAGE = "Done"
def __init__(self):
super(FilesUpdateManager, self).__init__(
file_update_scripts,
TvbProfile.current.version.DATA_CHECKED_TO_VERSION,
TvbProfile.current.version.DATA_VERSION)
self.files_helper = FilesHelper()
def get_file_data_version(self, file_path):
"""
Return the data version for the given file.
:param file_path: the path on disk to the file for which you need the TVB data version
:returns: a number representing the data version for which the input file was written
"""
manager = self._get_manager(file_path)
return manager.get_file_data_version()
def is_file_up_to_date(self, file_path):
"""
Returns True only if the data version of the file is equal with the
data version specified into the TVB configuration file.
"""
try:
file_version = self.get_file_data_version(file_path)
except MissingDataFileException as ex:
self.log.exception(ex)
return False
except FileStructureException as ex:
self.log.exception(ex)
return False
if file_version == TvbProfile.current.version.DATA_VERSION:
return True
return False
def upgrade_file(self, input_file_name, datatype=None):
"""
Upgrades the given file to the latest data version. The file will be upgraded
sequentially, up until the current version from tvb.basic.config.settings.VersionSettings.DB_STRUCTURE_VERSION
:param input_file_name the path to the file which needs to be upgraded
:return True, when update was needed and running it was successful.
False, the the file is already up to date.
"""
if self.is_file_up_to_date(input_file_name):
# Avoid running the DB update of size, when H5 is not being changed, to speed-up
return False
file_version = self.get_file_data_version(input_file_name)
self.log.info("Updating from version %s , file: %s " %
(file_version, input_file_name))
for script_name in self.get_update_scripts(file_version):
self.run_update_script(script_name, input_file=input_file_name)
if datatype:
# Compute and update the disk_size attribute of the DataType in DB:
datatype.disk_size = self.files_helper.compute_size_on_disk(
input_file_name)
dao.store_entity(datatype)
return True
def __upgrade_datatype_list(self, datatypes):
"""
Upgrade a list of DataTypes to the current version.
:param datatypes: The list of DataTypes that should be upgraded.
:returns: (nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault) a two-tuple of integers representing
the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which
some kind of fault occurred
"""
nr_of_dts_upgraded_fine = 0
nr_of_dts_upgraded_fault = 0
no_of_dts_ignored = 0
for datatype in datatypes:
try:
from tvb.basic.traits.types_mapped import MappedType
specific_datatype = dao.get_datatype_by_gid(datatype.gid,
load_lazy=False)
if specific_datatype is None:
datatype.invalid = True
dao.store_entity(datatype)
nr_of_dts_upgraded_fault += 1
elif isinstance(specific_datatype, MappedType):
update_was_needed = self.upgrade_file(
specific_datatype.get_storage_file_path(),
specific_datatype)
if update_was_needed:
nr_of_dts_upgraded_fine += 1
else:
no_of_dts_ignored += 1
else:
# Ignore DataTypeGroups
self.log.debug("We will ignore, due to type: " +
str(specific_datatype))
no_of_dts_ignored += 1
except Exception as ex:
# The file/class is missing for some reason. Just mark the DataType as invalid.
datatype.invalid = True
dao.store_entity(datatype)
nr_of_dts_upgraded_fault += 1
self.log.exception(ex)
return nr_of_dts_upgraded_fine, nr_of_dts_upgraded_fault, no_of_dts_ignored
def run_all_updates(self):
"""
Upgrades all the data types from TVB storage to the latest data version.
:returns: a two entry tuple (status, message) where status is a boolean that is True in case
the upgrade was successfully for all DataTypes and False otherwise, and message is a status
update message.
"""
if TvbProfile.current.version.DATA_CHECKED_TO_VERSION < TvbProfile.current.version.DATA_VERSION:
total_count = dao.count_all_datatypes()
self.log.info(
"Starting to run H5 file updates from version %d to %d, for %d datatypes"
% (TvbProfile.current.version.DATA_CHECKED_TO_VERSION,
TvbProfile.current.version.DATA_VERSION, total_count))
# Keep track of how many DataTypes were properly updated and how many
# were marked as invalid due to missing files or invalid manager.
no_ok = 0
no_error = 0
no_ignored = 0
start_time = datetime.now()
# Read DataTypes in pages to limit the memory consumption
for current_idx in range(0, total_count,
self.DATA_TYPES_PAGE_SIZE):
datatypes_for_page = dao.get_all_datatypes(
current_idx, self.DATA_TYPES_PAGE_SIZE)
count_ok, count_error, count_ignored = self.__upgrade_datatype_list(
datatypes_for_page)
no_ok += count_ok
no_error += count_error
no_ignored += count_ignored
self.log.info(
"Updated H5 files so far: %d [fine:%d, error:%d, ignored:%d of total:%d, in: %s min]"
% (current_idx + len(datatypes_for_page), no_ok, no_error,
no_ignored, total_count,
int((datetime.now() - start_time).seconds / 60)))
# Now update the configuration file since update was done
config_file_update_dict = {
stored.KEY_LAST_CHECKED_FILE_VERSION:
TvbProfile.current.version.DATA_VERSION
}
if no_error == 0:
# Everything went fine
config_file_update_dict[
stored.KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_VALID
FilesUpdateManager.STATUS = True
FilesUpdateManager.MESSAGE = (
"File upgrade finished successfully for all %s entries. "
"Thank you for your patience!" % total_count)
self.log.info(FilesUpdateManager.MESSAGE)
else:
# Something went wrong
config_file_update_dict[
stored.
KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_INVALID
FilesUpdateManager.STATUS = False
FilesUpdateManager.MESSAGE = (
"Out of %s stored DataTypes, %s were upgraded successfully, but %s had "
"faults and were marked invalid" %
(total_count, no_ok, no_error))
self.log.warning(FilesUpdateManager.MESSAGE)
TvbProfile.current.version.DATA_CHECKED_TO_VERSION = TvbProfile.current.version.DATA_VERSION
TvbProfile.current.manager.add_entries_to_config_file(
config_file_update_dict)
@staticmethod
def _get_manager(file_path):
"""
Returns a storage manager.
"""
folder, file_name = os.path.split(file_path)
return HDF5StorageManager(folder, file_name)
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# model.Algorithm instance that will be set for each adapter class created by in build_adapter method
stored_adapter = None
launch_mode = AdapterLaunchModeEnum.ASYNC_DIFF_MEM
def __init__(self):
self.generic_attributes = GenericAttributes()
self.generic_attributes.subject = DataTypeMetaData.DEFAULT_SUBJECT
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.submitted_form = None
self.log = get_logger(self.__class__.__module__)
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
def submit_form(self, form):
self.submitted_form = form
# TODO separate usage of get_form_class (returning a class) and return of a submitted instance
def get_form(self):
if self.submitted_form is not None:
return self.submitted_form
return self.get_form_class()
@abstractmethod
def get_form_class(self):
return None
def get_adapter_fragments(self, view_model):
"""
The result will be used for introspecting and checking operation changed input
params from the defaults, to show in web gui.
:return: a list of ABCAdapterForm classes, in case the current Adapter GUI
will be composed of multiple sub-forms.
"""
return {}
def get_view_model_class(self):
return self.get_form_class().get_view_model()
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, view_model):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, view_model):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, view_model):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self, view_model):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Takes a ViewModel with data, dependency direction is: Adapter -> Form -> ViewModel
Any returned DataType will be stored in DB, by the Framework.
:param view_model: the data model corresponding to the current adapter
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
def extract_operation_data(self, operation):
operation = dao.get_operation_by_id(operation.id)
project = dao.get_project_by_id(operation.fk_launched_in)
self.storage_path = self.file_handler.get_project_folder(project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
def _ensure_enough_resources(self, available_disk_space, view_model):
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(view_model)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(view_model)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
return required_disk_space
def _update_operation_entity(self, operation, required_disk_space):
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
@nan_not_allowed()
def _prelaunch(self, operation, view_model, uid=None, available_disk_space=0):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.extract_operation_data(operation)
self.generic_attributes.fill_from(view_model.generic_attributes)
self.configure(view_model)
required_disk_size = self._ensure_enough_resources(available_disk_space, view_model)
self._update_operation_entity(operation, required_disk_size)
if not self.generic_attributes.user_tag_1:
self.generic_attributes.user_tag_1 = uid
else:
self.generic_attributes.user_tag_2 = uid
result = self.launch(view_model)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result)
def _capture_operation_results(self, result):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
count_stored = 0
group_type = None # In case of a group, the first not-none type is sufficient to memorize here
for res in result:
if res is None:
continue
if not res.fixed_generic_attributes:
res.fill_from_generic_attributes(self.generic_attributes)
res.fk_from_operation = self.operation_id
res.fk_datatype_group = data_type_group_id
associated_file = h5.path_for_stored_index(res)
if os.path.exists(associated_file):
if not res.fixed_generic_attributes:
with H5File.from_file(associated_file) as f:
f.store_generic_attributes(self.generic_attributes)
# Compute size-on disk, in case file-storage is used
res.disk_size = self.file_handler.compute_size_on_disk(associated_file)
dao.store_entity(res)
res.after_store()
group_type = res.type
count_stored += 1
if count_stored > 0 and self._is_group_launch():
# Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(group_type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', count_stored
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
for result_entity in result:
if result_entity is None:
continue
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected output DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
# Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
def _get_output_path(self):
return self.storage_path
def load_entity_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> DataType
"""
Load a generic DataType, specified by GID.
"""
idx = load_entity_by_gid(data_gid)
if idx and self.generic_attributes.parent_burst is None:
# Only in case the BurstConfiguration references hasn't been set already, take it from the current DT
self.generic_attributes.parent_burst = idx.fk_parent_burst
return idx
def load_traited_by_gid(self, data_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
"""
Load a generic HasTraits instance, specified by GID.
"""
index = self.load_entity_by_gid(data_gid)
return h5.load_from_index(index)
def load_with_references(self, dt_gid):
# type: (typing.Union[uuid.UUID, str]) -> HasTraits
dt_index = self.load_entity_by_gid(dt_gid)
h5_path = h5.path_for_stored_index(dt_index)
dt, _ = h5.load_with_references(h5_path)
return dt
def view_model_to_has_traits(self, view_model):
# type: (ViewModel) -> HasTraits
has_traits_class = view_model.linked_has_traits
has_traits = has_traits_class()
view_model_class = type(view_model)
if not has_traits_class:
raise Exception("There is no linked HasTraits for this ViewModel {}".format(type(view_model)))
for attr_name in has_traits_class.declarative_attrs:
view_model_class_attr = getattr(view_model_class, attr_name)
view_model_attr = getattr(view_model, attr_name)
if isinstance(view_model_class_attr, DataTypeGidAttr) and view_model_attr:
attr_value = self.load_with_references(view_model_attr)
elif isinstance(view_model_class_attr, Attr) and isinstance(view_model_attr, ViewModel):
attr_value = self.view_model_to_has_traits(view_model_attr)
elif isinstance(view_model_class_attr, List) and len(view_model_attr) > 0 and isinstance(view_model_attr[0],
ViewModel):
attr_value = list()
for view_model_elem in view_model_attr:
elem = self.view_model_to_has_traits(view_model_elem)
attr_value.append(elem)
else:
attr_value = view_model_attr
setattr(has_traits, attr_name, attr_value)
return has_traits
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception as excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
@staticmethod
def determine_adapter_class(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Determine the class of an adapter based on module and classname strings from stored_adapter
:param stored_adapter: Algorithm or AlgorithmDTO type
:return: a subclass of ABCAdapter
"""
ad_module = importlib.import_module(stored_adapter.module)
adapter_class = getattr(ad_module, stored_adapter.classname)
return adapter_class
@staticmethod
def build_adapter(stored_adapter):
# type: (Algorithm) -> ABCAdapter
"""
Having a module and a class name, create an instance of ABCAdapter.
"""
try:
adapter_class = ABCAdapter.determine_adapter_class(stored_adapter)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception:
msg = "Could not load Adapter Instance for Stored row %s" % stored_adapter
LOGGER.exception(msg)
raise IntrospectionException(msg)
def load_view_model(self, operation):
storage_path = self.file_handler.get_project_folder(operation.project, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
def array_size2kb(self, size):
"""
:param size: size in bytes
:return: size in kB
"""
return size * TvbProfile.current.MAGIC_NUMBER / 8 / 2 ** 10
class FilesUpdateManager(UpdateManager):
"""
Manager for updating H5 files version, when code gets changed.
"""
UPDATE_SCRIPTS_SUFFIX = "_update_files"
PROJECTS_PAGE_SIZE = 20
DATA_TYPES_PAGE_SIZE = 500
STATUS = True
MESSAGE = "Done"
def __init__(self):
super(FilesUpdateManager, self).__init__(file_update_scripts,
TvbProfile.current.version.DATA_CHECKED_TO_VERSION,
TvbProfile.current.version.DATA_VERSION)
self.files_helper = FilesHelper()
def get_file_data_version(self, file_path):
"""
Return the data version for the given file.
:param file_path: the path on disk to the file for which you need the TVB data version
:returns: a number representing the data version for which the input file was written
"""
manager = self._get_manager(file_path)
return manager.get_file_data_version()
def is_file_up_to_date(self, file_path):
"""
Returns True only if the data version of the file is equal with the
data version specified into the TVB configuration file.
"""
try:
file_version = self.get_file_data_version(file_path)
except MissingDataFileException as ex:
self.log.exception(ex)
return False
except FileStructureException as ex:
self.log.exception(ex)
return False
if file_version == TvbProfile.current.version.DATA_VERSION:
return True
return False
def upgrade_file(self, input_file_name, datatype=None, burst_match_dict=None):
"""
Upgrades the given file to the latest data version. The file will be upgraded
sequentially, up until the current version from tvb.basic.config.settings.VersionSettings.DB_STRUCTURE_VERSION
:param input_file_name the path to the file which needs to be upgraded
:return True when update was successful and False when it resulted in an error.
"""
if self.is_file_up_to_date(input_file_name):
# Avoid running the DB update of size, when H5 is not being changed, to speed-up
return True
file_version = self.get_file_data_version(input_file_name)
self.log.info("Updating from version %s , file: %s " % (file_version, input_file_name))
for script_name in self.get_update_scripts(file_version):
temp_file_path = os.path.join(TvbProfile.current.TVB_TEMP_FOLDER,
os.path.basename(input_file_name) + '.tmp')
self.files_helper.copy_file(input_file_name, temp_file_path)
try:
self.run_update_script(script_name, input_file=input_file_name, burst_match_dict=burst_match_dict)
except FileMigrationException as excep:
self.files_helper.copy_file(temp_file_path, input_file_name)
os.remove(temp_file_path)
self.log.error(excep)
return False
if datatype:
# Compute and update the disk_size attribute of the DataType in DB:
datatype.disk_size = self.files_helper.compute_size_on_disk(input_file_name)
dao.store_entity(datatype)
return True
def __upgrade_h5_list(self, h5_files):
"""
Upgrade a list of DataTypes to the current version.
:returns: (nr_of_dts_upgraded_fine, nr_of_dts_ignored) a two-tuple of integers representing
the number of DataTypes for which the upgrade worked fine, and the number of DataTypes for which
the upgrade has failed.
"""
nr_of_dts_upgraded_fine = 0
nr_of_dts_failed = 0
burst_match_dict = {}
for path in h5_files:
update_result = self.upgrade_file(path, burst_match_dict=burst_match_dict)
if update_result:
nr_of_dts_upgraded_fine += 1
else:
nr_of_dts_failed += 1
return nr_of_dts_upgraded_fine, nr_of_dts_failed
# TO DO: We should migrate the older scripts to Python 3 if we want to support migration for versions < 4
def run_all_updates(self):
"""
Upgrades all the data types from TVB storage to the latest data version.
:returns: a two entry tuple (status, message) where status is a boolean that is True in case
the upgrade was successfully for all DataTypes and False otherwise, and message is a status
update message.
"""
if TvbProfile.current.version.DATA_CHECKED_TO_VERSION < TvbProfile.current.version.DATA_VERSION:
total_count = dao.count_all_datatypes()
self.log.info("Starting to run H5 file updates from version %d to %d, for %d datatypes" % (
TvbProfile.current.version.DATA_CHECKED_TO_VERSION,
TvbProfile.current.version.DATA_VERSION, total_count))
# Keep track of how many DataTypes were properly updated and how many
# were marked as invalid due to missing files or invalid manager.
start_time = datetime.now()
file_paths = self.get_all_h5_paths()
total_count = len(file_paths)
no_ok, no_error = self.__upgrade_h5_list(file_paths)
self.log.info("Updated H5 files in total: %d [fine:%d, failed:%d in: %s min]" % (
total_count, no_ok, no_error, int((datetime.now() - start_time).seconds / 60)))
delete_old_burst_table_after_migration()
# Now update the configuration file since update was done
config_file_update_dict = {stored.KEY_LAST_CHECKED_FILE_VERSION: TvbProfile.current.version.DATA_VERSION}
if no_error == 0:
# Everything went fine
config_file_update_dict[stored.KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_VALID
FilesUpdateManager.STATUS = True
FilesUpdateManager.MESSAGE = ("File upgrade finished successfully for all %s entries. "
"Thank you for your patience!" % total_count)
self.log.info(FilesUpdateManager.MESSAGE)
else:
# Something went wrong
config_file_update_dict[stored.KEY_FILE_STORAGE_UPDATE_STATUS] = FILE_STORAGE_INVALID
FilesUpdateManager.STATUS = False
FilesUpdateManager.MESSAGE = ("Out of %s stored DataTypes, %s were upgraded successfully, but %s had "
"faults and were marked invalid" % (total_count, no_ok, no_error))
self.log.warning(FilesUpdateManager.MESSAGE)
TvbProfile.current.version.DATA_CHECKED_TO_VERSION = TvbProfile.current.version.DATA_VERSION
TvbProfile.current.manager.add_entries_to_config_file(config_file_update_dict)
@staticmethod
def _get_manager(file_path):
"""
Returns a storage manager.
"""
folder, file_name = os.path.split(file_path)
return HDF5StorageManager(folder, file_name)
@staticmethod
def get_all_h5_paths():
"""
This method returns a list of all h5 files and it is used in the migration from version 4 to 5.
The h5 files inside a certain project are retrieved in numerical order (1, 2, 3 etc.).
"""
h5_files = []
projects_folder = FilesHelper().get_projects_folder()
for project_path in os.listdir(projects_folder):
# Getting operation folders inside the current project
project_full_path = os.path.join(projects_folder, project_path)
try:
project_operations = os.listdir(project_full_path)
except NotADirectoryError:
continue
project_operations_base_names = [os.path.basename(op) for op in project_operations]
for op_folder in project_operations_base_names:
try:
int(op_folder)
op_folder_path = os.path.join(project_full_path, op_folder)
for file in os.listdir(op_folder_path):
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(op_folder_path, file)
try:
if FilesUpdateManager._is_empty_file(h5_file):
continue
h5_files.append(h5_file)
except FileStructureException:
continue
except ValueError:
pass
# Sort all h5 files based on their creation date stored in the files themselves
sorted_h5_files = sorted(h5_files, key=lambda h5_path: FilesUpdateManager._get_create_date_for_sorting(
h5_path) or datetime.now())
return sorted_h5_files
@staticmethod
def _is_empty_file(h5_file):
return H5File.get_metadata_param(h5_file, 'Create_date') is None
@staticmethod
def _get_create_date_for_sorting(h5_file):
create_date_str = str(H5File.get_metadata_param(h5_file, 'Create_date'), 'utf-8')
create_date = string2date(create_date_str, date_format='datetime:%Y-%m-%d %H:%M:%S.%f')
return create_date
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# todo this constants copy is not nice
TYPE_SELECT = input_tree.TYPE_SELECT
TYPE_MULTIPLE = input_tree.TYPE_MULTIPLE
STATIC_ACCEPTED_TYPES = input_tree.STATIC_ACCEPTED_TYPES
KEY_TYPE = input_tree.KEY_TYPE
KEY_OPTIONS = input_tree.KEY_OPTIONS
KEY_ATTRIBUTES = input_tree.KEY_ATTRIBUTES
KEY_NAME = input_tree.KEY_NAME
KEY_DESCRIPTION = input_tree.KEY_DESCRIPTION
KEY_VALUE = input_tree.KEY_VALUE
KEY_LABEL = input_tree.KEY_LABEL
KEY_DEFAULT = input_tree.KEY_DEFAULT
KEY_DATATYPE = input_tree.KEY_DATATYPE
KEY_DTYPE = input_tree.KEY_DTYPE
KEY_DISABLED = input_tree.KEY_DISABLED
KEY_ALL = input_tree.KEY_ALL
KEY_CONDITION = input_tree.KEY_CONDITION
KEY_FILTERABLE = input_tree.KEY_FILTERABLE
KEY_REQUIRED = input_tree.KEY_REQUIRED
KEY_ID = input_tree.KEY_ID
KEY_UI_HIDE = input_tree.KEY_UI_HIDE
# TODO: move everything related to parameters PRE + POST into parameters_factory
KEYWORD_PARAMS = input_tree.KEYWORD_PARAMS
KEYWORD_SEPARATOR = input_tree.KEYWORD_SEPARATOR
KEYWORD_OPTION = input_tree.KEYWORD_OPTION
INTERFACE_ATTRIBUTES_ONLY = interface.INTERFACE_ATTRIBUTES_ONLY
INTERFACE_ATTRIBUTES = interface.INTERFACE_ATTRIBUTES
# model.Algorithm instance that will be set for each adapter created by in build_adapter method
stored_adapter = None
__metaclass__ = ABCMeta
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {
DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT
}
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.tree_manager = InputTreeManager()
@classmethod
def get_group_name(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "name"):
return cls._ui_group.name
return None
@classmethod
def get_group_description(cls):
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "description"):
return cls._ui_group.description
return None
@classmethod
def get_ui_name(cls):
if hasattr(cls, "_ui_name"):
return cls._ui_name
else:
return cls.__name__
@classmethod
def get_ui_description(cls):
if hasattr(cls, "_ui_description"):
return cls._ui_description
@classmethod
def get_ui_subsection(cls):
if hasattr(cls, "_ui_subsection"):
return cls._ui_subsection
if hasattr(cls, "_ui_group") and hasattr(cls._ui_group, "subsection"):
return cls._ui_group.subsection
@staticmethod
def can_be_active():
"""
To be overridden where needed (e.g. Matlab dependent adapters).
:return: By default True, and False when the current Adapter can not be executed in the current env
for various reasons (e.g. no Matlab or Octave installed)
"""
return True
@abstractmethod
def get_input_tree(self):
"""
Describes inputs and outputs of the launch method.
"""
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, **kwargs):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, **kwargs):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Any returned DataType will be stored in DB, by the Framework.
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
@nan_not_allowed()
def _prelaunch(self,
operation,
uid=None,
available_disk_space=0,
**kwargs):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.meta_data.update(json.loads(operation.meta_data))
self.storage_path = self.file_handler.get_project_folder(
operation.project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
self.configure(**kwargs)
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory(
).free
total_existent_memory = psutil.virtual_memory(
).total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(**kwargs)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(
msg %
(adapter_required_memory / 2**30, memory_reference / 2**30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(**kwargs)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg %
(-available_disk_space / 2**10))
if available_disk_space < required_disk_space:
msg = (
"You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(
msg %
(available_disk_space / 2**20, required_disk_space / 2**20))
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
result = self.launch(**kwargs)
if not isinstance(result, (list, tuple)):
result = [
result,
]
self.__check_integrity(result)
return self._capture_operation_results(result, uid)
def _capture_operation_results(self, result, user_tag=None):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
results_to_store = []
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(
datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(
operation.fk_operation_group).id
# All entities will have the same subject and state
subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT]
state = self.meta_data[DataTypeMetaData.KEY_STATE]
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
perpetuated_identifier = None
if DataTypeMetaData.KEY_TAG_1 in self.meta_data:
perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1]
for res in result:
if res is None:
continue
res.subject = str(subject)
res.state = state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
if not res.user_tag_1:
res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier
else:
res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier
res.fk_datatype_group = data_type_group_id
## Compute size-on disk, in case file-storage is used
if hasattr(res, 'storage_path') and hasattr(
res, 'get_storage_file_name'):
associated_file = os.path.join(res.storage_path,
res.get_storage_file_name())
res.close_file()
res.disk_size = self.file_handler.compute_size_on_disk(
associated_file)
res = dao.store_entity(res)
# Write metaData
res.persist_full_metadata()
results_to_store.append(res)
del result[0:len(result)]
result.extend(results_to_store)
if len(result) and self._is_group_launch():
## Update the operation group name
operation_group = dao.get_operationgroup_by_id(
operation.fk_operation_group)
operation_group.fill_operationgroup_name(result[0].type)
dao.store_entity(operation_group)
return 'Operation ' + str(
self.operation_id) + ' has finished.', len(results_to_store)
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
entity_id = self.__module__ + '.' + self.__class__.__name__
for result_entity in result:
if type(result_entity) == list and len(result_entity) > 0:
#### Determine the first element not None
first_item = None
for res in result_entity:
if res is not None:
first_item = res
break
if first_item is None:
return
#### All list items are None
#### Now check if the first item has a supported type
if not self.__is_data_in_supported_types(first_item):
msg = "Unexpected DataType %s"
raise InvalidParameterException(msg % type(first_item))
first_item_type = type(first_item)
for res in result_entity:
if not isinstance(res, first_item_type):
msg = '%s-Heterogeneous types (%s).Expected %s list.'
raise InvalidParameterException(
msg % (entity_id, type(res), first_item_type))
else:
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected DataType %s"
raise InvalidParameterException(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
"""
This method checks if the provided data is one of the adapter supported return types
"""
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
##### Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
@staticmethod
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
return load_entity_by_gid(data_gid)
@staticmethod
def build_adapter_from_class(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException(
"Invalid data type: It should extend adapters.ABCAdapter!")
try:
stored_adapter = dao.get_algorithm_by_module(
adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.stored_adapter = stored_adapter
return adapter_instance
except Exception, excep:
LOGGER.exception(excep)
raise IntrospectionException(str(excep))
class ABCAdapter(object):
"""
Root Abstract class for all TVB Adapters.
"""
# todo this constants copy is not nice
TYPE_SELECT = input_tree.TYPE_SELECT
TYPE_MULTIPLE = input_tree.TYPE_MULTIPLE
STATIC_ACCEPTED_TYPES = input_tree.STATIC_ACCEPTED_TYPES
KEY_TYPE = input_tree.KEY_TYPE
KEY_OPTIONS = input_tree.KEY_OPTIONS
KEY_ATTRIBUTES = input_tree.KEY_ATTRIBUTES
KEY_NAME = input_tree.KEY_NAME
KEY_DESCRIPTION = input_tree.KEY_DESCRIPTION
KEY_VALUE = input_tree.KEY_VALUE
KEY_LABEL = input_tree.KEY_LABEL
KEY_DEFAULT = input_tree.KEY_DEFAULT
KEY_DATATYPE = input_tree.KEY_DATATYPE
KEY_DTYPE = input_tree.KEY_DTYPE
KEY_DISABLED = input_tree.KEY_DISABLED
KEY_ALL = input_tree.KEY_ALL
KEY_CONDITION = input_tree.KEY_CONDITION
KEY_FILTERABLE = input_tree.KEY_FILTERABLE
KEY_REQUIRED = input_tree.KEY_REQUIRED
KEY_ID = input_tree.KEY_ID
KEY_UI_HIDE = input_tree.KEY_UI_HIDE
# TODO: move everything related to parameters PRE + POST into parameters_factory
KEYWORD_PARAMS = input_tree.KEYWORD_PARAMS
KEYWORD_SEPARATOR = input_tree.KEYWORD_SEPARATOR
KEYWORD_OPTION = input_tree.KEYWORD_OPTION
INTERFACE_ATTRIBUTES_ONLY = interface.INTERFACE_ATTRIBUTES_ONLY
INTERFACE_ATTRIBUTES = interface.INTERFACE_ATTRIBUTES
# Group that will be set for each adapter created by in build_adapter method
algorithm_group = None
_ui_display = 1
__metaclass__ = ABCMeta
def __init__(self):
# It will be populate with key from DataTypeMetaData
self.meta_data = {DataTypeMetaData.KEY_SUBJECT: DataTypeMetaData.DEFAULT_SUBJECT}
self.file_handler = FilesHelper()
self.storage_path = '.'
# Will be populate with current running operation's identifier
self.operation_id = None
self.user_id = None
self.log = get_logger(self.__class__.__module__)
self.tree_manager = InputTreeManager()
@abstractmethod
def get_input_tree(self):
"""
Describes inputs and outputs of the launch method.
"""
@abstractmethod
def get_output(self):
"""
Describes inputs and outputs of the launch method.
"""
def configure(self, **kwargs):
"""
To be implemented in each Adapter that requires any specific configurations
before the actual launch.
"""
@abstractmethod
def get_required_memory_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter.
"""
@abstractmethod
def get_required_disk_size(self, **kwargs):
"""
Abstract method to be implemented in each adapter. Should return the required memory
for launching the adapter in kilo-Bytes.
"""
def get_execution_time_approximation(self, **kwargs):
"""
Method should approximate based on input arguments, the time it will take for the operation
to finish (in seconds).
"""
return -1
@abstractmethod
def launch(self):
"""
To be implemented in each Adapter.
Will contain the logic of the Adapter.
Any returned DataType will be stored in DB, by the Framework.
"""
def add_operation_additional_info(self, message):
"""
Adds additional info on the operation to be displayed in the UI. Usually a warning message.
"""
current_op = dao.get_operation_by_id(self.operation_id)
current_op.additional_info = message
dao.store_entity(current_op)
@nan_not_allowed()
def _prelaunch(self, operation, uid=None, available_disk_space=0, **kwargs):
"""
Method to wrap LAUNCH.
Will prepare data, and store results on return.
"""
self.meta_data.update(json.loads(operation.meta_data))
self.storage_path = self.file_handler.get_project_folder(operation.project, str(operation.id))
self.operation_id = operation.id
self.current_project_id = operation.project.id
self.user_id = operation.fk_launched_by
self.configure(**kwargs)
# Compare the amount of memory the current algorithms states it needs,
# with the average between the RAM available on the OS and the free memory at the current moment.
# We do not consider only the free memory, because some OSs are freeing late and on-demand only.
total_free_memory = psutil.virtual_memory().free + psutil.swap_memory().free
total_existent_memory = psutil.virtual_memory().total + psutil.swap_memory().total
memory_reference = (total_free_memory + total_existent_memory) / 2
adapter_required_memory = self.get_required_memory_size(**kwargs)
if adapter_required_memory > memory_reference:
msg = "Machine does not have enough RAM memory for the operation (expected %.2g GB, but found %.2g GB)."
raise NoMemoryAvailableException(msg % (adapter_required_memory / 2 ** 30, memory_reference / 2 ** 30))
# Compare the expected size of the operation results with the HDD space currently available for the user
# TVB defines a quota per user.
required_disk_space = self.get_required_disk_size(**kwargs)
if available_disk_space < 0:
msg = "You have exceeded you HDD space quota by %.2f MB Stopping execution."
raise NoMemoryAvailableException(msg % (- available_disk_space / 2 ** 10))
if available_disk_space < required_disk_space:
msg = ("You only have %.2f GB of disk space available but the operation you "
"launched might require %.2f Stopping execution...")
raise NoMemoryAvailableException(msg % (available_disk_space / 2 ** 20, required_disk_space / 2 ** 20))
operation.start_now()
operation.estimated_disk_size = required_disk_space
dao.store_entity(operation)
result = self.launch(**kwargs)
if not isinstance(result, (list, tuple)):
result = [result, ]
self.__check_integrity(result)
return self._capture_operation_results(result, uid)
def _capture_operation_results(self, result, user_tag=None):
"""
After an operation was finished, make sure the results are stored
in DB storage and the correct meta-data,IDs are set.
"""
results_to_store = []
data_type_group_id = None
operation = dao.get_operation_by_id(self.operation_id)
if operation.user_group is None or len(operation.user_group) == 0:
operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT)
operation = dao.store_entity(operation)
if self._is_group_launch():
data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id
# All entities will have the same subject and state
subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT]
state = self.meta_data[DataTypeMetaData.KEY_STATE]
burst_reference = None
if DataTypeMetaData.KEY_BURST in self.meta_data:
burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST]
perpetuated_identifier = None
if DataTypeMetaData.KEY_TAG_1 in self.meta_data:
perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1]
for res in result:
if res is None:
continue
res.subject = str(subject)
res.state = state
res.fk_parent_burst = burst_reference
res.fk_from_operation = self.operation_id
res.framework_metadata = self.meta_data
if not res.user_tag_1:
res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier
else:
res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier
res.fk_datatype_group = data_type_group_id
## Compute size-on disk, in case file-storage is used
if hasattr(res, 'storage_path') and hasattr(res, 'get_storage_file_name'):
associated_file = os.path.join(res.storage_path, res.get_storage_file_name())
res.close_file()
res.disk_size = self.file_handler.compute_size_on_disk(associated_file)
res = dao.store_entity(res)
# Write metaData
res.persist_full_metadata()
results_to_store.append(res)
del result[0:len(result)]
result.extend(results_to_store)
if len(result) and self._is_group_launch():
## Update the operation group name
operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group)
operation_group.fill_operationgroup_name(result[0].type)
dao.store_entity(operation_group)
return 'Operation ' + str(self.operation_id) + ' has finished.', len(results_to_store)
def __check_integrity(self, result):
"""
Check that the returned parameters for LAUNCH operation
are of the type specified in the adapter's interface.
"""
entity_id = self.__module__ + '.' + self.__class__.__name__
for result_entity in result:
if type(result_entity) == list and len(result_entity) > 0:
#### Determine the first element not None
first_item = None
for res in result_entity:
if res is not None:
first_item = res
break
if first_item is None:
return
#### All list items are None
#### Now check if the first item has a supported type
if not self.__is_data_in_supported_types(first_item):
msg = "Unexpected DataType %s"
raise Exception(msg % type(first_item))
first_item_type = type(first_item)
for res in result_entity:
if not isinstance(res, first_item_type):
msg = '%s-Heterogeneous types (%s).Expected %s list.'
raise Exception(msg % (entity_id, type(res), first_item_type))
else:
if not self.__is_data_in_supported_types(result_entity):
msg = "Unexpected DataType %s"
raise Exception(msg % type(result_entity))
def __is_data_in_supported_types(self, data):
"""
This method checks if the provided data is one of the adapter supported return types
"""
if data is None:
return True
for supported_type in self.get_output():
if isinstance(data, supported_type):
return True
##### Data can't be mapped on any supported type !!
return False
def _is_group_launch(self):
"""
Return true if this adapter is launched from a group of operations
"""
operation = dao.get_operation_by_id(self.operation_id)
return operation.fk_operation_group is not None
@staticmethod
def load_entity_by_gid(data_gid):
"""
Load a generic DataType, specified by GID.
"""
return load_entity_by_gid(data_gid)
@staticmethod
def prepare_adapter(adapter_class):
"""
Having a subclass of ABCAdapter, prepare an instance for launching an operation with it.
"""
try:
if not issubclass(adapter_class, ABCAdapter):
raise IntrospectionException("Invalid data type: It should extend adapters.ABCAdapter!")
algo_group = dao.find_group(adapter_class.__module__, adapter_class.__name__)
adapter_instance = adapter_class()
adapter_instance.algorithm_group = algo_group
return adapter_instance
except Exception, excep:
get_logger("ABCAdapter").exception(excep)
raise IntrospectionException(str(excep))
