def _convert_to_array(self, input_data, row):
"""
Method used when the type of an input is array, to parse or read.
If the user set an equation for computing a model parameter then the
value of that parameter will be a dictionary which contains all the data
needed for computing that parameter for each vertex from the used surface.
"""
if KEY_EQUATION in str(input_data) and KEY_FOCAL_POINTS in str(
input_data) and KEY_SURFACE_GID in str(input_data):
try:
input_data = eval(str(input_data))
# TODO move at a different level
equation_type = input_data.get(KEY_DTYPE)
if equation_type is None:
self.log.warning(
"Cannot figure out type of equation from input dictionary: %s. "
"Returning []." % input_data)
return []
eq_class = get_class_by_name(equation_type)
equation = eq_class.from_json(input_data[KEY_EQUATION])
focal_points = json.loads(input_data[KEY_FOCAL_POINTS])
surface_gid = input_data[KEY_SURFACE_GID]
surface = load_entity_by_gid(surface_gid)
return surface.compute_equation(focal_points, equation)
except Exception:
self.log.exception(
"The parameter %s was ignored. None value was returned.",
row['name'])
return None
dtype = None
if KEY_DTYPE in row:
dtype = row[KEY_DTYPE]
return string2array(str(input_data), ",", dtype)
def _convert_to_array(self, input_data, row):
"""
Method used when the type of an input is array, to parse or read.
If the user set an equation for computing a model parameter then the
value of that parameter will be a dictionary which contains all the data
needed for computing that parameter for each vertex from the used surface.
"""
if KEY_EQUATION in str(input_data) and KEY_FOCAL_POINTS in str(input_data) and KEY_SURFACE_GID in str(input_data):
try:
input_data = eval(str(input_data))
# TODO move at a different level
equation_type = input_data.get(KEY_DTYPE)
if equation_type is None:
self.log.warning("Cannot figure out type of equation from input dictionary: %s. "
"Returning []." % input_data)
return []
eq_class = get_class_by_name(equation_type)
equation = eq_class.from_json(input_data[KEY_EQUATION])
focal_points = json.loads(input_data[KEY_FOCAL_POINTS])
surface_gid = input_data[KEY_SURFACE_GID]
surface = load_entity_by_gid(surface_gid)
return surface.compute_equation(focal_points, equation)
except Exception:
self.log.exception("The parameter %s was ignored. None value was returned.", row['name'])
return None
dtype = None
if KEY_DTYPE in row:
dtype = row[KEY_DTYPE]
return string2array(str(input_data), ",", dtype)
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> Operation
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS)
datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries)
time_series_index = dao.get_generic_entity(datatype_index, sim_operation.id, 'fk_from_operation')[0]
ga = self.prepare_metadata(metric_algo.algorithm_category, time_series_index.fk_parent_burst)
ga.visible = False
view_model = get_class_by_name("{}.{}".format(MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(ALGORITHMS.keys())
view_model.generic_attributes = ga
parent_burst = dao.get_generic_entity(BurstConfiguration, time_series_index.fk_parent_burst, 'gid')[0]
metric_op_group = dao.get_operationgroup_by_id(parent_burst.fk_metric_operation_group)
metric_operation_group_id = parent_burst.fk_metric_operation_group
range_values = sim_operation.range_values
view_model.operation_group_gid = uuid.UUID(metric_op_group.gid)
view_model.ranges = json.dumps(parent_burst.ranges)
view_model.range_values = range_values
view_model.is_metric_operation = True
metric_operation = Operation(view_model.gid.hex, sim_operation.fk_launched_by, sim_operation.fk_launched_in,
metric_algo.id, user_group=ga.operation_tag, op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(DataTypeGroup, metric_operation_group_id,
'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
dao.store_entity(metrics_datatype_group)
self.store_view_model(metric_operation, sim_operation.project, view_model)
return metric_operation
def populate_option_values_for_dtype(self, project_id, type_name, filter_condition=None,
category_key=None, complex_dt_attributes=None):
'''
Converts all datatypes that match the project_id, type_name and filter_condition
to a {name: , value:} dict used to populate options in the input tree ui
'''
data_type_cls = get_class_by_name(type_name)
#todo: send category instead of category_key to avoid redundant queries
#NOTE these functions are coupled via data_list, _populate_values makes no sense without _get_available_datatypes
data_list, total_count = get_filtered_datatypes(project_id, data_type_cls,
filter_condition)
values = self._populate_values(data_list, data_type_cls,
category_key, complex_dt_attributes)
return values, total_count
def _get_backend_client(adapter_instance):
# type: (ABCAdapter) -> BackendClient
# For the moment run only simulations on HPC
if TvbProfile.current.hpc.IS_HPC_RUN and type(adapter_instance) is get_class_by_name(
"{}.{}".format(SIMULATOR_MODULE, SIMULATOR_CLASS)):
if not TvbProfile.current.hpc.CAN_RUN_HPC:
raise InvalidSettingsException("We can not enable HPC run. Most probably pyunicore is not installed!")
# Return an entity capable to submit jobs to HPC.
return HPCSchedulerClient()
if TvbProfile.current.cluster.IS_DEPLOY:
# Return an entity capable to submit jobs to the cluster.
return ClusterSchedulerClient()
# Return a thread launcher.
return StandAloneClient()
def review_operation_inputs(self, parameters, flat_interface):
"""
Find out which of the submitted parameters are actually DataTypes and
return a list holding all the dataTypes in parameters.
:returns: list of dataTypes and changed parameters.
"""
inputs_datatypes = []
changed_parameters = dict()
for field_dict in flat_interface:
eq_flat_interface_name = self._find_field_submitted_name(
parameters, field_dict[KEY_NAME])
if eq_flat_interface_name is not None:
is_datatype = False
if field_dict.get(KEY_DATATYPE):
eq_datatype = load_entity_by_gid(
parameters.get(str(eq_flat_interface_name)))
if eq_datatype is not None:
inputs_datatypes.append(eq_datatype)
is_datatype = True
elif isinstance(field_dict[KEY_TYPE], basestring):
try:
class_entity = get_class_by_name(field_dict[KEY_TYPE])
if issubclass(class_entity, MappedType):
data_gid = parameters.get(str(
field_dict[KEY_NAME]))
data_type = load_entity_by_gid(data_gid)
if data_type:
inputs_datatypes.append(data_type)
is_datatype = True
except ImportError:
pass
if is_datatype:
changed_parameters[field_dict[
KEY_LABEL]] = inputs_datatypes[-1].display_name
else:
if field_dict[KEY_NAME] in parameters and (
KEY_DEFAULT not in field_dict
or str(field_dict[KEY_DEFAULT]) != str(
parameters[field_dict[KEY_NAME]])):
changed_parameters[field_dict[KEY_LABEL]] = str(
parameters[field_dict[KEY_NAME]])
return inputs_datatypes, changed_parameters
def populate_option_values_for_dtype(self, project_id, type_name, filter_condition=None,
category_key=None):
'''
Converts all datatypes that match the project_id, type_name and filter_condition
to a {name: , value:} dict used to populate options in the input tree ui
'''
# todo: normalize all itree[KEY_TYPE] to be a python type, not a str, not a None etc
if isinstance(type_name, basestring):
data_type_cls = get_class_by_name(type_name)
else:
data_type_cls = type_name
#todo: send category instead of category_key to avoid redundant queries
#NOTE these functions are coupled via data_list, _populate_values makes no sense without _get_available_datatypes
data_list, total_count = get_filtered_datatypes(project_id, data_type_cls,
filter_condition)
values = self._populate_values(data_list, data_type_cls, category_key)
return values, total_count
def populate_option_values_for_dtype(self,
project_id,
type_name,
filter_condition=None,
category_key=None,
complex_dt_attributes=None):
'''
Converts all datatypes that match the project_id, type_name and filter_condition
to a {name: , value:} dict used to populate options in the input tree ui
'''
data_type_cls = get_class_by_name(type_name)
#todo: send category instead of category_key to avoid redundant queries
#NOTE these functions are coupled via data_list, _populate_values makes no sense without _get_available_datatypes
data_list, total_count = get_filtered_datatypes(
project_id, data_type_cls, filter_condition)
values = self._populate_values(data_list, data_type_cls, category_key,
complex_dt_attributes)
return values, total_count
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> Operation
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries)
time_series_index = dao.get_generic_entity(datatype_index,
sim_operation.id,
'fk_from_operation')[0]
view_model = get_class_by_name("{}.{}".format(
MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(choices.values())
range_values = sim_operation.range_values
metadata = {
DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst
}
metadata, user_group = self._prepare_metadata(
metadata, metric_algo.algorithm_category, None, {})
meta_str = json.dumps(metadata)
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
metric_operation = Operation(sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
json.dumps({'gid': view_model.gid.hex}),
meta_str,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
stored_metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
self._store_view_model(stored_metric_operation, sim_operation.project,
view_model)
return stored_metric_operation
def review_operation_inputs(self, parameters, flat_interface):
"""
Find out which of the submitted parameters are actually DataTypes and
return a list holding all the dataTypes in parameters.
:returns: list of dataTypes and changed parameters.
"""
inputs_datatypes = []
changed_parameters = dict()
for field_dict in flat_interface:
eq_flat_interface_name = self._find_field_submitted_name(parameters, field_dict[KEY_NAME])
if eq_flat_interface_name is not None:
is_datatype = False
if field_dict.get(KEY_DATATYPE):
eq_datatype = load_entity_by_gid(parameters.get(str(eq_flat_interface_name)))
if eq_datatype is not None:
inputs_datatypes.append(eq_datatype)
is_datatype = True
elif isinstance(field_dict[KEY_TYPE], basestring):
try:
class_entity = get_class_by_name(field_dict[KEY_TYPE])
if issubclass(class_entity, MappedType):
data_gid = parameters.get(str(field_dict[KEY_NAME]))
data_type = load_entity_by_gid(data_gid)
if data_type:
inputs_datatypes.append(data_type)
is_datatype = True
except ImportError:
pass
if is_datatype:
changed_parameters[field_dict[KEY_LABEL]] = inputs_datatypes[-1].display_name
else:
if field_dict[KEY_NAME] in parameters and (KEY_DEFAULT not in field_dict
or str(field_dict[KEY_DEFAULT]) != str(parameters[field_dict[KEY_NAME]])):
changed_parameters[field_dict[KEY_LABEL]] = str(parameters[field_dict[KEY_NAME]])
return inputs_datatypes, changed_parameters
def _load_entity(self, row, datatype_gid, kwargs, metadata_out):
"""
Load specific DataType entities, as specified in DATA_TYPE table.
Check if the GID is for the correct DataType sub-class, otherwise throw an exception.
Updates metadata_out with the metadata of this entity
"""
entity = load_entity_by_gid(datatype_gid)
if entity is None:
## Validate required DT one more time, after actual retrieval from DB:
if row.get(KEY_REQUIRED):
raise InvalidParameterException(
"Empty DataType value for required parameter %s [%s]" %
(row[KEY_LABEL], row[KEY_NAME]))
return None
expected_dt_class = row[KEY_TYPE]
if isinstance(expected_dt_class, basestring):
expected_dt_class = get_class_by_name(expected_dt_class)
if not isinstance(entity, expected_dt_class):
raise InvalidParameterException(
"Expected param %s [%s] of type %s but got type %s." %
(row[KEY_LABEL], row[KEY_NAME], expected_dt_class.__name__,
entity.__class__.__name__))
result = entity
## Step 2 of updating Meta-data from parent DataType.
if entity.fk_parent_burst:
## Link just towards the last Burst identified.
metadata_out[DataTypeMetaData.KEY_BURST] = entity.fk_parent_burst
if entity.user_tag_1 and DataTypeMetaData.KEY_TAG_1 not in metadata_out:
metadata_out[DataTypeMetaData.KEY_TAG_1] = entity.user_tag_1
current_subject = metadata_out[DataTypeMetaData.KEY_SUBJECT]
if current_subject == DataTypeMetaData.DEFAULT_SUBJECT:
metadata_out[DataTypeMetaData.KEY_SUBJECT] = entity.subject
else:
if entity.subject != current_subject and entity.subject not in current_subject.split(
','):
metadata_out[
DataTypeMetaData.
KEY_SUBJECT] = current_subject + ',' + entity.subject
## End Step 2 - Meta-data Updates
## Validate current entity to be compliant with specified ROW filters.
dt_filter = row.get(KEY_CONDITION)
if dt_filter is not None and entity is not None and not dt_filter.get_python_filter_equivalent(
entity):
## If a filter is declared, check that the submitted DataType is in compliance to it.
raise InvalidParameterException(
"Field %s [%s] did not pass filters." %
(row[KEY_LABEL], row[KEY_NAME]))
# In case a specific field in entity is to be used, use it
if KEY_FIELD in row:
# note: this cannot be replaced by getattr(entity, row[KEY_FIELD])
# at least BCT has 'fields' like scaled_weights()
result = eval('entity.' + row[KEY_FIELD])
if ATT_METHOD in row:
# The 'shape' attribute of an arraywrapper is overridden by us
# the following check is made only to improve performance
# (to find data in the dictionary with O(1)) on else the data is found in O(n)
prefix = row[KEY_NAME] + "_" + row[ATT_PARAMETERS]
if hasattr(entity, 'shape'):
param_dict = {}
for i in range(1, len(entity.shape)):
param_key = prefix + "_" + str(i - 1)
if param_key in kwargs:
param_dict[param_key] = kwargs[param_key]
else:
param_dict = dict(
(k, v) for k, v in kwargs.items() if k.startswith(prefix))
result = getattr(entity, row[ATT_METHOD])(param_dict)
return result
def _load_entity(self, row, datatype_gid, kwargs, metadata_out):
"""
Load specific DataType entities, as specified in DATA_TYPE table.
Check if the GID is for the correct DataType sub-class, otherwise throw an exception.
Updates metadata_out with the metadata of this entity
"""
entity = load_entity_by_gid(datatype_gid)
if entity is None:
## Validate required DT one more time, after actual retrieval from DB:
if row.get(KEY_REQUIRED):
raise InvalidParameterException("Empty DataType value for required parameter %s [%s]" % (
row[KEY_LABEL], row[KEY_NAME]))
return None
expected_dt_class = row[KEY_TYPE]
if isinstance(expected_dt_class, basestring):
expected_dt_class = get_class_by_name(expected_dt_class)
if not isinstance(entity, expected_dt_class):
raise InvalidParameterException("Expected param %s [%s] of type %s but got type %s." % (
row[KEY_LABEL], row[KEY_NAME], expected_dt_class.__name__, entity.__class__.__name__))
result = entity
## Step 2 of updating Meta-data from parent DataType.
if entity.fk_parent_burst:
## Link just towards the last Burst identified.
metadata_out[DataTypeMetaData.KEY_BURST] = entity.fk_parent_burst
if entity.user_tag_1 and DataTypeMetaData.KEY_TAG_1 not in metadata_out:
metadata_out[DataTypeMetaData.KEY_TAG_1] = entity.user_tag_1
current_subject = metadata_out[DataTypeMetaData.KEY_SUBJECT]
if current_subject == DataTypeMetaData.DEFAULT_SUBJECT:
metadata_out[DataTypeMetaData.KEY_SUBJECT] = entity.subject
else:
if entity.subject != current_subject and entity.subject not in current_subject.split(','):
metadata_out[DataTypeMetaData.KEY_SUBJECT] = current_subject + ',' + entity.subject
## End Step 2 - Meta-data Updates
## Validate current entity to be compliant with specified ROW filters.
dt_filter = row.get(KEY_CONDITION)
if dt_filter is not None and entity is not None and not dt_filter.get_python_filter_equivalent(entity):
## If a filter is declared, check that the submitted DataType is in compliance to it.
raise InvalidParameterException("Field %s [%s] did not pass filters." % (row[KEY_LABEL],
row[KEY_NAME]))
# In case a specific field in entity is to be used, use it
if KEY_FIELD in row:
# note: this cannot be replaced by getattr(entity, row[KEY_FIELD])
# at least BCT has 'fields' like scaled_weights()
result = eval('entity.' + row[KEY_FIELD])
if ATT_METHOD in row:
# The 'shape' attribute of an arraywrapper is overridden by us
# the following check is made only to improve performance
# (to find data in the dictionary with O(1)) on else the data is found in O(n)
prefix = row[KEY_NAME] + "_" + row[ATT_PARAMETERS]
if hasattr(entity, 'shape'):
param_dict = {}
for i in xrange(1, len(entity.shape)):
param_key = prefix + "_" + str(i - 1)
if param_key in kwargs:
param_dict[param_key] = kwargs[param_key]
else:
param_dict = dict((k, v) for k, v in kwargs.items() if k.startswith(prefix))
result = getattr(entity, row[ATT_METHOD])(param_dict)
return result
def remove_datatype(self, project_id, datatype_gid, skip_validation=False, existing_dt_links=None):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid)
return
if datatype.parent_operation.fk_launched_in != int(project_id):
self.logger.warning("Datatype with GUID [%s] has been moved to another project and does "
"not need to be deleted anymore." % datatype_gid)
return
is_datatype_group = False
datatype_group = None
new_dt_links = []
# Datatype Groups were already handled when the first DatatypeMeasureIndex has been found
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
datatype_group = datatype
# Found the first DatatypeMeasureIndex from a group
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
# We load it this way to make sure we have the 'fk_operation_group' in every case
datatype_group_gid = dao.get_datatype_by_id(datatype.fk_datatype_group).gid
datatype_group = h5.load_entity_by_gid(datatype_group_gid)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
operations_set = [datatype_group.fk_from_operation]
self.logger.debug("Removing datatype group %s" % datatype_group)
datatypes = self.get_all_datatypes_from_data(datatype_group)
first_datatype = datatypes[0]
if hasattr(first_datatype, 'fk_source_gid'):
ts = h5.load_entity_by_gid(first_datatype.fk_source_gid)
ts_group = dao.get_datatypegroup_by_op_group_id(ts.parent_operation.fk_operation_group)
dm_group = datatype_group
else:
dt_measure_index = get_class_by_name("{}.{}".format(DATATYPE_MEASURE_INDEX_MODULE,
DATATYPE_MEASURE_INDEX_CLASS))
dm_group = dao.get_datatype_measure_group_from_ts_from_pse(first_datatype.gid, dt_measure_index)
ts_group = datatype_group
links = []
if ts_group:
links.extend(dao.get_links_for_datatype(ts_group.id))
correct = correct and self._remove_operation_group(ts_group.fk_operation_group, project_id,
skip_validation, operations_set, links)
if dm_group:
links.extend(dao.get_links_for_datatype(dm_group.id))
correct = correct and self._remove_operation_group(dm_group.fk_operation_group, project_id,
skip_validation, operations_set, links)
if len(links) > 0:
# We want to get the links for the first TSIndex directly
# This code works for all cases
datatypes = dao.get_datatype_in_group(ts_group.id)
ts = datatypes[0]
new_dt_links = self._add_links_for_datatype_references(ts, links[0].fk_to_project, links[0].id,
existing_dt_links)
else:
self.logger.debug("Removing datatype %s" % datatype)
links = dao.get_links_for_datatype(datatype.id)
if len(links) > 0:
new_dt_links = self._add_links_for_datatype_references(datatype, links[0].fk_to_project, links[0].id,
existing_dt_links)
self._remove_project_node_files(project_id, datatype.gid, links, skip_validation)
# Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation")
if len(dependent_dt) > 0:
# Do not remove Operation in case DataType still exist referring it.
continue
correct = correct and dao.remove_entity(Operation, operation_id)
# Make sure Operation folder is removed
self.storage_interface.remove_operation_data(project.name, operation_id)
self.storage_interface.push_folder_to_sync(project.name)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid))
return new_dt_links
