def create_datatype_group(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ):
"""
This method creates, stores and returns a DataTypeGroup entity.
"""
group = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group = dao.store_entity(group)
group_ms = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)])
group_ms = dao.store_entity(group_ms)
datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
datatype_group.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = model.DataTypeGroup(group_ms, subject=subject, state=state, operation_id=self.operation.id)
# Set storage path, before setting data
dt_group_ms.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id))
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in self.RANGE_1[1]:
for range_val2 in self.RANGE_2[1]:
operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
operation.fk_operation_group = group.id
operation = dao.store_entity(operation)
datatype = self.create_datatype_with_storage(operation_id=operation.id)
datatype.number1 = range_val1
datatype.number2 = range_val2
datatype.fk_datatype_group = datatype_group.id
datatype.set_operation_id(operation.id)
dao.store_entity(datatype)
op_ms = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters',
meta=json.dumps(self.meta), status=model.STATUS_FINISHED,
range_values=json.dumps({self.RANGE_1[0]: range_val1,
self.RANGE_2[0]: range_val2}))
op_ms.fk_operation_group = group_ms.id
op_ms = dao.store_entity(op_ms)
self.create_datatype_measure(datatype, op_ms,
FilesHelper().get_project_folder(self.project, str(op_ms.id)))
return datatype_group
def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id,
project_id, group, sim_operations):
"""
Create and store Operation entities from a list of Workflow Steps.
Will be generated workflows x workflow_step_list Operations.
For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created
(in case of PSE).
"""
for step in workflow_step_list:
operation_group = None
if (group is not None) and not isinstance(step, model.WorkflowStepView):
operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references)
operation_group = dao.store_entity(operation_group)
operation = None
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
algo_category = dao.get_algorithm_by_id(step.fk_algorithm)
if algo_category is not None:
algo_category = algo_category.algorithm_category
for wf_idx, workflow in enumerate(workflows):
cloned_w_step = step.clone()
cloned_w_step.fk_workflow = workflow.id
dynamic_params = cloned_w_step.dynamic_param
op_params = cloned_w_step.static_param
op_params.update(dynamic_params)
range_values = None
group_id = None
if operation_group is not None:
group_id = operation_group.id
range_values = sim_operations[wf_idx].range_values
if not isinstance(step, model.WorkflowStepView):
## For visualization steps, do not create operations, as those are not really needed.
metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params)
operation = model.Operation(user_id, project_id, step.fk_algorithm,
json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder),
meta=json.dumps(metadata),
op_group_id=group_id, range_values=range_values, user_group=user_group)
operation.visible = step.step_visible
operation = dao.store_entity(operation)
cloned_w_step.fk_operation = operation.id
dao.store_entity(cloned_w_step)
if operation_group is not None and operation is not None:
datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
def prepare_operations(self, user_id, project_id, algorithm, category, metadata,
visible=True, existing_dt_group=None, **kwargs):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
:param metadata: Initial MetaData with potential Burst identification inside.
"""
operations = []
available_args, group = self._prepare_group(project_id, existing_dt_group, kwargs)
if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER:
raise LaunchException("Too big range specified. You should limit the"
" resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER)
else:
self.logger.debug("Launching a range with %d operations..." % len(available_args))
group_id = None
if group is not None:
group_id = group.id
metadata, user_group = self._prepare_metadata(metadata, category, group, kwargs)
self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")")
visible_operation = visible and category.display is False
meta_str = json.dumps(metadata)
for (one_set_of_args, range_vals) in available_args:
range_values = json.dumps(range_vals) if range_vals else None
operation = model.Operation(user_id, project_id, algorithm.id,
json.dumps(one_set_of_args, cls=MapAsJson.MapAsJsonEncoder), meta_str,
op_group_id=group_id, user_group=user_group, range_values=range_values)
operation.visible = visible_operation
operations.append(operation)
operations = dao.store_entities(operations)
if group is not None:
burst_id = None
if DataTypeMetaData.KEY_BURST in metadata:
burst_id = metadata[DataTypeMetaData.KEY_BURST]
if existing_dt_group is None:
datatype_group = model.DataTypeGroup(group, operation_id=operations[0].id, fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
else:
# Reset count
existing_dt_group.count_results = None
dao.store_entity(existing_dt_group)
return operations, group
def create_datatype_group(
self,
subject=USER_FULL_NAME,
state=DATATYPE_STATE,
):
"""
This method creates, stores and returns a DataTypeGroup entity.
"""
OPERATION_GROUP_RANGE = [json.dumps(["row1", ['a', 'b', 'c']])]
group = model.OperationGroup(self.project.id,
self.OPERATION_GROUP_NAME,
OPERATION_GROUP_RANGE)
group = dao.store_entity(group)
datatype_group = model.DataTypeGroup(group,
subject=subject,
state=state,
operation_id=self.operation.id)
# Set storage path, before setting data
datatype_group.storage_path = self.files_helper.get_project_folder(
self.project, str(self.operation.id))
datatype_group = dao.store_entity(datatype_group)
# Now create some data types and add them to group
for range_val in ['a', 'b', 'c']:
operation = model.Operation(self.user.id,
self.project.id,
self.algorithm.id,
'test parameters',
meta=json.dumps(self.meta),
status="FINISHED",
method_name=ABCAdapter.LAUNCH_METHOD,
range_values=json.dumps(
{'row1': range_val}))
operation.fk_operation_group = group.id
operation = dao.store_entity(operation)
datatype = self.create_datatype_with_storage(
operation_id=operation.id)
datatype.row1 = range_val
datatype.fk_datatype_group = datatype_group.id
datatype.set_operation_id(operation.id)
dao.store_entity(datatype)
return datatype_group
def __import_operation(operation_entity):
"""
Store a Operation entity.
"""
operation_entity = dao.store_entity(operation_entity)
operation_group_id = operation_entity.fk_operation_group
datatype_group = None
if operation_group_id is not None:
try:
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
except SQLAlchemyError:
# If no dataType group present for current op. group, create it.
operation_group = dao.get_operationgroup_by_id(operation_group_id)
datatype_group = model.DataTypeGroup(operation_group, operation_id=operation_entity.id)
datatype_group.state = ADAPTERS['Upload']['defaultdatastate']
datatype_group = dao.store_entity(datatype_group)
return operation_entity, datatype_group
