def create_datatype_group(self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ): """ This method creates, stores and returns a DataTypeGroup entity. """ group = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)]) group = dao.store_entity(group) group_ms = model.OperationGroup(self.project.id, ranges=[json.dumps(self.RANGE_1), json.dumps(self.RANGE_2)]) group_ms = dao.store_entity(group_ms) datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id) # Set storage path, before setting data datatype_group.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id)) datatype_group = dao.store_entity(datatype_group) dt_group_ms = model.DataTypeGroup(group_ms, subject=subject, state=state, operation_id=self.operation.id) # Set storage path, before setting data dt_group_ms.storage_path = self.files_helper.get_project_folder(self.project, str(self.operation.id)) dao.store_entity(dt_group_ms) # Now create some data types and add them to group for range_val1 in self.RANGE_1[1]: for range_val2 in self.RANGE_2[1]: operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, range_values=json.dumps({self.RANGE_1[0]: range_val1, self.RANGE_2[0]: range_val2})) operation.fk_operation_group = group.id operation = dao.store_entity(operation) datatype = self.create_datatype_with_storage(operation_id=operation.id) datatype.number1 = range_val1 datatype.number2 = range_val2 datatype.fk_datatype_group = datatype_group.id datatype.set_operation_id(operation.id) dao.store_entity(datatype) op_ms = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status=model.STATUS_FINISHED, range_values=json.dumps({self.RANGE_1[0]: range_val1, self.RANGE_2[0]: range_val2})) op_ms.fk_operation_group = group_ms.id op_ms = dao.store_entity(op_ms) self.create_datatype_measure(datatype, op_ms, FilesHelper().get_project_folder(self.project, str(op_ms.id))) return datatype_group
def prepare_operations_for_workflowsteps(self, workflow_step_list, workflows, user_id, burst_id, project_id, group, sim_operations): """ Create and store Operation entities from a list of Workflow Steps. Will be generated workflows x workflow_step_list Operations. For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created (in case of PSE). """ for step in workflow_step_list: operation_group = None if (group is not None) and not isinstance(step, model.WorkflowStepView): operation_group = model.OperationGroup(project_id=project_id, ranges=group.range_references) operation_group = dao.store_entity(operation_group) operation = None metadata = {DataTypeMetaData.KEY_BURST: burst_id} algo_category = dao.get_algorithm_by_id(step.fk_algorithm) if algo_category is not None: algo_category = algo_category.algorithm_category for wf_idx, workflow in enumerate(workflows): cloned_w_step = step.clone() cloned_w_step.fk_workflow = workflow.id dynamic_params = cloned_w_step.dynamic_param op_params = cloned_w_step.static_param op_params.update(dynamic_params) range_values = None group_id = None if operation_group is not None: group_id = operation_group.id range_values = sim_operations[wf_idx].range_values if not isinstance(step, model.WorkflowStepView): ## For visualization steps, do not create operations, as those are not really needed. metadata, user_group = self._prepare_metadata(metadata, algo_category, operation_group, op_params) operation = model.Operation(user_id, project_id, step.fk_algorithm, json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder), meta=json.dumps(metadata), op_group_id=group_id, range_values=range_values, user_group=user_group) operation.visible = step.step_visible operation = dao.store_entity(operation) cloned_w_step.fk_operation = operation.id dao.store_entity(cloned_w_step) if operation_group is not None and operation is not None: datatype_group = model.DataTypeGroup(operation_group, operation_id=operation.id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group)
def prepare_operations(self, user_id, project_id, algorithm, category, metadata, visible=True, existing_dt_group=None, **kwargs): """ Do all the necessary preparations for storing an operation. If it's the case of a range of values create an operation group and multiple operations for each possible instance from the range. :param metadata: Initial MetaData with potential Burst identification inside. """ operations = [] available_args, group = self._prepare_group(project_id, existing_dt_group, kwargs) if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER: raise LaunchException("Too big range specified. You should limit the" " resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER) else: self.logger.debug("Launching a range with %d operations..." % len(available_args)) group_id = None if group is not None: group_id = group.id metadata, user_group = self._prepare_metadata(metadata, category, group, kwargs) self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) + "," + str(metadata) + ",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")") visible_operation = visible and category.display is False meta_str = json.dumps(metadata) for (one_set_of_args, range_vals) in available_args: range_values = json.dumps(range_vals) if range_vals else None operation = model.Operation(user_id, project_id, algorithm.id, json.dumps(one_set_of_args, cls=MapAsJson.MapAsJsonEncoder), meta_str, op_group_id=group_id, user_group=user_group, range_values=range_values) operation.visible = visible_operation operations.append(operation) operations = dao.store_entities(operations) if group is not None: burst_id = None if DataTypeMetaData.KEY_BURST in metadata: burst_id = metadata[DataTypeMetaData.KEY_BURST] if existing_dt_group is None: datatype_group = model.DataTypeGroup(group, operation_id=operations[0].id, fk_parent_burst=burst_id, state=metadata[DataTypeMetaData.KEY_STATE]) dao.store_entity(datatype_group) else: # Reset count existing_dt_group.count_results = None dao.store_entity(existing_dt_group) return operations, group
def create_datatype_group( self, subject=USER_FULL_NAME, state=DATATYPE_STATE, ): """ This method creates, stores and returns a DataTypeGroup entity. """ OPERATION_GROUP_RANGE = [json.dumps(["row1", ['a', 'b', 'c']])] group = model.OperationGroup(self.project.id, self.OPERATION_GROUP_NAME, OPERATION_GROUP_RANGE) group = dao.store_entity(group) datatype_group = model.DataTypeGroup(group, subject=subject, state=state, operation_id=self.operation.id) # Set storage path, before setting data datatype_group.storage_path = self.files_helper.get_project_folder( self.project, str(self.operation.id)) datatype_group = dao.store_entity(datatype_group) # Now create some data types and add them to group for range_val in ['a', 'b', 'c']: operation = model.Operation(self.user.id, self.project.id, self.algorithm.id, 'test parameters', meta=json.dumps(self.meta), status="FINISHED", method_name=ABCAdapter.LAUNCH_METHOD, range_values=json.dumps( {'row1': range_val})) operation.fk_operation_group = group.id operation = dao.store_entity(operation) datatype = self.create_datatype_with_storage( operation_id=operation.id) datatype.row1 = range_val datatype.fk_datatype_group = datatype_group.id datatype.set_operation_id(operation.id) dao.store_entity(datatype) return datatype_group
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = model.DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = ADAPTERS['Upload']['defaultdatastate'] datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group