def import_workflow_steps(self, workflow, wf_steps, view_steps): """ Import all workflow steps for the given workflow. We create both wf_steps and view_steps in the same method, since if a wf_step has to be omited for some reason, we also need to omit that view step. :param workflow: a model.Workflow entity from which we need to add workflow steps :param wf_steps: a list of WorkflowStepInformation entities, from which we will rebuild \ the workflow steps :param view_steps: a list of WorkflowViewStepInformation entities, from which we will \ rebuild the workflow view steps """ for wf_step in wf_steps: algorithm = wf_step.get_algorithm() if algorithm is None: # The algorithm is invalid for some reason. Just remove also the view step. position = wf_step.index() for entry in view_steps: if entry.index() == position: view_steps.remove(entry) continue wf_step_entity = model.WorkflowStep(algorithm.id) wf_step_entity.from_dict(wf_step.data) wf_step_entity.fk_workflow = workflow.id wf_step_entity.fk_operation = wf_step.get_operagion().id dao.store_entity(wf_step_entity) for view_step in view_steps: algorithm = view_step.get_algorithm() if algorithm is None: continue view_step_entity = model.WorkflowStepView(algorithm.id) view_step_entity.from_dict(view_step.data) view_step_entity.fk_workflow = workflow.id view_step_entity.fk_portlet = view_step.get_portlet().id dao.store_entity(view_step_entity)
def load_burst_entity(self, json_burst, project_id): """ Load BurstConfiguration from JSON (possibly exported from a different machine). Nothing gets persisted in DB or on disk. :param json_burst: Burst JSON export :param project_id: Current project ID (it will be used if the user later starts this simulation) :return: BurstConfiguration filled from JSON """ burst_information = BurstInformation.load_from_dict(json_burst) burst_entity = model.BurstConfiguration(project_id) burst_entity.from_dict(burst_information.data) burst_entity.prepare_after_load() burst_entity.reset_tabs() workflow_info = burst_information.get_workflows()[0] workflow_entity = model.Workflow(project_id, None) workflow_entity.from_dict(workflow_info.data) view_steps = workflow_info.get_view_steps() analyze_steps = workflow_info.get_workflow_steps() for view_step in view_steps: try: algorithm = view_step.get_algorithm() portlet = view_step.get_portlet() view_step_entity = model.WorkflowStepView( algorithm.id, portlet_id=portlet.id) view_step_entity.from_dict(view_step.data) view_step_entity.workflow = workflow_entity ## For each visualize step, also load all of the analyze steps. analyzers = [] for an_step in analyze_steps: if (an_step.data["tab_index"] != view_step_entity.tab_index or an_step.data["index_in_tab"] != view_step_entity.index_in_tab): continue algorithm = an_step.get_algorithm() wf_step_entity = model.WorkflowStep(algorithm.id) wf_step_entity.from_dict(an_step.data) wf_step_entity.workflow = workflow_entity analyzers.append(wf_step_entity) portlet = PortletConfiguration(portlet.id) portlet.set_visualizer(view_step_entity) portlet.set_analyzers(analyzers) burst_entity.set_portlet(view_step_entity.tab_index, view_step_entity.index_in_tab, portlet) except Exception: # only log exception and ignore this step from loading self.logger.exception("Could not restore Workflow Step " + view_step.get_algorithm().name) return burst_entity
def create_and_store_workflow(project_id, burst_id, simulator_index, simulator_id, operations): """ Create and store the workflow given the project, user and burst in which the workflow is created. :param simulator_index: the index of the simulator in the workflow :param simulator_id: the id of the simulator adapter :param operations: a list with the operations created for the simulator steps """ workflows = [] for operation in operations: new_workflow = model.Workflow(project_id, burst_id) new_workflow = dao.store_entity(new_workflow) workflows.append(new_workflow) simulation_step = model.WorkflowStep( algorithm_id=simulator_id, workflow_id=new_workflow.id, step_index=simulator_index, static_param=operation.parameters) simulation_step.fk_operation = operation.id dao.store_entity(simulation_step) return workflows
def create_workflow_step(module, classname, static_kwargs=None, dynamic_kwargs=None, step_index=0, base_step=0, tab_index=0, index_in_tab=0, is_view_step=False): """ Build non-persisted WorkflowStep entity. """ if static_kwargs is None: static_kwargs = {} if dynamic_kwargs is None: dynamic_kwargs = {} algorithm = dao.get_algorithm_by_module(module, classname) second_step_configuration = wf_cfg(algorithm.id, static_kwargs, dynamic_kwargs) static_params = second_step_configuration.static_params dynamic_params = second_step_configuration.dynamic_params for entry in dynamic_params: dynamic_params[entry][wf_cfg.STEP_INDEX_KEY] += base_step if is_view_step: return model.WorkflowStepView(algorithm_id=algorithm.id, tab_index=tab_index, index_in_tab=index_in_tab, static_param=static_params, dynamic_param=dynamic_params) return model.WorkflowStep(algorithm_id=algorithm.id, step_index=step_index, tab_index=tab_index, index_in_tab=index_in_tab, static_param=static_params, dynamic_param=dynamic_params)
def _prepare_operations(self, burst_config, simulator_index, simulator_id, user_id): """ Prepare all required operations for burst launch. """ project_id = burst_config.fk_project burst_id = burst_config.id workflow_step_list = [] starting_index = simulator_index + 1 sim_algo = FlowService().get_algorithm_by_identifier(simulator_id) metadata = {DataTypeMetaData.KEY_BURST: burst_id} launch_data = burst_config.get_all_simulator_values()[0] operations, group = self.operation_service.prepare_operations( user_id, project_id, sim_algo, sim_algo.algo_group.group_category, metadata, **launch_data) group_launched = group is not None if group_launched: starting_index += 1 for tab in burst_config.tabs: for portlet_cfg in tab.portlets: ### For each portlet configuration stored, update the step index ### ### and also change the dynamic parameters step indexes to point ### ### to the simulator outputs. ## if portlet_cfg is not None: analyzers = portlet_cfg.analyzers visualizer = portlet_cfg.visualizer for entry in analyzers: entry.step_index = starting_index self.workflow_service.set_dynamic_step_references( entry, simulator_index) workflow_step_list.append(entry) starting_index += 1 ### Change the dynamic parameters to point to the last adapter from this portlet execution. visualizer.step_visible = False if len(workflow_step_list) > 0 and isinstance( workflow_step_list[-1], model.WorkflowStep): self.workflow_service.set_dynamic_step_references( visualizer, workflow_step_list[-1].step_index) else: self.workflow_service.set_dynamic_step_references( visualizer, simulator_index) ### Only for a single operation have the step of visualization, otherwise is useless. if not group_launched: workflow_step_list.append(visualizer) if group_launched: ### For a group of operations, make sure the metric for PSE view ### is also computed, immediately after the simulation. metric_algo, metric_group = FlowService( ).get_algorithm_by_module_and_class(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS) _, metric_interface = FlowService().prepare_adapter( project_id, metric_group) dynamics = {} for entry in metric_interface: # We have a select that should be the dataType and a select multiple with the # required metric algorithms to be evaluated. Only dynamic parameter should be # the select type. if entry[ABCAdapter.KEY_TYPE] == 'select': dynamics[entry[ABCAdapter.KEY_NAME]] = { WorkflowStepConfiguration.DATATYPE_INDEX_KEY: 0, WorkflowStepConfiguration.STEP_INDEX_KEY: simulator_index } metric_step = model.WorkflowStep(algorithm_id=metric_algo.id, step_index=simulator_index + 1, static_param={}, dynamic_param=dynamics) metric_step.step_visible = False workflow_step_list.insert(0, metric_step) workflows = self.workflow_service.create_and_store_workflow( project_id, burst_id, simulator_index, simulator_id, operations) self.operation_service.prepare_operations_for_workflowsteps( workflow_step_list, workflows, user_id, burst_id, project_id, group, operations) operation_ids = [operation.id for operation in operations] return operation_ids