def import_workflow_steps(self, workflow, wf_steps, view_steps):
"""
Import all workflow steps for the given workflow. We create both wf_steps and view_steps
in the same method, since if a wf_step has to be omited for some reason, we also need to
omit that view step.
:param workflow: a model.Workflow entity from which we need to add workflow steps
:param wf_steps: a list of WorkflowStepInformation entities, from which we will rebuild \
the workflow steps
:param view_steps: a list of WorkflowViewStepInformation entities, from which we will \
rebuild the workflow view steps
"""
for wf_step in wf_steps:
algorithm = wf_step.get_algorithm()
if algorithm is None:
# The algorithm is invalid for some reason. Just remove also the view step.
position = wf_step.index()
for entry in view_steps:
if entry.index() == position:
view_steps.remove(entry)
continue
wf_step_entity = model.WorkflowStep(algorithm.id)
wf_step_entity.from_dict(wf_step.data)
wf_step_entity.fk_workflow = workflow.id
wf_step_entity.fk_operation = wf_step.get_operagion().id
dao.store_entity(wf_step_entity)
for view_step in view_steps:
algorithm = view_step.get_algorithm()
if algorithm is None:
continue
view_step_entity = model.WorkflowStepView(algorithm.id)
view_step_entity.from_dict(view_step.data)
view_step_entity.fk_workflow = workflow.id
view_step_entity.fk_portlet = view_step.get_portlet().id
dao.store_entity(view_step_entity)
def load_burst_entity(self, json_burst, project_id):
"""
Load BurstConfiguration from JSON (possibly exported from a different machine).
Nothing gets persisted in DB or on disk.
:param json_burst: Burst JSON export
:param project_id: Current project ID (it will be used if the user later starts this simulation)
:return: BurstConfiguration filled from JSON
"""
burst_information = BurstInformation.load_from_dict(json_burst)
burst_entity = model.BurstConfiguration(project_id)
burst_entity.from_dict(burst_information.data)
burst_entity.prepare_after_load()
burst_entity.reset_tabs()
workflow_info = burst_information.get_workflows()[0]
workflow_entity = model.Workflow(project_id, None)
workflow_entity.from_dict(workflow_info.data)
view_steps = workflow_info.get_view_steps()
analyze_steps = workflow_info.get_workflow_steps()
for view_step in view_steps:
try:
algorithm = view_step.get_algorithm()
portlet = view_step.get_portlet()
view_step_entity = model.WorkflowStepView(
algorithm.id, portlet_id=portlet.id)
view_step_entity.from_dict(view_step.data)
view_step_entity.workflow = workflow_entity
## For each visualize step, also load all of the analyze steps.
analyzers = []
for an_step in analyze_steps:
if (an_step.data["tab_index"] != view_step_entity.tab_index
or an_step.data["index_in_tab"] !=
view_step_entity.index_in_tab):
continue
algorithm = an_step.get_algorithm()
wf_step_entity = model.WorkflowStep(algorithm.id)
wf_step_entity.from_dict(an_step.data)
wf_step_entity.workflow = workflow_entity
analyzers.append(wf_step_entity)
portlet = PortletConfiguration(portlet.id)
portlet.set_visualizer(view_step_entity)
portlet.set_analyzers(analyzers)
burst_entity.set_portlet(view_step_entity.tab_index,
view_step_entity.index_in_tab,
portlet)
except Exception:
# only log exception and ignore this step from loading
self.logger.exception("Could not restore Workflow Step " +
view_step.get_algorithm().name)
return burst_entity
def create_and_store_workflow(project_id, burst_id, simulator_index,
simulator_id, operations):
"""
Create and store the workflow given the project, user and burst in which the workflow is created.
:param simulator_index: the index of the simulator in the workflow
:param simulator_id: the id of the simulator adapter
:param operations: a list with the operations created for the simulator steps
"""
workflows = []
for operation in operations:
new_workflow = model.Workflow(project_id, burst_id)
new_workflow = dao.store_entity(new_workflow)
workflows.append(new_workflow)
simulation_step = model.WorkflowStep(
algorithm_id=simulator_id,
workflow_id=new_workflow.id,
step_index=simulator_index,
static_param=operation.parameters)
simulation_step.fk_operation = operation.id
dao.store_entity(simulation_step)
return workflows
def create_workflow_step(module, classname, static_kwargs=None, dynamic_kwargs=None,
step_index=0, base_step=0, tab_index=0, index_in_tab=0, is_view_step=False):
"""
Build non-persisted WorkflowStep entity.
"""
if static_kwargs is None:
static_kwargs = {}
if dynamic_kwargs is None:
dynamic_kwargs = {}
algorithm = dao.get_algorithm_by_module(module, classname)
second_step_configuration = wf_cfg(algorithm.id, static_kwargs, dynamic_kwargs)
static_params = second_step_configuration.static_params
dynamic_params = second_step_configuration.dynamic_params
for entry in dynamic_params:
dynamic_params[entry][wf_cfg.STEP_INDEX_KEY] += base_step
if is_view_step:
return model.WorkflowStepView(algorithm_id=algorithm.id, tab_index=tab_index, index_in_tab=index_in_tab,
static_param=static_params, dynamic_param=dynamic_params)
return model.WorkflowStep(algorithm_id=algorithm.id, step_index=step_index, tab_index=tab_index,
index_in_tab=index_in_tab, static_param=static_params, dynamic_param=dynamic_params)
def _prepare_operations(self, burst_config, simulator_index, simulator_id,
user_id):
"""
Prepare all required operations for burst launch.
"""
project_id = burst_config.fk_project
burst_id = burst_config.id
workflow_step_list = []
starting_index = simulator_index + 1
sim_algo = FlowService().get_algorithm_by_identifier(simulator_id)
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
launch_data = burst_config.get_all_simulator_values()[0]
operations, group = self.operation_service.prepare_operations(
user_id, project_id, sim_algo, sim_algo.algo_group.group_category,
metadata, **launch_data)
group_launched = group is not None
if group_launched:
starting_index += 1
for tab in burst_config.tabs:
for portlet_cfg in tab.portlets:
### For each portlet configuration stored, update the step index ###
### and also change the dynamic parameters step indexes to point ###
### to the simulator outputs. ##
if portlet_cfg is not None:
analyzers = portlet_cfg.analyzers
visualizer = portlet_cfg.visualizer
for entry in analyzers:
entry.step_index = starting_index
self.workflow_service.set_dynamic_step_references(
entry, simulator_index)
workflow_step_list.append(entry)
starting_index += 1
### Change the dynamic parameters to point to the last adapter from this portlet execution.
visualizer.step_visible = False
if len(workflow_step_list) > 0 and isinstance(
workflow_step_list[-1], model.WorkflowStep):
self.workflow_service.set_dynamic_step_references(
visualizer, workflow_step_list[-1].step_index)
else:
self.workflow_service.set_dynamic_step_references(
visualizer, simulator_index)
### Only for a single operation have the step of visualization, otherwise is useless.
if not group_launched:
workflow_step_list.append(visualizer)
if group_launched:
### For a group of operations, make sure the metric for PSE view
### is also computed, immediately after the simulation.
metric_algo, metric_group = FlowService(
).get_algorithm_by_module_and_class(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
_, metric_interface = FlowService().prepare_adapter(
project_id, metric_group)
dynamics = {}
for entry in metric_interface:
# We have a select that should be the dataType and a select multiple with the
# required metric algorithms to be evaluated. Only dynamic parameter should be
# the select type.
if entry[ABCAdapter.KEY_TYPE] == 'select':
dynamics[entry[ABCAdapter.KEY_NAME]] = {
WorkflowStepConfiguration.DATATYPE_INDEX_KEY: 0,
WorkflowStepConfiguration.STEP_INDEX_KEY:
simulator_index
}
metric_step = model.WorkflowStep(algorithm_id=metric_algo.id,
step_index=simulator_index + 1,
static_param={},
dynamic_param=dynamics)
metric_step.step_visible = False
workflow_step_list.insert(0, metric_step)
workflows = self.workflow_service.create_and_store_workflow(
project_id, burst_id, simulator_index, simulator_id, operations)
self.operation_service.prepare_operations_for_workflowsteps(
workflow_step_list, workflows, user_id, burst_id, project_id,
group, operations)
operation_ids = [operation.id for operation in operations]
return operation_ids
