def __init__(self,
trans,
workflow,
workflow_run_config,
workflow_invocation=None):
self.trans = trans
self.workflow = workflow
if workflow_invocation is None:
invocation_uuid = uuid.uuid1()
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.workflow = self.workflow
# In one way or another, following attributes will become persistent
# so they are available during delayed/revisited workflow scheduling.
workflow_invocation.uuid = invocation_uuid
workflow_invocation.history = workflow_run_config.target_history
self.workflow_invocation = workflow_invocation
else:
self.workflow_invocation = workflow_invocation
self.workflow_invocation.copy_inputs_to_history = workflow_run_config.copy_inputs_to_history
self.workflow_invocation.replacement_dict = workflow_run_config.replacement_dict
module_injector = modules.WorkflowModuleInjector(trans)
self.progress = WorkflowProgress(self.workflow_invocation,
workflow_run_config.inputs,
module_injector)
def workflow_run_config_to_request( trans, run_config, workflow ):
param_types = model.WorkflowRequestInputParameter.types
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.uuid = uuid.uuid1()
workflow_invocation.history = run_config.target_history
def add_parameter( name, value, type ):
parameter = model.WorkflowRequestInputParameter(
name=name,
value=value,
type=type,
)
workflow_invocation.input_parameters.append( parameter )
replacement_dict = run_config.replacement_dict
for name, value in replacement_dict.iteritems():
add_parameter(
name=name,
value=value,
type=param_types.REPLACEMENT_PARAMETERS,
)
for step_id, content in run_config.inputs.iteritems():
workflow_invocation.add_input( content, step_id )
for step in workflow.steps:
state = step.state
serializable_runtime_state = step.module.normalize_runtime_state( state )
step_state = model.WorkflowRequestStepState()
step_state.workflow_step = step
step_state.value = serializable_runtime_state
workflow_invocation.step_states.append( step_state )
add_parameter( "copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS )
return workflow_invocation
def invoke(self):
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.workflow = self.workflow
# Web controller will populate state on each step before calling
# invoke but not API controller. More work should be done to further
# harmonize these methods going forward if possible - if possible
# moving more web controller logic here.
state_populated = not self.workflow.steps or hasattr(
self.workflow.steps[0], "state")
if not state_populated:
self._populate_state()
for step in self.workflow.steps:
jobs = self._invoke_step(step)
for job in util.listify(jobs):
# Record invocation
workflow_invocation_step = model.WorkflowInvocationStep()
workflow_invocation_step.workflow_invocation = workflow_invocation
workflow_invocation_step.workflow_step = step
workflow_invocation_step.job = job
# All jobs ran successfully, so we can save now
self.trans.sa_session.add(workflow_invocation)
# Not flushing in here, because web controller may create multiple
# invokations.
return self.outputs
def workflow_run_config_to_request(trans, run_config, workflow):
param_types = model.WorkflowRequestInputParameter.types
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.uuid = uuid.uuid1()
workflow_invocation.history = run_config.target_history
def add_parameter(name, value, type):
parameter = model.WorkflowRequestInputParameter(
name=name,
value=value,
type=type,
)
workflow_invocation.input_parameters.append(parameter)
steps_by_id = {}
for step in workflow.steps:
steps_by_id[step.id] = step
state = step.state
serializable_runtime_state = step.module.normalize_runtime_state(state)
step_state = model.WorkflowRequestStepState()
step_state.workflow_step = step
log.info("Creating a step_state for step.id %s" % step.id)
step_state.value = serializable_runtime_state
workflow_invocation.step_states.append(step_state)
if step.type == "subworkflow":
subworkflow_run_config = WorkflowRunConfig(
target_history=run_config.target_history,
replacement_dict=run_config.replacement_dict,
copy_inputs_to_history=False,
inputs={},
param_map={},
allow_tool_state_corrections=run_config.
allow_tool_state_corrections)
subworkflow_invocation = workflow_run_config_to_request(
trans,
subworkflow_run_config,
step.subworkflow,
)
workflow_invocation.attach_subworkflow_invocation_for_step(
step,
subworkflow_invocation,
)
replacement_dict = run_config.replacement_dict
for name, value in replacement_dict.iteritems():
add_parameter(
name=name,
value=value,
type=param_types.REPLACEMENT_PARAMETERS,
)
for step_id, content in run_config.inputs.iteritems():
workflow_invocation.add_input(content, step_id)
add_parameter("copy_inputs_to_history",
"true" if run_config.copy_inputs_to_history else "false",
param_types.META_PARAMETERS)
return workflow_invocation
def __init__( self, trans, workflow, workflow_run_config ):
self.trans = trans
self.workflow = workflow
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.workflow = self.workflow
self.workflow_invocation = workflow_invocation
self.progress = WorkflowProgress( self.workflow_invocation, workflow_run_config.inputs )
invocation_uuid = uuid.uuid1().hex
# In one way or another, following attributes will become persistent
# so they are available during delayed/revisited workflow scheduling.
self.workflow_invocation.uuid = invocation_uuid
self.workflow_invocation.history = workflow_run_config.target_history
self.workflow_invocation.copy_inputs_to_history = workflow_run_config.copy_inputs_to_history
self.workflow_invocation.replacement_dict = workflow_run_config.replacement_dict
def __init__(self,
trans,
workflow,
workflow_run_config,
workflow_invocation=None,
progress=None):
self.trans = trans
self.workflow = workflow
if progress is not None:
assert workflow_invocation is None
workflow_invocation = progress.workflow_invocation
if workflow_invocation is None:
invocation_uuid = uuid.uuid1()
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.workflow = self.workflow
# In one way or another, following attributes will become persistent
# so they are available during delayed/revisited workflow scheduling.
workflow_invocation.uuid = invocation_uuid
workflow_invocation.history = workflow_run_config.target_history
self.workflow_invocation = workflow_invocation
else:
self.workflow_invocation = workflow_invocation
self.workflow_invocation.copy_inputs_to_history = workflow_run_config.copy_inputs_to_history
self.workflow_invocation.use_cached_job = workflow_run_config.use_cached_job
self.workflow_invocation.replacement_dict = workflow_run_config.replacement_dict
module_injector = modules.WorkflowModuleInjector(trans)
if progress is None:
progress = WorkflowProgress(
self.workflow_invocation,
workflow_run_config.inputs,
module_injector,
param_map=workflow_run_config.param_map,
jobs_per_scheduling_iteration=getattr(
trans.app.config,
"maximum_workflow_jobs_per_scheduling_iteration", -1),
)
self.progress = progress
def example_invocation(trans):
invocation = model.WorkflowInvocation()
workflow = yaml_to_model(TEST_WORKFLOW_YAML)
workflow.id = 342
invocation.id = 44
invocation.workflow = workflow
# TODO: fix this to use workflow id and eliminate hack.
stored_workflow = model.StoredWorkflow()
stored_workflow.id = 342
invocation.workflow.stored_workflow = stored_workflow
hda = model.HistoryDatasetAssociation(create_dataset=True, sa_session=trans.sa_session)
hda.id = 567
invocation.add_input(hda, step=workflow.steps[0])
out_hda = model.HistoryDatasetAssociation(create_dataset=True, sa_session=trans.sa_session)
out_hda.id = 563
wf_output = model.WorkflowOutput(workflow.steps[2], label="output_label")
invocation.add_output(wf_output, workflow.steps[2], out_hda)
return invocation
def test_workflows(self):
model = self.model
user = model.User(
email="*****@*****.**",
password="******"
)
def workflow_from_steps(steps):
stored_workflow = model.StoredWorkflow()
stored_workflow.user = user
workflow = model.Workflow()
workflow.steps = steps
workflow.stored_workflow = stored_workflow
return workflow
child_workflow = workflow_from_steps([])
self.persist(child_workflow)
workflow_step_1 = model.WorkflowStep()
workflow_step_1.order_index = 0
workflow_step_1.type = "data_input"
workflow_step_2 = model.WorkflowStep()
workflow_step_2.order_index = 1
workflow_step_2.type = "subworkflow"
workflow_step_2.subworkflow = child_workflow
workflow_step_1.get_or_add_input("moo1")
workflow_step_1.get_or_add_input("moo2")
workflow_step_2.get_or_add_input("moo")
workflow_step_1.add_connection("foo", "cow", workflow_step_2)
workflow = workflow_from_steps([workflow_step_1, workflow_step_2])
self.persist(workflow)
workflow_id = workflow.id
annotation = model.WorkflowStepAnnotationAssociation()
annotation.annotation = "Test Step Annotation"
annotation.user = user
annotation.workflow_step = workflow_step_1
self.persist(annotation)
assert workflow_step_1.id is not None
h1 = model.History(name="WorkflowHistory1", user=user)
invocation_uuid = uuid.uuid1()
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.uuid = invocation_uuid
workflow_invocation.history = h1
workflow_invocation_step1 = model.WorkflowInvocationStep()
workflow_invocation_step1.workflow_invocation = workflow_invocation
workflow_invocation_step1.workflow_step = workflow_step_1
subworkflow_invocation = model.WorkflowInvocation()
workflow_invocation.attach_subworkflow_invocation_for_step(workflow_step_2, subworkflow_invocation)
workflow_invocation_step2 = model.WorkflowInvocationStep()
workflow_invocation_step2.workflow_invocation = workflow_invocation
workflow_invocation_step2.workflow_step = workflow_step_2
workflow_invocation.workflow = workflow
d1 = self.new_hda(h1, name="1")
workflow_request_dataset = model.WorkflowRequestToInputDatasetAssociation()
workflow_request_dataset.workflow_invocation = workflow_invocation
workflow_request_dataset.workflow_step = workflow_step_1
workflow_request_dataset.dataset = d1
self.persist(workflow_invocation)
assert workflow_request_dataset is not None
assert workflow_invocation.id is not None
history_id = h1.id
self.expunge()
loaded_invocation = self.query(model.WorkflowInvocation).get(workflow_invocation.id)
assert loaded_invocation.uuid == invocation_uuid, "%s != %s" % (loaded_invocation.uuid, invocation_uuid)
assert loaded_invocation
assert loaded_invocation.history.id == history_id
step_1, step_2 = loaded_invocation.workflow.steps
assert not step_1.subworkflow
assert step_2.subworkflow
assert len(loaded_invocation.steps) == 2
subworkflow_invocation_assoc = loaded_invocation.get_subworkflow_invocation_association_for_step(step_2)
assert subworkflow_invocation_assoc is not None
assert isinstance(subworkflow_invocation_assoc.subworkflow_invocation, model.WorkflowInvocation)
assert isinstance(subworkflow_invocation_assoc.parent_workflow_invocation, model.WorkflowInvocation)
assert subworkflow_invocation_assoc.subworkflow_invocation.history.id == history_id
loaded_workflow = self.query(model.Workflow).get(workflow_id)
assert len(loaded_workflow.steps[0].annotations) == 1
copied_workflow = loaded_workflow.copy(user=user)
annotations = copied_workflow.steps[0].annotations
assert len(annotations) == 1
def setUp(self):
self.app = TestApp()
self.inputs_by_step_id = {}
self.invocation = model.WorkflowInvocation()
self.progress = {}
def _new_invocation(self):
invocation = model.WorkflowInvocation()
invocation.id = 1
invocation.create_time = now()
return invocation
def workflow_run_config_to_request(trans, run_config, workflow):
param_types = model.WorkflowRequestInputParameter.types
workflow_invocation = model.WorkflowInvocation()
workflow_invocation.uuid = uuid.uuid1()
workflow_invocation.history = run_config.target_history
def add_parameter(name, value, type):
parameter = model.WorkflowRequestInputParameter(
name=name,
value=value,
type=type,
)
workflow_invocation.input_parameters.append(parameter)
steps_by_id = {}
for step in workflow.steps:
steps_by_id[step.id] = step
serializable_runtime_state = step.module.encode_runtime_state(
step.state)
step_state = model.WorkflowRequestStepState()
step_state.workflow_step = step
log.info(f"Creating a step_state for step.id {step.id}")
step_state.value = serializable_runtime_state
workflow_invocation.step_states.append(step_state)
if step.type == "subworkflow":
subworkflow_run_config = WorkflowRunConfig(
target_history=run_config.target_history,
replacement_dict=run_config.replacement_dict,
copy_inputs_to_history=False,
use_cached_job=run_config.use_cached_job,
inputs={},
param_map=run_config.param_map.get(step.order_index, {}),
allow_tool_state_corrections=run_config.
allow_tool_state_corrections,
resource_params=run_config.resource_params)
subworkflow_invocation = workflow_run_config_to_request(
trans,
subworkflow_run_config,
step.subworkflow,
)
workflow_invocation.attach_subworkflow_invocation_for_step(
step,
subworkflow_invocation,
)
replacement_dict = run_config.replacement_dict
for name, value in replacement_dict.items():
add_parameter(
name=name,
value=value,
type=param_types.REPLACEMENT_PARAMETERS,
)
for step_id, content in run_config.inputs.items():
workflow_invocation.add_input(content, step_id)
for step_id, param_dict in run_config.param_map.items():
add_parameter(
name=step_id,
value=json.dumps(param_dict),
type=param_types.STEP_PARAMETERS,
)
resource_parameters = run_config.resource_params
for key, value in resource_parameters.items():
add_parameter(key, value, param_types.RESOURCE_PARAMETERS)
add_parameter("copy_inputs_to_history",
"true" if run_config.copy_inputs_to_history else "false",
param_types.META_PARAMETERS)
add_parameter("use_cached_job",
"true" if run_config.use_cached_job else "false",
param_types.META_PARAMETERS)
return workflow_invocation