def get_controller(wf_ex, wf_spec=None):
"""Gets a workflow controller instance by given workflow execution object.
:param wf_ex: Workflow execution object.
:param wf_spec: Workflow specification object. If passed, the method works
faster.
:returns: Workflow controller class.
"""
if not wf_spec:
wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id)
wf_type = wf_spec.get_type()
ctrl_cls = None
for cls in u.iter_subclasses(WorkflowController):
if cls.__workflow_type__ == wf_type:
ctrl_cls = cls
break
if not ctrl_cls:
raise exc.MistralError(
'Failed to find a workflow controller [type=%s]' % wf_type
)
return ctrl_cls(wf_ex, wf_spec)
def _build_action(action_ex):
if isinstance(action_ex, models.WorkflowExecution):
return actions.WorkflowAction(None, action_ex=action_ex)
wf_name = None
wf_spec_name = None
if action_ex.workflow_name:
wf_name = action_ex.workflow_name
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
action_ex.task_execution.workflow_execution_id)
wf_spec_name = wf_spec.get_name()
adhoc_action_name = action_ex.runtime_context.get('adhoc_action_name')
if adhoc_action_name:
action_def = actions.resolve_action_definition(adhoc_action_name,
wf_name, wf_spec_name)
return actions.AdHocAction(action_def, action_ex=action_ex)
action_def = actions.resolve_action_definition(action_ex.name, wf_name,
wf_spec_name)
return actions.PythonAction(action_def, action_ex=action_ex)
def schedule(self, input_dict, target, index=0, desc='', safe_rerun=False):
assert not self.action_ex
parent_wf_ex = self.task_ex.workflow_execution
parent_wf_spec = spec_parser.get_workflow_spec_by_execution_id(
parent_wf_ex.id)
task_spec = spec_parser.get_task_spec(self.task_ex.spec)
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(
parent_wf_ex.workflow_name, parent_wf_spec.get_name(),
wf_spec_name)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_params = {'task_execution_id': self.task_ex.id, 'index': index}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in list(input_dict.items()):
if k not in wf_spec.get_input():
wf_params[k] = v
del input_dict[k]
wf_handler.start_workflow(wf_def.id, input_dict,
"sub-workflow execution", wf_params)
def _build_action(action_ex):
if isinstance(action_ex, models.WorkflowExecution):
return actions.WorkflowAction(None, action_ex=action_ex)
wf_name = None
wf_spec_name = None
if action_ex.workflow_name:
wf_name = action_ex.workflow_name
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
action_ex.task_execution.workflow_execution_id
)
wf_spec_name = wf_spec.get_name()
adhoc_action_name = action_ex.runtime_context.get('adhoc_action_name')
if adhoc_action_name:
action_def = actions.resolve_action_definition(
adhoc_action_name,
wf_name,
wf_spec_name
)
return actions.AdHocAction(action_def, action_ex=action_ex)
action_def = actions.resolve_action_definition(
action_ex.name,
wf_name,
wf_spec_name
)
return actions.PythonAction(action_def, action_ex=action_ex)
def _on_action_complete(action_ex):
"""Handles action completion event.
:param action_ex: Action execution.
"""
task_ex = action_ex.task_execution
if not task_ex:
return
task_spec = spec_parser.get_task_spec(task_ex.spec)
wf_ex = task_ex.workflow_execution
task = _create_task(
wf_ex, spec_parser.get_workflow_spec_by_execution_id(wf_ex.id),
task_spec, task_ex.in_context, task_ex)
try:
task.on_action_complete(action_ex)
except exc.MistralException as e:
wf_ex = task_ex.workflow_execution
msg = ("Failed to handle action completion [error=%s, wf=%s, task=%s,"
" action=%s]:\n%s" %
(e, wf_ex.name, task_ex.name, action_ex.name, tb.format_exc()))
LOG.error(msg)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(wf_ex, msg)
return
def get_controller(wf_ex, wf_spec=None):
"""Gets a workflow controller instance by given workflow execution object.
:param wf_ex: Workflow execution object.
:param wf_spec: Workflow specification object. If passed, the method works
faster.
:returns: Workflow controller class.
"""
if not wf_spec:
wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id)
wf_type = wf_spec.get_type()
ctrl_cls = None
for cls in u.iter_subclasses(WorkflowController):
if cls.__workflow_type__ == wf_type:
ctrl_cls = cls
break
if not ctrl_cls:
raise exc.MistralError(
'Failed to find a workflow controller [type=%s]' % wf_type)
return ctrl_cls(wf_ex, wf_spec)
def continue_task(task_ex):
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
task = _build_task_from_execution(wf_spec, task_ex)
try:
task.set_state(states.RUNNING, None)
task.run()
except exc.MistralException as e:
wf_ex = task_ex.workflow_execution
msg = (
"Failed to run task [error=%s, wf=%s, task=%s]:\n%s" %
(e, wf_ex, task_ex.name, tb.format_exc())
)
LOG.error(msg)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(wf_ex, msg)
return
def continue_task(task_ex):
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
task = _build_task_from_execution(wf_spec, task_ex)
try:
task.set_state(states.RUNNING, None)
task.run()
except exc.MistralException as e:
wf_ex = task_ex.workflow_execution
msg = (
"Failed to run task [wf=%s, task=%s]: %s\n%s" %
(wf_ex, task_ex.name, e, tb.format_exc())
)
LOG.error(msg)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(wf_ex, msg)
return
def fail_task(task_ex, msg):
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
task = _build_task_from_execution(wf_spec, task_ex)
task.set_state(states.ERROR, msg)
wf_handler.fail_workflow(task_ex.workflow_execution, msg)
def __init__(self, wf_ex=None):
self.wf_ex = wf_ex
if wf_ex:
# We're processing a workflow that's already in progress.
self.wf_spec = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id)
else:
self.wf_spec = None
def __init__(self, wf_def, wf_ex=None):
self.wf_def = wf_def
self.wf_ex = wf_ex
if wf_ex:
# We're processing a workflow that's already in progress.
self.wf_spec = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id)
else:
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
def __init__(self, wf_ex, wf_spec=None):
"""Creates a new workflow controller.
:param wf_ex: Workflow execution.
:param wf_spec: Workflow specification.
"""
self.wf_ex = wf_ex
if wf_spec is None:
wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id)
self.wf_spec = wf_spec
def __init__(self, wf_ex, wf_spec=None):
"""Creates a new workflow controller.
:param wf_ex: Workflow execution.
:param wf_spec: Workflow specification.
"""
self.wf_ex = wf_ex
if wf_spec is None:
wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id)
self.wf_spec = wf_spec
def __init__(self, wf_def, wf_ex=None):
self.wf_def = wf_def
self.wf_ex = wf_ex
if wf_ex:
# We're processing a workflow that's already in progress.
self.wf_spec = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
else:
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.load_task_execution(task_ex_id)
if not task_ex:
return
wf_ex = task_ex.workflow_execution
if states.is_completed(wf_ex.state):
return
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
state, state_info, cardinality = wf_ctrl.get_logical_task_state(
task_ex
)
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
task = _build_task_from_execution(wf_spec, task_ex)
task.complete(state, state_info)
elif state == states.WAITING:
# Let's assume that a task takes 0.01 sec in average to complete
# and based on this assumption calculate a time of the next check.
# The estimation is very rough, of course, but this delay will be
# decreasing as task preconditions will be completing which will
# give a decent asymptotic approximation.
# For example, if a 'join' task has 100 inbound incomplete tasks
# then the next 'refresh_task_state' call will happen in 10
# seconds. For 500 tasks it will be 50 seconds. The larger the
# workflow is, the more beneficial this mechanism will be.
delay = int(cardinality * 0.01)
_schedule_refresh_task_state(task_ex, max(1, delay))
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex=%s, state=%s]' %
(task_ex, state)
)
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.load_task_execution(task_ex_id)
if not task_ex:
return
wf_ex = task_ex.workflow_execution
if states.is_completed(wf_ex.state):
return
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
state, state_info, cardinality = wf_ctrl.get_logical_task_state(
task_ex
)
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
task = _build_task_from_execution(wf_spec, task_ex)
task.complete(state, state_info)
elif state == states.WAITING:
# Let's assume that a task takes 0.01 sec in average to complete
# and based on this assumption calculate a time of the next check.
# The estimation is very rough, of course, but this delay will be
# decreasing as task preconditions will be completing which will
# give a decent asymptotic approximation.
# For example, if a 'join' task has 100 inbound incomplete tasks
# then the next 'refresh_task_state' call will happen in 10
# seconds. For 500 tasks it will be 50 seconds. The larger the
# workflow is, the more beneficial this mechanism will be.
delay = int(cardinality * 0.01)
_schedule_refresh_task_state(task_ex, max(1, delay))
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex=%s, state=%s]' %
(task_ex, state)
)
def complete_task(task_ex, state, state_info):
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
task = _build_task_from_execution(wf_spec, task_ex)
try:
task.complete(state, state_info)
except exc.MistralException as e:
wf_ex = task_ex.workflow_execution
msg = ("Failed to complete task [error=%s, wf=%s, task=%s]:\n%s" %
(e, wf_ex, task_ex.name, tb.format_exc()))
LOG.error(msg)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(wf_ex, msg)
return
def force_fail_task(task_ex, msg):
"""Forces the given task to fail.
This method implements the 'forced' task fail without giving a chance
to a workflow controller to handle the error. Its main purpose is to
reflect errors caused by workflow structure (errors 'publish', 'on-xxx'
clauses etc.) rather than failed actions. If such an error happens
we should also force the entire workflow to fail. I.e., this kind of
error must be propagated to a higher level, to the workflow.
:param task_ex: Task execution.
:param msg: Error message.
"""
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
task = _build_task_from_execution(wf_spec, task_ex)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(task_ex.workflow_execution, msg)
def _on_action_complete(action_ex):
"""Handles action completion event.
:param action_ex: Action execution.
"""
task_ex = action_ex.task_execution
if not task_ex:
return
task_spec = spec_parser.get_task_spec(task_ex.spec)
wf_ex = task_ex.workflow_execution
task = _create_task(
wf_ex,
spec_parser.get_workflow_spec_by_execution_id(wf_ex.id),
task_spec,
task_ex.in_context,
task_ex
)
try:
task.on_action_complete(action_ex)
except exc.MistralException as e:
wf_ex = task_ex.workflow_execution
msg = ("Failed to handle action completion [error=%s, wf=%s, task=%s,"
" action=%s]:\n%s" %
(e, wf_ex.name, task_ex.name, action_ex.name, tb.format_exc()))
LOG.error(msg)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(wf_ex, msg)
return
def force_fail_task(task_ex, msg):
"""Forces the given task to fail.
This method implements the 'forced' task fail without giving a chance
to a workflow controller to handle the error. Its main purpose is to
reflect errors caused by workflow structure (errors 'publish', 'on-xxx'
clauses etc.) rather than failed actions. If such an error happens
we should also force the entire workflow to fail. I.e., this kind of
error must be propagated to a higher level, to the workflow.
:param task_ex: Task execution.
:param msg: Error message.
"""
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
task = _build_task_from_execution(wf_spec, task_ex)
task.set_state(states.ERROR, msg)
wf_handler.force_fail_workflow(task_ex.workflow_execution, msg)
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
wf_ctrl = wf_base.get_controller(task_ex.workflow_execution, wf_spec)
state, state_info = wf_ctrl.get_logical_task_state(task_ex)
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
fail_task(task_ex, state_info)
elif state == states.WAITING:
# TODO(rakhmerov): Algorithm for increasing rescheduling delay.
_schedule_refresh_task_state(task_ex, 1)
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex=%s, state=%s]' %
(task_ex, state))
def test_cache_workflow_spec_by_execution_id(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wfs = wf_service.create_workflows(wf_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf_def = wfs[0]
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
self.assertEqual(1, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf_ex = db_api.create_workflow_execution({
'id': '1-2-3-4',
'name': 'wf',
'workflow_id': wf_def.id,
'spec': wf_spec.to_dict(),
'state': states.RUNNING
})
# Check that we can get a valid spec by execution id.
wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
self.assertEqual(1, len(wf_spec_by_exec_id.get_tasks()))
# Now update workflow definition and check that cache is updated too.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="1"
task2:
action: std.echo output="2"
"""
wfs = wf_service.update_workflows(wf_text)
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
self.assertEqual(2, len(wf_spec.get_tasks()))
self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_execution_spec_cache_size())
# Now finally update execution cache and check that we can
# get a valid spec by execution id.
spec_parser.cache_workflow_spec_by_execution_id(wf_ex.id, wf_spec)
wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
self.assertEqual(2, len(wf_spec_by_exec_id.get_tasks()))
