def _continue_workflow(self, task_ex=None, reset=True, env=None):
wf_ctrl = wf_base.get_controller(self.wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
filter(lambda c: not isinstance(c, commands.PauseWorkflow), cmds)
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
if not cmds:
self._check_and_complete()
def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param wf_ex_id: Workflow execution id.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
:raises
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_trace.info(
self.wf_ex, 'Starting workflow [name=%s, input=%s]' %
(wf_def.name, utils.cut(input_dict)))
self.validate_input(input_dict)
self._create_execution(wf_def, wf_ex_id,
self.prepare_input(input_dict), desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
dispatcher.dispatch_workflow_commands(self.wf_ex,
wf_ctrl.continue_workflow())
def start(self, input_dict, desc='', params=None):
"""Start workflow.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def)
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec)
self._create_execution(input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def _continue_workflow(self, task_ex=None, reset=True, env=None):
wf_ctrl = wf_base.get_controller(self.wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
filter(lambda c: not isinstance(c, commands.PauseWorkflow), cmds)
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
if not cmds:
self._check_and_complete()
def start(self, input_dict, desc='', params=None):
"""Start workflow.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def)
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec)
self._create_execution(input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def start(self, wf_def, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_trace.info(
self.wf_ex, "Starting workflow [name=%s, input=%s]" %
(wf_def.name, utils.cut(input_dict)))
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(wf_def, input_dict, self.wf_spec)
self._create_execution(wf_def, input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Ignore if task already completed.
if self.is_completed():
return
# If we were unable to change the task state it means that it was
# already changed by a concurrent process. In this case we need to
# skip all regular completion logic like scheduling new tasks,
# running engine commands and publishing.
if not self.set_state(state, state_info):
return
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=self.task_ex)
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Ignore if task already completed.
if states.is_completed(self.task_ex.state):
return
self.set_state(state, state_info)
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow()
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param wf_ex_id: Workflow execution id.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
:raises
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
wf_trace.info(
self.wf_ex,
'Starting workflow [name=%s, input=%s]' %
(wf_def.name, utils.cut(input_dict))
)
self.validate_input(input_dict)
self._create_execution(
wf_def,
wf_ex_id,
self.prepare_input(input_dict),
desc,
params
)
self.set_state(states.RUNNING)
# Publish event as soon as state is set to running.
self.notify(events.WORKFLOW_LAUNCHED)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
dispatcher.dispatch_workflow_commands(
self.wf_ex,
wf_ctrl.continue_workflow()
)
def _continue_workflow(self, cmds):
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
[c for c in cmds if not isinstance(c, commands.PauseWorkflow)])
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
if cmds or self._get_backlog():
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
else:
self.check_and_complete()
def _continue_workflow(self, cmds):
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
[c for c in cmds if not isinstance(c, commands.PauseWorkflow)]
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
if cmds:
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
else:
self.check_and_complete()
def _continue_workflow(wf_ex, task_ex=None, reset=True, env=None):
wf_ex = wf_service.update_workflow_execution_env(wf_ex, env)
wf_handler.set_workflow_state(
wf_ex,
states.RUNNING,
set_upstream=True
)
wf_ctrl = wf_base.get_controller(wf_ex)
# TODO(rakhmerov): Add error handling.
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
# TODO(rakhmerov): This all should be in workflow handler, it's too
# specific for engine level.
cmds = list(
filter(
lambda c: not isinstance(c, commands.PauseWorkflow),
cmds
)
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
dispatcher.dispatch_workflow_commands(wf_ex, cmds)
if not cmds:
wf_handler.check_workflow_completion(wf_ex)
return wf_ex.get_clone()
def start_workflow(self, wf_identifier, wf_input, description='',
**params):
with db_api.transaction():
# TODO(rakhmerov): It needs to be hidden in workflow_handler and
# Workflow abstraction.
# The new workflow execution will be in an IDLE
# state on initial record creation.
wf_ex, wf_spec = wf_ex_service.create_workflow_execution(
wf_identifier,
wf_input,
description,
params
)
wf_handler.set_workflow_state(wf_ex, states.RUNNING)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(wf_ex, cmds)
return wf_ex.get_clone()
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Record the current task state.
old_task_state = self.task_ex.state
# Ignore if task already completed.
if self.is_completed():
# Publish task event again so subscribers know
# task completed state is being processed again.
self.notify(old_task_state, self.task_ex.state)
return
# If we were unable to change the task state it means that it was
# already changed by a concurrent process. In this case we need to
# skip all regular completion logic like scheduling new tasks,
# running engine commands and publishing.
if not self.set_state(state, state_info):
return
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
# Publish task event even if the workflow is paused.
self.notify(old_task_state, self.task_ex.state)
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=self.task_ex)
# Check whether the task generated any next tasks.
if any([not commands.is_engine_command(c) for c in cmds]):
self.task_ex.has_next_tasks = True
# Check whether the error is handled.
if self.task_ex.state == states.ERROR:
self.task_ex.error_handled = any([c.handles_error for c in cmds])
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
self.register_workflow_completion_check()
self.save_finished_time()
# Publish task event.
self.notify(old_task_state, self.task_ex.state)
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Ignore if task already completed.
if self.is_completed():
return
# If we were unable to change the task state it means that it was
# already changed by a concurrent process. In this case we need to
# skip all regular completion logic like scheduling new tasks,
# running engine commands and publishing.
if not self.set_state(state, state_info):
return
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=self.task_ex)
# Save next task names in DB to avoid evaluating them again
# in the future.
self.task_ex.next_tasks = []
for c in cmds:
if commands.is_engine_command(c):
continue
event = c.triggered_by[0]['event'] if c.triggered_by else None
self.task_ex.next_tasks.append((c.task_spec.get_name(), event))
self.task_ex.has_next_tasks = bool(self.task_ex.next_tasks)
# Check whether the error is handled.
if self.task_ex.state == states.ERROR:
self.task_ex.error_handled = any([c.handles_error for c in cmds])
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
return
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
self.register_workflow_completion_check()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Record the current task state.
old_task_state = self.task_ex.state
# Ignore if task already completed.
if self.is_completed():
# Publish task event again so subscribers know
# task completed state is being processed again.
self.notify(old_task_state, self.task_ex.state)
return
# If we were unable to change the task state it means that it was
# already changed by a concurrent process. In this case we need to
# skip all regular completion logic like scheduling new tasks,
# running engine commands and publishing.
if not self.set_state(state, state_info):
return
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
# Publish task event even if the workflow is paused.
self.notify(old_task_state, self.task_ex.state)
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=self.task_ex)
# Check whether the task generated any next tasks.
if any([not commands.is_engine_command(c) for c in cmds]):
self.task_ex.has_next_tasks = True
# Check whether the error is handled.
if self.task_ex.state == states.ERROR:
self.task_ex.error_handled = any([c.handles_error for c in cmds])
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
self.register_workflow_completion_check()
self.save_finished_time()
# Publish task event.
self.notify(old_task_state, self.task_ex.state)
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
