def cancel(self):
# Cancel the target workflow.
wf_svc.request_cancellation(self.execution)
# Request cancellation of tasks that are workflows and still running.
for child_ex_id in self.execution.children:
child_ex = ex_db_access.ActionExecution.get(id=child_ex_id)
if (child_ex.runner['name'] in ac_const.WORKFLOW_RUNNER_TYPES and
child_ex.status in ac_const.LIVEACTION_CANCELABLE_STATES):
ac_svc.request_cancellation(
lv_db_access.LiveAction.get(id=child_ex.liveaction['id']),
self.context.get('user', None)
)
status = (
ac_const.LIVEACTION_STATUS_CANCELING
if ac_svc.is_children_active(self.liveaction.id)
else ac_const.LIVEACTION_STATUS_CANCELED
)
return (
status,
self.liveaction.result,
self.liveaction.context
)
def cancel(self):
mistral_ctx = self.context.get('mistral', dict())
if not mistral_ctx.get('execution_id'):
raise Exception(
'Unable to cancel because mistral execution_id is missing.')
# Cancels the main workflow execution. Any non-workflow tasks that are still
# running will be allowed to complete gracefully.
self._client.executions.update(mistral_ctx.get('execution_id'),
'CANCELLED')
# If workflow is executed under another parent workflow, cancel the corresponding
# action execution for the task in the parent workflow.
if 'parent' in getattr(self, 'context',
{}) and mistral_ctx.get('action_execution_id'):
mistral_action_ex_id = mistral_ctx.get('action_execution_id')
self._client.action_executions.update(mistral_action_ex_id,
'CANCELLED')
# Identify the list of action executions that are workflows and still running.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id)
if (child_exec.runner['name']
in action_constants.WORKFLOW_RUNNER_TYPES
and child_exec.status
in action_constants.LIVEACTION_CANCELABLE_STATES):
action_service.request_cancellation(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None))
return (action_constants.LIVEACTION_STATUS_CANCELING,
self.liveaction.result, self.liveaction.context)
def cancel(self):
# Cancel the target workflow.
wf_svc.request_cancellation(self.execution)
# Request cancellation of tasks that are workflows and still running.
for child_ex_id in self.execution.children:
child_ex = ex_db_access.ActionExecution.get(id=child_ex_id)
if (child_ex.runner['name'] in ac_const.WORKFLOW_RUNNER_TYPES and
child_ex.status in ac_const.LIVEACTION_CANCELABLE_STATES):
ac_svc.request_cancellation(
lv_db_access.LiveAction.get(id=child_ex.liveaction['id']),
self.context.get('user', None)
)
status = (
ac_const.LIVEACTION_STATUS_CANCELING
if ac_svc.is_children_active(self.liveaction.id)
else ac_const.LIVEACTION_STATUS_CANCELED
)
return (
status,
self.liveaction.result,
self.liveaction.context
)
def test_basic_cancel(self):
runner_run_result = (
action_constants.LIVEACTION_STATUS_RUNNING,
{
"data": "foobar"
},
None,
)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, "run",
mock_runner_run):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def request_cancellation(ac_ex_db):
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
if not wf_ex_dbs:
raise wf_exc.WorkflowExecutionNotFoundException(str(ac_ex_db.id))
if len(wf_ex_dbs) > 1:
raise wf_exc.AmbiguousWorkflowExecutionException(str(ac_ex_db.id))
wf_ex_db = wf_ex_dbs[0]
if wf_ex_db.status in states.COMPLETED_STATES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor = deserialize_conductor(wf_ex_db)
if conductor.get_workflow_state() in states.COMPLETED_STATES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor.set_workflow_state(states.CANCELED)
# Write the updated workflow state and task flow to the database.
wf_ex_db.status = conductor.get_workflow_state()
wf_ex_db.flow = conductor.flow.serialize()
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db, publish=False)
# Cascade the cancellation up to the root of the workflow.
root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db)
if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES:
root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction['id'])
ac_svc.request_cancellation(root_lv_ac_db, None)
return wf_ex_db
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_failed_cancel(self):
runner_run_result = (
action_constants.LIVEACTION_STATUS_RUNNING,
{
"data": "foobar"
},
None,
)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, "run",
mock_runner_run):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
runners.ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_cancel_delayed_execution_with_parent(self):
liveaction = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': uuid.uuid4().hex
}})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_basic_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertDictEqual(liveaction.result, MOCK_RESULT)
def test_basic_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertDictEqual(liveaction.result, MOCK_RESULT)
def test_failed_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_failed_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_noop_cancel(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def cancel(self):
result = None
wf_ex_db = None
# Try to cancel the target workflow execution.
try:
wf_ex_db = wf_svc.request_cancellation(self.execution)
# If workflow execution is not found because the action execution is cancelled
# before the workflow execution is created or if the workflow execution is
# already completed, then ignore the exception and proceed with cancellation.
except (
wf_svc_exc.WorkflowExecutionNotFoundException,
wf_svc_exc.WorkflowExecutionIsCompletedException,
):
pass
# If there is an unknown exception, then log the error. Continue with the
# cancelation sequence below to cancel children and determine final status.
# If we rethrow the exception here, the workflow will be stuck in a canceling
# state with no options for user to clean up. It is safer to continue with
# the cancel then to revert back to some other statuses because the workflow
# execution will be in an unknown state.
except Exception:
_, ex, tb = sys.exc_info()
msg = "Error encountered when canceling workflow execution."
LOG.exception("[%s] %s", str(self.execution.id), msg)
msg = "Error encountered when canceling workflow execution. %s"
wf_svc.update_progress(wf_ex_db, msg % str(ex), log=False)
result = {
"error": msg % str(ex),
"traceback": "".join(traceback.format_tb(tb, 20)),
}
# Request cancellation of tasks that are workflows and still running.
for child_ex_id in self.execution.children:
child_ex = ex_db_access.ActionExecution.get(id=child_ex_id)
if self.task_cancelable(child_ex):
ac_svc.request_cancellation(
lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]),
self.context.get("user", None),
)
status = (ac_const.LIVEACTION_STATUS_CANCELING
if ac_svc.is_children_active(self.liveaction.id) else
ac_const.LIVEACTION_STATUS_CANCELED)
return (
status,
result if result else self.liveaction.result,
self.liveaction.context,
)
def test_basic_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def purge_orphaned_workflow_executions(logger):
"""
Purge workflow executions that are idled and identified as orphans.
"""
# Cancel workflow executions that are identified as orphans. The workflow executions are
# marked as canceled instead of failed because error handling during normal operation
# failed and the system does not know what state the workflow execution is in. A failed
# workflow execution can be rerun from failed task(s). Since we do not know what state
# the workflow execution is in because correct data may not be recorded in the database
# as a result of the original failure, the garbage collection routine here cancels
# the workflow execution so it cannot be rerun from failed task(s).
for ac_ex_db in workflow_service.identify_orphaned_workflows():
lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction['id'])
action_service.request_cancellation(lv_ac_db, None)
def test_noop_cancel(self):
liveaction = LiveActionDB(action='executions.local', parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_basic_cancel(self):
runner_cls = self.get_runner_class('runner')
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
runner_cls.run = mock.Mock(return_value=runner_run_result)
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def cancel(self):
# Identify the list of action executions that are workflows and cascade pause.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id,
raise_exception=True)
if (child_exec.runner['name']
in action_constants.WORKFLOW_RUNNER_TYPES
and child_exec.status
in action_constants.LIVEACTION_CANCELABLE_STATES):
action_service.request_cancellation(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None))
return (action_constants.LIVEACTION_STATUS_CANCELING,
self.liveaction.result, self.liveaction.context)
def test_cancel_subworkflow_action(self):
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction1, execution1 = action_service.request_cancellation(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'CANCELLED'),
mock.call(WF1_EXEC.get('id'), 'CANCELLED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
def cancel(self):
# Identify the list of action executions that are workflows and cascade pause.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id, raise_exception=True)
if (child_exec.runner['name'] in action_constants.WORKFLOW_RUNNER_TYPES and
child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES):
action_service.request_cancellation(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None)
)
return (
action_constants.LIVEACTION_STATUS_CANCELING,
self.liveaction.result,
self.liveaction.context
)
def test_cancel_retry_exhausted(self):
MistralRunner.entry_point = mock.PropertyMock(
return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(
liveaction, requester)
calls = [call(WF1_EXEC.get('id'), 'PAUSED') for i in range(0, 2)]
executions.ExecutionManager.update.assert_has_calls(calls)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def test_chain_cancel_cascade_to_subworkflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_cancel_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to cancel.
liveaction, execution = action_service.request_cancellation(liveaction, USERNAME)
# Wait until the liveaction is canceling.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceling.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is canceled.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is canceled.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_on_status(task1_live, action_constants.LIVEACTION_STATUS_CANCELED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_CANCELED)
def test_cancel_unexpected_exception(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(
lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result
)
# Cancel the action execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = ac_svc.request_cancellation(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
# Make sure request cancellation is called.
self.assertTrue(wf_svc.request_cancellation.called)
# Make sure the live action and action execution still has a canceled
# status despite of cancelation failure. The other option would be
# to raise an exception and the records will be stuck in a canceling
# status and user is unable to easily clean up.
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
self.assertIn(
"Error encountered when canceling", lv_ac_db.result.get("error", "")
)
def test_cancel_retry_exhausted(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
MockLiveActionPublisherNonBlocking.wait_all()
eventlet.sleep(4)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(
liveaction, requester)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
expected_call_count = 2
self._wait_on_call_count(executions.ExecutionManager.update,
expected_call_count)
calls = [
call(WF1_EXEC.get('id'), 'CANCELLED')
for i in range(0, expected_call_count)
]
executions.ExecutionManager.update.assert_has_calls(calls)
def test_cancel_subworkflow_action(self):
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction1, execution1 = action_service.request_cancellation(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'CANCELLED'),
mock.call(WF1_EXEC.get('id'), 'CANCELLED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
def test_cancel_workflow_cascade_down_to_subworkflow(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the subworkflow.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(len(wf_ex_dbs), 1)
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 1)
tk_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))
self.assertEqual(len(tk_ac_ex_dbs), 1)
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Cancel the main workflow.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = ac_svc.request_cancellation(lv_ac_db, requester)
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert the subworkflow is canceled.
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk_lv_ac_db.id))
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
# Assert the main workflow is canceled.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
def test_cancel_workflow_cascade_down_to_subworkflow(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the subworkflow.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(len(wf_ex_dbs), 1)
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 1)
tk_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))
self.assertEqual(len(tk_ac_ex_dbs), 1)
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Cancel the main workflow.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = ac_svc.request_cancellation(lv_ac_db, requester)
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert the subworkflow is canceled.
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk_lv_ac_db.id))
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
# Assert the main workflow is canceled.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
def delete(self, id, requester_user, **kwargs):
"""
Stops a single execution.
Handles requests:
DELETE /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
LOG.info('Action %s already in "canceled" state; \
returning execution object.' % liveaction_db.id)
return execution_api
if liveaction_db.status not in LIVEACTION_CANCELABLE_STATES:
abort(
http_client.OK, 'Action cannot be canceled. State = %s.' %
liveaction_db.status)
try:
(liveaction_db,
execution_db) = action_service.request_cancellation(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
except:
LOG.exception('Failed requesting cancellation for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed canceling execution.')
from_model_kwargs = self._get_from_model_kwargs_for_request(**kwargs)
return ActionExecutionAPI.from_model(execution_db, from_model_kwargs)
def request_cancellation(ac_ex_db):
wf_ac_ex_id = str(ac_ex_db.id)
LOG.info('[%s] Processing cancelation request for workflow.', wf_ac_ex_id)
wf_ex_dbs = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))
if not wf_ex_dbs:
raise wf_exc.WorkflowExecutionNotFoundException(str(ac_ex_db.id))
if len(wf_ex_dbs) > 1:
raise wf_exc.AmbiguousWorkflowExecutionException(str(ac_ex_db.id))
wf_ex_db = wf_ex_dbs[0]
if wf_ex_db.status in statuses.COMPLETED_STATUSES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor = deserialize_conductor(wf_ex_db)
if conductor.get_workflow_status() in statuses.COMPLETED_STATUSES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor.request_workflow_status(statuses.CANCELED)
# Write the updated workflow status and task flow to the database.
wf_ex_db.status = conductor.get_workflow_status()
wf_ex_db.state = conductor.workflow_state.serialize()
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db, publish=False)
# Cascade the cancellation up to the root of the workflow.
root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db)
if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES:
LOG.info('[%s] Cascading cancelation request to parent workflow.',
wf_ac_ex_id)
root_lv_ac_db = lv_db_access.LiveAction.get(
id=root_ac_ex_db.liveaction['id'])
ac_svc.request_cancellation(root_lv_ac_db, None)
LOG.debug('[%s] %s', wf_ac_ex_id, conductor.serialize())
LOG.info('[%s] Completed processing cancelation request for workflow.',
wf_ac_ex_id)
return wf_ex_db
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_failed_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction,
action_constants.LIVEACTION_STATUS_RUNNING
)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
runners.ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def cancel(self):
mistral_ctx = self.context.get('mistral', dict())
if not mistral_ctx.get('execution_id'):
raise Exception('Unable to cancel because mistral execution_id is missing.')
# Cancels the main workflow execution. Any non-workflow tasks that are still
# running will be allowed to complete gracefully.
self._client.executions.update(mistral_ctx.get('execution_id'), 'CANCELLED')
# Identify the list of action executions that are workflows and still running.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id)
if (child_exec.runner['name'] == self.runner_type_db.name and
child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES):
action_service.request_cancellation(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None)
)
def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflows.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the subworkflow.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(len(wf_ex_dbs), 1)
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 2)
tk1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))
self.assertEqual(len(tk1_ac_ex_dbs), 1)
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
tk2_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[1].id))
self.assertEqual(len(tk2_ac_ex_dbs), 1)
tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Cancel the subworkflow which should cascade up to the root.
requester = cfg.CONF.system_user.user
tk1_lv_ac_db, tk1_ac_ex_db = ac_svc.request_cancellation(tk1_lv_ac_db, requester)
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert the main workflow is canceling.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert both subworkflows are canceled.
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk1_lv_ac_db.id))
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk2_lv_ac_db.id))
self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
# Manually handle action execution completion for one of the tasks.
tk1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(tk1_ac_ex_db.id))
self.assertEqual(tk1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
# Manually handle action execution completion for the other task.
tk2_ac_ex_db = tk2_ac_ex_dbs[0]
tk2_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(tk2_ac_ex_db.id))
self.assertEqual(tk2_ac_ex_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
wf_svc.handle_action_execution_completion(tk2_ac_ex_db)
# Assert the main workflow is canceling.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflows.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the subworkflow.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(len(wf_ex_dbs), 1)
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 2)
tk1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))
self.assertEqual(len(tk1_ac_ex_dbs), 1)
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
tk2_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[1].id))
self.assertEqual(len(tk2_ac_ex_dbs), 1)
tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Cancel the subworkflow which should cascade up to the root.
requester = cfg.CONF.system_user.user
tk1_lv_ac_db, tk1_ac_ex_db = ac_svc.request_cancellation(tk1_lv_ac_db, requester)
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert the main workflow is canceling.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
# Assert both subworkflows are canceled.
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk1_lv_ac_db.id))
self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk2_lv_ac_db.id))
self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
# Manually handle action execution completion for one of the tasks.
tk1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(tk1_ac_ex_db.id))
self.assertEqual(tk1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
# Manually handle action execution completion for the other task.
tk2_ac_ex_db = tk2_ac_ex_dbs[0]
tk2_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(tk2_ac_ex_db.id))
self.assertEqual(tk2_ac_ex_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
wf_svc.handle_action_execution_completion(tk2_ac_ex_db)
# Assert the main workflow is canceling.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELED)
def test_cancel(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = ac_svc.request_cancellation(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
def test_basic_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction,
action_constants.LIVEACTION_STATUS_RUNNING
)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = self._wait_on_status(
liveaction,
action_constants.LIVEACTION_STATUS_CANCELED
)
def test_cancel(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = ac_svc.request_cancellation(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_CANCELING)
def request_cancellation(ac_ex_db):
wf_ac_ex_id = str(ac_ex_db.id)
LOG.info('[%s] Processing cancelation request for workflow.', wf_ac_ex_id)
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
if not wf_ex_dbs:
raise wf_exc.WorkflowExecutionNotFoundException(str(ac_ex_db.id))
if len(wf_ex_dbs) > 1:
raise wf_exc.AmbiguousWorkflowExecutionException(str(ac_ex_db.id))
wf_ex_db = wf_ex_dbs[0]
if wf_ex_db.status in statuses.COMPLETED_STATUSES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor = deserialize_conductor(wf_ex_db)
if conductor.get_workflow_status() in statuses.COMPLETED_STATUSES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor.request_workflow_status(statuses.CANCELED)
# Write the updated workflow status and task flow to the database.
wf_ex_db.status = conductor.get_workflow_status()
wf_ex_db.state = conductor.workflow_state.serialize()
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db, publish=False)
# Cascade the cancellation up to the root of the workflow.
root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db)
if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES:
LOG.info('[%s] Cascading cancelation request to parent workflow.', wf_ac_ex_id)
root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction['id'])
ac_svc.request_cancellation(root_lv_ac_db, None)
LOG.debug('[%s] %s', wf_ac_ex_id, conductor.serialize())
LOG.info('[%s] Completed processing cancelation request for workflow.', wf_ac_ex_id)
return wf_ex_db
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request will be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_failed_cancel(self):
runner_cls = self.get_runner_class('runner')
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING,
'foobar', None)
runner_cls.run = mock.Mock(return_value=runner_run_result)
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
runners.ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def delete(self, id, requester_user, show_secrets=False):
"""
Stops a single execution.
Handles requests:
DELETE /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(id=id, requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_CANCELED:
LOG.info(
'Action %s already in "canceled" state; \
returning execution object.' % liveaction_db.id
)
return execution_api
if liveaction_db.status not in action_constants.LIVEACTION_CANCELABLE_STATES:
abort(http_client.OK, 'Action cannot be canceled. State = %s.' % liveaction_db.status)
try:
(liveaction_db, execution_db) = action_service.request_cancellation(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
except:
LOG.exception('Failed requesting cancellation for liveaction %s.', liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR, 'Failed canceling execution.')
return ActionExecutionAPI.from_model(execution_db,
mask_secrets=from_model_kwargs['mask_secrets'])
def delete(self, exec_id):
"""
Stops a single execution.
Handles requests:
DELETE /executions/<id>
"""
execution_api = self._get_one(id=exec_id)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % exec_id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
abort(http_client.OK, 'Action is already in "canceled" state.')
if liveaction_db.status not in CANCELABLE_STATES:
abort(
http_client.OK, 'Action cannot be canceled. State = %s.' %
liveaction_db.status)
try:
(liveaction_db,
execution_db) = action_service.request_cancellation(
liveaction_db, self._get_requester())
except:
LOG.exception('Failed requesting cancellation for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed canceling execution.')
from_model_kwargs = self._get_from_model_kwargs_for_request(
request=pecan.request)
return ActionExecutionAPI.from_model(execution_db, from_model_kwargs)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], "stanley")
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_cancel_retry(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'CANCELLED')
def cancel(self):
mistral_ctx = self.context.get('mistral', dict())
if not mistral_ctx.get('execution_id'):
raise Exception('Unable to cancel because mistral execution_id is missing.')
# Cancels the main workflow execution. Any non-workflow tasks that are still
# running will be allowed to complete gracefully.
self._client.executions.update(mistral_ctx.get('execution_id'), 'CANCELLED')
# If workflow is executed under another parent workflow, cancel the corresponding
# action execution for the task in the parent workflow.
if 'parent' in getattr(self, 'context', {}) and mistral_ctx.get('action_execution_id'):
mistral_action_ex_id = mistral_ctx.get('action_execution_id')
self._client.action_executions.update(mistral_action_ex_id, 'CANCELLED')
# Identify the list of action executions that are workflows and still running.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id)
if (child_exec.runner['name'] in action_constants.WORKFLOW_RUNNER_TYPES and
child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES):
action_service.request_cancellation(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None)
)
status = (
action_constants.LIVEACTION_STATUS_CANCELING
if action_service.is_children_active(self.liveaction.id)
else action_constants.LIVEACTION_STATUS_CANCELED
)
return (
status,
self.liveaction.result,
self.liveaction.context
)
def request_cancellation(ac_ex_db):
wf_ex_dbs = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))
if not wf_ex_dbs:
raise wf_exc.WorkflowExecutionNotFoundException(str(ac_ex_db.id))
if len(wf_ex_dbs) > 1:
raise wf_exc.AmbiguousWorkflowExecutionException(str(ac_ex_db.id))
wf_ex_db = wf_ex_dbs[0]
if wf_ex_db.status in states.COMPLETED_STATES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor = deserialize_conductor(wf_ex_db)
if conductor.get_workflow_state() in states.COMPLETED_STATES:
raise wf_exc.WorkflowExecutionIsCompletedException(str(wf_ex_db.id))
conductor.set_workflow_state(states.CANCELED)
# Write the updated workflow state and task flow to the database.
wf_ex_db.status = conductor.get_workflow_state()
wf_ex_db.flow = conductor.flow.serialize()
wf_ex_db = wf_db_access.WorkflowExecution.update(wf_ex_db, publish=False)
# Cascade the cancellation up to the root of the workflow.
root_ac_ex_db = ac_svc.get_root_execution(ac_ex_db)
if root_ac_ex_db != ac_ex_db and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES:
root_lv_ac_db = lv_db_access.LiveAction.get(
id=root_ac_ex_db.liveaction['id'])
ac_svc.request_cancellation(root_lv_ac_db, None)
return wf_ex_db
def test_cancel_retry(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'CANCELLED')
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_cancel_retry(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED')
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def delete(self, exec_id):
"""
Stops a single execution.
Handles requests:
DELETE /executions/<id>
"""
execution_api = self._get_one(id=exec_id)
if not execution_api:
abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % exec_id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
LOG.info(
'Action %s already in "canceled" state; \
returning execution object.' % liveaction_db.id
)
return execution_api
if liveaction_db.status not in LIVEACTION_CANCELABLE_STATES:
abort(http_client.OK, 'Action cannot be canceled. State = %s.' % liveaction_db.status)
try:
(liveaction_db, execution_db) = action_service.request_cancellation(
liveaction_db, get_requester())
except:
LOG.exception('Failed requesting cancellation for liveaction %s.', liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR, 'Failed canceling execution.')
from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request)
return ActionExecutionAPI.from_model(execution_db, from_model_kwargs)
def test_cancel_retry_exhausted(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
calls = [call(WF1_EXEC.get('id'), 'CANCELLED') for i in range(0, 2)]
executions.ExecutionManager.update.assert_has_calls(calls)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def test_cancel_retry_exhausted(self):
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
MockLiveActionPublisherNonBlocking.wait_all()
eventlet.sleep(4)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_cancellation(liveaction, requester)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
expected_call_count = 2
self._wait_on_call_count(executions.ExecutionManager.update, expected_call_count)
calls = [call(WF1_EXEC.get('id'), 'CANCELLED') for i in range(0, expected_call_count)]
executions.ExecutionManager.update.assert_has_calls(calls)
def _submit_cancellation(self, execution):
execution, _ = action_service.request_cancellation(execution, USERNAME)
execution = action_db.get_liveaction_by_id(execution.id)
return execution
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(id=id, requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST, 'Execution is already in completed state.')
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, 'result', None)
liveaction_db = action_service.update_status(liveaction_db, status, result)
actionexecution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
try:
if (liveaction_db.status == action_constants.LIVEACTION_STATUS_CANCELING and
liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_CANCELING
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
elif (liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELING or
liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELED):
liveaction_db, actionexecution_db = action_service.request_cancellation(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
elif (liveaction_db.status == action_constants.LIVEACTION_STATUS_PAUSING and
liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_PAUSING
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
elif (liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSING or
liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
else:
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % six.text_type(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % six.text_type(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(
http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.'
)
mask_secrets = self._get_mask_secrets(requester_user, show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def test_with_items_concurrency_cancellation(self):
concurrency = 2
wf_input = {'concurrency': concurrency}
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'with-items-concurrency.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'], parameters=wf_input)
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert the workflow execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING)
query_filters = {'workflow_execution': str(wf_ex_db.id), 'task_id': 'task1'}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
self.assertEqual(t1_ex_db.status, wf_statuses.RUNNING)
self.assertEqual(len(t1_ac_ex_dbs), concurrency)
# Reset the action executions to running status.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_RUNNING
)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_RUNNING
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
# Cancel the workflow execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = action_service.request_cancellation(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_CANCELING)
# Manually succeed the action executions and process completion.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_SUCCEEDED
)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_SUCCEEDED
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
for t1_ac_ex_db in t1_ac_ex_dbs:
workflows.get_engine().process(t1_ac_ex_db)
# Check that the workflow execution is canceled.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.CANCELED)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_CANCELED)
