def resume(self):
mistral_ctx = self.context.get('mistral', dict())
if not mistral_ctx.get('execution_id'):
raise Exception('Unable to resume because mistral execution_id is missing.')
# If workflow is executed under another parent workflow, resume the corresponding
# action execution for the task in the parent workflow.
if 'parent' in getattr(self, 'context', {}) and mistral_ctx.get('action_execution_id'):
mistral_action_ex_id = mistral_ctx.get('action_execution_id')
self._client.action_executions.update(mistral_action_ex_id, 'RUNNING')
# Pause the main workflow execution. Any non-workflow tasks that are still
# running will be allowed to complete gracefully.
self._client.executions.update(mistral_ctx.get('execution_id'), 'RUNNING')
# Identify the list of action executions that are workflows and cascade resume.
for child_exec_id in self.execution.children:
child_exec = ActionExecution.get(id=child_exec_id, raise_exception=True)
if (child_exec.runner['name'] in action_constants.WORKFLOW_RUNNER_TYPES and
child_exec.status == action_constants.LIVEACTION_STATUS_PAUSED):
action_service.request_resume(
LiveAction.get(id=child_exec.liveaction['id']),
self.context.get('user', None)
)
return (
action_constants.LIVEACTION_STATUS_RUNNING,
self.execution.result,
self.execution.context
)
def purge_inquiries(logger):
"""Purge Inquiries that have exceeded their configured TTL
At the moment, Inquiries do not have their own database model, so this function effectively
is another, more specialized GC for executions. It will look for executions with a 'pending'
status that use the 'inquirer' runner, which is the current definition for an Inquiry.
Then it will mark those that have a nonzero TTL have existed longer than their TTL as
"timed out". It will then request that the parent workflow(s) resume, where the failure
can be handled as the user desires.
"""
# Get all existing Inquiries
filters = {'runner__name': 'inquirer', 'status': action_constants.LIVEACTION_STATUS_PENDING}
inquiries = list(ActionExecution.query(**filters))
gc_count = 0
# Inspect each Inquiry, and determine if TTL is expired
for inquiry in inquiries:
ttl = int(inquiry.result.get('ttl'))
if ttl <= 0:
logger.debug("Inquiry %s has a TTL of %s. Skipping." % (inquiry.id, ttl))
continue
min_since_creation = int(
(get_datetime_utc_now() - inquiry.start_timestamp).total_seconds() / 60
)
logger.debug("Inquiry %s has a TTL of %s and was started %s minute(s) ago" % (
inquiry.id, ttl, min_since_creation))
if min_since_creation > ttl:
gc_count += 1
logger.info("TTL expired for Inquiry %s. Marking as timed out." % inquiry.id)
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_TIMED_OUT,
result=inquiry.result,
liveaction_id=inquiry.liveaction.get('id'))
executions.update_execution(liveaction_db)
# Call Inquiry runner's post_run to trigger callback to workflow
action_db = get_action_by_ref(liveaction_db.action)
invoke_post_run(liveaction_db=liveaction_db, action_db=action_db)
if liveaction_db.context.get("parent"):
# Request that root workflow resumes
root_liveaction = action_service.get_root_liveaction(liveaction_db)
action_service.request_resume(
root_liveaction,
UserDB(cfg.CONF.system_user.user)
)
logger.info('Marked %s ttl-expired Inquiries as "timed out".' % (gc_count))
def respond(inquiry, response, requester=None):
# Set requester to system user is not provided.
if not requester:
requester = cfg.CONF.system_user.user
# Retrieve the liveaction from the database.
liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction.get('id'))
# Resume the parent workflow first. If the action execution for the inquiry is updated first,
# it triggers handling of the action execution completion which will interact with the paused
# parent workflow. The resuming logic that is executed here will then race with the completion
# of the inquiry action execution, which will randomly result in the parent workflow stuck in
# paused state.
if liveaction_db.context.get('parent'):
LOG.debug('Resuming workflow parent(s) for inquiry "%s".' % str(inquiry.id))
# For action execution under Action Chain and Mistral workflows, request the entire
# workflow to resume. Orquesta handles resume differently and so does not require root
# to resume. Orquesta allows for specifc branches to resume while other is paused. When
# there is no other paused branches, the conductor will resume the rest of the workflow.
resume_target = (
action_service.get_parent_liveaction(liveaction_db)
if workflow_service.is_action_execution_under_workflow_context(liveaction_db)
else action_service.get_root_liveaction(liveaction_db)
)
if resume_target.status in action_constants.LIVEACTION_PAUSE_STATES:
action_service.request_resume(resume_target, requester)
# Succeed the liveaction and update result with the inquiry response.
LOG.debug('Updating response for inquiry "%s".' % str(inquiry.id))
result = copy.deepcopy(inquiry.result)
result['response'] = response
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
end_timestamp=date_utils.get_datetime_utc_now(),
runner_info=sys_info_utils.get_process_info(),
result=result,
liveaction_id=str(liveaction_db.id)
)
# Sync the liveaction with the corresponding action execution.
execution_service.update_execution(liveaction_db)
# Invoke inquiry post run to trigger a callback to parent workflow.
LOG.debug('Invoking post run for inquiry "%s".' % str(inquiry.id))
runner_container = container.get_runner_container()
action_db = action_utils.get_action_by_ref(liveaction_db.action)
runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])
runner = runner_container._get_runner(runnertype_db, action_db, liveaction_db)
runner.post_run(status=action_constants.LIVEACTION_STATUS_SUCCEEDED, result=result)
return liveaction_db
def test_resume(self):
# Launch the workflow execution.
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_pause(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED')
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually update the liveaction from pausing to paused. The paused state
# is usually updated by the mistral querier.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
liveaction, execution = action_service.request_resume(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'RUNNING')
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
def resume(self):
# Resume the target workflow.
wf_ex_db = wf_svc.request_resume(self.execution)
# Request resume of tasks that are workflows and still running.
for child_ex_id in self.execution.children:
child_ex = ex_db_access.ActionExecution.get(id=child_ex_id)
if self.task_resumeable(child_ex):
ac_svc.request_resume(
lv_db_access.LiveAction.get(id=child_ex.liveaction['id']),
self.context.get('user', None)
)
return (
wf_ex_db.status if wf_ex_db else ac_const.LIVEACTION_STATUS_RUNNING,
self.liveaction.result,
self.liveaction.context
)
def resume(self):
# Resume the target workflow.
wf_svc.request_resume(self.execution)
# Request resume of tasks that are workflows and still running.
for child_ex_id in self.execution.children:
child_ex = ex_db_access.ActionExecution.get(id=child_ex_id)
if (child_ex.runner['name'] in ac_const.WORKFLOW_RUNNER_TYPES and
child_ex.status == ac_const.LIVEACTION_STATUS_PAUSED):
ac_svc.request_resume(
lv_db_access.LiveAction.get(id=child_ex.liveaction['id']),
self.context.get('user', None)
)
return (
ac_const.LIVEACTION_STATUS_RUNNING,
self.liveaction.result,
self.liveaction.context
)
def test_chain_pause_resume_last_task_failed_with_no_next_task(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_last_task_failed_with_no_next_task'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_FAILED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
self.assertEqual(
liveaction.result['tasks'][0]['state'],
action_constants.LIVEACTION_STATUS_FAILED
)
def test_chain_pause_resume_status_change(self):
# Tests context_result is updated when last task's status changes between pause and resume
action = TEST_PACK + '.' + 'test_pause_resume_context_result'
liveaction = LiveActionDB(action=action)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
last_task_liveaction_id = liveaction.result['tasks'][-1]['liveaction_id']
action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
end_timestamp=date_utils.get_datetime_utc_now(),
result={'foo': 'bar'},
liveaction_id=last_task_liveaction_id
)
# Request action chain to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(
liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED,
str(liveaction)
)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
def _resume_action(self, liveaction, wait_for_completion=True, sleep_delay=1.0):
"""
:param sleep_delay: Number of seconds to wait during "is completed" polls.
:type sleep_delay: ``float``
"""
try:
user = self.context.get('user', None)
liveaction, _ = action_service.request_resume(liveaction, user)
except Exception as e:
liveaction.status = action_constants.LIVEACTION_STATUS_FAILED
LOG.exception('Failed to schedule liveaction.')
raise e
while (wait_for_completion and liveaction.status not in (
action_constants.LIVEACTION_COMPLETED_STATES +
[action_constants.LIVEACTION_STATUS_PAUSED])):
eventlet.sleep(sleep_delay)
liveaction = action_db_util.get_liveaction_by_id(liveaction.id)
return liveaction
def test_resume(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Pause the workflow.
lv_ac_db, ac_ex_db = ac_svc.request_pause(lv_ac_db, cfg.CONF.system_user.user)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING)
# Identify the records for the running task(s) and manually complete it.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 1)
tk_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction['id'])
self.assertEqual(tk_ac_ex_dbs[0].status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
wf_svc.handle_action_execution_completion(tk_ac_ex_dbs[0])
# Ensure the workflow is paused.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSED, lv_ac_db.result)
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(wf_ex_dbs[0].status, wf_statuses.PAUSED)
# Resume the workflow.
lv_ac_db, ac_ex_db = ac_svc.request_resume(lv_ac_db, cfg.CONF.system_user.user)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(wf_ex_dbs[0].status, wf_statuses.RUNNING)
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_dbs[0].id))
self.assertEqual(len(tk_ex_dbs), 2)
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
extra_info = str(task1_live)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSING, extra_info)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(task1_live)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
extra_info = str(liveaction)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED, extra_info)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 2)
self.assertEqual(subworkflow['state'], action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_resume_from_subworkflow_when_parent_is_running(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflows.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the main workflow.
wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
self.assertEqual(len(tk_ex_dbs), 2)
# Identify the records for the subworkflows.
t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))[0]
t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction['id'])
t1_wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(t1_ac_ex_db.id))[0]
self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
self.assertEqual(t1_wf_ex_db.status, wf_statuses.RUNNING)
t2_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[1].id))[0]
t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction['id'])
t2_wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(t2_ac_ex_db.id))[0]
self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
self.assertEqual(t2_wf_ex_db.status, wf_statuses.RUNNING)
# Pause the subworkflow.
t1_lv_ac_db, t1_ac_ex_db = ac_svc.request_pause(t1_lv_ac_db, cfg.CONF.system_user.user)
self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING)
# Assert the main workflow is still running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Assert the other subworkflow is still running.
t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction['id'])
self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Manually notify action execution completion for the task in the subworkflow.
t1_t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[0]
t1_t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t1_ex_db.id))[0]
workflows.get_engine().process(t1_t1_ac_ex_db)
# Assert the subworkflow is paused and manually notify the paused of the
# corresponding action execution in the main workflow.
t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(t1_lv_ac_db.id))
self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSED)
t1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(t1_ac_ex_db.id)
self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_PAUSED)
workflows.get_engine().process(t1_ac_ex_db)
# Assert the main workflow is still running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Assert the other subworkflow is still running.
t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction['id'])
self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Resume the subworkflow and assert it is running.
t1_lv_ac_db, t1_ac_ex_db = ac_svc.request_resume(t1_lv_ac_db, cfg.CONF.system_user.user)
t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(t1_lv_ac_db.id))
self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Assert the main workflow is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Assert the other subworkflow is still running.
t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction['id'])
self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Manually notify action execution completion for the tasks in the subworkflow.
t1_t2_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[1]
t1_t2_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t2_ex_db.id))[0]
workflows.get_engine().process(t1_t2_ac_ex_db)
t1_t3_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[2]
t1_t3_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t3_ex_db.id))[0]
workflows.get_engine().process(t1_t3_ac_ex_db)
t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(t1_lv_ac_db.id))
self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
# Assert the subworkflow is completed and manually notify the
# completion to the corresponding action execution in the main workflow.
t1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(t1_ac_ex_db.id)
workflows.get_engine().process(t1_ac_ex_db)
# Manually notify action execution completion for the tasks in the other subworkflow.
t2_t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t2_wf_ex_db.id))[0]
t2_t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t2_t1_ex_db.id))[0]
workflows.get_engine().process(t2_t1_ac_ex_db)
t2_t2_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t2_wf_ex_db.id))[1]
t2_t2_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t2_t2_ex_db.id))[0]
workflows.get_engine().process(t2_t2_ac_ex_db)
t2_t3_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t2_wf_ex_db.id))[2]
t2_t3_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t2_t3_ex_db.id))[0]
workflows.get_engine().process(t2_t3_ac_ex_db)
t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(t2_lv_ac_db.id))
self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
# Assert this other subworkflow is completed and manually notify the
# completion to the corresponding action execution in the main workflow.
t2_ac_ex_db = ex_db_access.ActionExecution.get_by_id(t2_ac_ex_db.id)
workflows.get_engine().process(t2_ac_ex_db)
# Assert task3 has started and completed.
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
self.assertEqual(len(tk_ex_dbs), 3)
t3_ex_db_qry = {'workflow_execution': str(wf_ex_db.id), 'task_id': 'task3'}
t3_ex_db = wf_db_access.TaskExecution.query(**t3_ex_db_qry)[0]
t3_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t3_ex_db.id))[0]
t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction['id'])
self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
wf_svc.handle_action_execution_completion(t3_ac_ex_db)
# Assert the main workflow is completed.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
def test_resume_cascade_to_subworkflow(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the main workflow.
wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))
self.assertEqual(len(wf_ex_dbs), 1)
wf_ex_db = wf_ex_dbs[0]
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
self.assertEqual(len(tk_ex_dbs), 1)
tk_ex_db = tk_ex_dbs[0]
tk_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_db.id))
self.assertEqual(len(tk_ac_ex_dbs), 1)
tk_ac_ex_db = tk_ac_ex_dbs[0]
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction['id'])
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Identify the records for the subworkflow.
sub_wf_ex_dbs = wf_db_access.WorkflowExecution.query(action_execution=str(tk_ac_ex_db.id))
self.assertEqual(len(sub_wf_ex_dbs), 1)
sub_wf_ex_db = sub_wf_ex_dbs[0]
sub_tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(sub_wf_ex_db.id))
self.assertEqual(len(sub_tk_ex_dbs), 1)
sub_tk_ex_db = sub_tk_ex_dbs[0]
sub_tk_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(sub_tk_ex_db.id))
self.assertEqual(len(sub_tk_ac_ex_dbs), 1)
# Pause the main workflow and assert it is pausing because subworkflow is still running.
lv_ac_db, ac_ex_db = ac_svc.request_pause(lv_ac_db, cfg.CONF.system_user.user)
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING)
# Assert the subworkflow is pausing.
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk_lv_ac_db.id))
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING)
# Manually handle action execution completion for the task in the subworkflow.
sub_tk_ac_ex_db = sub_tk_ac_ex_dbs[0]
self.assertEqual(sub_tk_ac_ex_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED)
workflows.get_engine().process(sub_tk_ac_ex_db)
# Assert the subworkflow is paused and manually process the execution update.
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk_lv_ac_db.id))
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSED)
tk_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(tk_ac_ex_db.id))
self.assertEqual(tk_ac_ex_db.status, ac_const.LIVEACTION_STATUS_PAUSED)
workflows.get_engine().process(tk_ac_ex_db)
# Assert the main workflow is paused.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSED)
# Resume the main workflow and assert it is running.
lv_ac_db, ac_ex_db = ac_svc.request_resume(lv_ac_db, cfg.CONF.system_user.user)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
# Assert the subworkflow is running.
tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(str(tk_lv_ac_db.id))
self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
if (getattr(liveaction_api, 'result', None) is not None
and liveaction_api.status in [
action_constants.LIVEACTION_STATUS_PAUSING,
action_constants.LIVEACTION_STATUS_PAUSED,
action_constants.LIVEACTION_STATUS_RESUMING
]):
abort(
http_client.BAD_REQUEST,
'The result is not applicable for pausing and resuming execution.'
)
try:
if (liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
else:
liveaction_db = action_service.update_status(
liveaction_db,
liveaction_api.status,
result=getattr(liveaction_api, 'result', None))
actionexecution_db = ActionExecution.get(
liveaction__id=str(liveaction_db.id))
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.')
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
"mask_secrets":
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP,
)
if not execution_api:
abort(http_client.NOT_FOUND,
"Execution with id %s not found." % id)
liveaction_id = execution_api.liveaction["id"]
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
"Execution object missing link to liveaction %s." %
liveaction_id,
)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
"Execution object missing link to liveaction %s." %
liveaction_id,
)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
"Execution is already in completed state.")
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, "result", None)
liveaction_db = action_service.update_status(liveaction_db,
status,
result,
set_result_size=True)
actionexecution_db = ActionExecution.get(
liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
try:
if (liveaction_db.status
== action_constants.LIVEACTION_STATUS_CANCELING
and liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_CANCELING
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
elif (liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_CANCELED):
liveaction_db, actionexecution_db = action_service.request_cancellation(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif (liveaction_db.status
== action_constants.LIVEACTION_STATUS_PAUSING
and liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_PAUSING
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
elif (liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
else:
liveaction_db, actionexecution_db = update_status(
liveaction_api, liveaction_db)
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception("Failed updating liveaction %s. %s",
liveaction_db.id, six.text_type(e))
abort(
http_client.BAD_REQUEST,
"Failed updating execution. %s" % six.text_type(e),
)
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception("Failed updating liveaction %s. %s",
liveaction_db.id, six.text_type(e))
abort(
http_client.BAD_REQUEST,
"Failed updating execution. %s" % six.text_type(e),
)
except Exception as e:
LOG.exception("Failed updating liveaction %s. %s",
liveaction_db.id, six.text_type(e))
abort(
http_client.INTERNAL_SERVER_ERROR,
"Failed updating execution due to unexpected error.",
)
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def put(self, inquiry_id, response_data, requester_user):
"""Provide response data to an Inquiry
In general, provided the response data validates against the provided
schema, and the user has the appropriate permissions to respond,
this will set the Inquiry execution to a successful status, and resume
the parent workflow.
Handles requests:
PUT /inquiries/<inquiry id>
"""
LOG.debug("Inquiry %s received response payload: %s" % (inquiry_id, response_data.response))
# Retrieve details of the inquiry via ID (i.e. params like schema)
inquiry = self._get_one_by_id(
id=inquiry_id,
requester_user=requester_user,
permission_type=PermissionType.INQUIRY_RESPOND
)
sanity_result, msg = self._inquiry_sanity_check(inquiry)
if not sanity_result:
abort(http_client.BAD_REQUEST, msg)
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
# Determine permission of this user to respond to this Inquiry
if not self._can_respond(inquiry, requester_user):
abort(
http_client.FORBIDDEN,
'Requesting user does not have permission to respond to inquiry %s.' % inquiry_id
)
# Validate the body of the response against the schema parameter for this inquiry
schema = inquiry.schema
LOG.debug("Validating inquiry response: %s against schema: %s" %
(response_data.response, schema))
try:
util_schema.validate(instance=response_data.response, schema=schema,
cls=util_schema.CustomValidator, use_default=True,
allow_default_none=True)
except Exception as e:
LOG.debug("Failed to validate response data against provided schema: %s" % e.message)
abort(http_client.BAD_REQUEST, 'Response did not pass schema validation.')
# Update inquiry for completion
new_result = copy.deepcopy(inquiry.result)
new_result["response"] = response_data.response
liveaction_db = self._mark_inquiry_complete(
inquiry.liveaction.get('id'),
new_result
)
# We only want to request a workflow resume if this has a parent
if liveaction_db.context.get("parent"):
# Request that root workflow resumes
root_liveaction = action_service.get_root_liveaction(liveaction_db)
action_service.request_resume(
root_liveaction,
requester_user
)
return {
"id": inquiry_id,
"response": response_data.response
}
def _submit_resume(self, execution):
execution, _ = action_service.request_resume(execution, USERNAME)
execution = action_db.get_liveaction_by_id(execution.id)
return execution
def test_with_items_concurrency_pause_and_resume(self):
num_items = 3
concurrency = 2
wf_input = {'concurrency': concurrency}
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'with-items-concurrency.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'], parameters=wf_input)
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert the workflow execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING)
query_filters = {'workflow_execution': str(wf_ex_db.id), 'task_id': 'task1'}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
self.assertEqual(t1_ex_db.status, wf_statuses.RUNNING)
self.assertEqual(len(t1_ac_ex_dbs), concurrency)
# Reset the action executions to running status.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_RUNNING
)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_RUNNING
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = action_service.request_pause(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually succeed the action executions and process completion.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_SUCCEEDED
)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_SUCCEEDED
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
for t1_ac_ex_db in t1_ac_ex_dbs:
workflows.get_engine().process(t1_ac_ex_db)
# Check that the workflow execution is paused.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.PAUSED)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = action_service.request_resume(lv_ac_db, requester)
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RESUMING)
# Check that the workflow execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Check new set of action execution is scheduled.
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))
self.assertEqual(len(t1_ac_ex_dbs), num_items)
# Manually process the last action execution.
workflows.get_engine().process(t1_ac_ex_dbs[2])
# Check that the workflow execution is completed.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(id=id, requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST, 'Execution is already in completed state.')
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, 'result', None)
liveaction_db = action_service.update_status(liveaction_db, status, result)
actionexecution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
try:
if (liveaction_db.status == action_constants.LIVEACTION_STATUS_CANCELING and
liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_CANCELING
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
elif (liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELING or
liveaction_api.status == action_constants.LIVEACTION_STATUS_CANCELED):
liveaction_db, actionexecution_db = action_service.request_cancellation(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
elif (liveaction_db.status == action_constants.LIVEACTION_STATUS_PAUSING and
liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSED):
if action_service.is_children_active(liveaction_id):
liveaction_api.status = action_constants.LIVEACTION_STATUS_PAUSING
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
elif (liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSING or
liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name or cfg.CONF.system_user.user)
else:
liveaction_db, actionexecution_db = update_status(liveaction_api, liveaction_db)
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % six.text_type(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % six.text_type(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, six.text_type(e))
abort(
http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.'
)
mask_secrets = self._get_mask_secrets(requester_user, show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(actionexecution_db, mask_secrets=mask_secrets)
return execution_api
def test_chain_pause_resume_last_task_failed_with_no_next_task(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_last_task_failed_with_no_next_task'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request action chain to pause.
liveaction, execution = action_service.request_pause(
liveaction, USERNAME)
# Wait until the liveaction is pausing.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Request action chain to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_FAILED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
self.assertEqual(liveaction.result['tasks'][0]['state'],
action_constants.LIVEACTION_STATUS_FAILED)
def test_with_items_concurrency_pause_and_resume(self):
num_items = 3
concurrency = 2
wf_input = {'concurrency': concurrency}
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH,
'with-items-concurrency.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'],
parameters=wf_input)
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert the workflow execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task1'
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))
self.assertEqual(t1_ex_db.status, wf_statuses.RUNNING)
self.assertEqual(len(t1_ac_ex_dbs), concurrency)
# Reset the action executions to running status.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_RUNNING)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_RUNNING
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = action_service.request_pause(lv_ac_db, requester)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Manually succeed the action executions and process completion.
for ac_ex in t1_ac_ex_dbs:
self.set_execution_status(
ac_ex.liveaction['id'],
action_constants.LIVEACTION_STATUS_SUCCEEDED)
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))
status = [
ac_ex.status == action_constants.LIVEACTION_STATUS_SUCCEEDED
for ac_ex in t1_ac_ex_dbs
]
self.assertTrue(all(status))
for t1_ac_ex_db in t1_ac_ex_dbs:
workflows.get_engine().process(t1_ac_ex_db)
# Check that the workflow execution is paused.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.PAUSED)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
requester = cfg.CONF.system_user.user
lv_ac_db, ac_ex_db = action_service.request_resume(lv_ac_db, requester)
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RESUMING)
# Check that the workflow execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Check new set of action execution is scheduled.
t1_ac_ex_dbs = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))
self.assertEqual(len(t1_ac_ex_dbs), num_items)
# Manually process the last action execution.
workflows.get_engine().process(t1_ac_ex_dbs[2])
# Check that the workflow execution is completed.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_resume_missing_subworkflow_action(self):
requester = cfg.CONF.system_user.user
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status,
action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'],
WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF2_EXEC.get('workflow_name'))
# Pause the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_pause(
liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED')
]
executions.ExecutionManager.update.assert_has_calls(
calls, any_order=False)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status,
action_constants.LIVEACTION_STATUS_PAUSING)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Manually set the liveaction status to PAUSED.
action_service.update_status(
liveaction2, action_constants.LIVEACTION_STATUS_PAUSED)
action_service.update_status(
liveaction1, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status,
action_constants.LIVEACTION_STATUS_PAUSED)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [uuid.uuid4().hex]
# Resume the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_resume(
liveaction1, requester)
# Includes the previous calls.
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 3)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED'),
mock.call(WF2_EXEC.get('id'), 'RUNNING'),
]
executions.ExecutionManager.update.assert_has_calls(
calls, any_order=False)
# The workflow execution will fail because the liveaction for the subworkflow
# should not be missing and we do not know what state it is in.
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status,
action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('not a valid ObjectId',
liveaction1.result.get('error', ''))
def purge_inquiries(logger):
"""Purge Inquiries that have exceeded their configured TTL
At the moment, Inquiries do not have their own database model, so this function effectively
is another, more specialized GC for executions. It will look for executions with a 'pending'
status that use the 'inquirer' runner, which is the current definition for an Inquiry.
Then it will mark those that have a nonzero TTL have existed longer than their TTL as
"timed out". It will then request that the parent workflow(s) resume, where the failure
can be handled as the user desires.
"""
# Get all existing Inquiries
filters = {'runner__name': 'inquirer', 'status': action_constants.LIVEACTION_STATUS_PENDING}
inquiries = list(ActionExecution.query(**filters))
gc_count = 0
# Inspect each Inquiry, and determine if TTL is expired
for inquiry in inquiries:
ttl = int(inquiry.result.get('ttl'))
if ttl <= 0:
logger.debug("Inquiry %s has a TTL of %s. Skipping." % (inquiry.id, ttl))
continue
min_since_creation = int(
(get_datetime_utc_now() - inquiry.start_timestamp).total_seconds() / 60
)
logger.debug("Inquiry %s has a TTL of %s and was started %s minute(s) ago" % (
inquiry.id, ttl, min_since_creation))
if min_since_creation > ttl:
gc_count += 1
logger.info("TTL expired for Inquiry %s. Marking as timed out." % inquiry.id)
liveaction_db = action_utils.update_liveaction_status(
status=action_constants.LIVEACTION_STATUS_TIMED_OUT,
result=inquiry.result,
liveaction_id=inquiry.liveaction.get('id'))
executions.update_execution(liveaction_db)
# Call Inquiry runner's post_run to trigger callback to workflow
runner_container = get_runner_container()
action_db = get_action_by_ref(liveaction_db.action)
runnertype_db = get_runnertype_by_name(action_db.runner_type['name'])
runner = runner_container._get_runner(runnertype_db, action_db, liveaction_db)
runner.post_run(status=action_constants.LIVEACTION_STATUS_TIMED_OUT,
result=inquiry.result)
if liveaction_db.context.get("parent"):
# Request that root workflow resumes
root_liveaction = action_service.get_root_liveaction(liveaction_db)
action_service.request_resume(
root_liveaction,
UserDB(cfg.CONF.system_user.user)
)
logger.info('Marked %s ttl-expired Inquiries as "timed out".' % (gc_count))
def test_resume_missing_subworkflow_action(self):
requester = cfg.CONF.system_user.user
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = self._wait_on_status(liveaction1, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = self._wait_on_status(liveaction2, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
# Pause the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_pause(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually set the liveaction status to PAUSED.
action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED)
action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [uuid.uuid4().hex]
# Resume the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_resume(liveaction1, requester)
# Includes the previous calls.
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 3)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED'),
mock.call(WF2_EXEC.get('id'), 'RUNNING'),
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
# The workflow execution will fail because the liveaction for the subworkflow
# should not be missing and we do not know what state it is in.
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def test_chain_pause_resume_cascade_to_parent_workflow(self):
# A temp file is created during test setup. Ensure the temp file exists.
# The test action chain will stall until this file is deleted. This gives
# the unit test a moment to run any test related logic.
path = self.temp_file_path
self.assertTrue(os.path.exists(path))
action = TEST_PACK + '.' + 'test_pause_resume_with_subworkflow'
params = {'tempfile': path, 'message': 'foobar'}
liveaction = LiveActionDB(action=action, parameters=params)
liveaction, execution = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
# Wait until the liveaction is running.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Wait for subworkflow to register.
execution = self._wait_for_children(execution)
self.assertEqual(len(execution.children), 1)
# Wait until the subworkflow is running.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_RUNNING)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request subworkflow to pause.
task1_live, task1_exec = action_service.request_pause(
task1_live, USERNAME)
# Wait until the subworkflow is pausing.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSING)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Delete the temporary file that the action chain is waiting on.
os.remove(path)
self.assertFalse(os.path.exists(path))
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait until the parent liveaction is paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(len(execution.children), 1)
# Wait for non-blocking threads to complete. Ensure runner is not running.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request subworkflow to resume.
task1_live, task1_exec = action_service.request_resume(
task1_live, USERNAME)
# Wait until the subworkflow is paused.
task1_exec = ActionExecution.get_by_id(execution.children[0])
task1_live = LiveAction.get_by_id(task1_exec.liveaction['id'])
task1_live = self._wait_for_status(
task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(task1_live.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# The parent workflow will stay paused.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result of the parent, which should stay the same
# because only the subworkflow was resumed.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 1)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 1)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_PAUSED)
# Request parent workflow to resume.
liveaction, execution = action_service.request_resume(
liveaction, USERNAME)
# Wait until the liveaction is completed.
liveaction = self._wait_for_status(
liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
# Wait for non-blocking threads to complete.
MockLiveActionPublisherNonBlocking.wait_all()
# Check liveaction result.
self.assertIn('tasks', liveaction.result)
self.assertEqual(len(liveaction.result['tasks']), 2)
subworkflow = liveaction.result['tasks'][0]
self.assertEqual(len(subworkflow['result']['tasks']), 2)
self.assertEqual(subworkflow['state'],
action_constants.LIVEACTION_STATUS_SUCCEEDED)
