def test_resume(self):
# Launch the workflow execution.
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_pause(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED')
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually update the liveaction from pausing to paused. The paused state
# is usually updated by the mistral querier.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
liveaction, execution = action_service.request_resume(liveaction, requester)
executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'RUNNING')
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def recover_delayed_executions():
coordinator = coordination.get_coordinator()
dt_now = date_utils.get_datetime_utc_now()
dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery)
dt_timeout = dt_now - dt_delta
with coordinator.get_lock('st2-rescheduling-delayed-executions'):
liveactions = LiveAction.query(status=action_constants.LIVEACTION_STATUS_DELAYED,
start_timestamp__lte=dt_timeout,
order_by=['start_timestamp'])
if not liveactions:
return
LOG.info('There are %d liveactions that have been delayed for longer than %d seconds.',
len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery)
# Update status to requested and publish status for each liveactions.
rescheduled = 0
for instance in liveactions:
try:
action_service.update_status(instance,
action_constants.LIVEACTION_STATUS_REQUESTED,
publish=True)
rescheduled += 1
except:
LOG.exception('Unable to reschedule liveaction. <LiveAction.id=%s>', instance.id)
LOG.info('Rescheduled %d out of %d delayed liveactions.', len(liveactions), rescheduled)
def test_over_threshold(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
self.assertGreater(policy_db.parameters["threshold"], 0)
self.assertIn("actionstr", policy_db.parameters["attributes"])
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_retry_policy_applied_on_workflow_failure(self):
wf_name = 'sequential'
wf_ac_ref = TEST_PACK + '.' + wf_name
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, wf_name + '.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Ensure there is only one execution recorded.
self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 1)
# Identify the records for the workflow and task.
wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0]
# Manually set the status to fail.
ac_svc.update_status(t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED)
t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0]
self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
notifier.get_notifier().process(t1_ac_ex_db)
workflows.get_engine().process(t1_ac_ex_db)
# Assert the main workflow is completed.
ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
notifier.get_notifier().process(ac_ex_db)
# Ensure execution is retried.
self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 2)
def _apply_after(self, target):
# Schedule the oldest delayed executions.
filters = self._get_filters(target)
filters["status"] = action_constants.LIVEACTION_STATUS_DELAYED
requests = action_access.LiveAction.query(order_by=["start_timestamp"], limit=1, **filters)
if requests:
action_service.update_status(requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _apply_after(self, target):
# Schedule the oldest delayed executions.
requests = action_access.LiveAction.query(action=target.action,
status=action_constants.LIVEACTION_STATUS_DELAYED,
order_by=['start_timestamp'], limit=1)
if requests:
action_service.update_status(
requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)
def test_request_cancellation_uncancelable_state(self):
request, execution = self._submit_request()
self.assertIsNotNone(execution)
self.assertEqual(execution.id, request.id)
self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Update execution status to FAILED.
action_service.update_status(execution, action_constants.LIVEACTION_STATUS_FAILED, False)
execution = action_db.get_liveaction_by_id(execution.id)
self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_FAILED)
# Request cancellation.
self.assertRaises(Exception, action_service.request_cancellation, execution)
def test_request_cancellation(self):
request, execution = self._submit_request()
self.assertIsNotNone(execution)
self.assertEqual(execution.id, request.id)
self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Update execution status to RUNNING.
action_service.update_status(execution, action_constants.LIVEACTION_STATUS_RUNNING, False)
execution = action_db.get_liveaction_by_id(execution.id)
self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Request cancellation.
execution = self._submit_cancellation(execution)
self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_CANCELING)
def _apply_before(self, target):
# Get the count of scheduled instances of the action.
scheduled = action_access.LiveAction.count(
action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED)
# Get the count of running instances of the action.
running = action_access.LiveAction.count(
action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING)
count = scheduled + running
# Mark the execution as scheduled if threshold is not reached or delayed otherwise.
if count < self.threshold:
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is not reached. Action execution will be scheduled.',
count, target.action, self._policy_ref)
status = action_constants.LIVEACTION_STATUS_SCHEDULED
else:
action = 'delayed' if self.policy_action == 'delay' else 'canceled'
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is reached. Action execution will be %s.',
count, target.action, self._policy_ref, action)
status = self._get_status_for_policy_action(action=self.policy_action)
# Update the status in the database. Publish status for cancellation so the
# appropriate runner can cancel the execution. Other statuses are not published
# because they will be picked up by the worker(s) to be processed again,
# leading to duplicate action executions.
publish = (status == action_constants.LIVEACTION_STATUS_CANCELING)
target = action_service.update_status(target, status, publish=publish)
return target
def _apply_before(self, target):
# Get the count of scheduled and running instances of the action.
filters = self._get_filters(target)
# Get the count of scheduled instances of the action.
filters['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
scheduled = action_access.LiveAction.count(**filters)
# Get the count of running instances of the action.
filters['status'] = action_constants.LIVEACTION_STATUS_RUNNING
running = action_access.LiveAction.count(**filters)
count = scheduled + running
# Mark the execution as scheduled if threshold is not reached or delayed otherwise.
if count < self.threshold:
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is not reached. Action execution will be scheduled.',
count, target.action, self._policy_ref)
status = action_constants.LIVEACTION_STATUS_SCHEDULED
else:
action = 'delayed' if self.policy_action == 'delay' else 'canceled'
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is reached. Action execution will be %s.',
count, target.action, self._policy_ref, action)
status = self._get_status_for_policy_action(action=self.policy_action)
# Update the status in the database but do not publish.
target = action_service.update_status(target, status, publish=False)
return target
def _update_to_scheduled(liveaction_db, execution_queue_item_db):
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
# Update liveaction status to "scheduled".
LOG.info('Liveaction (%s) Status Update to Scheduled 1: %s (%s)',
liveaction_id, liveaction_db.status, queue_item_id, extra=extra)
if liveaction_db.status in [action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_DELAYED]:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
extra['liveaction_status'] = liveaction_db.status
# Delete execution queue entry only after status is published.
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
LOG.info('Liveaction (%s) Status Update to Scheduled 2: %s (%s)',
liveaction_id, liveaction_db.status, queue_item_id)
def _apply_before(self, target):
# Get the count of scheduled instances of the action.
scheduled = action_access.LiveAction.count(
action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED)
# Get the count of running instances of the action.
running = action_access.LiveAction.count(
action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING)
count = scheduled + running
# Mark the execution as scheduled if threshold is not reached or delayed otherwise.
if count < self.threshold:
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is not reached. Action execution will be scheduled.',
count, target.action, self._policy_ref)
status = action_constants.LIVEACTION_STATUS_SCHEDULED
else:
LOG.debug('There are %s instances of %s in scheduled or running status. '
'Threshold of %s is reached. Action execution will be delayed.',
count, target.action, self._policy_ref)
status = action_constants.LIVEACTION_STATUS_DELAYED
# Update the status in the database but do not publish.
target = action_service.update_status(target, status, publish=False)
return target
def resume(self):
# Restore runner and action parameters since they are not provided on resume.
runner_parameters, action_parameters = param_utils.render_final_params(
self.runner_type.runner_parameters,
self.action.parameters,
self.liveaction.parameters,
self.liveaction.context
)
# Assign runner parameters needed for pre-run.
if runner_parameters:
self.runner_parameters = runner_parameters
# Restore chain holder if it is not initialized.
if not self.chain_holder:
self.pre_run()
# Change the status of the liveaction from resuming to running.
self.liveaction = action_service.update_status(
self.liveaction,
action_constants.LIVEACTION_STATUS_RUNNING,
publish=False
)
# Run the action chain.
return self._run_chain(action_parameters, resuming=True)
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def _invoke_action(self, action_db, runnertype_db, params, context=None,
additional_contexts=None):
"""
Schedule an action execution.
:type action_exec_spec: :class:`ActionExecutionSpecDB`
:param params: Partially rendered parameters to execute the action with.
:type params: ``dict``
:rtype: :class:`LiveActionDB` on successful scheduling, None otherwise.
"""
action_ref = action_db.ref
runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])
liveaction_db = LiveActionDB(action=action_ref, context=context, parameters=params)
try:
liveaction_db.parameters = self.get_resolved_parameters(
runnertype_db=runnertype_db,
action_db=action_db,
params=liveaction_db.parameters,
context=liveaction_db.context,
additional_contexts=additional_contexts)
except param_exc.ParamException as e:
# We still need to create a request, so liveaction_db is assigned an ID
liveaction_db, execution_db = action_service.create_request(liveaction_db)
# By this point the execution is already in the DB therefore need to mark it failed.
_, e, tb = sys.exc_info()
action_service.update_status(
liveaction=liveaction_db,
new_status=action_constants.LIVEACTION_STATUS_FAILED,
result={'error': six.text_type(e),
'traceback': ''.join(traceback.format_tb(tb, 20))})
# Might be a good idea to return the actual ActionExecution rather than bubble up
# the exception.
raise validation_exc.ValueValidationException(six.text_type(e))
liveaction_db, execution_db = action_service.request(liveaction_db)
return execution_db
def _schedule_execution(self, liveaction, user=None):
# Initialize execution context if it does not exist.
if not hasattr(liveaction, 'context'):
liveaction.context = dict()
liveaction.context['user'] = user
LOG.debug('User is: %s' % liveaction.context['user'])
# Retrieve other st2 context from request header.
if 'st2-context' in pecan.request.headers and pecan.request.headers['st2-context']:
context = jsonify.try_loads(pecan.request.headers['st2-context'])
if not isinstance(context, dict):
raise ValueError('Unable to convert st2-context from the headers into JSON.')
liveaction.context.update(context)
# Schedule the action execution.
liveaction_db = LiveActionAPI.to_model(liveaction)
liveaction_db, actionexecution_db = action_service.create_request(liveaction_db)
action_db = action_utils.get_action_by_ref(liveaction_db.action)
runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])
try:
liveaction_db.parameters = param_utils.render_live_params(
runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters,
liveaction_db.context)
except ParamException:
# By this point the execution is already in the DB therefore need to mark it failed.
_, e, tb = sys.exc_info()
action_service.update_status(
liveaction=liveaction_db,
new_status=LIVEACTION_STATUS_FAILED,
result={'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20))})
# Might be a good idea to return the actual ActionExecution rather than bubble up
# the execption.
raise ValueValidationException(str(e))
liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False)
_, actionexecution_db = action_service.publish_request(liveaction_db, actionexecution_db)
from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request)
return ActionExecutionAPI.from_model(actionexecution_db, from_model_kwargs)
def test_no_retry_policy_applied_on_task_failure(self):
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)
# Identify the records for the main workflow.
wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
self.assertEqual(len(tk_ex_dbs), 1)
# Identify the records for the tasks.
t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))[0]
t1_wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(t1_ac_ex_db.id))[0]
self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
self.assertEqual(t1_wf_ex_db.status, wf_statuses.RUNNING)
# Ensure there is only one execution for the task.
tk_ac_ref = TEST_PACK + '.' + 'sequential'
self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1)
# Fail the subtask of the subworkflow.
t1_t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[0]
t1_t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_t1_ex_db.id))[0]
ac_svc.update_status(t1_t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED)
t1_t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t1_ex_db.id))[0]
self.assertEqual(t1_t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
notifier.get_notifier().process(t1_t1_ac_ex_db)
workflows.get_engine().process(t1_t1_ac_ex_db)
# Ensure the task execution is not retried.
self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1)
# Process the failure of the subworkflow.
t1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(t1_ac_ex_db.id))
self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
workflows.get_engine().process(t1_ac_ex_db)
# Assert the main workflow is completed.
ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
def process(self, request):
"""Schedules the LiveAction and publishes the request
to the appropriate action runner(s).
LiveAction in statuses other than "requested" are ignored.
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__,
type(request),
request.id,
request.status,
)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(request.id)
except StackStormDBObjectNotFoundError:
LOG.exception("Failed to find liveaction %s in the database.", request.id)
raise
# Apply policies defined for the action.
liveaction_db = self._apply_pre_run_policies(liveaction_db=liveaction_db)
# Exit if the status of the request is no longer runnable.
# The status could have be changed by one of the policies.
if liveaction_db.status not in [
action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_SCHEDULED,
]:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status after policies are applied.',
self.__class__.__name__,
type(request),
request.id,
liveaction_db.status,
)
return
# Update liveaction status to "scheduled".
if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False
)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
def test_put_resume_with_result(self):
# Add the runner type to the list of runners that support pause and resume.
action_constants.WORKFLOW_RUNNER_TYPES.append(ACTION_1['runner_type'])
try:
post_resp = self._do_post(LIVE_ACTION_1)
self.assertEqual(post_resp.status_int, 201)
execution_id = self._get_actionexecution_id(post_resp)
updates = {'status': 'running'}
put_resp = self._do_put(execution_id, updates)
self.assertEqual(put_resp.status_int, 200)
self.assertEqual(put_resp.json['status'], 'running')
updates = {'status': 'pausing'}
put_resp = self._do_put(execution_id, updates)
self.assertEqual(put_resp.status_int, 200)
self.assertEqual(put_resp.json['status'], 'pausing')
self.assertIsNone(put_resp.json.get('result'))
# Manually change the status to paused because only the runner pause method should
# set the paused status directly to the liveaction and execution database objects.
liveaction_id = self._get_liveaction_id(post_resp)
liveaction = action_db_util.get_liveaction_by_id(liveaction_id)
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
get_resp = self._do_get_one(execution_id)
self.assertEqual(get_resp.status_int, 200)
self.assertEqual(get_resp.json['status'], 'paused')
self.assertIsNone(get_resp.json.get('result'))
updates = {'status': 'resuming', 'result': {'stdout': 'foobar'}}
put_resp = self._do_put(execution_id, updates, expect_errors=True)
self.assertEqual(put_resp.status_int, 400)
self.assertIn('result is not applicable', put_resp.json['faultstring'])
finally:
action_constants.WORKFLOW_RUNNER_TYPES.remove(ACTION_1['runner_type'])
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_cancel_delayed_execution_with_parent(self):
liveaction = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {'execution_id': uuid.uuid4().hex}}
)
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def process(self, request):
"""Schedules the LiveAction and publishes the request
to the appropriate action runner(s).
LiveAction in statuses other than "requested" are ignored.
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__, type(request), request.id,
request.status)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(request.id)
except StackStormDBObjectNotFoundError:
LOG.exception('Failed to find liveaction %s in the database.',
request.id)
raise
# Apply policies defined for the action.
liveaction_db = self._apply_pre_run_policies(
liveaction_db=liveaction_db)
# Exit if the status of the request is no longer runnable.
# The status could have be changed by one of the policies.
if liveaction_db.status not in [
action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_SCHEDULED
]:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status after policies are applied.',
self.__class__.__name__, type(request), request.id,
liveaction_db.status)
return
# Update liveaction status to "scheduled".
if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_SCHEDULED,
publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
def test_req_resume(self):
# Add the runner type to the list of runners that support pause and resume.
action_constants.WORKFLOW_RUNNER_TYPES.append(ACTION['runner_type'])
try:
req, ex = self._submit_request()
self.assertIsNotNone(ex)
self.assertEqual(ex.id, req.id)
self.assertEqual(ex.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Update ex status to RUNNING.
action_service.update_status(
ex, action_constants.LIVEACTION_STATUS_RUNNING, False)
ex = action_db.get_liveaction_by_id(ex.id)
self.assertEqual(ex.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Request pause.
ex = self._submit_pause(ex)
self.assertEqual(ex.status,
action_constants.LIVEACTION_STATUS_PAUSING)
# Update ex status to PAUSED.
action_service.update_status(
ex, action_constants.LIVEACTION_STATUS_PAUSED, False)
ex = action_db.get_liveaction_by_id(ex.id)
self.assertEqual(ex.status,
action_constants.LIVEACTION_STATUS_PAUSED)
# Request resume.
ex = self._submit_resume(ex)
self.assertEqual(ex.status,
action_constants.LIVEACTION_STATUS_RESUMING)
finally:
action_constants.WORKFLOW_RUNNER_TYPES.remove(
ACTION['runner_type'])
def test_resume(self):
# Launch the workflow execution.
liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, execution = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'],
WF1_EXEC.get('workflow_name'))
# Pause the workflow execution.
requester = cfg.CONF.system_user.user
liveaction, execution = action_service.request_pause(
liveaction, requester)
executions.ExecutionManager.update.assert_called_with(
WF1_EXEC.get('id'), 'PAUSED')
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually update the liveaction from pausing to paused. The paused state
# is usually updated by the mistral querier.
action_service.update_status(liveaction,
action_constants.LIVEACTION_STATUS_PAUSED)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
# Resume the workflow execution.
liveaction, execution = action_service.request_resume(
liveaction, requester)
executions.ExecutionManager.update.assert_called_with(
WF1_EXEC.get('id'), 'RUNNING')
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def recover_delayed_executions():
coordinator = coordination.get_coordinator()
dt_now = date_utils.get_datetime_utc_now()
dt_delta = datetime.timedelta(
seconds=cfg.CONF.scheduler.delayed_execution_recovery)
dt_timeout = dt_now - dt_delta
with coordinator.get_lock('st2-rescheduling-delayed-executions'):
liveactions = LiveAction.query(
status=action_constants.LIVEACTION_STATUS_DELAYED,
start_timestamp__lte=dt_timeout,
order_by=['start_timestamp'])
if not liveactions:
return
LOG.info(
'There are %d liveactions that have been delayed for longer than %d seconds.',
len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery)
# Update status to requested and publish status for each liveactions.
rescheduled = 0
for instance in liveactions:
try:
action_service.update_status(
instance,
action_constants.LIVEACTION_STATUS_REQUESTED,
publish=True)
rescheduled += 1
except:
LOG.exception(
'Unable to reschedule liveaction. <LiveAction.id=%s>',
instance.id)
LOG.info('Rescheduled %d out of %d delayed liveactions.',
len(liveactions), rescheduled)
def _cleanup_policy_delayed(self):
"""
Clean up any action execution in the deprecated policy-delayed status. Associated
entries in the scheduling queue will be removed and the action execution will be
moved back into requested status.
"""
policy_delayed_liveaction_dbs = LiveAction.query(status='policy-delayed') or []
for liveaction_db in policy_delayed_liveaction_dbs:
ex_que_qry = {'liveaction_id': str(liveaction_db.id), 'handling': False}
execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**ex_que_qry) or []
for execution_queue_item_db in execution_queue_item_dbs:
# Mark the entry in the scheduling queue for handling.
try:
execution_queue_item_db.handling = True
execution_queue_item_db = ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
msg = (
'[%s] Item "%s" is currently being processed by another scheduler.' %
(execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id))
)
LOG.error(msg)
raise Exception(msg)
# Delete the entry from the scheduling queue.
LOG.info(
'[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id)
)
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
# Update the status of the liveaction and execution to requested.
LOG.info(
'[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
execution_queue_item_db.action_execution_id,
str(execution_queue_item_db.id)
)
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_REQUESTED)
execution_service.update_execution(liveaction_db)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)',
liveaction_id,
liveaction_db.status,
queue_item_id,
extra=extra)
if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id)
return
if (liveaction_db.status
in action_constants.LIVEACTION_COMPLETED_STATES
or liveaction_db.status
in action_constants.LIVEACTION_CANCEL_STATES):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def process(self, request):
"""
Adds execution into execution_scheduling database for scheduling
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__,
type(request),
request.id,
request.status,
)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
except StackStormDBObjectNotFoundError:
LOG.exception("Failed to find liveaction %s in the database.",
str(request.id))
raise
query = {
"liveaction_id": str(liveaction_db.id),
}
queued_requests = ActionExecutionSchedulingQueue.query(**query)
if len(queued_requests) > 0:
# Particular execution is already being scheduled
return queued_requests[0]
if liveaction_db.delay and liveaction_db.delay > 0:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
liveaction_db, delay=liveaction_db.delay)
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db,
publish=False)
return execution_queue_item_db
def _apply_before(self, target):
# Get the count of scheduled and running instances of the action.
filters = self._get_filters(target)
# Get the count of scheduled instances of the action.
filters["status"] = action_constants.LIVEACTION_STATUS_SCHEDULED
scheduled = action_access.LiveAction.count(**filters)
# Get the count of running instances of the action.
filters["status"] = action_constants.LIVEACTION_STATUS_RUNNING
running = action_access.LiveAction.count(**filters)
count = scheduled + running
# Mark the execution as scheduled if threshold is not reached or delayed otherwise.
if count < self.threshold:
LOG.debug(
"There are %s instances of %s in scheduled or running status. "
"Threshold of %s is not reached. Action execution will be scheduled.",
count,
target.action,
self._policy_ref,
)
status = action_constants.LIVEACTION_STATUS_REQUESTED
else:
action = "delayed" if self.policy_action == "delay" else "canceled"
LOG.debug(
"There are %s instances of %s in scheduled or running status. "
"Threshold of %s is reached. Action execution will be %s.",
count,
target.action,
self._policy_ref,
action,
)
status = self._get_status_for_policy_action(
action=self.policy_action)
# Update the status in the database. Publish status for cancellation so the
# appropriate runner can cancel the execution. Other statuses are not published
# because they will be picked up by the worker(s) to be processed again,
# leading to duplicate action executions.
publish = status == action_constants.LIVEACTION_STATUS_CANCELING
target = action_service.update_status(target, status, publish=publish)
return target
def _delay(self, liveaction_db, execution_queue_item_db):
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False)
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
try:
execution_queue_item_db.handling = False
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id)
def _delay(self, liveaction_db, execution_queue_item_db):
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
)
try:
execution_queue_item_db.handling = False
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id
)
def process(self, request):
"""
Adds execution into execution_scheduling database for scheduling
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__, type(request), request.id, request.status)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
except StackStormDBObjectNotFoundError:
LOG.exception('Failed to find liveaction %s in the database.', str(request.id))
raise
query = {
'liveaction_id': str(liveaction_db.id),
}
queued_requests = ActionExecutionSchedulingQueue.query(**query)
if len(queued_requests) > 0:
# Particular execution is already being scheduled
return queued_requests[0]
if liveaction_db.delay and liveaction_db.delay > 0:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_DELAYED,
publish=False
)
execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
liveaction_db,
delay=liveaction_db.delay
)
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
return execution_queue_item_db
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)', liveaction_id, liveaction_db.status,
queue_item_id, extra=extra)
if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
)
try:
ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
'Execution queue item update conflict during scheduling: %s',
execution_queue_item_db.id
)
return
if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or
liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def resume(self):
# Restore runner and action parameters since they are not provided on resume.
runner_parameters, action_parameters = param_utils.render_final_params(
self.runner_type.runner_parameters, self.action.parameters,
self.liveaction.parameters, self.liveaction.context)
# Assign runner parameters needed for pre-run.
if runner_parameters:
self.runner_parameters = runner_parameters
# Restore chain holder if it is not initialized.
if not self.chain_holder:
self.pre_run()
# Change the status of the liveaction from resuming to running.
self.liveaction = action_service.update_status(
self.liveaction,
action_constants.LIVEACTION_STATUS_RUNNING,
publish=False)
# Run the action chain.
return self._run_chain(action_parameters, resuming=True)
def _update_to_scheduled(liveaction_db, execution_queue_item_db):
liveaction_id = str(liveaction_db.id)
queue_item_id = str(execution_queue_item_db.id)
extra = {
'liveaction_id': liveaction_id,
'liveaction_status': liveaction_db.status,
'queue_item_id': queue_item_id
}
# Update liveaction status to "scheduled".
LOG.info('Liveaction (%s) Status Update to Scheduled 1: %s (%s)',
liveaction_id,
liveaction_db.status,
queue_item_id,
extra=extra)
if liveaction_db.status in [
action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_DELAYED
]:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_SCHEDULED,
publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
extra['liveaction_status'] = liveaction_db.status
# Delete execution queue entry only after status is published.
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
LOG.info('Liveaction (%s) Status Update to Scheduled 2: %s (%s)',
liveaction_id, liveaction_db.status, queue_item_id)
def _update_to_scheduled(liveaction_db, execution_queue_item_db):
action_execution_id = str(execution_queue_item_db.action_execution_id)
liveaction_id = str(execution_queue_item_db.liveaction_id)
queue_item_id = str(execution_queue_item_db.id)
extra = {'queue_item_id': queue_item_id}
# Update liveaction status to "scheduled".
LOG.info(
'[%s] Liveaction "%s" with status "%s" is updated to status "scheduled."',
action_execution_id, liveaction_id, liveaction_db.status, extra=extra
)
if liveaction_db.status in [action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_DELAYED]:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
# Delete execution queue entry only after status is published.
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
def test_include_result_to_error_log(self):
username = "******"
wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
wf_input = {"who": "Thanos"}
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"],
parameters=wf_input)
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING,
lv_ac_db.result)
wf_ex_dbs = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))
wf_ex_db = wf_ex_dbs[0]
# Assert task1 is already completed.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.context.get("user"), username)
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
# Manually override and fail the action execution and write some result.
# Action execution result can contain dotted notation so ensure this is tested.
result = {"127.0.0.1": {"hostname": "foobar"}}
ac_svc.update_status(
tk1_lv_ac_db,
ac_const.LIVEACTION_STATUS_FAILED,
result=result,
publish=False,
)
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_FAILED)
self.assertDictEqual(tk1_lv_ac_db.result, result)
# Manually handle action execution completion.
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
# Assert task and workflow failed.
tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id)
self.assertEqual(tk1_ex_db.status, wf_statuses.FAILED)
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
# Assert result is included in the error log.
expected_errors = [{
"message": "Execution failed. See result for details.",
"type": "error",
"task_id": "task1",
"result": {
"127.0.0.1": {
"hostname": "foobar"
}
},
}]
self.assertListEqual(wf_ex_db.errors, expected_errors)
def tearDown(self):
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo-last'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
action_execution_id = str(execution_queue_item_db.action_execution_id)
liveaction_id = str(execution_queue_item_db.liveaction_id)
queue_item_id = str(execution_queue_item_db.id)
extra = {"queue_item_id": queue_item_id}
LOG.info(
'[%s] Liveaction "%s" has status "%s" before applying policies.',
action_execution_id,
liveaction_id,
liveaction_db.status,
extra=extra,
)
# Apply policies defined for the action.
liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)
LOG.info(
'[%s] Liveaction "%s" has status "%s" after applying policies.',
action_execution_id,
liveaction_id,
liveaction_db.status,
extra=extra,
)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED:
LOG.info(
'[%s] Liveaction "%s" is delayed and scheduling queue is updated.',
action_execution_id,
liveaction_id,
extra=extra,
)
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
)
execution_queue_item_db.handling = False
execution_queue_item_db.scheduled_start_timestamp = (
date.append_milliseconds_to_time(
date.get_datetime_utc_now(),
POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS,
)
)
try:
ActionExecutionSchedulingQueue.add_or_update(
execution_queue_item_db, publish=False
)
except db_exc.StackStormDBObjectWriteConflictError:
LOG.warning(
"[%s] Database write conflict on updating scheduling queue.",
action_execution_id,
extra=extra,
)
return
if (
liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES
or liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES
):
ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
return
self._schedule(liveaction_db, execution_queue_item_db)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True
)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo-last'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True
)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as scheduled.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Check the status changes.
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
expected_status_changes = ['requested', 'delayed', 'requested', 'scheduled', 'running']
actual_status_changes = [entry['status'] for entry in execution.log]
self.assertListEqual(actual_status_changes, expected_status_changes)
def test_resume_missing_subworkflow_action(self):
requester = cfg.CONF.system_user.user
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = self._wait_on_status(liveaction1, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = self._wait_on_status(liveaction2, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
# Pause the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_pause(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually set the liveaction status to PAUSED.
action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED)
action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [uuid.uuid4().hex]
# Resume the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_resume(liveaction1, requester)
# Includes the previous calls.
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 3)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED'),
mock.call(WF2_EXEC.get('id'), 'RUNNING'),
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
# The workflow execution will fail because the liveaction for the subworkflow
# should not be missing and we do not know what state it is in.
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo-last"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as scheduled.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Check the status changes.
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
expected_status_changes = [
"requested",
"delayed",
"requested",
"scheduled",
"running",
]
actual_status_changes = [entry["status"] for entry in execution.log]
self.assertListEqual(actual_status_changes, expected_status_changes)
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
try:
liveaction_db = action_service.update_status(liveaction_db,
liveaction_api.status,
result=getattr(
liveaction_api,
'result', None))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution.')
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
return execution_api
def update_status(liveaction_api, liveaction_db):
status = liveaction_api.status
result = getattr(liveaction_api, 'result', None)
liveaction_db = action_service.update_status(liveaction_db, status, result)
actionexecution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id))
return (liveaction_db, actionexecution_db)
def tearDown(self):
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
self.assertGreater(policy_db.parameters["threshold"], 0)
self.assertIn("actionstr", policy_db.parameters["attributes"])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(
action="wolfpack.action-1", parameters={"actionstr": "foo"}
)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(
action="wolfpack.action-1", parameters={"actionstr": "foo"}
)
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED
)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(
action="wolfpack.action-1", parameters={"actionstr": "bar"}
)
liveaction, _ = action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True
)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _schedule_execution(self,
liveaction,
requester_user,
user=None,
context_string=None,
show_secrets=False):
# Initialize execution context if it does not exist.
if not hasattr(liveaction, 'context'):
liveaction.context = dict()
liveaction.context['user'] = user
LOG.debug('User is: %s' % liveaction.context['user'])
# Retrieve other st2 context from request header.
if context_string:
context = try_loads(context_string)
if not isinstance(context, dict):
raise ValueError(
'Unable to convert st2-context from the headers into JSON.'
)
liveaction.context.update(context)
# Schedule the action execution.
liveaction_db = LiveActionAPI.to_model(liveaction)
liveaction_db, actionexecution_db = action_service.create_request(
liveaction_db)
action_db = action_utils.get_action_by_ref(liveaction_db.action)
runnertype_db = action_utils.get_runnertype_by_name(
action_db.runner_type['name'])
try:
liveaction_db.parameters = param_utils.render_live_params(
runnertype_db.runner_parameters, action_db.parameters,
liveaction_db.parameters, liveaction_db.context)
except ParamException:
# By this point the execution is already in the DB therefore need to mark it failed.
_, e, tb = sys.exc_info()
action_service.update_status(liveaction=liveaction_db,
new_status=LIVEACTION_STATUS_FAILED,
result={
'error':
str(e),
'traceback':
''.join(
traceback.format_tb(tb, 20))
})
# Might be a good idea to return the actual ActionExecution rather than bubble up
# the execption.
raise ValueValidationException(str(e))
liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False)
_, actionexecution_db = action_service.publish_request(
liveaction_db, actionexecution_db)
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return Response(json=execution_api, status=http_client.CREATED)
def test_resume_missing_subworkflow_action(self):
requester = cfg.CONF.system_user.user
liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction1, execution1 = action_service.request(liveaction1)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING)
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [execution2.id]
mistral_context = liveaction1.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name'))
# Pause the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_pause(liveaction1, requester)
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 2)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED')
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING)
# Manually set the liveaction status to PAUSED.
action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED)
action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED)
# Mock the children of the parent execution to make this
# test case has subworkflow execution.
with mock.patch.object(
ActionExecutionDB, 'children',
new_callable=mock.PropertyMock) as action_ex_children_mock:
action_ex_children_mock.return_value = [uuid.uuid4().hex]
# Resume the parent liveaction and check that the request is cascaded down.
liveaction1, execution1 = action_service.request_resume(liveaction1, requester)
# Includes the previous calls.
self.assertTrue(executions.ExecutionManager.update.called)
self.assertEqual(executions.ExecutionManager.update.call_count, 3)
calls = [
mock.call(WF2_EXEC.get('id'), 'PAUSED'),
mock.call(WF1_EXEC.get('id'), 'PAUSED'),
mock.call(WF2_EXEC.get('id'), 'RUNNING'),
]
executions.ExecutionManager.update.assert_has_calls(calls, any_order=False)
# The workflow execution will fail because the liveaction for the subworkflow
# should not be missing and we do not know what state it is in.
liveaction1 = LiveAction.get_by_id(str(liveaction1.id))
self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED)
self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def tearDown(self):
# Ensure all liveactions are canceled at end of each test.
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def tearDown(self):
# Ensure all liveactions are canceled at end of each test.
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def _schedule_execution(self,
liveaction,
requester_user,
user=None,
context_string=None,
show_secrets=False,
pack=None):
# Initialize execution context if it does not exist.
if not hasattr(liveaction, 'context'):
liveaction.context = dict()
liveaction.context['user'] = user
liveaction.context['pack'] = pack
LOG.debug('User is: %s' % liveaction.context['user'])
# Retrieve other st2 context from request header.
if context_string:
context = try_loads(context_string)
if not isinstance(context, dict):
raise ValueError(
'Unable to convert st2-context from the headers into JSON.'
)
liveaction.context.update(context)
# Include RBAC context (if RBAC is available and enabled)
if cfg.CONF.rbac.enable:
user_db = UserDB(name=user)
role_dbs = rbac_service.get_roles_for_user(user_db=user_db,
include_remote=True)
roles = [role_db.name for role_db in role_dbs]
liveaction.context['rbac'] = {'user': user, 'roles': roles}
# Schedule the action execution.
liveaction_db = LiveActionAPI.to_model(liveaction)
action_db = action_utils.get_action_by_ref(liveaction_db.action)
runnertype_db = action_utils.get_runnertype_by_name(
action_db.runner_type['name'])
try:
liveaction_db.parameters = param_utils.render_live_params(
runnertype_db.runner_parameters, action_db.parameters,
liveaction_db.parameters, liveaction_db.context)
except param_exc.ParamException:
# We still need to create a request, so liveaction_db is assigned an ID
liveaction_db, actionexecution_db = action_service.create_request(
liveaction_db)
# By this point the execution is already in the DB therefore need to mark it failed.
_, e, tb = sys.exc_info()
action_service.update_status(
liveaction=liveaction_db,
new_status=action_constants.LIVEACTION_STATUS_FAILED,
result={
'error': str(e),
'traceback': ''.join(traceback.format_tb(tb, 20))
})
# Might be a good idea to return the actual ActionExecution rather than bubble up
# the exception.
raise validation_exc.ValueValidationException(str(e))
# The request should be created after the above call to render_live_params
# so any templates in live parameters have a chance to render.
liveaction_db, actionexecution_db = action_service.create_request(
liveaction_db)
liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False)
_, actionexecution_db = action_service.publish_request(
liveaction_db, actionexecution_db)
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return Response(json=execution_api, status=http_client.CREATED)
def put(self, id, liveaction_api, requester_user, show_secrets=False):
"""
Updates a single execution.
Handles requests:
PUT /executions/<id>
"""
if not requester_user:
requester_user = UserDB(cfg.CONF.system_user.user)
from_model_kwargs = {
'mask_secrets':
self._get_mask_secrets(requester_user, show_secrets=show_secrets)
}
execution_api = self._get_one_by_id(
id=id,
requester_user=requester_user,
from_model_kwargs=from_model_kwargs,
permission_type=PermissionType.EXECUTION_STOP)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % id)
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES:
abort(http_client.BAD_REQUEST,
'Execution is already in completed state.')
if (getattr(liveaction_api, 'result', None) is not None
and liveaction_api.status in [
action_constants.LIVEACTION_STATUS_PAUSING,
action_constants.LIVEACTION_STATUS_PAUSED,
action_constants.LIVEACTION_STATUS_RESUMING
]):
abort(
http_client.BAD_REQUEST,
'The result is not applicable for pausing and resuming execution.'
)
try:
if (liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSING
or liveaction_api.status
== action_constants.LIVEACTION_STATUS_PAUSED):
liveaction_db, actionexecution_db = action_service.request_pause(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING:
liveaction_db, actionexecution_db = action_service.request_resume(
liveaction_db, requester_user.name
or cfg.CONF.system_user.user)
else:
liveaction_db = action_service.update_status(
liveaction_db,
liveaction_api.status,
result=getattr(liveaction_api, 'result', None))
actionexecution_db = ActionExecution.get(
liveaction__id=str(liveaction_db.id))
except runner_exc.InvalidActionRunnerOperationError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except runner_exc.UnexpectedActionExecutionStatusError as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.BAD_REQUEST,
'Failed updating execution. %s' % str(e))
except Exception as e:
LOG.exception('Failed updating liveaction %s. %s',
liveaction_db.id, str(e))
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed updating execution due to unexpected error.')
mask_secrets = self._get_mask_secrets(requester_user,
show_secrets=show_secrets)
execution_api = ActionExecutionAPI.from_model(
actionexecution_db, mask_secrets=mask_secrets)
return execution_api
