def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': 'abcde'
}})
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_max_retries_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
live_action_db.context['policies']['retry'] = {'retry_count': 2}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since max retries has been reached
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_no_retry_since_no_timeout_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which succeeds
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
execution_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should only be 1 object since the action didn't timeout and therefore it wasn't
# retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_SUCCEEDED)
def test_no_retry_on_non_applicable_statuses(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action in various statuses in which we shouldn't retry
non_retry_statuses = [
LIVEACTION_STATUS_REQUESTED,
LIVEACTION_STATUS_SCHEDULED,
LIVEACTION_STATUS_DELAYED,
LIVEACTION_STATUS_CANCELING,
LIVEACTION_STATUS_CANCELED,
]
action_ref = 'wolfpack.action-1'
for status in non_retry_statuses:
liveaction = LiveActionDB(action=action_ref,
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = status
execution_db.status = status
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# None of the actions should have been retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), len(non_retry_statuses))
self.assertEqual(len(action_execution_dbs), len(non_retry_statuses))
def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {'execution_id': 'abcde'}}
)
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_non_applicable_statuses(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action in various statuses in which we shouldn't retry
non_retry_statuses = [
LIVEACTION_STATUS_REQUESTED,
LIVEACTION_STATUS_SCHEDULED,
LIVEACTION_STATUS_DELAYED,
LIVEACTION_STATUS_CANCELING,
LIVEACTION_STATUS_CANCELED,
]
action_ref = 'wolfpack.action-1'
for status in non_retry_statuses:
liveaction = LiveActionDB(action=action_ref, parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = status
execution_db.status = status
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# None of the actions should have been retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), len(non_retry_statuses))
self.assertEqual(len(action_execution_dbs), len(non_retry_statuses))
def test_retry_on_timeout_max_retries_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
live_action_db.context['policies']['retry'] = {'retry_count': 2}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since max retries has been reached
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_retry_on_timeout_first_retry_is_successful(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status,
LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[0].liveaction["id"]
context = action_execution_dbs[1].context
self.assertIn("policies", context)
self.assertEqual(context["policies"]["retry"]["retry_count"], 1)
self.assertEqual(context["policies"]["retry"]["applied_policy"],
"test_policy")
self.assertEqual(
context["policies"]["retry"]["retried_liveaction_id"],
original_liveaction_id,
)
# Simulate success of second action so no it shouldn't be retried anymore
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be no new object since action succeeds so no retry was attempted
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(live_action_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(live_action_dbs[1].status,
LIVEACTION_STATUS_SUCCEEDED)
def delete(self, exec_id):
"""
Stops a single execution.
Handles requests:
DELETE /actionexecutions/<id>
"""
execution_api = self._get_one(id=exec_id)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % exec_id)
return
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
return
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
abort(http_client.OK, 'Action is already in "canceled" state.')
if liveaction_db.status not in CANCELABLE_STATES:
abort(
http_client.OK, 'Action cannot be canceled. State = %s.' %
liveaction_db.status)
return
liveaction_db.status = 'canceled'
liveaction_db.end_timestamp = isotime.add_utc_tz(
datetime.datetime.utcnow())
liveaction_db.result = {'message': 'Action canceled by user.'}
try:
LiveAction.add_or_update(liveaction_db)
except:
LOG.exception(
'Failed updating status to canceled for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed canceling execution.')
return
execution_db = execution_service.update_execution(liveaction_db)
return ActionExecutionAPI.from_model(execution_db)
def test_retry_on_timeout_first_retry_is_successful(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[0].liveaction['id']
context = action_execution_dbs[1].context
self.assertTrue('policies' in context)
self.assertEqual(context['policies']['retry']['retry_count'], 1)
self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy')
self.assertEqual(context['policies']['retry']['retried_liveaction_id'],
original_liveaction_id)
# Simulate success of second action so no it shouldn't be retried anymore
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be no new object since action succeeds so no retry was attempted
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(live_action_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(live_action_dbs[1].status, LIVEACTION_STATUS_SUCCEEDED)
def delete(self, exec_id):
"""
Stops a single execution.
Handles requests:
DELETE /actionexecutions/<id>
"""
execution_api = self._get_one(id=exec_id)
if not execution_api:
abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % exec_id)
return
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' % liveaction_id)
return
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
abort(http_client.OK,
'Action is already in "canceled" state.')
if liveaction_db.status not in CANCELABLE_STATES:
abort(http_client.OK,
'Action cannot be canceled. State = %s.' % liveaction_db.status)
return
liveaction_db.status = 'canceled'
liveaction_db.end_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow())
liveaction_db.result = {'message': 'Action canceled by user.'}
try:
LiveAction.add_or_update(liveaction_db)
except:
LOG.exception('Failed updating status to canceled for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR, 'Failed canceling execution.')
return
execution_db = execution_service.update_execution(liveaction_db)
return ActionExecutionAPI.from_model(execution_db)
def test_execution_creation_action_triggered_by_rule(self):
# Wait for the action execution to complete and then confirm outcome.
trigger_type = self.MODELS['triggertypes']['triggertype2.json']
trigger = self.MODELS['triggers']['trigger2.json']
trigger_instance = self.MODELS['triggerinstances']['trigger_instance_1.json']
test_liveaction = self.FIXTURES['liveactions']['liveaction3.json']
rule = self.MODELS['rules']['rule3.json']
# Setup LiveAction to point to right rule and trigger_instance.
# XXX: We need support for dynamic fixtures.
test_liveaction['context']['rule']['id'] = str(rule.id)
test_liveaction['context']['trigger_instance']['id'] = str(trigger_instance.id)
test_liveaction_api = LiveActionAPI(**test_liveaction)
test_liveaction = LiveAction.add_or_update(LiveActionAPI.to_model(test_liveaction_api))
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
self.assertEqual(liveaction.status, LIVEACTION_STATUS_SCHEDULED)
executions_util.create_execution_object(liveaction)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEquals(execution.liveaction['id'], str(liveaction.id))
def setup_action_models(cls):
action_db = ActionDB()
action_db.name = 'action-1'
action_db.description = 'awesomeness'
action_db.enabled = True
action_db.pack = 'wolfpack'
action_db.ref = ResourceReference(name=action_db.name, pack=action_db.pack).ref
action_db.entry_point = ''
action_db.runner_type = {'name': 'test-runner'}
action_db.parameters = {
'actionstr': {'type': 'string', 'position': 1, 'required': True},
'actionint': {'type': 'number', 'default': 10, 'position': 0},
'runnerdummy': {'type': 'string', 'default': 'actiondummy'}
}
ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)
def test_liveaction_crud_no_notify(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
self.assertEqual(retrieved.notify, None)
# Test update
self.assertTrue(retrieved.end_timestamp is None)
retrieved.end_timestamp = isotime.add_utc_tz(
datetime.datetime.utcnow())
updated = LiveAction.add_or_update(retrieved)
self.assertTrue(updated.end_timestamp == retrieved.end_timestamp)
# Test delete
LiveActionModelTest._delete([retrieved])
try:
retrieved = LiveAction.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_retry_on_timeout_first_retry_is_successful(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status,
LIVEACTION_STATUS_REQUESTED)
# Simulate success of second action so no it shouldn't be retried anymore
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be no new object since action succeeds so no retry was attempted
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(live_action_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(live_action_dbs[1].status,
LIVEACTION_STATUS_SUCCEEDED)
def _update_action_results(self, execution_id, status, results):
liveaction_db = LiveAction.get_by_id(execution_id)
if not liveaction_db:
raise Exception('No DB model for liveaction_id: %s' % execution_id)
liveaction_db.result = results
liveaction_db.status = status
# update liveaction, update actionexecution and then publish update.
updated_liveaction = LiveAction.add_or_update(liveaction_db, publish=False)
executions.update_execution(updated_liveaction)
LiveAction.publish_update(updated_liveaction)
return updated_liveaction
def test_dispatch_runner_failure(self):
runner_container = get_runner_container()
params = {'actionstr': 'bar'}
liveaction_db = self._get_failingaction_exec_db_model(params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
runner_container.dispatch(liveaction_db)
# pickup updated liveaction_db
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
self.assertTrue('message' in liveaction_db.result)
self.assertTrue('traceback' in liveaction_db.result)
def _update_action_results(self, execution_id, status, results):
liveaction_db = LiveAction.get_by_id(execution_id)
if not liveaction_db:
raise Exception('No DB model for liveaction_id: %s' % execution_id)
liveaction_db.result = results
liveaction_db.status = status
# update liveaction, update actionexecution and then publish update.
updated_liveaction = LiveAction.add_or_update(liveaction_db,
publish=False)
executions.update_execution(updated_liveaction)
LiveAction.publish_update(updated_liveaction)
return updated_liveaction
def schedule(liveaction):
"""
Schedule an action to be run.
:return: (liveaction, execution)
:rtype: tuple
"""
# Use the user context from the parent action execution. Subtasks in a workflow
# action can be invoked by a system user and so we want to use the user context
# from the original workflow action.
if getattr(liveaction, 'context', None) and 'parent' in liveaction.context:
parent = LiveAction.get_by_id(liveaction.context['parent'])
liveaction.context['user'] = getattr(parent, 'context', dict()).get('user')
# Validate action.
action_db = action_utils.get_action_by_ref(liveaction.action)
if not action_db:
raise ValueError('Action "%s" cannot be found.' % liveaction.action)
if not action_db.enabled:
raise ValueError('Unable to execute. Action "%s" is disabled.' % liveaction.action)
runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])
if not hasattr(liveaction, 'parameters'):
liveaction.parameters = dict()
# Validate action parameters.
schema = util_schema.get_parameter_schema(action_db)
validator = util_schema.get_validator()
jsonschema.validate(liveaction.parameters, schema, validator)
# validate that no immutable params are being overriden. Although possible to
# ignore the override it is safer to inform the user to avoid surprises.
immutables = _get_immutable_params(action_db.parameters)
immutables.extend(_get_immutable_params(runnertype_db.runner_parameters))
overridden_immutables = [p for p in six.iterkeys(liveaction.parameters) if p in immutables]
if len(overridden_immutables) > 0:
raise ValueError('Override of immutable parameter(s) %s is unsupported.'
% str(overridden_immutables))
# Write to database and send to message queue.
liveaction.status = LIVEACTION_STATUS_SCHEDULED
liveaction.start_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow())
# Publish creation after both liveaction and actionexecution are created.
liveaction = LiveAction.add_or_update(liveaction, publish=False)
execution = executions.create_execution_object(liveaction, publish=False)
# assume that this is a creation.
LiveAction.publish_create(liveaction)
ActionExecution.publish_create(execution)
LOG.audit('Action execution scheduled. LiveAction=%s. ActionExecution=%s', liveaction,
execution)
return liveaction, execution
def test_dispatch_override_default_action_params(self):
runner_container = get_runner_container()
params = {'actionstr': 'foo', 'actionint': 20}
liveaction_db = self._get_action_exec_db_model(
RunnerContainerTest.action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran successfully.
runner_container.dispatch(liveaction_db)
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
result = liveaction_db.result
self.assertTrue(result.get('action_params').get('actionint') == 20)
self.assertTrue(result.get('action_params').get('actionstr') == 'foo')
def test_dispatch_runner_failure(self):
runner_container = get_runner_container()
params = {
'actionstr': 'bar'
}
liveaction_db = self._get_failingaction_exec_db_model(params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
runner_container.dispatch(liveaction_db)
# pickup updated liveaction_db
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
self.assertTrue('message' in liveaction_db.result)
self.assertTrue('traceback' in liveaction_db.result)
def update_liveaction_status(status=None,
result=None,
context=None,
end_timestamp=None,
liveaction_id=None,
runner_info=None,
liveaction_db=None):
"""
Update the status of the specified LiveAction to the value provided in
new_status.
The LiveAction may be specified using either liveaction_id, or as an
liveaction_db instance.
"""
if (liveaction_id is None) and (liveaction_db is None):
raise ValueError(
'Must specify an liveaction_id or an liveaction_db when '
'calling update_LiveAction_status')
if liveaction_db is None:
liveaction_db = get_liveaction_by_id(liveaction_id)
if status not in LIVEACTION_STATUSES:
raise ValueError(
'Attempting to set status for LiveAction "%s" '
'to unknown status string. Unknown status is "%s"', liveaction_db,
status)
LOG.debug('Updating ActionExection: "%s" with status="%s"', liveaction_db,
status)
liveaction_db.status = status
if result:
liveaction_db.result = result
if context:
liveaction_db.context.update(context)
if end_timestamp:
liveaction_db.end_timestamp = end_timestamp
if runner_info:
liveaction_db.runner_info = runner_info
liveaction_db = LiveAction.add_or_update(liveaction_db)
LOG.debug('Updated status for LiveAction object: %s', liveaction_db)
return liveaction_db
def test_dispatch_override_default_action_params(self):
runner_container = get_runner_container()
params = {
'actionstr': 'foo',
'actionint': 20
}
liveaction_db = self._get_action_exec_db_model(RunnerContainerTest.action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran successfully.
runner_container.dispatch(liveaction_db)
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
result = liveaction_db.result
self.assertTrue(result.get('action_params').get('actionint') == 20)
self.assertTrue(result.get('action_params').get('actionstr') == 'foo')
def test_retry_on_timeout_no_retry_since_no_timeout_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which succeeds
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
execution_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should only be 1 object since the action didn't timeout and therefore it wasn't
# retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_SUCCEEDED)
def test_delayed_executions_recovery_before_timeout(self):
# Create a live action that's delayed but has not passed the timeout.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
start_timestamp=date_utils.get_datetime_utc_now(),
status=action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.add_or_update(liveaction, publish=False)
executions.create_execution_object(liveaction, publish=False)
# Run the rescheduling routine.
scheduler.recover_delayed_executions()
# The live action is expected to stay "delayed".
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
def test_delayed_executions_recovery_before_timeout(self):
# Create a live action that's delayed but has not passed the timeout.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
start_timestamp=date_utils.get_datetime_utc_now(),
status=action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.add_or_update(liveaction, publish=False)
executions.create_execution_object(liveaction, publish=False)
# Run the rescheduling routine.
scheduler.recover_delayed_executions()
# The live action is expected to stay "delayed".
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
def test_dispatch(self):
runner_container = get_runner_container()
params = {'actionstr': 'bar'}
liveaction_db = self._get_action_exec_db_model(
RunnerContainerTest.action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran successfully.
runner_container.dispatch(liveaction_db)
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
result = liveaction_db.result
self.assertTrue(result.get('action_params').get('actionint') == 10)
self.assertTrue(result.get('action_params').get('actionstr') == 'bar')
# Assert that context is written correctly.
context = {'user': '******', 'third_party_system': {'ref_id': '1234'}}
self.assertDictEqual(liveaction_db.context, context)
def test_update_LiveAction_status_invalid(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
# Update by id.
self.assertRaises(ValueError, action_db_utils.update_liveaction_status,
status='mea culpa', liveaction_id=liveaction_db.id)
def test_state_db_creation_async_actions(self):
runner_container = get_runner_container()
params = {'actionstr': 'foo', 'actionint': 20, 'async_test': True}
liveaction_db = self._get_action_exec_db_model(
RunnerContainerTest.async_action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran without exceptions.
runner_container.dispatch(liveaction_db)
states = ActionExecutionState.get_all()
found = None
for state in states:
if state.execution_id == liveaction_db.id:
found = state
self.assertTrue(found is not None,
'There should be a state db object.')
self.assertTrue(found.query_context is not None)
self.assertTrue(found.query_module is not None)
def update_liveaction_status(status=None, result=None, context=None, end_timestamp=None,
liveaction_id=None, runner_info=None, liveaction_db=None):
"""
Update the status of the specified LiveAction to the value provided in
new_status.
The LiveAction may be specified using either liveaction_id, or as an
liveaction_db instance.
"""
if (liveaction_id is None) and (liveaction_db is None):
raise ValueError('Must specify an liveaction_id or an liveaction_db when '
'calling update_LiveAction_status')
if liveaction_db is None:
liveaction_db = get_liveaction_by_id(liveaction_id)
if status not in LIVEACTION_STATUSES:
raise ValueError('Attempting to set status for LiveAction "%s" '
'to unknown status string. Unknown status is "%s"',
liveaction_db, status)
LOG.debug('Updating ActionExection: "%s" with status="%s"', liveaction_db, status)
liveaction_db.status = status
if result:
liveaction_db.result = result
if context:
liveaction_db.context.update(context)
if end_timestamp:
liveaction_db.end_timestamp = end_timestamp
if runner_info:
liveaction_db.runner_info = runner_info
liveaction_db = LiveAction.add_or_update(liveaction_db)
LOG.debug('Updated status for LiveAction object: %s', liveaction_db)
return liveaction_db
def test_update_LiveAction_status_invalid(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
# Update by id.
self.assertRaises(ValueError,
action_db_utils.update_liveaction_status,
status='mea culpa',
liveaction_id=liveaction_db.id)
def test_delayed_executions_recovery(self):
# Create a live action that's already delayed pass the allowed timeout.
dt_now = date_utils.get_datetime_utc_now()
dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery)
dt_timeout = dt_now - dt_delta
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
start_timestamp=dt_timeout,
status=action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.add_or_update(liveaction, publish=False)
executions.create_execution_object(liveaction, publish=False)
# Run the rescheduling routine.
scheduler.recover_delayed_executions()
# The live action is expected to complete.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_delayed_executions_recovery(self):
# Create a live action that's already delayed pass the allowed timeout.
dt_now = date_utils.get_datetime_utc_now()
dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery)
dt_timeout = dt_now - dt_delta
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
start_timestamp=dt_timeout,
status=action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.add_or_update(liveaction, publish=False)
executions.create_execution_object(liveaction, publish=False)
# Run the rescheduling routine.
scheduler.recover_delayed_executions()
# The live action is expected to complete.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
def test_state_db_creation_async_actions(self):
runner_container = get_runner_container()
params = {
'actionstr': 'foo',
'actionint': 20,
'async_test': True
}
liveaction_db = self._get_action_exec_db_model(RunnerContainerTest.async_action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran without exceptions.
runner_container.dispatch(liveaction_db)
states = ActionExecutionState.get_all()
found = None
for state in states:
if state.execution_id == liveaction_db.id:
found = state
self.assertTrue(found is not None, 'There should be a state db object.')
self.assertTrue(found.query_context is not None)
self.assertTrue(found.query_module is not None)
def test_update_liveaction_status(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
origliveaction_db = copy.copy(liveaction_db)
# Update by id.
newliveaction_db = action_db_utils.update_liveaction_status(
status='running', liveaction_id=liveaction_db.id)
# Verify id didn't change.
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, 'running')
# Update status, result, context, and end timestamp.
now = datetime.datetime.utcnow()
status = 'succeeded'
result = 'Work is done.'
context = {'third_party_id': uuid.uuid4().hex}
newliveaction_db = action_db_utils.update_liveaction_status(
status=status,
result=result,
context=context,
end_timestamp=now,
liveaction_id=liveaction_db.id)
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, status)
self.assertEqual(newliveaction_db.result, result)
self.assertDictEqual(newliveaction_db.context, context)
self.assertEqual(newliveaction_db.end_timestamp, now)
def setup_action_models(cls):
action_db = ActionDB()
action_db.name = 'action-1'
action_db.description = 'awesomeness'
action_db.enabled = True
action_db.pack = 'wolfpack'
action_db.ref = ResourceReference(name=action_db.name,
pack=action_db.pack).ref
action_db.entry_point = ''
action_db.runner_type = {'name': 'test-runner'}
action_db.parameters = {
'actionstr': {
'type': 'string',
'position': 1,
'required': True
},
'actionint': {
'type': 'number',
'default': 10,
'position': 0
},
'runnerdummy': {
'type': 'string',
'default': 'actiondummy'
}
}
ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(
liveaction_db)
def test_execution_creation_action_triggered_by_rule(self):
# Wait for the action execution to complete and then confirm outcome.
trigger_type = self.MODELS['triggertypes']['triggertype2.json']
trigger = self.MODELS['triggers']['trigger2.json']
trigger_instance = self.MODELS['triggerinstances'][
'trigger_instance_1.json']
test_liveaction = self.FIXTURES['liveactions']['liveaction3.json']
rule = self.MODELS['rules']['rule3.json']
# Setup LiveAction to point to right rule and trigger_instance.
# XXX: We need support for dynamic fixtures.
test_liveaction['context']['rule']['id'] = str(rule.id)
test_liveaction['context']['trigger_instance']['id'] = str(
trigger_instance.id)
test_liveaction_api = LiveActionAPI(**test_liveaction)
test_liveaction = LiveAction.add_or_update(
LiveActionAPI.to_model(test_liveaction_api))
liveaction = LiveAction.get(
context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
self.assertEqual(liveaction.status, LIVEACTION_STATUS_SCHEDULED)
executions_util.create_execution_object(liveaction)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger,
vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type,
vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(
execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEquals(execution.liveaction['id'], str(liveaction.id))
def test_update_liveaction_status(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
origliveaction_db = copy.copy(liveaction_db)
# Update by id.
newliveaction_db = action_db_utils.update_liveaction_status(
status='running', liveaction_id=liveaction_db.id)
# Verify id didn't change.
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, 'running')
# Update status, result, context, and end timestamp.
now = datetime.datetime.utcnow()
status = 'succeeded'
result = 'Work is done.'
context = {'third_party_id': uuid.uuid4().hex}
newliveaction_db = action_db_utils.update_liveaction_status(
status=status, result=result, context=context, end_timestamp=now,
liveaction_id=liveaction_db.id)
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, status)
self.assertEqual(newliveaction_db.result, result)
self.assertDictEqual(newliveaction_db.context, context)
self.assertEqual(newliveaction_db.end_timestamp, now)
def test_dispatch(self):
runner_container = get_runner_container()
params = {
'actionstr': 'bar'
}
liveaction_db = self._get_action_exec_db_model(RunnerContainerTest.action_db, params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
# Assert that execution ran successfully.
runner_container.dispatch(liveaction_db)
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
result = liveaction_db.result
self.assertTrue(result.get('action_params').get('actionint') == 10)
self.assertTrue(result.get('action_params').get('actionstr') == 'bar')
# Assert that context is written correctly.
context = {
'user': '******',
'third_party_system': {
'ref_id': '1234'
}
}
self.assertDictEqual(liveaction_db.context, context)
def test_liveaction_crud_no_notify(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
self.assertEqual(retrieved.notify, None)
# Test update
self.assertTrue(retrieved.end_timestamp is None)
retrieved.end_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow())
updated = LiveAction.add_or_update(retrieved)
self.assertTrue(updated.end_timestamp == retrieved.end_timestamp)
# Test delete
LiveActionModelTest._delete([retrieved])
try:
retrieved = LiveAction.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def schedule(liveaction):
"""
Schedule an action to be run.
:return: (liveaction, execution)
:rtype: tuple
"""
# Use the user context from the parent action execution. Subtasks in a workflow
# action can be invoked by a system user and so we want to use the user context
# from the original workflow action.
if getattr(liveaction, 'context', None) and 'parent' in liveaction.context:
parent = LiveAction.get_by_id(liveaction.context['parent'])
liveaction.context['user'] = getattr(parent, 'context', dict()).get('user')
# Validate action.
action_db = action_utils.get_action_by_ref(liveaction.action)
if not action_db:
raise ValueError('Action "%s" cannot be found.' % liveaction.action)
if not action_db.enabled:
raise ValueError('Unable to execute. Action "%s" is disabled.' % liveaction.action)
runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])
if not hasattr(liveaction, 'parameters'):
liveaction.parameters = dict()
# Validate action parameters.
schema = util_schema.get_parameter_schema(action_db)
validator = util_schema.get_validator()
util_schema.validate(liveaction.parameters, schema, validator, use_default=True)
# validate that no immutable params are being overriden. Although possible to
# ignore the override it is safer to inform the user to avoid surprises.
immutables = _get_immutable_params(action_db.parameters)
immutables.extend(_get_immutable_params(runnertype_db.runner_parameters))
overridden_immutables = [p for p in six.iterkeys(liveaction.parameters) if p in immutables]
if len(overridden_immutables) > 0:
raise ValueError('Override of immutable parameter(s) %s is unsupported.'
% str(overridden_immutables))
# Set notification settings for action.
# XXX: There are cases when we don't want notifications to be sent for a particular
# execution. So we should look at liveaction.parameters['notify']
# and not set liveaction.notify.
if action_db.notify:
liveaction.notify = action_db.notify
else:
print(action_db)
# Write to database and send to message queue.
liveaction.status = LIVEACTION_STATUS_SCHEDULED
liveaction.start_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow())
# Publish creation after both liveaction and actionexecution are created.
liveaction = LiveAction.add_or_update(liveaction, publish=False)
execution = executions.create_execution_object(liveaction, publish=False)
# assume that this is a creation.
LiveAction.publish_create(liveaction)
ActionExecution.publish_create(execution)
extra = {'liveaction_db': liveaction, 'execution_db': execution}
LOG.audit('Action execution scheduled. LiveAction.id=%s, ActionExecution.id=%s' %
(liveaction.id, execution.id), extra=extra)
return liveaction, execution
def test_retry_on_timeout_policy_is_retried_twice(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[0].liveaction['id']
context = action_execution_dbs[1].context
self.assertTrue('policies' in context)
self.assertEqual(context['policies']['retry']['retry_count'], 1)
self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy')
self.assertEqual(context['policies']['retry']['retried_liveaction_id'],
original_liveaction_id)
# Simulate timeout of second action which should cause another retry
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
execution_db = action_execution_dbs[1]
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be three objects - original execution and 2 retried executions
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 3)
self.assertEqual(len(action_execution_dbs), 3)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[2].status, LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[1].liveaction['id']
context = action_execution_dbs[2].context
self.assertTrue('policies' in context)
self.assertEqual(context['policies']['retry']['retry_count'], 2)
self.assertEqual(context['policies']['retry']['applied_policy'], 'test_policy')
self.assertEqual(context['policies']['retry']['retried_liveaction_id'],
original_liveaction_id)
def _save_liveaction(liveaction):
return LiveAction.add_or_update(liveaction)
def _save_liveaction(liveaction):
return LiveAction.add_or_update(liveaction)
def test_retry_on_timeout_policy_is_retried_twice(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status,
LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[0].liveaction['id']
context = action_execution_dbs[1].context
self.assertTrue('policies' in context)
self.assertEqual(context['policies']['retry']['retry_count'], 1)
self.assertEqual(context['policies']['retry']['applied_policy'],
'test_policy')
self.assertEqual(context['policies']['retry']['retried_liveaction_id'],
original_liveaction_id)
# Simulate timeout of second action which should cause another retry
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
execution_db = action_execution_dbs[1]
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be three objects - original execution and 2 retried executions
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 3)
self.assertEqual(len(action_execution_dbs), 3)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[2].status,
LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[1].liveaction['id']
context = action_execution_dbs[2].context
self.assertTrue('policies' in context)
self.assertEqual(context['policies']['retry']['retry_count'], 2)
self.assertEqual(context['policies']['retry']['applied_policy'],
'test_policy')
self.assertEqual(context['policies']['retry']['retried_liveaction_id'],
original_liveaction_id)
