def test_cancel_delayed_execution_with_parent(self):
liveaction = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': uuid.uuid4().hex
}})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def test_over_threshold(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
self.assertGreater(policy_db.parameters["threshold"], 0)
self.assertIn("actionstr", policy_db.parameters["attributes"])
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
eventlet.spawn(action_service.request, liveaction)
# Sleep here to let the threads above schedule the action execution.
eventlet.sleep(1)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Sleep here to let the threads above complete the action execution.
eventlet.sleep(RUN_DELAY + 1)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_basic_execution(self):
liveaction = LiveActionDB(action='core.local',
parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.schedule(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, {})
self.assertDictEqual(execution.trigger_type, {})
self.assertDictEqual(execution.trigger_instance, {})
self.assertDictEqual(execution.rule, {})
action = action_utils.get_action_by_ref('core.local')
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_chained_executions(self):
liveaction = LiveActionDB(action='core.chain')
liveaction, _ = action_service.schedule(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref('core.chain')
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action['name'], 'local')
self.assertEqual(record.runner['name'], 'run-local')
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_chained_executions(self):
liveaction = LiveActionDB(action='core.chain')
liveaction, _ = action_service.schedule(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref('core.chain')
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action['name'], 'local')
self.assertEqual(record.runner['name'], 'run-local')
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_liveaction_crud_no_notify(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
self.assertEqual(retrieved.notify, None)
# Test update
self.assertTrue(retrieved.end_timestamp is None)
retrieved.end_timestamp = isotime.add_utc_tz(
datetime.datetime.utcnow())
updated = LiveAction.add_or_update(retrieved)
self.assertTrue(updated.end_timestamp == retrieved.end_timestamp)
# Test delete
LiveActionModelTest._delete([retrieved])
try:
retrieved = LiveAction.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_triggered_execution(self):
docs = {
'trigger_type':
copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger':
copy.deepcopy(fixture.ARTIFACTS['trigger']),
'rule':
copy.deepcopy(fixture.ARTIFACTS['rule']),
'trigger_instance':
copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])
}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type'])))
trigger = Trigger.add_or_update(
TriggerAPI.to_model(TriggerAPI(**docs['trigger'])))
rule = RuleAPI.to_model(RuleAPI(**docs['rule']))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(
TriggerInstanceAPI(**docs['trigger_instance'])))
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(
context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger,
vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type,
vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(
execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into cancel state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in [
action_constants.LIVEACTION_STATUS_CANCELING,
action_constants.LIVEACTION_STATUS_CANCELED]:
break
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into cancel state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in [
action_constants.LIVEACTION_STATUS_CANCELING,
action_constants.LIVEACTION_STATUS_CANCELED]:
break
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_bootstrap(self):
tracker = results_tracker.ResultsTracker()
tracker._bootstrap()
eventlet.sleep(0.2)
exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
tracker.shutdown()
def test_bootstrap(self):
tracker = results_tracker.ResultsTracker()
tracker._bootstrap()
eventlet.sleep(0.2)
exec_id = str(ResultsTrackerTests.states['state1.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
exec_id = str(ResultsTrackerTests.states['state2.yaml'].execution_id)
exec_db = LiveAction.get_by_id(exec_id)
self.assertTrue(exec_db.result['called_with'][exec_id] is not None,
exec_db.result)
tracker.shutdown()
def test_basic_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_basic_cancel(self):
runner_cls = self.get_runner_class('runner')
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None)
runner_cls.run = mock.Mock(return_value=runner_run_result)
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_noop_cancel(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_noop_cancel(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel")
self.assertEqual(policy_db.parameters["action"], "cancel")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(
action="wolfpack.action-2", parameters={"actionstr": "foo"}
)
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
expected_num_exec += 0 # This request will not be scheduled for execution.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workflow_mistral_offline(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def test_launch_workflow_with_many_workflows(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH)
execution = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_liveaction_create_with_notify_on_success_only(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
notify_db = NotificationSchema()
notify_sub_schema = NotificationSubSchema()
notify_sub_schema.message = 'Action succeeded.'
notify_sub_schema.data = {
'foo': 'bar',
'bar': 1,
'baz': {'k1': 'v1'}
}
notify_db.on_success = notify_sub_schema
created.notify = notify_db
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(notify_sub_schema.message,
retrieved.notify.on_success.message)
self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data)
self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels)
self.assertEqual(retrieved.notify.on_failure, None)
self.assertEqual(retrieved.notify.on_complete, None)
def _purge_action_models(execution_db):
liveaction_id = execution_db.liveaction['id']
if not liveaction_id:
print('Invalid LiveAction id. Skipping delete: %s', execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
print('LiveAction with id: %s not found. Skipping delete.',
liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except Exception as e:
print('Exception deleting Execution model: %s, exception: %s',
execution_db, str(e))
else:
try:
LiveAction.delete(liveaction_db)
except Exception as e:
print('Zombie LiveAction left in db: %s. Exception: %s',
liveaction_db, str(e))
def test_launch_workbook_with_many_workflows_no_default(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH)
execution = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_launch_workbook_with_many_workflows_no_default(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH)
execution = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_execution_creation_action_triggered_by_rule(self):
# Wait for the action execution to complete and then confirm outcome.
trigger_type = self.MODELS['triggertypes']['triggertype2.json']
trigger = self.MODELS['triggers']['trigger2.json']
trigger_instance = self.MODELS['triggerinstances']['trigger_instance_1.json']
test_liveaction = self.FIXTURES['liveactions']['liveaction3.json']
rule = self.MODELS['rules']['rule3.json']
# Setup LiveAction to point to right rule and trigger_instance.
# XXX: We need support for dynamic fixtures.
test_liveaction['context']['rule']['id'] = str(rule.id)
test_liveaction['context']['trigger_instance']['id'] = str(trigger_instance.id)
test_liveaction_api = LiveActionAPI(**test_liveaction)
test_liveaction = LiveAction.add_or_update(LiveActionAPI.to_model(test_liveaction_api))
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
self.assertEqual(liveaction.status, LIVEACTION_STATUS_SCHEDULED)
executions_util.create_execution_object(liveaction)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEquals(execution.liveaction['id'], str(liveaction.id))
def _purge_action_models(execution_db):
liveaction_id = execution_db.liveaction['id']
if not liveaction_id:
print('Invalid LiveAction id. Skipping delete: %s', execution_db)
liveaction_db = None
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
print('LiveAction with id: %s not found. Skipping delete.', liveaction_id)
else:
global DELETED_COUNT
DELETED_COUNT += 1
try:
ActionExecution.delete(execution_db)
except Exception as e:
print('Exception deleting Execution model: %s, exception: %s',
execution_db, str(e))
else:
try:
LiveAction.delete(liveaction_db)
except Exception as e:
print('Zombie LiveAction left in db: %s. Exception: %s',
liveaction_db, str(e))
def test_launch_workflow_with_many_workflows(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH)
execution = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_failed_cancel(self):
runner_run_result = (
action_constants.LIVEACTION_STATUS_RUNNING,
{
"data": "foobar"
},
None,
)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, "run",
mock_runner_run):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
runners.ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
def test_launch_workflow_mistral_offline(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_liveaction_create_with_notify_on_success_only(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
notify_db = NotificationSchema()
notify_sub_schema = NotificationSubSchema()
notify_sub_schema.message = 'Action succeeded.'
notify_sub_schema.data = {'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'}}
notify_db.on_success = notify_sub_schema
created.notify = notify_db
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(notify_sub_schema.message,
retrieved.notify.on_success.message)
self.assertDictEqual(notify_sub_schema.data,
retrieved.notify.on_success.data)
self.assertListEqual(notify_sub_schema.channels,
retrieved.notify.on_success.channels)
self.assertEqual(retrieved.notify.on_failure, None)
self.assertEqual(retrieved.notify.on_complete, None)
def test_enforce(self):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': 'abcde'
}})
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {'execution_id': 'abcde'}}
)
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref(
'wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_launch_workflow_with_auth(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ACTION_CONTEXT)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
workflow_input = copy.deepcopy(ACTION_PARAMS)
workflow_input.update({'count': '3'})
env = {
'__actions': {
'st2.action': {
'st2_context': {
'auth_token': TOKEN_DB.token,
'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
'parent': str(liveaction.id)
}
}
}
}
executions.ExecutionManager.create.assert_called_with(
WF1_NAME, workflow_input=workflow_input, env=env)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workbook_name_mismatch(self):
action_ref = 'generic.workbook_v2_name_mismatch'
MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH)
execution = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_launch_workbook_name_mismatch(self):
action_ref = 'generic.workbook_v2_name_mismatch'
MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH)
execution = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_apply(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
FakeConcurrencyApplicator.apply_before.assert_called_once_with(liveaction)
RaiseExceptionApplicator.apply_before.assert_called_once_with(liveaction)
FakeConcurrencyApplicator.apply_after.assert_called_once_with(liveaction)
RaiseExceptionApplicator.apply_after.assert_called_once_with(liveaction)
def delete(self, exec_id):
"""
Stops a single execution.
Handles requests:
DELETE /actionexecutions/<id>
"""
execution_api = self._get_one(id=exec_id)
if not execution_api:
abort(http_client.NOT_FOUND,
'Execution with id %s not found.' % exec_id)
return
liveaction_id = execution_api.liveaction['id']
if not liveaction_id:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
try:
liveaction_db = LiveAction.get_by_id(liveaction_id)
except:
abort(
http_client.INTERNAL_SERVER_ERROR,
'Execution object missing link to liveaction %s.' %
liveaction_id)
return
if liveaction_db.status == LIVEACTION_STATUS_CANCELED:
abort(http_client.OK, 'Action is already in "canceled" state.')
if liveaction_db.status not in CANCELABLE_STATES:
abort(
http_client.OK, 'Action cannot be canceled. State = %s.' %
liveaction_db.status)
return
liveaction_db.status = 'canceled'
liveaction_db.end_timestamp = isotime.add_utc_tz(
datetime.datetime.utcnow())
liveaction_db.result = {'message': 'Action canceled by user.'}
try:
LiveAction.add_or_update(liveaction_db)
except:
LOG.exception(
'Failed updating status to canceled for liveaction %s.',
liveaction_db.id)
abort(http_client.INTERNAL_SERVER_ERROR,
'Failed canceling execution.')
return
execution_db = execution_service.update_execution(liveaction_db)
return ActionExecutionAPI.from_model(execution_db)
def test_triggered_execution(self):
docs = {
'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type'])))
trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger'])))
rule = RuleAPI.to_model(RuleAPI(**docs['rule']))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance'])))
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_launch_when_workbook_not_exists(self):
execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def test_launch_when_workbook_not_exists(self):
execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_callback(self):
execution = LiveActionDB(
action='core.local', parameters={'cmd': 'uname -a'},
callback={'source': 'mistral', 'url': 'http://localhost:8989/v2/tasks/12345'})
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_SUCCEEDED)
requests.request.assert_called_with('PUT', liveaction.callback['url'],
data=json.dumps({'state': 'SUCCESS',
'result': NON_EMPTY_RESULT}),
headers={'content-type': 'application/json'})
def test_launch_when_workbook_definition_changed(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH)
execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def _update_action_results(self, execution_id, status, results):
liveaction_db = LiveAction.get_by_id(execution_id)
if not liveaction_db:
raise Exception('No DB model for liveaction_id: %s' % execution_id)
liveaction_db.result = results
liveaction_db.status = status
# update liveaction, update actionexecution and then publish update.
updated_liveaction = LiveAction.add_or_update(liveaction_db, publish=False)
executions.update_execution(updated_liveaction)
LiveAction.publish_update(updated_liveaction)
return updated_liveaction
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request will be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_dispatch_runner_failure(self):
runner_container = get_runner_container()
params = {
'actionstr': 'bar'
}
liveaction_db = self._get_failingaction_exec_db_model(params)
liveaction_db = LiveAction.add_or_update(liveaction_db)
executions.create_execution_object(liveaction_db)
runner_container.dispatch(liveaction_db)
# pickup updated liveaction_db
liveaction_db = LiveAction.get_by_id(liveaction_db.id)
self.assertTrue('message' in liveaction_db.result)
self.assertTrue('traceback' in liveaction_db.result)
