def test_disabled_policy_not_applied_on_pre_run(self):
##########
# First test a scenario where policy is enabled
##########
self.assertTrue(self.policy_db.enabled)
# Post run hasn't been called yet, call count should be 0
self.assertEqual(policies.get_driver.call_count, 0)
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
policy_service.apply_pre_run_policies(live_action_db)
# Ony policy has been applied so call count should be 1
self.assertEqual(policies.get_driver.call_count, 1)
##########
# Now a scenaro with disabled policy
##########
policies.get_driver.call_count = 0
self.policy_db.enabled = False
self.policy_db = Policy.add_or_update(self.policy_db)
self.assertFalse(self.policy_db.enabled)
self.assertEqual(policies.get_driver.call_count, 0)
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
policy_service.apply_pre_run_policies(live_action_db)
# Policy is disabled so call_count should stay the same as before as no policies have been
# applied
self.assertEqual(policies.get_driver.call_count, 0)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _run_action(self,
action_node,
parent_execution_id,
params,
wait_for_completion=True):
liveaction = LiveActionDB(action=action_node.ref)
liveaction.parameters = action_param_utils.cast_params(
action_ref=action_node.ref, params=params)
if action_node.notify:
liveaction.notify = NotificationsHelper.to_model(
action_node.notify)
liveaction.context = {
'parent': str(parent_execution_id),
'chain': vars(action_node)
}
liveaction, _ = action_service.schedule(liveaction)
while (wait_for_completion
and liveaction.status != LIVEACTION_STATUS_SUCCEEDED
and liveaction.status != LIVEACTION_STATUS_FAILED):
eventlet.sleep(1)
liveaction = action_db_util.get_liveaction_by_id(liveaction.id)
return liveaction
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref(
'wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def setup_action_models(cls):
action_db = ActionDB()
action_db.name = 'action-1'
action_db.description = 'awesomeness'
action_db.enabled = True
action_db.pack = 'wolfpack'
action_db.ref = ResourceReference(name=action_db.name, pack=action_db.pack).ref
action_db.entry_point = ''
action_db.runner_type = {'name': 'test-runner'}
action_db.parameters = {
'actionstr': {'type': 'string', 'position': 1, 'required': True},
'actionint': {'type': 'number', 'default': 10, 'position': 0},
'runnerdummy': {'type': 'string', 'default': 'actiondummy'}
}
ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel")
self.assertEqual(policy_db.parameters["action"], "cancel")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(
action="wolfpack.action-2", parameters={"actionstr": "foo"}
)
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
expected_num_exec += 0 # This request will not be scheduled for execution.
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(
expected_num_pubs, LiveActionPublisher.publish_state.call_count
)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': 'abcde'
}})
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_workflow_task(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
live_action_db = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {'execution_id': 'abcde'}}
)
live_action_db, execution_db = action_service.request(live_action_db)
live_action_db = LiveAction.get_by_id(str(live_action_db.id))
self.assertEqual(live_action_db.status, LIVEACTION_STATUS_REQUESTED)
# Expire the workflow instance.
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since live action is under the context of a workflow.
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status, LIVEACTION_STATUS_TIMED_OUT)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def _get_action_exec_db_model(self, params):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(name=ParamsUtilsTest.action_db.name,
pack=ParamsUtilsTest.action_db.pack).ref
liveaction_db.parameters = params
return liveaction_db
def _get_execution_db_model(self, status=action_constants.LIVEACTION_STATUS_SCHEDULED):
live_action_db = LiveActionDB()
live_action_db.status = status
live_action_db.start_timestamp = datetime.datetime.utcnow()
live_action_db.action = ResourceReference(
name='test_action',
pack='test_pack').ref
live_action_db.parameters = None
return action.LiveAction.add_or_update(live_action_db, publish=False)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into cancel state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in [
action_constants.LIVEACTION_STATUS_CANCELING,
action_constants.LIVEACTION_STATUS_CANCELED]:
break
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_launch_workflow_with_many_workflows(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF2_YAML_FILE_PATH)
execution = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Multiple workflows is not supported.', liveaction.result['message'])
def test_launch_workflow_mistral_offline(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WF1_YAML_FILE_PATH)
execution = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Failed to connect to mistral', liveaction.result['message'])
def _run_action(action_node, parent_execution_id, params, wait_for_completion=True):
execution = LiveActionDB(action=action_node.ref)
execution.parameters = action_param_utils.cast_params(action_ref=action_node.ref,
params=params)
execution.context = {
'parent': str(parent_execution_id),
'chain': vars(action_node)
}
liveaction, _ = action_service.schedule(execution)
while (wait_for_completion and
liveaction.status != LIVEACTION_STATUS_SUCCEEDED and
liveaction.status != LIVEACTION_STATUS_FAILED):
eventlet.sleep(1)
liveaction = action_db_util.get_liveaction_by_id(liveaction.id)
return liveaction
def test_launch_workbook_with_many_workflows_no_default(self):
MistralRunner.entry_point = mock.PropertyMock(return_value=WB3_YAML_FILE_PATH)
execution = LiveActionDB(action=WB3_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Default workflow cannot be determined.', liveaction.result['message'])
def test_basic_execution(self):
liveaction = LiveActionDB(action='core.local',
parameters={'cmd': 'uname -a'})
liveaction, _ = action_service.schedule(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger, {})
self.assertDictEqual(execution.trigger_type, {})
self.assertDictEqual(execution.trigger_instance, {})
self.assertDictEqual(execution.rule, {})
action = action_utils.get_action_by_ref('core.local')
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def test_chained_executions(self):
liveaction = LiveActionDB(action='core.chain')
liveaction, _ = action_service.schedule(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
action = action_utils.get_action_by_ref('core.chain')
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
self.assertGreater(len(execution.children), 0)
for child in execution.children:
record = ActionExecution.get(id=child, raise_exception=True)
self.assertEqual(record.parent, str(execution.id))
self.assertEqual(record.action['name'], 'local')
self.assertEqual(record.runner['name'], 'run-local')
def test_retry_on_timeout_max_retries_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
live_action_db.context['policies'] = {}
live_action_db.context['policies']['retry'] = {'retry_count': 2}
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# Note: There should be no new objects since max retries has been reached
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
def test_no_retry_on_non_applicable_statuses(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action in various statuses in which we shouldn't retry
non_retry_statuses = [
LIVEACTION_STATUS_REQUESTED,
LIVEACTION_STATUS_SCHEDULED,
LIVEACTION_STATUS_DELAYED,
LIVEACTION_STATUS_CANCELING,
LIVEACTION_STATUS_CANCELED,
]
action_ref = 'wolfpack.action-1'
for status in non_retry_statuses:
liveaction = LiveActionDB(action=action_ref,
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = status
execution_db.status = status
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# None of the actions should have been retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), len(non_retry_statuses))
self.assertEqual(len(action_execution_dbs), len(non_retry_statuses))
def test_enforce(self):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def test_retry_on_timeout_no_retry_since_no_timeout_reached(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which succeeds
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
execution_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should only be 1 object since the action didn't timeout and therefore it wasn't
# retried
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 1)
self.assertEqual(len(action_execution_dbs), 1)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_SUCCEEDED)
def test_launch_workbook_name_mismatch(self):
action_ref = 'generic.workbook_v2_name_mismatch'
MistralRunner.entry_point = mock.PropertyMock(return_value=WB1_YAML_FILE_PATH)
execution = LiveActionDB(action=action_ref, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_FAILED)
self.assertIn('Name of the workbook must be the same', liveaction.result['message'])
def test_enforce(self):
liveaction = LiveActionDB(
action="wolfpack.action-1", parameters={"actionstr": "foo"}
)
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED
)
def test_schedule_disabled_action(self):
self.actiondb.enabled = False
Action.add_or_update(self.actiondb)
parameters = {'hosts': 'localhost', 'cmd': 'uname -a'}
execution = LiveActionDB(action=ACTION_REF, parameters=parameters)
self.assertRaises(ValueError, action_service.schedule, execution)
self.actiondb.enabled = True
Action.add_or_update(self.actiondb)
def test_retry_on_timeout_first_retry_is_successful(self):
# Verify initial state
self.assertSequenceEqual(LiveAction.get_all(), [])
self.assertSequenceEqual(ActionExecution.get_all(), [])
# Start a mock action which times out
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
live_action_db, execution_db = action_service.request(liveaction)
live_action_db.status = LIVEACTION_STATUS_TIMED_OUT
execution_db.status = LIVEACTION_STATUS_TIMED_OUT
LiveAction.add_or_update(live_action_db)
ActionExecution.add_or_update(execution_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be two objects - original execution and retried execution
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(action_execution_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(action_execution_dbs[1].status,
LIVEACTION_STATUS_REQUESTED)
# Verify retried execution contains policy related context
original_liveaction_id = action_execution_dbs[0].liveaction["id"]
context = action_execution_dbs[1].context
self.assertIn("policies", context)
self.assertEqual(context["policies"]["retry"]["retry_count"], 1)
self.assertEqual(context["policies"]["retry"]["applied_policy"],
"test_policy")
self.assertEqual(
context["policies"]["retry"]["retried_liveaction_id"],
original_liveaction_id,
)
# Simulate success of second action so no it shouldn't be retried anymore
live_action_db = live_action_dbs[1]
live_action_db.status = LIVEACTION_STATUS_SUCCEEDED
LiveAction.add_or_update(live_action_db)
# Simulate policy "apply_after" run
self.policy.apply_after(target=live_action_db)
# There should be no new object since action succeeds so no retry was attempted
live_action_dbs = LiveAction.get_all()
action_execution_dbs = ActionExecution.get_all()
self.assertEqual(len(live_action_dbs), 2)
self.assertEqual(len(action_execution_dbs), 2)
self.assertEqual(live_action_dbs[0].status,
LIVEACTION_STATUS_TIMED_OUT)
self.assertEqual(live_action_dbs[1].status,
LIVEACTION_STATUS_SUCCEEDED)
def test_liveaction_create_with_notify_on_success_only(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
notify_db = NotificationSchema()
notify_sub_schema = NotificationSubSchema()
notify_sub_schema.message = 'Action succeeded.'
notify_sub_schema.data = {'foo': 'bar', 'bar': 1, 'baz': {'k1': 'v1'}}
notify_db.on_success = notify_sub_schema
created.notify = notify_db
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(notify_sub_schema.message,
retrieved.notify.on_success.message)
self.assertDictEqual(notify_sub_schema.data,
retrieved.notify.on_success.data)
self.assertListEqual(notify_sub_schema.channels,
retrieved.notify.on_success.channels)
self.assertEqual(retrieved.notify.on_failure, None)
self.assertEqual(retrieved.notify.on_complete, None)
def test_apply(self):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
FakeConcurrencyApplicator.apply_before.assert_called_once_with(liveaction)
RaiseExceptionApplicator.apply_before.assert_called_once_with(liveaction)
FakeConcurrencyApplicator.apply_after.assert_called_once_with(liveaction)
RaiseExceptionApplicator.apply_after.assert_called_once_with(liveaction)
def _run_action(action_node,
parent_execution_id,
params,
wait_for_completion=True):
execution = LiveActionDB(action=action_node.ref)
execution.parameters = action_param_utils.cast_params(
action_ref=action_node.ref, params=params)
execution.context = {
'parent': str(parent_execution_id),
'chain': vars(action_node)
}
execution, _ = action_service.schedule(execution)
while (wait_for_completion
and execution.status != LIVEACTION_STATUS_SUCCEEDED
and execution.status != LIVEACTION_STATUS_FAILED):
eventlet.sleep(1)
execution = action_db_util.get_liveaction_by_id(execution.id)
return execution
def test_update_LiveAction_status_invalid(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
# Update by id.
self.assertRaises(ValueError, action_db_utils.update_liveaction_status,
status='mea culpa', liveaction_id=liveaction_db.id)
def test_launch_when_workbook_not_exists(self):
execution = LiveActionDB(action=WB1_NAME, parameters=ACTION_PARAMS)
liveaction, _ = action_service.schedule(execution)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, LIVEACTION_STATUS_RUNNING)
mistral_context = liveaction.context.get('mistral', None)
self.assertIsNotNone(mistral_context)
self.assertEqual(mistral_context['execution_id'], WB1_EXEC.get('id'))
self.assertEqual(mistral_context['workflow_name'], WB1_EXEC.get('workflow_name'))
def _get_action_exec_db_model(self, action_db, params):
liveaction_db = LiveActionDB()
liveaction_db.status = action_constants.LIVEACTION_STATUS_SCHEDULED
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(name=action_db.name,
pack=action_db.pack).ref
liveaction_db.parameters = params
liveaction_db.context = {'user': cfg.CONF.system_user.user}
return liveaction_db
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_liveaction_create_with_notify_on_success_only(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
notify_db = NotificationSchema()
notify_sub_schema = NotificationSubSchema()
notify_sub_schema.message = 'Action succeeded.'
notify_sub_schema.data = {
'foo': 'bar',
'bar': 1,
'baz': {'k1': 'v1'}
}
notify_db.on_success = notify_sub_schema
created.notify = notify_db
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(notify_sub_schema.message,
retrieved.notify.on_success.message)
self.assertDictEqual(notify_sub_schema.data, retrieved.notify.on_success.data)
self.assertListEqual(notify_sub_schema.channels, retrieved.notify.on_success.channels)
self.assertEqual(retrieved.notify.on_failure, None)
self.assertEqual(retrieved.notify.on_complete, None)
def test_update_liveaction_status(self):
liveaction_db = LiveActionDB()
liveaction_db.status = 'initializing'
liveaction_db.start_timestamp = datetime.datetime.utcnow()
liveaction_db.action = ResourceReference(
name=ActionDBUtilsTestCase.action_db.name,
pack=ActionDBUtilsTestCase.action_db.pack).ref
params = {
'actionstr': 'foo',
'some_key_that_aint_exist_in_action_or_runner': 'bar',
'runnerint': 555
}
liveaction_db.parameters = params
liveaction_db = LiveAction.add_or_update(liveaction_db)
origliveaction_db = copy.copy(liveaction_db)
# Update by id.
newliveaction_db = action_db_utils.update_liveaction_status(
status='running', liveaction_id=liveaction_db.id)
# Verify id didn't change.
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, 'running')
# Update status, result, context, and end timestamp.
now = datetime.datetime.utcnow()
status = 'succeeded'
result = 'Work is done.'
context = {'third_party_id': uuid.uuid4().hex}
newliveaction_db = action_db_utils.update_liveaction_status(
status=status, result=result, context=context, end_timestamp=now,
liveaction_id=liveaction_db.id)
self.assertEqual(origliveaction_db.id, newliveaction_db.id)
self.assertEqual(newliveaction_db.status, status)
self.assertEqual(newliveaction_db.result, result)
self.assertDictEqual(newliveaction_db.context, context)
self.assertEqual(newliveaction_db.end_timestamp, now)
def test_liveaction_crud_no_notify(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
self.assertEqual(retrieved.notify, None)
# Test update
self.assertTrue(retrieved.end_timestamp is None)
retrieved.end_timestamp = isotime.add_utc_tz(datetime.datetime.utcnow())
updated = LiveAction.add_or_update(retrieved)
self.assertTrue(updated.end_timestamp == retrieved.end_timestamp)
# Test delete
LiveActionModelTest._delete([retrieved])
try:
retrieved = LiveAction.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def _get_failingaction_exec_db_model(self, params):
liveaction_db = LiveActionDB()
liveaction_db.status = action_constants.LIVEACTION_STATUS_SCHEDULED
liveaction_db.start_timestamp = datetime.datetime.now()
liveaction_db.action = ResourceReference(
name=RunnerContainerTest.failingaction_db.name,
pack=RunnerContainerTest.failingaction_db.pack).ref
liveaction_db.parameters = params
liveaction_db.context = {'user': cfg.CONF.system_user.user}
return liveaction_db
def test_notify_triggers(self):
liveaction = LiveActionDB(action='core.local')
liveaction.description = ''
liveaction.status = 'succeeded'
liveaction.parameters = {}
on_success = NotificationSubSchema(message='Action succeeded.')
on_failure = NotificationSubSchema(message='Action failed.')
liveaction.notify = NotificationSchema(on_success=on_success,
on_failure=on_failure)
liveaction.start_timestamp = datetime.datetime.utcnow()
dispatcher = NotifierTestCase.MockDispatcher(self)
notifier = Notifier(q_connection=None,
trigger_dispatcher=dispatcher)
notifier.handle_action_complete(liveaction)
def test_liveaction_create_with_notify_both_on_success_and_on_error(self):
created = LiveActionDB()
created.action = 'core.local'
created.description = ''
created.status = 'running'
created.parameters = {}
on_success = NotificationSubSchema(message='Action succeeded.')
on_failure = NotificationSubSchema(message='Action failed.')
created.notify = NotificationSchema(on_success=on_success,
on_failure=on_failure)
saved = LiveActionModelTest._save_liveaction(created)
retrieved = LiveAction.get_by_id(saved.id)
self.assertEqual(saved.action, retrieved.action,
'Same triggertype was not returned.')
# Assert notify settings saved are right.
self.assertEqual(on_success.message, retrieved.notify.on_success.message)
self.assertEqual(on_failure.message, retrieved.notify.on_failure.message)
self.assertEqual(retrieved.notify.on_complete, None)
FIXTURES_1 = {
'runners': ['testrunner1.yaml', 'testrunner2.yaml'],
'actions': ['action1.yaml', 'a2.yaml'],
'triggertypes': ['triggertype1.yaml'],
'triggers': ['trigger1.yaml']
}
FIXTURES_2 = {
'rules': ['rule1.yaml', 'rule2.yaml']
}
MOCK_TRIGGER_INSTANCE = TriggerInstanceDB()
MOCK_TRIGGER_INSTANCE.id = 'triggerinstance-test'
MOCK_TRIGGER_INSTANCE.payload = {'t1_p': 't1_p_v'}
MOCK_TRIGGER_INSTANCE.occurrence_time = datetime.datetime.utcnow()
MOCK_LIVEACTION = LiveActionDB()
MOCK_LIVEACTION.id = 'liveaction-test-1.id'
MOCK_LIVEACTION.name = 'liveaction-test-1.name'
MOCK_LIVEACTION.status = 'scheduled'
class EnforceTest(DbTestCase):
models = None
@classmethod
def setUpClass(cls):
super(EnforceTest, cls).setUpClass()
# Create TriggerTypes before creation of Rule to avoid failure. Rule requires the
# Trigger and therefore TriggerType to be created prior to rule creation.
cls.models = FixturesLoader().save_fixtures_to_db(
