PACK = 'generic'
LOADER = FixturesLoader()
FIXTURES = LOADER.load_fixtures(fixtures_pack=PACK,
fixtures_dict=TEST_FIXTURES)
@mock.patch.object(
CUDPublisher, 'publish_update',
mock.MagicMock(side_effect=MockExecutionPublisher.publish_update))
@mock.patch.object(CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state))
@mock.patch('st2common.runners.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
@mock.patch('st2actions.container.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
class SchedulingPolicyTest(ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
super(SchedulingPolicyTest, cls).setUpClass()
# Register runners
runners_registrar.register_runners()
for _, fixture in six.iteritems(FIXTURES['actions']):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policytypes']):
PACK = 'generic'
LOADER = FixturesLoader()
FIXTURES = LOADER.load_fixtures(fixtures_pack=PACK, fixtures_dict=TEST_FIXTURES)
@mock.patch.object(
CUDPublisher, 'publish_update',
mock.MagicMock(side_effect=MockExecutionPublisher.publish_update))
@mock.patch.object(
CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state))
@mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner()))
@mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner()))
class SchedulingPolicyTest(ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
super(SchedulingPolicyTest, cls).setUpClass()
# Register runners
runners_registrar.register_runners()
for _, fixture in six.iteritems(FIXTURES['actions']):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policytypes']):
class ConcurrencyPolicyTestCase(EventletTestCase, ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
EventletTestCase.setUpClass()
DbTestCase.setUpClass()
# Override the coordinator to use the noop driver otherwise the tests will be blocked.
tests_config.parse_args(coordinator_noop=True)
coordination.COORDINATOR = None
# Register runners
runners_registrar.register_runners()
# Register common policy types
register_policy_types(st2common)
loader = FixturesLoader()
loader.save_fixtures_to_db(fixtures_pack=PACK,
fixtures_dict=TEST_FIXTURES)
@classmethod
def tearDownClass(cls):
# Reset the coordinator.
coordination.coordinator_teardown(coordination.COORDINATOR)
coordination.COORDINATOR = None
super(ConcurrencyPolicyTestCase, cls).tearDownClass()
# NOTE: This monkey patch needs to happen again here because during tests for some reason this
# method gets unpatched (test doing reload() or similar)
@mock.patch(
"st2actions.container.base.get_runner",
mock.Mock(return_value=runner.get_runner()),
)
def tearDown(self):
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
@staticmethod
def _process_scheduling_queue():
for queued_req in ActionExecutionSchedulingQueue.get_all():
scheduling_queue.get_handler()._handle_execution(queued_req)
@mock.patch.object(
runner.MockActionRunner,
"run",
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE),
)
@mock.patch.object(
LiveActionPublisher,
"publish_state",
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state),
)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo-last"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as scheduled.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Check the status changes.
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
expected_status_changes = [
"requested",
"delayed",
"requested",
"scheduled",
"running",
]
actual_status_changes = [entry["status"] for entry in execution.log]
self.assertListEqual(actual_status_changes, expected_status_changes)
@mock.patch.object(
runner.MockActionRunner,
"run",
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE),
)
@mock.patch.object(
LiveActionPublisher,
"publish_state",
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state),
)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel")
self.assertEqual(policy_db.parameters["action"], "cancel")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-2",
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-2",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
expected_num_exec += 0 # This request will not be scheduled for execution.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
@mock.patch.object(
runner.MockActionRunner,
"run",
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE),
)
@mock.patch.object(
LiveActionPublisher,
"publish_state",
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state),
)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], "stanley")
expected_num_pubs += 2 # Tally the canceling and canceled states.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
'policy_3.yaml',
'policy_7.yaml'
]
}
NON_EMPTY_RESULT = 'non-empty'
MOCK_RUN_RETURN_VALUE = (action_constants.LIVEACTION_STATUS_RUNNING, NON_EMPTY_RESULT, None)
SCHEDULED_STATES = [
action_constants.LIVEACTION_STATUS_SCHEDULED,
action_constants.LIVEACTION_STATUS_RUNNING,
action_constants.LIVEACTION_STATUS_SUCCEEDED
]
@mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner()))
@mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner()))
@mock.patch.object(
CUDPublisher, 'publish_update',
mock.MagicMock(side_effect=MockExecutionPublisher.publish_update))
@mock.patch.object(
CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
class ConcurrencyByAttributePolicyTestCase(EventletTestCase, ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
EventletTestCase.setUpClass()
ExecutionDbTestCase.setUpClass()
# Override the coordinator to use the noop driver otherwise the tests will be blocked.
NON_EMPTY_RESULT = "non-empty"
MOCK_RUN_RETURN_VALUE = (
action_constants.LIVEACTION_STATUS_RUNNING,
NON_EMPTY_RESULT,
None,
)
SCHEDULED_STATES = [
action_constants.LIVEACTION_STATUS_SCHEDULED,
action_constants.LIVEACTION_STATUS_RUNNING,
action_constants.LIVEACTION_STATUS_SUCCEEDED,
]
@mock.patch(
"st2common.runners.base.get_runner", mock.Mock(return_value=runner.get_runner())
)
@mock.patch(
"st2actions.container.base.get_runner", mock.Mock(return_value=runner.get_runner())
)
@mock.patch.object(
CUDPublisher,
"publish_update",
mock.MagicMock(side_effect=MockExecutionPublisher.publish_update),
)
@mock.patch.object(CUDPublisher, "publish_create", mock.MagicMock(return_value=None))
class ConcurrencyByAttributePolicyTestCase(EventletTestCase, ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
EventletTestCase.setUpClass()
ExecutionDbTestCase.setUpClass()
class ExecutionCancellationTestCase(ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
super(ExecutionCancellationTestCase, cls).setUpClass()
for _, fixture in six.iteritems(FIXTURES['actions']):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
runners_registrar.register_runners()
def tearDown(self):
# Ensure all liveactions are canceled at end of each test.
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
@classmethod
def get_runner_class(cls, runner_name):
return runners.get_runner(runner_name).__class__
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(
side_effect=MockLiveActionPublisherNonBlocking.publish_state))
@mock.patch('st2common.runners.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
@mock.patch('st2actions.container.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
def test_basic_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING,
'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run',
mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
@mock.patch.object(
CUDPublisher, 'publish_create',
mock.MagicMock(side_effect=MockLiveActionPublisher.publish_create))
@mock.patch.object(
CUDPublisher, 'publish_update',
mock.MagicMock(side_effect=MockExecutionPublisher.publish_update))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state))
@mock.patch.object(
runners.ActionRunner, 'cancel',
mock.MagicMock(side_effect=Exception('Mock cancellation failure.')))
@mock.patch('st2common.runners.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
@mock.patch('st2actions.container.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
def test_failed_cancel(self):
runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING,
'foobar', None)
mock_runner_run = mock.Mock(return_value=runner_run_result)
with mock.patch.object(runner.MockActionRunner, 'run',
mock_runner_run):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancellation failed and execution state remains "canceling".
runners.ActionRunner.cancel.assert_called_once_with()
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
@mock.patch.object(CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
@mock.patch.object(LiveActionPublisher, 'publish_state',
mock.MagicMock(return_value=None))
@mock.patch.object(runners.ActionRunner, 'cancel',
mock.MagicMock(return_value=None))
def test_noop_cancel(self):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
@mock.patch.object(CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
@mock.patch.object(LiveActionPublisher, 'publish_state',
mock.MagicMock(return_value=None))
@mock.patch.object(runners.ActionRunner, 'cancel',
mock.MagicMock(return_value=None))
def test_cancel_delayed_execution(self):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
self.assertFalse(runners.ActionRunner.cancel.called)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
@mock.patch.object(CUDPublisher, 'publish_create',
mock.MagicMock(return_value=None))
@mock.patch.object(LiveActionPublisher, 'publish_state',
mock.MagicMock(return_value=None))
@mock.patch.object(trace_service, 'get_trace_db_by_live_action',
mock.MagicMock(return_value=(None, None)))
def test_cancel_delayed_execution_with_parent(self):
liveaction = LiveActionDB(
action='wolfpack.action-1',
parameters={'actionstr': 'foo'},
context={'parent': {
'execution_id': uuid.uuid4().hex
}})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_REQUESTED)
# Manually update the liveaction from requested to delayed to mock concurrency policy.
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(liveaction,
cfg.CONF.system_user.user)
# Cancel is only called when liveaction is still in running state.
# Otherwise, the cancellation is only a state change.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELING)
class ConcurrencyByAttributePolicyTestCase(EventletTestCase,
ExecutionDbTestCase):
@classmethod
def setUpClass(cls):
EventletTestCase.setUpClass()
ExecutionDbTestCase.setUpClass()
# Override the coordinator to use the noop driver otherwise the tests will be blocked.
tests_config.parse_args(coordinator_noop=True)
coordination.COORDINATOR = None
# Register runners
runners_registrar.register_runners()
# Register common policy types
register_policy_types(st2common)
loader = FixturesLoader()
loader.save_fixtures_to_db(fixtures_pack=PACK,
fixtures_dict=TEST_FIXTURES)
@classmethod
def tearDownClass(cls):
# Reset the coordinator.
coordination.COORDINATOR = None
super(ConcurrencyByAttributePolicyTestCase, cls).tearDownClass()
# NOTE: This monkey patch needs to happen again here because during tests for some reason this
# method gets unpatched (test doing reload() or similar)
@mock.patch('st2actions.container.base.get_runner',
mock.Mock(return_value=runner.get_runner()))
def tearDown(self):
for liveaction in LiveAction.get_all():
action_service.update_status(
liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
@staticmethod
def _process_scheduling_queue():
for queued_req in ActionExecutionSchedulingQueue.get_all():
scheduling_queue.get_handler()._handle_execution(queued_req)
@mock.patch.object(runner.MockActionRunner, 'run',
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state))
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
# Assert the action is delayed.
delayed = liveaction
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
@mock.patch.object(runner.MockActionRunner, 'run',
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state))
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref(
'wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status,
action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
@mock.patch.object(runner.MockActionRunner, 'run',
mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE))
@mock.patch.object(
LiveActionPublisher, 'publish_state',
mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly.
publish_state))
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
# Assert the action is delayed.
delayed = liveaction
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = LiveAction.get_by_id(str(delayed.id))
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)