PACK = 'generic' LOADER = FixturesLoader() FIXTURES = LOADER.load_fixtures(fixtures_pack=PACK, fixtures_dict=TEST_FIXTURES) @mock.patch.object( CUDPublisher, 'publish_update', mock.MagicMock(side_effect=MockExecutionPublisher.publish_update)) @mock.patch.object(CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state)) @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) class SchedulingPolicyTest(ExecutionDbTestCase): @classmethod def setUpClass(cls): super(SchedulingPolicyTest, cls).setUpClass() # Register runners runners_registrar.register_runners() for _, fixture in six.iteritems(FIXTURES['actions']): instance = ActionAPI(**fixture) Action.add_or_update(ActionAPI.to_model(instance)) for _, fixture in six.iteritems(FIXTURES['policytypes']):
PACK = 'generic' LOADER = FixturesLoader() FIXTURES = LOADER.load_fixtures(fixtures_pack=PACK, fixtures_dict=TEST_FIXTURES) @mock.patch.object( CUDPublisher, 'publish_update', mock.MagicMock(side_effect=MockExecutionPublisher.publish_update)) @mock.patch.object( CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state)) @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) class SchedulingPolicyTest(ExecutionDbTestCase): @classmethod def setUpClass(cls): super(SchedulingPolicyTest, cls).setUpClass() # Register runners runners_registrar.register_runners() for _, fixture in six.iteritems(FIXTURES['actions']): instance = ActionAPI(**fixture) Action.add_or_update(ActionAPI.to_model(instance)) for _, fixture in six.iteritems(FIXTURES['policytypes']):
class ConcurrencyPolicyTestCase(EventletTestCase, ExecutionDbTestCase): @classmethod def setUpClass(cls): EventletTestCase.setUpClass() DbTestCase.setUpClass() # Override the coordinator to use the noop driver otherwise the tests will be blocked. tests_config.parse_args(coordinator_noop=True) coordination.COORDINATOR = None # Register runners runners_registrar.register_runners() # Register common policy types register_policy_types(st2common) loader = FixturesLoader() loader.save_fixtures_to_db(fixtures_pack=PACK, fixtures_dict=TEST_FIXTURES) @classmethod def tearDownClass(cls): # Reset the coordinator. coordination.coordinator_teardown(coordination.COORDINATOR) coordination.COORDINATOR = None super(ConcurrencyPolicyTestCase, cls).tearDownClass() # NOTE: This monkey patch needs to happen again here because during tests for some reason this # method gets unpatched (test doing reload() or similar) @mock.patch( "st2actions.container.base.get_runner", mock.Mock(return_value=runner.get_runner()), ) def tearDown(self): for liveaction in LiveAction.get_all(): action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED) @staticmethod def _process_scheduling_queue(): for queued_req in ActionExecutionSchedulingQueue.get_all(): scheduling_queue.get_handler()._handle_execution(queued_req) @mock.patch.object( runner.MockActionRunner, "run", mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE), ) @mock.patch.object( LiveActionPublisher, "publish_state", mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state), ) def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-1", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo-last"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as scheduled. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", "requested", "scheduled", "running", ] actual_status_changes = [entry["status"] for entry in execution.log] self.assertListEqual(actual_status_changes, expected_status_changes) @mock.patch.object( runner.MockActionRunner, "run", mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE), ) @mock.patch.object( LiveActionPublisher, "publish_state", mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state), ) def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel") self.assertEqual(policy_db.parameters["action"], "cancel") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-2", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Assert the canceling state is being published. calls = [ call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) ] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. expected_num_exec += 0 # This request will not be scheduled for execution. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) @mock.patch.object( runner.MockActionRunner, "run", mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE), ) @mock.patch.object( LiveActionPublisher, "publish_state", mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state), ) def test_on_cancellation(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-1", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], "stanley") expected_num_pubs += 2 # Tally the canceling and canceled states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
'policy_3.yaml', 'policy_7.yaml' ] } NON_EMPTY_RESULT = 'non-empty' MOCK_RUN_RETURN_VALUE = (action_constants.LIVEACTION_STATUS_RUNNING, NON_EMPTY_RESULT, None) SCHEDULED_STATES = [ action_constants.LIVEACTION_STATUS_SCHEDULED, action_constants.LIVEACTION_STATUS_RUNNING, action_constants.LIVEACTION_STATUS_SUCCEEDED ] @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch.object( CUDPublisher, 'publish_update', mock.MagicMock(side_effect=MockExecutionPublisher.publish_update)) @mock.patch.object( CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) class ConcurrencyByAttributePolicyTestCase(EventletTestCase, ExecutionDbTestCase): @classmethod def setUpClass(cls): EventletTestCase.setUpClass() ExecutionDbTestCase.setUpClass() # Override the coordinator to use the noop driver otherwise the tests will be blocked.
NON_EMPTY_RESULT = "non-empty" MOCK_RUN_RETURN_VALUE = ( action_constants.LIVEACTION_STATUS_RUNNING, NON_EMPTY_RESULT, None, ) SCHEDULED_STATES = [ action_constants.LIVEACTION_STATUS_SCHEDULED, action_constants.LIVEACTION_STATUS_RUNNING, action_constants.LIVEACTION_STATUS_SUCCEEDED, ] @mock.patch( "st2common.runners.base.get_runner", mock.Mock(return_value=runner.get_runner()) ) @mock.patch( "st2actions.container.base.get_runner", mock.Mock(return_value=runner.get_runner()) ) @mock.patch.object( CUDPublisher, "publish_update", mock.MagicMock(side_effect=MockExecutionPublisher.publish_update), ) @mock.patch.object(CUDPublisher, "publish_create", mock.MagicMock(return_value=None)) class ConcurrencyByAttributePolicyTestCase(EventletTestCase, ExecutionDbTestCase): @classmethod def setUpClass(cls): EventletTestCase.setUpClass() ExecutionDbTestCase.setUpClass()
class ExecutionCancellationTestCase(ExecutionDbTestCase): @classmethod def setUpClass(cls): super(ExecutionCancellationTestCase, cls).setUpClass() for _, fixture in six.iteritems(FIXTURES['actions']): instance = ActionAPI(**fixture) Action.add_or_update(ActionAPI.to_model(instance)) runners_registrar.register_runners() def tearDown(self): # Ensure all liveactions are canceled at end of each test. for liveaction in LiveAction.get_all(): action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED) @classmethod def get_runner_class(cls, runner_name): return runners.get_runner(runner_name).__class__ @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock( side_effect=MockLiveActionPublisherNonBlocking.publish_state)) @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) def test_basic_cancel(self): runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None) mock_runner_run = mock.Mock(return_value=runner_run_result) with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED) @mock.patch.object( CUDPublisher, 'publish_create', mock.MagicMock(side_effect=MockLiveActionPublisher.publish_create)) @mock.patch.object( CUDPublisher, 'publish_update', mock.MagicMock(side_effect=MockExecutionPublisher.publish_update)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisher.publish_state)) @mock.patch.object( runners.ActionRunner, 'cancel', mock.MagicMock(side_effect=Exception('Mock cancellation failure.'))) @mock.patch('st2common.runners.base.get_runner', mock.Mock(return_value=runner.get_runner())) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) def test_failed_cancel(self): runner_run_result = (action_constants.LIVEACTION_STATUS_RUNNING, 'foobar', None) mock_runner_run = mock.Mock(return_value=runner_run_result) with mock.patch.object(runner.MockActionRunner, 'run', mock_runner_run): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancellation failed and execution state remains "canceling". runners.ActionRunner.cancel.assert_called_once_with() liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING) @mock.patch.object(CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) @mock.patch.object(LiveActionPublisher, 'publish_state', mock.MagicMock(return_value=None)) @mock.patch.object(runners.ActionRunner, 'cancel', mock.MagicMock(return_value=None)) def test_noop_cancel(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) @mock.patch.object(CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) @mock.patch.object(LiveActionPublisher, 'publish_state', mock.MagicMock(return_value=None)) @mock.patch.object(runners.ActionRunner, 'cancel', mock.MagicMock(return_value=None)) def test_cancel_delayed_execution(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) @mock.patch.object(CUDPublisher, 'publish_create', mock.MagicMock(return_value=None)) @mock.patch.object(LiveActionPublisher, 'publish_state', mock.MagicMock(return_value=None)) @mock.patch.object(trace_service, 'get_trace_db_by_live_action', mock.MagicMock(return_value=(None, None))) def test_cancel_delayed_execution_with_parent(self): liveaction = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': { 'execution_id': uuid.uuid4().hex }}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
class ConcurrencyByAttributePolicyTestCase(EventletTestCase, ExecutionDbTestCase): @classmethod def setUpClass(cls): EventletTestCase.setUpClass() ExecutionDbTestCase.setUpClass() # Override the coordinator to use the noop driver otherwise the tests will be blocked. tests_config.parse_args(coordinator_noop=True) coordination.COORDINATOR = None # Register runners runners_registrar.register_runners() # Register common policy types register_policy_types(st2common) loader = FixturesLoader() loader.save_fixtures_to_db(fixtures_pack=PACK, fixtures_dict=TEST_FIXTURES) @classmethod def tearDownClass(cls): # Reset the coordinator. coordination.COORDINATOR = None super(ConcurrencyByAttributePolicyTestCase, cls).tearDownClass() # NOTE: This monkey patch needs to happen again here because during tests for some reason this # method gets unpatched (test doing reload() or similar) @mock.patch('st2actions.container.base.get_runner', mock.Mock(return_value=runner.get_runner())) def tearDown(self): for liveaction in LiveAction.get_all(): action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED) @staticmethod def _process_scheduling_queue(): for queued_req in ActionExecutionSchedulingQueue.get_all(): scheduling_queue.get_handler()._handle_execution(queued_req) @mock.patch.object(runner.MockActionRunner, 'run', mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state)) def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) # Assert the action is delayed. delayed = liveaction self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) @mock.patch.object(runner.MockActionRunner, 'run', mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state)) def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref( 'wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Assert the canceling state is being published. calls = [ call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) ] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. canceled = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) @mock.patch.object(runner.MockActionRunner, 'run', mock.MagicMock(return_value=MOCK_RUN_RETURN_VALUE)) @mock.patch.object( LiveActionPublisher, 'publish_state', mock.MagicMock(side_effect=MockLiveActionPublisherSchedulingQueueOnly. publish_state)) def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) # Assert the action is delayed. delayed = liveaction self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = LiveAction.get_by_id(str(delayed.id)) liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)