def test_register_policies(self): # Note: Only one policy should be registered since second one fails validation pack_dir = os.path.join(fixturesloader.get_fixtures_base_path(), "dummy_pack_1") self.assertEqual(register_policies(pack_dir=pack_dir), 1) p1 = Policy.get_by_ref("dummy_pack_1.test_policy_1") self.assertEqual(p1.name, "test_policy_1") self.assertEqual(p1.pack, "dummy_pack_1") self.assertEqual(p1.resource_ref, "dummy_pack_1.local") self.assertEqual(p1.policy_type, "action.concurrency") p2 = Policy.get_by_ref("dummy_pack_1.test_policy_2") self.assertEqual(p2, None)
def test_register_policies_from_pack(self): pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1') self.assertEqual(register_policies(pack_dir=pack_dir), 2) p1 = Policy.get_by_ref('dummy_pack_1.test_policy_1') self.assertEqual(p1.name, 'test_policy_1') self.assertEqual(p1.pack, 'dummy_pack_1') self.assertEqual(p1.resource_ref, 'dummy_pack_1.local') self.assertEqual(p1.policy_type, 'action.concurrency') # Verify that a default value for parameter "action" which isn't provided in the file is set self.assertEqual(p1.parameters['action'], 'delay') p2 = Policy.get_by_ref('dummy_pack_1.test_policy_2') self.assertEqual(p2, None)
def test_register_policies(self): # Note: Only one policy should be registered since second one fails validation pack_dir = os.path.join(fixturesloader.get_fixtures_base_path(), 'dummy_pack_1') self.assertEqual(register_policies(pack_dir=pack_dir), 1) p1 = Policy.get_by_ref('dummy_pack_1.test_policy_1') self.assertEqual(p1.name, 'test_policy_1') self.assertEqual(p1.pack, 'dummy_pack_1') self.assertEqual(p1.resource_ref, 'dummy_pack_1.local') self.assertEqual(p1.policy_type, 'action.concurrency') p2 = Policy.get_by_ref('dummy_pack_1.test_policy_2') self.assertEqual(p2, None)
def test_register_policies_from_pack(self): pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1") self.assertEqual(register_policies(pack_dir=pack_dir), 2) p1 = Policy.get_by_ref("dummy_pack_1.test_policy_1") self.assertEqual(p1.name, "test_policy_1") self.assertEqual(p1.pack, "dummy_pack_1") self.assertEqual(p1.resource_ref, "dummy_pack_1.local") self.assertEqual(p1.policy_type, "action.concurrency") # Verify that a default value for parameter "action" which isn't provided in the file is set self.assertEqual(p1.parameters["action"], "delay") self.assertEqual(p1.metadata_file, "policies/policy_1.yaml") p2 = Policy.get_by_ref("dummy_pack_1.test_policy_2") self.assertEqual(p2, None)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-2.concurrency.cancel") self.assertEqual(policy_db.parameters["action"], "cancel") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-2", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB( action="wolfpack.action-2", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. expected_num_exec += 0 # This request will not be scheduled for execution. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) # Assert the canceling state is being published. calls = [ call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING) ] LiveActionPublisher.publish_state.assert_has_calls(calls) # Assert the action is canceled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_register_policies(self): pack_dir = os.path.join(fixturesloader.get_fixtures_base_path(), 'dummy_pack_1') self.assertEqual(register_policies(pack_dir=pack_dir), 2) p1 = Policy.get_by_ref('dummy_pack_1.test_policy_1') self.assertEqual(p1.name, 'test_policy_1') self.assertEqual(p1.pack, 'dummy_pack_1') self.assertEqual(p1.resource_ref, 'dummy_pack_1.local') self.assertEqual(p1.policy_type, 'action.concurrency') p2 = Policy.get_by_ref('dummy_pack_1.test_policy_2') self.assertEqual(p2.name, 'test_policy_2') self.assertEqual(p2.pack, 'dummy_pack_1') self.assertEqual(p2.resource_ref, 'dummy_pack_1.local') self.assertEqual(p2.policy_type, 'action.mock_policy_error') self.assertEqual(p2.resource_ref, 'dummy_pack_1.local')
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) eventlet.spawn(action_service.request, liveaction) # Sleep here to let the threads above schedule the action execution. eventlet.sleep(1) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Sleep here to let the threads above complete the action execution. eventlet.sleep(RUN_DELAY + 1) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr") self.assertGreater(policy_db.parameters["threshold"], 0) self.assertIn("actionstr", policy_db.parameters["attributes"]) for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_get_driver(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") policy = get_driver(policy_db.ref, policy_db.policy_type, **policy_db.parameters) self.assertIsInstance(policy, ResourcePolicyApplicator) self.assertEqual(policy._policy_ref, policy_db.ref) self.assertEqual(policy._policy_type, policy_db.policy_type) self.assertTrue(hasattr(policy, "threshold")) self.assertEqual(policy.threshold, 3)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_cancel_on_task_action_concurrency_by_attr(self): # Delete other policies in the test pack to avoid conflicts. required_policy = 'mistral_tests.cancel_on_concurrency_by_attr' self._drop_all_other_policies(required_policy) # Get threshold from the policy. policy = Policy.get_by_ref(required_policy) threshold = policy.parameters.get('threshold', 0) self.assertGreater(threshold, 0) params = {'friend': 'grande animalerie'} # Launch instances of the workflow up to threshold. for i in range(0, threshold): liveaction = LiveActionDB(action=WF1_NAME, parameters=params) liveaction, execution1 = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING) # Check number of running instances running = LiveAction.count( action=WF1_NAME, status=action_constants.LIVEACTION_STATUS_RUNNING, parameters__friend=params['friend']) self.assertEqual(running, threshold) # Mock the mistral runner cancel method to assert cancel is called. mistral_runner_cls = self.get_runner_class('mistral_v2') with mock.patch.object(mistral_runner_cls, 'cancel', mock.MagicMock(return_value=None)): # Launch another instance of the workflow with mistral callback defined # to indicate that this is executed under a workflow. callback = { 'source': MISTRAL_RUNNER_NAME, 'url': 'http://127.0.0.1:8989/v2/action_executions/12345' } liveaction2 = LiveActionDB(action=WF1_NAME, parameters=params, callback=callback) liveaction2, execution2 = action_service.request(liveaction2) action_executions.ActionExecutionManager.update.assert_called_once_with( '12345', output='{"error": "Execution canceled by user."}', state='CANCELLED') liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_CANCELED) # Assert cancel has been called. mistral_runner_cls.cancel.assert_called_once_with()
def test_get_by_ref(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertIsNotNone(policy_db) self.assertEqual(policy_db.pack, "wolfpack") self.assertEqual(policy_db.name, "action-1.concurrency") policy_type_db = PolicyType.get_by_ref(policy_db.policy_type) self.assertIsNotNone(policy_type_db) self.assertEqual(policy_type_db.resource_type, "action") self.assertEqual(policy_type_db.name, "concurrency")
def test_get_by_ref(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertIsNotNone(policy_db) self.assertEqual(policy_db.pack, 'wolfpack') self.assertEqual(policy_db.name, 'action-1.concurrency') policy_type_db = PolicyType.get_by_ref(policy_db.policy_type) self.assertIsNotNone(policy_type_db) self.assertEqual(policy_type_db.resource_type, 'action') self.assertEqual(policy_type_db.name, 'concurrency')
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into cancel state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in [ action_constants.LIVEACTION_STATUS_CANCELING, action_constants.LIVEACTION_STATUS_CANCELED]: break # Assert the canceling state is being published. calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)] LiveActionPublisher.publish_state.assert_has_calls(calls) expected_num_pubs += 2 # Tally canceling and canceled state changes. # Assert the action is canceled. canceled = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_cancel_on_task_action_concurrency_by_attr(self): # Delete other policies in the test pack to avoid conflicts. required_policy = 'mistral_tests.cancel_on_concurrency_by_attr' self._drop_all_other_policies(required_policy) # Get threshold from the policy. policy = Policy.get_by_ref(required_policy) threshold = policy.parameters.get('threshold', 0) self.assertGreater(threshold, 0) params = {'friend': 'grande animalerie'} # Launch instances of the workflow up to threshold. for i in range(0, threshold): liveaction = LiveActionDB(action=WF1_NAME, parameters=params) liveaction, execution1 = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING ) # Check number of running instances running = LiveAction.count( action=WF1_NAME, status=action_constants.LIVEACTION_STATUS_RUNNING, parameters__friend=params['friend']) self.assertEqual(running, threshold) # Mock the mistral runner cancel method to assert cancel is called. mistral_runner_cls = runners.get_runner('mistral-v2').__class__ mock_cancel_return_value = (action_constants.LIVEACTION_STATUS_CANCELING, None, None) mock_cancel = mock.MagicMock(return_value=mock_cancel_return_value) with mock.patch.object(mistral_runner_cls, 'cancel', mock_cancel): # Launch another instance of the workflow with mistral callback defined # to indicate that this is executed under a workflow. callback = { 'source': MISTRAL_RUNNER_NAME, 'url': 'http://127.0.0.1:8989/v2/action_executions/12345' } liveaction2 = LiveActionDB(action=WF1_NAME, parameters=params, callback=callback) liveaction2, execution2 = action_service.request(liveaction2) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) # Assert cancel has been called. liveaction2 = self._wait_on_status( liveaction2, action_constants.LIVEACTION_STATUS_CANCELING ) mistral_runner_cls.cancel.assert_called_once_with()
def test_cancel_on_task_action_concurrency(self): # Delete other policies in the test pack to avoid conflicts. required_policy = 'mistral_tests.cancel_on_concurrency' self._drop_all_other_policies(required_policy) # Get threshold from the policy. policy = Policy.get_by_ref(required_policy) threshold = policy.parameters.get('threshold', 0) self.assertGreater(threshold, 0) # Launch instances of the workflow up to threshold. for i in range(0, threshold): liveaction = LiveActionDB(action=WF1_NAME, parameters={'friend': 'friend' + str(i)}) liveaction, execution1 = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING ) # Check number of running instances running = LiveAction.count( action=WF1_NAME, status=action_constants.LIVEACTION_STATUS_RUNNING) self.assertEqual(running, threshold) # Mock the mistral runner cancel method to assert cancel is called. mistral_runner_cls = runners.get_runner('mistral-v2').__class__ mock_cancel_return_value = (action_constants.LIVEACTION_STATUS_CANCELING, None, None) mock_cancel = mock.MagicMock(return_value=mock_cancel_return_value) with mock.patch.object(mistral_runner_cls, 'cancel', mock_cancel): # Launch another instance of the workflow with mistral callback defined # to indicate that this is executed under a workflow. callback = { 'source': MISTRAL_RUNNER_NAME, 'url': 'http://127.0.0.1:8989/v2/action_executions/12345' } params = {'friend': 'grande animalerie'} liveaction2 = LiveActionDB(action=WF1_NAME, parameters=params, callback=callback) liveaction2, execution2 = action_service.request(liveaction2) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) # Assert cancel has been called. liveaction2 = self._wait_on_status( liveaction2, action_constants.LIVEACTION_STATUS_CANCELING ) mistral_runner_cls.cancel.assert_called_once_with()
def test_over_threshold_cancel_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel') self.assertEqual(policy_db.parameters['action'], 'cancel') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be canceled since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request will be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo-last'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo-last'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as scheduled. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = ['requested', 'delayed', 'requested', 'scheduled', 'running'] actual_status_changes = [entry['status'] for entry in execution.log] self.assertListEqual(actual_status_changes, expected_status_changes)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-1", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], "stanley") expected_num_pubs += 2 # Tally the canceling and canceled states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-1", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo-last"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as scheduled. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", "requested", "scheduled", "running", ] actual_status_changes = [entry["status"] for entry in execution.log] self.assertListEqual(actual_status_changes, expected_status_changes)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr") self.assertGreater(policy_db.parameters["threshold"], 0) self.assertIn("actionstr", policy_db.parameters["attributes"]) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "foo"} ) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED ) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "bar"} ) liveaction, _ = action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request will be scheduled for execution. expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED: break # Assert the action is delayed. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) delayed = liveaction expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = LiveAction.get_by_id(str(delayed.id)) liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
def test_on_cancellation(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED: break # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Cancel execution. action_service.request_cancellation(scheduled[0], 'stanley') expected_num_pubs += 2 # Tally the canceling and canceled states. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)