def _register_policy(self, pack, policy):
content = self._meta_loader.load(policy)
pack_field = content.get('pack', None)
if not pack_field:
content['pack'] = pack
pack_field = pack
if pack_field != pack:
raise Exception('Model is in pack "%s" but field "pack" is different: %s' %
(pack, pack_field))
policy_api = PolicyAPI(**content)
policy_api.validate()
policy_db = PolicyAPI.to_model(policy_api)
try:
policy_db.id = Policy.get_by_name(policy_api.name).id
except ValueError:
LOG.debug('Policy "%s" is not found. Creating new entry.', policy)
try:
policy_db = Policy.add_or_update(policy_db)
extra = {'policy_db': policy_db}
LOG.audit('Policy "%s" is updated.', policy_db.ref, extra=extra)
except Exception:
LOG.exception('Failed to create policy %s.', policy_api.name)
raise
def _register_policy(self, pack, policy):
content = self._meta_loader.load(policy)
pack_field = content.get('pack', None)
if not pack_field:
content['pack'] = pack
pack_field = pack
if pack_field != pack:
raise Exception('Model is in pack "%s" but field "pack" is different: %s' %
(pack, pack_field))
# Add in "metadata_file" attribute which stores path to the pack metadata file relative to
# the pack directory
metadata_file = content_utils.get_relative_path_to_pack_file(pack_ref=pack,
file_path=policy,
use_pack_cache=True)
content['metadata_file'] = metadata_file
policy_api = PolicyAPI(**content)
policy_api = policy_api.validate()
policy_db = PolicyAPI.to_model(policy_api)
try:
policy_db.id = Policy.get_by_name(policy_api.name).id
except StackStormDBObjectNotFoundError:
LOG.debug('Policy "%s" is not found. Creating new entry.', policy)
try:
policy_db = Policy.add_or_update(policy_db)
extra = {'policy_db': policy_db}
LOG.audit('Policy "%s" is updated.', policy_db.ref, extra=extra)
except Exception:
LOG.exception('Failed to create policy %s.', policy_api.name)
raise
def test_register_all_policies(self):
policies_dbs = Policy.get_all()
self.assertEqual(len(policies_dbs), 0)
packs_base_path = get_fixtures_packs_base_path()
count = policies_registrar.register_policies(packs_base_paths=[packs_base_path])
# Verify PolicyDB objects have been created
policies_dbs = Policy.get_all()
policies = {
policies_db.name: {
'pack': policies_db.pack,
'type': policies_db.policy_type,
'parameters': policies_db.parameters
}
for policies_db in policies_dbs
}
expected_policies = {
'test_policy_1': {
'pack': 'dummy_pack_1',
'type': 'action.concurrency',
'parameters': {
'action': 'delay',
'threshold': 3
}
},
'test_policy_3': {
'pack': 'dummy_pack_1',
'type': 'action.retry',
'parameters': {
'retry_on': 'timeout',
'max_retry_count': 5
}
},
'cancel_on_concurrency': {
'pack': 'mistral_tests',
'type': 'action.concurrency',
'parameters': {
'action': 'cancel',
'threshold': 3
}
},
'cancel_on_concurrency_by_attr': {
'pack': 'mistral_tests',
'type': 'action.concurrency.attr',
'parameters': {
'action': 'cancel',
'threshold': 1,
'attributes': ['friend']
}
}
}
self.assertEqual(len(expected_policies), count)
self.assertEqual(len(expected_policies), len(policies_dbs))
self.assertDictEqual(expected_policies, policies)
def setUpClass(cls):
super(PolicyControllerTest, cls).setUpClass()
for _, fixture in six.iteritems(FIXTURES['policytypes']):
instance = PolicyTypeAPI(**fixture)
PolicyType.add_or_update(PolicyTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policies']):
instance = PolicyAPI(**fixture)
Policy.add_or_update(PolicyAPI.to_model(instance))
def test_run(self):
pack = 'dummy_pack_1'
# Verify all the resources are there
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(action_dbs), 1)
self.assertEqual(len(alias_dbs), 2)
self.assertEqual(len(rule_dbs), 1)
self.assertEqual(len(sensor_dbs), 3)
self.assertEqual(len(trigger_type_dbs), 4)
self.assertEqual(len(policy_dbs), 2)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(len(config_dbs), 1)
# Run action
action = self.get_action_instance()
action.run(packs=[pack])
# Make sure all resources have been removed from the db
pack_dbs = Pack.query(ref=pack)
action_dbs = Action.query(pack=pack)
alias_dbs = ActionAlias.query(pack=pack)
rule_dbs = Rule.query(pack=pack)
sensor_dbs = Sensor.query(pack=pack)
trigger_type_dbs = TriggerType.query(pack=pack)
policy_dbs = Policy.query(pack=pack)
config_schema_dbs = ConfigSchema.query(pack=pack)
config_dbs = Config.query(pack=pack)
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(action_dbs), 0)
self.assertEqual(len(alias_dbs), 0)
self.assertEqual(len(rule_dbs), 0)
self.assertEqual(len(sensor_dbs), 0)
self.assertEqual(len(trigger_type_dbs), 0)
self.assertEqual(len(policy_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
self.assertEqual(len(config_dbs), 0)
def test_register_policies(self):
# Note: Only one policy should be registered since second one fails validation
pack_dir = os.path.join(fixturesloader.get_fixtures_base_path(), "dummy_pack_1")
self.assertEqual(register_policies(pack_dir=pack_dir), 1)
p1 = Policy.get_by_ref("dummy_pack_1.test_policy_1")
self.assertEqual(p1.name, "test_policy_1")
self.assertEqual(p1.pack, "dummy_pack_1")
self.assertEqual(p1.resource_ref, "dummy_pack_1.local")
self.assertEqual(p1.policy_type, "action.concurrency")
p2 = Policy.get_by_ref("dummy_pack_1.test_policy_2")
self.assertEqual(p2, None)
def test_delete_sys_pack(self):
instance = self.__create_instance()
instance['pack'] = 'core'
post_resp = self.__do_post(instance)
self.assertEqual(post_resp.status_int, http_client.CREATED)
del_resp = self.__do_delete(self.__get_obj_id(post_resp))
self.assertEqual(del_resp.status_int, http_client.BAD_REQUEST)
self.assertEqual(del_resp.json['faultstring'],
"Resources belonging to system level packs can't be manipulated")
# Clean up manually since API won't delete object in sys pack.
Policy.delete(Policy.get_by_id(self.__get_obj_id(post_resp)))
def test_register_policies_from_pack(self):
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
self.assertEqual(register_policies(pack_dir=pack_dir), 2)
p1 = Policy.get_by_ref('dummy_pack_1.test_policy_1')
self.assertEqual(p1.name, 'test_policy_1')
self.assertEqual(p1.pack, 'dummy_pack_1')
self.assertEqual(p1.resource_ref, 'dummy_pack_1.local')
self.assertEqual(p1.policy_type, 'action.concurrency')
# Verify that a default value for parameter "action" which isn't provided in the file is set
self.assertEqual(p1.parameters['action'], 'delay')
p2 = Policy.get_by_ref('dummy_pack_1.test_policy_2')
self.assertEqual(p2, None)
def test_over_threshold(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
self.assertGreater(policy_db.parameters["threshold"], 0)
self.assertIn("actionstr", policy_db.parameters["attributes"])
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
eventlet.spawn(action_service.request, liveaction)
# Sleep here to let the threads above schedule the action execution.
eventlet.sleep(1)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Sleep here to let the threads above complete the action execution.
eventlet.sleep(RUN_DELAY + 1)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = LiveAction.get_all()
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
for liveaction in scheduled:
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_register_policies(self):
pack_dir = os.path.join(fixturesloader.get_fixtures_base_path(), 'dummy_pack_1')
self.assertEqual(register_policies(pack_dir=pack_dir), 2)
p1 = Policy.get_by_ref('dummy_pack_1.test_policy_1')
self.assertEqual(p1.name, 'test_policy_1')
self.assertEqual(p1.pack, 'dummy_pack_1')
self.assertEqual(p1.resource_ref, 'dummy_pack_1.local')
self.assertEqual(p1.policy_type, 'action.concurrency')
p2 = Policy.get_by_ref('dummy_pack_1.test_policy_2')
self.assertEqual(p2.name, 'test_policy_2')
self.assertEqual(p2.pack, 'dummy_pack_1')
self.assertEqual(p2.resource_ref, 'dummy_pack_1.local')
self.assertEqual(p2.policy_type, 'action.mock_policy_error')
self.assertEqual(p2.resource_ref, 'dummy_pack_1.local')
def test_disabled_policy_not_applied_on_pre_run(self, mock_policies):
scheduler_worker = scheduler.get_scheduler()
##########
# First test a scenario where policy is enabled
##########
self.assertTrue(self.policy_db.enabled)
# Post run hasn't been called yet, call count should be 0
self.assertEqual(mock_policies.get_driver.call_count, 0)
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
scheduler_worker._apply_pre_run_policies(liveaction_db=live_action_db)
# Ony policy has been applied so call count should be 1
self.assertEqual(mock_policies.get_driver.call_count, 1)
##########
# Now a scenaro with disabled policy
##########
mock_policies.get_driver.call_count = 0
self.policy_db.enabled = False
self.policy_db = Policy.add_or_update(self.policy_db)
self.assertFalse(self.policy_db.enabled)
self.assertEqual(mock_policies.get_driver.call_count, 0)
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
live_action_db, execution_db = action_service.request(liveaction)
scheduler_worker._apply_pre_run_policies(liveaction_db=live_action_db)
# Policy is disabled so call_count should stay the same as before as no policies have been
# applied
self.assertEqual(mock_policies.get_driver.call_count, 0)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_get_driver(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
policy = get_driver(policy_db.ref, policy_db.policy_type, **policy_db.parameters)
self.assertIsInstance(policy, ResourcePolicyApplicator)
self.assertEqual(policy._policy_ref, policy_db.ref)
self.assertEqual(policy._policy_type, policy_db.policy_type)
self.assertTrue(hasattr(policy, "threshold"))
self.assertEqual(policy.threshold, 3)
def setUpClass(cls):
super(SchedulingPolicyTest, cls).setUpClass()
# Register runners
runners_registrar.register_runners()
for _, fixture in six.iteritems(FIXTURES['actions']):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policytypes']):
instance = PolicyTypeAPI(**fixture)
PolicyType.add_or_update(PolicyTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policies']):
instance = PolicyAPI(**fixture)
Policy.add_or_update(PolicyAPI.to_model(instance))
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_register_all_policies(self):
policies_dbs = Policy.get_all()
self.assertEqual(len(policies_dbs), 0)
packs_base_path = get_fixtures_packs_base_path()
count = policies_registrar.register_policies(packs_base_paths=[packs_base_path])
self.assertEqual(count, 2)
# Verify PolicyDB objects have been created
policies_dbs = Policy.get_all()
self.assertEqual(len(policies_dbs), 2)
self.assertEqual(policies_dbs[0].name, 'test_policy_1')
self.assertEqual(policies_dbs[0].policy_type, 'action.concurrency')
# Verify that a default value for parameter "action" which isn't provided in the file is set
self.assertEqual(policies_dbs[0].parameters['action'], 'delay')
self.assertEqual(policies_dbs[0].parameters['threshold'], 3)
def setUpClass(cls):
super(PolicyTest, cls).setUpClass()
for _, fixture in six.iteritems(FIXTURES['runners']):
instance = RunnerTypeAPI(**fixture)
RunnerType.add_or_update(RunnerTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['actions']):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policytypes']):
instance = PolicyTypeAPI(**fixture)
PolicyType.add_or_update(PolicyTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES['policies']):
instance = PolicyAPI(**fixture)
Policy.add_or_update(PolicyAPI.to_model(instance))
def setUp(self):
EventletTestCase.setUpClass()
DbTestCase.setUpClass()
for _, fixture in six.iteritems(FIXTURES["runners"]):
instance = RunnerTypeAPI(**fixture)
RunnerType.add_or_update(RunnerTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES["actions"]):
instance = ActionAPI(**fixture)
Action.add_or_update(ActionAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES["policytypes"]):
instance = PolicyTypeAPI(**fixture)
PolicyType.add_or_update(PolicyTypeAPI.to_model(instance))
for _, fixture in six.iteritems(FIXTURES["policies"]):
instance = PolicyAPI(**fixture)
Policy.add_or_update(PolicyAPI.to_model(instance))
def test_get_by_ref(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertIsNotNone(policy_db)
self.assertEqual(policy_db.pack, 'wolfpack')
self.assertEqual(policy_db.name, 'action-1.concurrency')
policy_type_db = PolicyType.get_by_ref(policy_db.policy_type)
self.assertIsNotNone(policy_type_db)
self.assertEqual(policy_type_db.resource_type, 'action')
self.assertEqual(policy_type_db.name, 'concurrency')
def test_get_by_ref(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertIsNotNone(policy_db)
self.assertEqual(policy_db.pack, "wolfpack")
self.assertEqual(policy_db.name, "action-1.concurrency")
policy_type_db = PolicyType.get_by_ref(policy_db.policy_type)
self.assertIsNotNone(policy_type_db)
self.assertEqual(policy_type_db.resource_type, "action")
self.assertEqual(policy_type_db.name, "concurrency")
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into cancel state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in [
action_constants.LIVEACTION_STATUS_CANCELING,
action_constants.LIVEACTION_STATUS_CANCELED]:
break
# Assert the canceling state is being published.
calls = [call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status, action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _apply_post_run_policies(self, liveaction=None, execution_id=None):
# Apply policies defined for the action.
for policy_db in Policy.query(resource_ref=liveaction.action):
driver = policies.get_driver(policy_db.ref,
policy_db.policy_type,
**policy_db.parameters)
try:
liveaction = driver.apply_after(liveaction)
except:
LOG.exception('An exception occurred while applying policy "%s".', policy_db.ref)
def test_cancel_on_task_action_concurrency_by_attr(self):
# Delete other policies in the test pack to avoid conflicts.
required_policy = 'mistral_tests.cancel_on_concurrency_by_attr'
self._drop_all_other_policies(required_policy)
# Get threshold from the policy.
policy = Policy.get_by_ref(required_policy)
threshold = policy.parameters.get('threshold', 0)
self.assertGreater(threshold, 0)
params = {'friend': 'grande animalerie'}
# Launch instances of the workflow up to threshold.
for i in range(0, threshold):
liveaction = LiveActionDB(action=WF1_NAME, parameters=params)
liveaction, execution1 = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
liveaction = self._wait_on_status(
liveaction,
action_constants.LIVEACTION_STATUS_RUNNING
)
# Check number of running instances
running = LiveAction.count(
action=WF1_NAME, status=action_constants.LIVEACTION_STATUS_RUNNING,
parameters__friend=params['friend'])
self.assertEqual(running, threshold)
# Mock the mistral runner cancel method to assert cancel is called.
mistral_runner_cls = runners.get_runner('mistral-v2').__class__
mock_cancel_return_value = (action_constants.LIVEACTION_STATUS_CANCELING, None, None)
mock_cancel = mock.MagicMock(return_value=mock_cancel_return_value)
with mock.patch.object(mistral_runner_cls, 'cancel', mock_cancel):
# Launch another instance of the workflow with mistral callback defined
# to indicate that this is executed under a workflow.
callback = {
'source': MISTRAL_RUNNER_NAME,
'url': 'http://127.0.0.1:8989/v2/action_executions/12345'
}
liveaction2 = LiveActionDB(action=WF1_NAME, parameters=params, callback=callback)
liveaction2, execution2 = action_service.request(liveaction2)
liveaction2 = LiveAction.get_by_id(str(liveaction2.id))
# Assert cancel has been called.
liveaction2 = self._wait_on_status(
liveaction2,
action_constants.LIVEACTION_STATUS_CANCELING
)
mistral_runner_cls.cancel.assert_called_once_with()
def process(self, request):
"""Schedules the LiveAction and publishes the request
to the appropriate action runner(s).
LiveAction in statuses other than "requested" are ignored.
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__, type(request), request.id, request.status)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(request.id)
except StackStormDBObjectNotFoundError:
LOG.exception('Failed to find liveaction %s in the database.', request.id)
raise
# Apply policies defined for the action.
for policy_db in Policy.query(resource_ref=liveaction_db.action):
driver = policies.get_driver(policy_db.ref,
policy_db.policy_type,
**policy_db.parameters)
try:
liveaction_db = driver.apply_before(liveaction_db)
except:
LOG.exception('An exception occurred while applying policy "%s".', policy_db.ref)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Exit if the status of the request is no longer runnable.
# The status could have be changed by one of the policies.
if liveaction_db.status not in [action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_SCHEDULED]:
LOG.info('%s is ignoring %s (id=%s) with "%s" status after policies are applied.',
self.__class__.__name__, type(request), request.id, liveaction_db.status)
return
# Update liveaction status to "scheduled".
if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED:
liveaction_db = action_service.update_status(
liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
def _apply_post_run_policies(self, liveaction=None, execution_id=None):
# Apply policies defined for the action.
policy_dbs = Policy.query(resource_ref=liveaction.action)
LOG.debug("Applying %s post_run policies" % (len(policy_dbs)))
for policy_db in policy_dbs:
driver = policies.get_driver(policy_db.ref, policy_db.policy_type, **policy_db.parameters)
try:
LOG.debug(
'Applying post_run policy "%s" (%s) for liveaction %s'
% (policy_db.ref, policy_db.policy_type, str(liveaction.id))
)
liveaction = driver.apply_after(liveaction)
except:
LOG.exception('An exception occurred while applying policy "%s".', policy_db.ref)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
action_service.request(liveaction)
scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2', parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
MockLiveActionPublisherNonBlocking.wait_all()
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
if len(scheduled) == policy_db.parameters['threshold']:
break
MockLiveActionPublisherNonBlocking.wait_all()
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency")
self.assertGreater(policy_db.parameters["threshold"], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
parameters = {"actionstr": "foo-" + str(i)}
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo-last"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as scheduled.
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Check the status changes.
execution = ActionExecution.get(liveaction__id=str(liveaction.id))
expected_status_changes = [
"requested",
"delayed",
"requested",
"scheduled",
"running",
]
actual_status_changes = [entry["status"] for entry in execution.log]
self.assertListEqual(actual_status_changes, expected_status_changes)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-2.concurrency.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be canceled since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into cancel state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in [
action_constants.LIVEACTION_STATUS_CANCELING,
action_constants.LIVEACTION_STATUS_CANCELED
]:
break
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
# Assert the action is canceled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_CANCELED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Since states are being processed asynchronously, wait for the
# liveactions to go into scheduled states.
for i in range(0, 100):
eventlet.sleep(1)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
if len(scheduled) == policy_db.parameters['threshold']:
break
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Assert the action is delayed.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(liveaction.status,
action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
for i in range(0, 100):
eventlet.sleep(1)
liveaction = LiveAction.get_by_id(str(liveaction.id))
if liveaction.status in SCHEDULED_STATES:
break
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
self.assertGreater(policy_db.parameters["threshold"], 0)
self.assertIn("actionstr", policy_db.parameters["attributes"])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters["threshold"]):
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "foo"})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
delayed = liveaction
expected_num_exec += 0 # This request will not be scheduled for execution.
expected_num_pubs += 0 # The delayed status change should not be published.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action="wolfpack.action-1",
parameters={"actionstr": "bar"})
liveaction, _ = action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Cancel execution.
action_service.request_cancellation(scheduled[0], "stanley")
expected_num_pubs += 2 # Tally the canceling and canceled states.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 2 # Tally scheduled and running state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Since states are being processed asynchronously, wait for the
# liveaction to go into scheduled state.
liveaction = LiveAction.get_by_id(str(delayed.id))
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
def _drop_all_other_policies(self, test_policy):
policy_dbs = [policy_db for policy_db in Policy.get_all() if policy_db.ref != test_policy]
for policy_db in policy_dbs:
Policy.delete(policy_db, publish=False)
def test_over_threshold_delay_executions(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
action_service.request(liveaction)
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'fu'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
# Assert the action is delayed.
delayed = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(delayed.status,
action_constants.LIVEACTION_STATUS_DELAYED)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be scheduled since concurrency threshold is not reached.
# The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'bar'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running states.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Mark one of the execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally succeeded state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_exec += 1 # The delayed request is expected to be executed.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(delayed.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def test_over_threshold_cancel_executions(self):
policy_db = Policy.get_by_ref(
'wolfpack.action-2.concurrency.attr.cancel')
self.assertEqual(policy_db.parameters['action'], 'cancel')
self.assertGreater(policy_db.parameters['threshold'], 0)
self.assertIn('actionstr', policy_db.parameters['attributes'])
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-2',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_pubs += 1 # Tally requested state.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Assert the canceling state is being published.
calls = [
call(liveaction, action_constants.LIVEACTION_STATUS_CANCELING)
]
LiveActionPublisher.publish_state.assert_has_calls(calls)
expected_num_pubs += 2 # Tally canceling and canceled state changes.
expected_num_exec += 0 # This request will not be scheduled for execution.
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Assert the action is canceled.
canceled = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(canceled.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def process(self, request):
"""Schedules the LiveAction and publishes the request
to the appropriate action runner(s).
LiveAction in statuses other than "requested" are ignored.
:param request: Action execution request.
:type request: ``st2common.models.db.liveaction.LiveActionDB``
"""
if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
self.__class__.__name__, type(request), request.id,
request.status)
return
try:
liveaction_db = action_utils.get_liveaction_by_id(request.id)
except StackStormDBObjectNotFoundError:
LOG.exception('Failed to find liveaction %s in the database.',
request.id)
raise
# Apply policies defined for the action.
for policy_db in Policy.query(resource_ref=liveaction_db.action):
driver = policies.get_driver(policy_db.ref, policy_db.policy_type,
**policy_db.parameters)
try:
liveaction_db = driver.apply_before(liveaction_db)
except:
LOG.exception(
'An exception occurred while applying policy "%s".',
policy_db.ref)
if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED:
break
# Exit if the status of the request is no longer runnable.
# The status could have be changed by one of the policies.
if liveaction_db.status not in [
action_constants.LIVEACTION_STATUS_REQUESTED,
action_constants.LIVEACTION_STATUS_SCHEDULED
]:
LOG.info(
'%s is ignoring %s (id=%s) with "%s" status after policies are applied.',
self.__class__.__name__, type(request), request.id,
liveaction_db.status)
return
# Update liveaction status to "scheduled".
if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED:
liveaction_db = action_service.update_status(
liveaction_db,
action_constants.LIVEACTION_STATUS_SCHEDULED,
publish=False)
# Publish the "scheduled" status here manually. Otherwise, there could be a
# race condition with the update of the action_execution_db if the execution
# of the liveaction completes first.
LiveAction.publish_status(liveaction_db)
def test_over_threshold_delay_executions(self):
# Ensure the concurrency policy is accurate.
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# Assert the correct number of published states and action executions. This is to avoid
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo-last'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request is expected to be executed.
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
# Mark one of the scheduled/running execution as completed.
action_service.update_status(
scheduled[0],
action_constants.LIVEACTION_STATUS_SUCCEEDED,
publish=True)
expected_num_pubs += 1 # Tally requested state.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to be scheduled.
liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
def test_register_all_policies(self):
policies_dbs = Policy.get_all()
self.assertEqual(len(policies_dbs), 0)
packs_base_path = get_fixtures_packs_base_path()
count = policies_registrar.register_policies(
packs_base_paths=[packs_base_path])
# Verify PolicyDB objects have been created
policies_dbs = Policy.get_all()
policies = {
policies_db.name: {
'pack': policies_db.pack,
'type': policies_db.policy_type,
'parameters': policies_db.parameters
}
for policies_db in policies_dbs
}
expected_policies = {
'test_policy_1': {
'pack': 'dummy_pack_1',
'type': 'action.concurrency',
'parameters': {
'action': 'delay',
'threshold': 3
}
},
'test_policy_3': {
'pack': 'dummy_pack_1',
'type': 'action.retry',
'parameters': {
'retry_on': 'timeout',
'max_retry_count': 5
}
},
'cancel_on_concurrency': {
'pack': 'mistral_tests',
'type': 'action.concurrency',
'parameters': {
'action': 'cancel',
'threshold': 3
}
},
'cancel_on_concurrency_by_attr': {
'pack': 'mistral_tests',
'type': 'action.concurrency.attr',
'parameters': {
'action': 'cancel',
'threshold': 1,
'attributes': ['friend']
}
},
'sequential.retry_on_failure': {
'pack': 'orquesta_tests',
'type': 'action.retry',
'parameters': {
'retry_on': 'failure',
'max_retry_count': 1
}
}
}
self.assertEqual(len(expected_policies), count)
self.assertEqual(len(expected_policies), len(policies_dbs))
self.assertDictEqual(expected_policies, policies)
def test_on_cancellation(self):
policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
self.assertGreater(policy_db.parameters['threshold'], 0)
# Launch action executions until the expected threshold is reached.
for i in range(0, policy_db.parameters['threshold']):
parameters = {'actionstr': 'foo-' + str(i)}
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters=parameters)
action_service.request(liveaction)
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Check the number of action executions in scheduled state.
scheduled = [
item for item in LiveAction.get_all()
if item.status in SCHEDULED_STATES
]
self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
# duplicate executions caused by accidental publishing of state in the concurrency policies.
# num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
expected_num_exec = len(scheduled)
expected_num_pubs = expected_num_exec * 3
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)
# Execution is expected to be delayed since concurrency threshold is reached.
liveaction = LiveActionDB(action='wolfpack.action-1',
parameters={'actionstr': 'foo'})
liveaction, _ = action_service.request(liveaction)
expected_num_exec += 1 # This request will be scheduled for execution.
expected_num_pubs += 1 # Tally requested state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Since states are being processed async, wait for the liveaction to go into delayed state.
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
# Cancel execution.
action_service.request_cancellation(scheduled[0], 'stanley')
expected_num_pubs += 2 # Tally the canceling and canceled states.
# Once capacity freed up, the delayed execution is published as requested again.
expected_num_pubs += 3 # Tally requested, scheduled, and running state.
# Run the scheduler to schedule action executions.
self._process_scheduling_queue()
# Execution is expected to be rescheduled.
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertIn(liveaction.status, SCHEDULED_STATES)
self.assertEqual(expected_num_pubs,
LiveActionPublisher.publish_state.call_count)
self.assertEqual(expected_num_exec,
runner.MockActionRunner.run.call_count)