def test_purge(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={ 'hola': 'hi', 'kuraci': 'chicken' }, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={ 'hola': 'hi', 'kuraci': 'chicken' }, occurrence_time=now - timedelta(days=5), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 2) purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20)) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) purge_trigger_instances() self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20)) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_trigger_instances, logger=LOG, timestamp=None) self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_trigger_instances, logger=LOG, timestamp=None) self.assertEqual(len(TriggerInstance.get_all()), 1)
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now(), status=TRIGGER_INSTANCE_PROCESSED) return TriggerInstance.add_or_update(created)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger) if not trigger_db: LOG.debug("No trigger in db for %s", trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError("Trigger not found for %s" % trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger) if not trigger_db: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s' % trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'type' and 'parameters'. :param trigger: Dictionary with trigger query filters. :type trigger: ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: type_ = trigger.get('type', None) parameters = trigger.get('parameters', {}) trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=type_, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'type' and 'parameters'. :param trigger: Dictionary with trigger query filters. :type trigger: ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: type_ = trigger.get('type', None) parameters = trigger.get('parameters', {}) trigger_db = TriggerService.get_trigger_db_given_type_and_params( type=type_, parameters=parameters) if trigger_db is None: LOG.info('No trigger in db for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time return TriggerInstance.add_or_update(trigger_instance)
def _create_trigger_instance(cls, trigger_ref, payload): trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = datetime.datetime.utcnow() created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now(), status=TRIGGER_INSTANCE_PROCESSED, ) return TriggerInstance.add_or_update(created)
def _create_trigger_instance(cls, trigger_ref, payload): trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = date_utils.get_datetime_utc_now() created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: # If id / uid is available we try to look up Trigger by id. This way we can avoid bug in # pymongo / mongoengine related to "parameters" dictionary lookups trigger_id = trigger.get('id', None) trigger_uid = trigger.get('uid', None) # TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary # passed to this method always contains id or uid if trigger_id: LOG.debug('Looking up TriggerDB by id: %s', trigger_id) trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id) elif trigger_uid: LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid) trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid) else: # Last resort - look it up by parameters trigger_type = trigger.get('type', None) parameters = trigger.get('parameters', {}) LOG.debug( 'Looking up TriggerDB by type and parameters: type=%s, parameters=%s', trigger_type, parameters) trigger_db = TriggerService.get_trigger_db_given_type_and_params( type=trigger_type, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def test_purge(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=5), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 2) purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(TriggerInstance.get_all()), 1)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: # If id / uid is available we try to look up Trigger by id. This way we can avoid bug in # pymongo / mongoengine related to "parameters" dictionary lookups trigger_id = trigger.get('id', None) trigger_uid = trigger.get('uid', None) # TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary # passed to this method always contains id or uid if trigger_id: LOG.debug('Looking up TriggerDB by id: %s', trigger_id) trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id) elif trigger_uid: LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid) trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid) else: # Last resort - look it up by parameters trigger_type = trigger.get('type', None) parameters = trigger.get('parameters', {}) LOG.debug('Looking up TriggerDB by type and parameters: type=%s, parameters=%s', trigger_type, parameters) trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=trigger_type, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def _create_trigger_instance(cls, trigger_ref, payload, seconds): # Note: We use 1 second intervals between occurence time to prevent # occasional test failures occurrence_time = date_utils.get_datetime_utc_now() occurrence_time = occurrence_time + datetime.timedelta(seconds=seconds) trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def _create_trigger_instance(cls, trigger_ref, payload, seconds): # Note: We use 1 second intervals between occurence time to prevent # occasional test failures occurrence_time = date_utils.get_datetime_utc_now() occurrence_time = occurrence_time + datetime.timedelta(seconds=seconds) trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def test_triggered_execution(self): docs = { 'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']), 'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']), 'rule': copy.deepcopy(fixture.ARTIFACTS['rule']), 'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])} # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type']))) trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger']))) rule = RuleAPI.to_model(RuleAPI(**docs['rule'])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance']))) trace_service.add_or_update_given_trace_context( trace_context={'trace_tag': 'test_triggered_execution_trace'}, trigger_instances=[str(trigger_instance.id)]) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED) execution = self._get_action_execution( liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual(execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance))) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB(trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now()) return TriggerInstance.add_or_update(created)
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now()) return TriggerInstance.add_or_update(created)
def test_triggered_execution(self): docs = { "trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]), "trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]), "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "trigger_instance": copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]), } # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs["trigger_type"]))) trigger = Trigger.add_or_update( TriggerAPI.to_model(TriggerAPI(**docs["trigger"]))) rule = RuleAPI.to_model(RuleAPI(**docs["rule"])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model( TriggerInstanceAPI(**docs["trigger_instance"]))) trace_service.add_or_update_given_trace_context( trace_context={"trace_tag": "test_triggered_execution_trace"}, trigger_instances=[str(trigger_instance.id)], ) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get( context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual( execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance)), ) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) # NOTE: Timestamp of liveaction and execution may be a bit different, depending on how long # it takes to persist each object in the database self.assertEqual( execution.end_timestamp.replace(microsecond=0), liveaction.end_timestamp.replace(microsecond=0), ) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction["callback"], liveaction.callback) self.assertEqual(execution.liveaction["action"], liveaction.action)
def update_trigger_instance_status(trigger_instance, status): trigger_instance.status = status return TriggerInstance.add_or_update(trigger_instance)
def test_migrate_triggers(self): TriggerInstanceDB._meta["allow_inheritance"] = True class TriggerInstanceDB_OldFieldType(TriggerInstanceDB): payload = stormbase.EscapedDictField() trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(trigger_instance_dbs), 0) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 0) # 1. Insert data in old format trigger_instance_1_db = TriggerInstanceDB_OldFieldType() trigger_instance_1_db.payload = MOCK_PAYLOAD_1 trigger_instance_1_db.status = TRIGGER_INSTANCE_PROCESSED trigger_instance_1_db.occurrence_time = datetime.datetime.utcnow() trigger_instance_1_db = TriggerInstance.add_or_update( trigger_instance_1_db, publish=False) trigger_instance_2_db = TriggerInstanceDB_OldFieldType() trigger_instance_2_db.payload = MOCK_PAYLOAD_2 trigger_instance_2_db.status = TRIGGER_INSTANCE_PENDING trigger_instance_2_db.occurrence_time = datetime.datetime.utcnow() trigger_instance_2_db = TriggerInstance.add_or_update( trigger_instance_2_db, publish=False) # This object is older than the default threshold so it should not be migrated trigger_instance_3_db = TriggerInstanceDB_OldFieldType() trigger_instance_3_db.payload = MOCK_PAYLOAD_2 trigger_instance_3_db.status = TRIGGER_INSTANCE_PROCESSED trigger_instance_3_db.occurrence_time = datetime.datetime.utcfromtimestamp( 0) trigger_instance_3_db = TriggerInstance.add_or_update( trigger_instance_3_db, publish=False) # Verify data has been inserted in old format trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(trigger_instance_dbs), 3) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 3) # Update inserted documents and remove special _cls field added by mongoengine. We need to # do that here due to how mongoengine works with subclasses. TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }).update(set___cls="TriggerInstanceDB") # 2. Run migration start_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2) end_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) migration_module.migrate_triggers(start_dt=start_dt, end_dt=end_dt) # 3. Verify data has been migrated - only 1 item should have been migrated since it's in a # completed state trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) # TODO: Also verify raw as_pymongo() bin field value self.assertEqual(len(trigger_instance_dbs), 2) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 2) trigger_instance_1_db_retrieved = TriggerInstance.get_by_id( trigger_instance_1_db.id) self.assertEqual(trigger_instance_1_db_retrieved.payload, MOCK_PAYLOAD_1) trigger_instance_2_db_retrieved = TriggerInstance.get_by_id( trigger_instance_2_db.id) self.assertEqual(trigger_instance_2_db_retrieved.payload, MOCK_PAYLOAD_2)
def update_trigger_instance_status(trigger_instance, status): trigger_instance.status = status return TriggerInstance.add_or_update(trigger_instance)