def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) query_filters = {'occurrence_time__lt': isotime.parse(timestamp)} # TODO: Update this code to return statistics on deleted objects once we # upgrade to newer version of MongoDB where delete_by_query actually returns # some data try: TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete trigger instances: %s' 'Please contact support.' % (query_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception('Deleting instances using query_filters %s failed.', query_filters) # Print stats logger.info( 'All trigger instance models older than timestamp %s were deleted.', timestamp)
def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) query_filters = {'occurrence_time__lt': isotime.parse(timestamp)} # TODO: Update this code to return statistics on deleted objects once we # upgrade to newer version of MongoDB where delete_by_query actually returns # some data try: TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete trigger instances: %s' 'Please contact support.' % (query_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception('Deleting instances using query_filters %s failed.', query_filters) # Print stats logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def _purge_model(instance): try: TriggerInstance.delete(instance) except: LOG.exception('Exception deleting instance %s.', instance) else: global DELETED_COUNT DELETED_COUNT += 1
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20)) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) purge_trigger_instances() self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_triggerinstance_crud(self): triggertype = ReactorModelTest._create_save_triggertype() trigger = ReactorModelTest._create_save_trigger(triggertype) saved = ReactorModelTest._create_save_triggerinstance(trigger) retrieved = TriggerInstance.get_by_id(saved.id) self.assertIsNotNone(retrieved, 'No triggerinstance created.') ReactorModelTest._delete([retrieved, trigger, triggertype]) try: retrieved = TriggerInstance.get_by_id(saved.id) except StackStormDBObjectNotFoundError: retrieved = None self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_triggerinstance_crud(self): triggertype = ReactorModelTest._create_save_triggertype() trigger = ReactorModelTest._create_save_trigger(triggertype) saved = ReactorModelTest._create_save_triggerinstance(trigger) retrieved = TriggerInstance.get_by_id(saved.id) self.assertIsNotNone(retrieved, "No triggerinstance created.") ReactorModelTest._delete([retrieved, trigger, triggertype]) try: retrieved = TriggerInstance.get_by_id(saved.id) except StackStormDBObjectNotFoundError: retrieved = None self.assertIsNone(retrieved, "managed to retrieve after failure.")
def test_triggerinstance_crud(self): triggertype = ReactorModelTest._create_save_triggertype() trigger = ReactorModelTest._create_save_trigger(triggertype) saved = ReactorModelTest._create_save_triggerinstance(trigger) retrieved = TriggerInstance.get_by_id(saved.id) self.assertIsNotNone(retrieved, 'No triggerinstance created.') ReactorModelTest._delete([retrieved, trigger, triggertype]) try: retrieved = TriggerInstance.get_by_id(saved.id) except ValueError: retrieved = None self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20)) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_trigger_instances, logger=LOG, timestamp=None) self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_no_timestamp_doesnt_delete(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 1) expected_msg = 'Specify a valid timestamp' self.assertRaisesRegexp(ValueError, expected_msg, purge_trigger_instances, logger=LOG, timestamp=None) self.assertEqual(len(TriggerInstance.get_all()), 1)
def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) query_filters = {'occurrence_time__lt': isotime.parse(timestamp)} try: deleted_count = TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete trigger instances: %s' 'Please contact support.' % (query_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception('Deleting instances using query_filters %s failed.', query_filters) else: logger.info('Deleted %s trigger instance objects' % (deleted_count)) # Print stats logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger) if not trigger_db: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s' % trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'type' and 'parameters'. :param trigger: Dictionary with trigger query filters. :type trigger: ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: type_ = trigger.get('type', None) parameters = trigger.get('parameters', {}) trigger_db = TriggerService.get_trigger_db_given_type_and_params( type=type_, parameters=parameters) if trigger_db is None: LOG.info('No trigger in db for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time return TriggerInstance.add_or_update(trigger_instance)
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now(), status=TRIGGER_INSTANCE_PROCESSED) return TriggerInstance.add_or_update(created)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'type' and 'parameters'. :param trigger: Dictionary with trigger query filters. :type trigger: ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: type_ = trigger.get('type', None) parameters = trigger.get('parameters', {}) trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=type_, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger) if not trigger_db: LOG.debug("No trigger in db for %s", trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError("Trigger not found for %s" % trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError("Specify a valid timestamp to purge.") logger.info("Purging trigger instances older than timestamp: %s" % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) query_filters = {"occurrence_time__lt": isotime.parse(timestamp)} try: deleted_count = TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ("Bad query (%s) used to delete trigger instances: %s" "Please contact support." % ( query_filters, six.text_type(e), )) raise InvalidQueryError(msg) except: logger.exception("Deleting instances using query_filters %s failed.", query_filters) else: logger.info("Deleted %s trigger instance objects" % (deleted_count)) # Print stats logger.info( "All trigger instance models older than timestamp %s were deleted.", timestamp)
def _create_trigger_instance(cls, trigger_ref, payload): trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = datetime.datetime.utcnow() created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now(), status=TRIGGER_INSTANCE_PROCESSED, ) return TriggerInstance.add_or_update(created)
def _create_trigger_instance(cls, trigger_ref, payload): trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = date_utils.get_datetime_utc_now() created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: # If id / uid is available we try to look up Trigger by id. This way we can avoid bug in # pymongo / mongoengine related to "parameters" dictionary lookups trigger_id = trigger.get('id', None) trigger_uid = trigger.get('uid', None) # TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary # passed to this method always contains id or uid if trigger_id: LOG.debug('Looking up TriggerDB by id: %s', trigger_id) trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id) elif trigger_uid: LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid) trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid) else: # Last resort - look it up by parameters trigger_type = trigger.get('type', None) parameters = trigger.get('parameters', {}) LOG.debug( 'Looking up TriggerDB by type and parameters: type=%s, parameters=%s', trigger_type, parameters) trigger_db = TriggerService.get_trigger_db_given_type_and_params( type=trigger_type, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def get_all(self, **kw): """ List all triggerinstances. Handles requests: GET /triggerinstances/ """ trigger_instance_apis = [TriggerInstanceAPI.from_model(trigger_instance_db) for trigger_instance_db in TriggerInstance.get_all(**kw)] return trigger_instance_apis
def create_execution_object(liveaction, action_db=None, runnertype_db=None, publish=True): if not action_db: action_db = action_utils.get_action_by_ref(liveaction.action) if not runnertype_db: runnertype_db = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runnertype_db)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] # TODO: This object initialization takes 20-30or so ms execution = ActionExecutionDB(**attrs) # TODO: Do 100% research this is fully safe and unique in distributed setups execution.id = ObjectId() execution.web_url = _get_web_url_for_execution(str(execution.id)) # NOTE: User input data is already validate as part of the API request, # other data is set by us. Skipping validation here makes operation 10%-30% faster execution = ActionExecution.add_or_update(execution, publish=publish, validate=False) if parent and str(execution.id) not in parent.children: values = {} values['push__children'] = str(execution.id) ActionExecution.update(parent, **values) return execution
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False): """ This creates a trigger instance object given trigger and payload. Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or 'uid' or type' and 'parameters' keys. :param trigger: Trigger reference or dictionary with trigger query filters. :type trigger: ``str`` or ``dict`` :param payload: Trigger payload. :type payload: ``dict`` """ # TODO: This is nasty, this should take a unique reference and not a dict if isinstance(trigger, six.string_types): trigger_db = TriggerService.get_trigger_db_by_ref(trigger) else: # If id / uid is available we try to look up Trigger by id. This way we can avoid bug in # pymongo / mongoengine related to "parameters" dictionary lookups trigger_id = trigger.get('id', None) trigger_uid = trigger.get('uid', None) # TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary # passed to this method always contains id or uid if trigger_id: LOG.debug('Looking up TriggerDB by id: %s', trigger_id) trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id) elif trigger_uid: LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid) trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid) else: # Last resort - look it up by parameters trigger_type = trigger.get('type', None) parameters = trigger.get('parameters', {}) LOG.debug('Looking up TriggerDB by type and parameters: type=%s, parameters=%s', trigger_type, parameters) trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=trigger_type, parameters=parameters) if trigger_db is None: LOG.debug('No trigger in db for %s', trigger) if raise_on_no_trigger: raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger) return None trigger_ref = trigger_db.get_reference().ref trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time trigger_instance.status = TRIGGER_INSTANCE_PENDING return TriggerInstance.add_or_update(trigger_instance)
def test_purge(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={ 'hola': 'hi', 'kuraci': 'chicken' }, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={ 'hola': 'hi', 'kuraci': 'chicken' }, occurrence_time=now - timedelta(days=5), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 2) purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(TriggerInstance.get_all()), 1)
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref( db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars( TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=False) # Update the web_url field in execution. Unfortunately, we need # the execution id for constructing the URL which we only get # after the model is written to disk. execution.web_url = _get_web_url_for_execution(str(execution.id)) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def _create_trigger_instance(cls, trigger_ref, payload, seconds): # Note: We use 1 second intervals between occurence time to prevent # occasional test failures occurrence_time = date_utils.get_datetime_utc_now() occurrence_time = occurrence_time + datetime.timedelta(seconds=seconds) trigger_instance = TriggerInstanceDB() trigger_instance.trigger = trigger_ref trigger_instance.payload = payload trigger_instance.occurrence_time = occurrence_time created = TriggerInstance.add_or_update(trigger_instance) cls.triggerinstance_count += 1 return created
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) attrs['log'] = [_create_execution_log_entry(liveaction['status'])] execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=False) # Update the web_url field in execution. Unfortunately, we need # the execution id for constructing the URL which we only get # after the model is written to disk. execution.web_url = _get_web_url_for_execution(str(execution.id)) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def test_triggered_execution(self): docs = { 'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']), 'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']), 'rule': copy.deepcopy(fixture.ARTIFACTS['rule']), 'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])} # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type']))) trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger']))) rule = RuleAPI.to_model(RuleAPI(**docs['rule'])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance']))) trace_service.add_or_update_given_trace_context( trace_context={'trace_tag': 'test_triggered_execution_trace'}, trigger_instances=[str(trigger_instance.id)]) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED) execution = self._get_action_execution( liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual(execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance))) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type['name']) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) self.assertEqual(execution.end_timestamp, liveaction.end_timestamp) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction['callback'], liveaction.callback) self.assertEqual(execution.liveaction['action'], liveaction.action)
def get_one(self, id): """ List triggerinstance by id. Handle: GET /triggerinstances/1 """ try: trigger_instance_db = TriggerInstance.get_by_id(id) except (ValueError, ValidationError): LOG.exception('Database lookup for id="%s" resulted in exception.', id) abort(http_client.NOT_FOUND) return trigger_instance_api = TriggerInstanceAPI.from_model(trigger_instance_db) return trigger_instance_api
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref( db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars( TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = ActionExecution.get( liveaction__id=liveaction.context.get('parent', '')) if parent: attrs['parent'] = str(parent.id) execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def create_execution_object(liveaction, publish=True): action_db = action_utils.get_action_by_ref(liveaction.action) runner = RunnerType.get_by_name(action_db.runner_type['name']) attrs = { 'action': vars(ActionAPI.from_model(action_db)), 'parameters': liveaction['parameters'], 'runner': vars(RunnerTypeAPI.from_model(runner)) } attrs.update(_decompose_liveaction(liveaction)) if 'rule' in liveaction.context: rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {})) attrs['rule'] = vars(RuleAPI.from_model(rule)) if 'trigger_instance' in liveaction.context: trigger_instance_id = liveaction.context.get('trigger_instance', {}) trigger_instance_id = trigger_instance_id.get('id', None) trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger = reference.get_model_by_resource_ref(db_api=Trigger, ref=trigger_instance.trigger) trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType, ref=trigger.type) trigger_instance = reference.get_model_from_ref( TriggerInstance, liveaction.context.get('trigger_instance', {})) attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance)) attrs['trigger'] = vars(TriggerAPI.from_model(trigger)) attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type)) parent = _get_parent_execution(liveaction) if parent: attrs['parent'] = str(parent.id) execution = ActionExecutionDB(**attrs) execution = ActionExecution.add_or_update(execution, publish=publish) if parent: if str(execution.id) not in parent.children: parent.children.append(str(execution.id)) ActionExecution.add_or_update(parent) return execution
def purge_trigger_instances(timestamp=None): if not timestamp: LOG.error('Specify a valid timestamp to purge.') return LOG.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) # XXX: Think about paginating this call. filters = {'occurrence_time__lt': isotime.parse(timestamp)} instances = TriggerInstance.query(**filters) LOG.info('#### Total number of trigger instances to delete: %d' % len(instances)) # Purge execution and liveaction models now for instance in instances: _purge_model(instance) # Print stats LOG.info('#### Total trigger instances deleted: %d' % DELETED_COUNT)
def test_purge(self): now = date_utils.get_datetime_utc_now() instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=20), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger', payload={'hola': 'hi', 'kuraci': 'chicken'}, occurrence_time=now - timedelta(days=5), status=TRIGGER_INSTANCE_PROCESSED) TriggerInstance.add_or_update(instance_db) self.assertEqual(len(TriggerInstance.get_all()), 2) purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10)) self.assertEqual(len(TriggerInstance.get_all()), 1)
def _refire_trigger_instance(trigger_instance_id, log_): trigger_instance = TriggerInstance.get_by_id(trigger_instance_id) trigger_dispatcher = TriggerDispatcher(log_) trigger_dispatcher.dispatch(trigger=trigger_instance.trigger, payload=trigger_instance.payload)
def test_migrate_triggers(self): TriggerInstanceDB._meta["allow_inheritance"] = True class TriggerInstanceDB_OldFieldType(TriggerInstanceDB): payload = stormbase.EscapedDictField() trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(trigger_instance_dbs), 0) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 0) # 1. Insert data in old format trigger_instance_1_db = TriggerInstanceDB_OldFieldType() trigger_instance_1_db.payload = MOCK_PAYLOAD_1 trigger_instance_1_db.status = TRIGGER_INSTANCE_PROCESSED trigger_instance_1_db.occurrence_time = datetime.datetime.utcnow() trigger_instance_1_db = TriggerInstance.add_or_update( trigger_instance_1_db, publish=False) trigger_instance_2_db = TriggerInstanceDB_OldFieldType() trigger_instance_2_db.payload = MOCK_PAYLOAD_2 trigger_instance_2_db.status = TRIGGER_INSTANCE_PENDING trigger_instance_2_db.occurrence_time = datetime.datetime.utcnow() trigger_instance_2_db = TriggerInstance.add_or_update( trigger_instance_2_db, publish=False) # This object is older than the default threshold so it should not be migrated trigger_instance_3_db = TriggerInstanceDB_OldFieldType() trigger_instance_3_db.payload = MOCK_PAYLOAD_2 trigger_instance_3_db.status = TRIGGER_INSTANCE_PROCESSED trigger_instance_3_db.occurrence_time = datetime.datetime.utcfromtimestamp( 0) trigger_instance_3_db = TriggerInstance.add_or_update( trigger_instance_3_db, publish=False) # Verify data has been inserted in old format trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) self.assertEqual(len(trigger_instance_dbs), 3) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 3) # Update inserted documents and remove special _cls field added by mongoengine. We need to # do that here due to how mongoengine works with subclasses. TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }).update(set___cls="TriggerInstanceDB") # 2. Run migration start_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2) end_dt = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) migration_module.migrate_triggers(start_dt=start_dt, end_dt=end_dt) # 3. Verify data has been migrated - only 1 item should have been migrated since it's in a # completed state trigger_instance_dbs = TriggerInstance.query( __raw__={"payload": { "$not": { "$type": "binData", }, }}) # TODO: Also verify raw as_pymongo() bin field value self.assertEqual(len(trigger_instance_dbs), 2) trigger_instance_dbs = TriggerInstance.query(__raw__={ "payload": { "$type": "object", }, }) self.assertEqual(len(trigger_instance_dbs), 2) trigger_instance_1_db_retrieved = TriggerInstance.get_by_id( trigger_instance_1_db.id) self.assertEqual(trigger_instance_1_db_retrieved.payload, MOCK_PAYLOAD_1) trigger_instance_2_db_retrieved = TriggerInstance.get_by_id( trigger_instance_2_db.id) self.assertEqual(trigger_instance_2_db_retrieved.payload, MOCK_PAYLOAD_2)
def test_triggered_execution(self): docs = { "trigger_type": copy.deepcopy(fixture.ARTIFACTS["trigger_type"]), "trigger": copy.deepcopy(fixture.ARTIFACTS["trigger"]), "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "trigger_instance": copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]), } # Trigger an action execution. trigger_type = TriggerType.add_or_update( TriggerTypeAPI.to_model(TriggerTypeAPI(**docs["trigger_type"]))) trigger = Trigger.add_or_update( TriggerAPI.to_model(TriggerAPI(**docs["trigger"]))) rule = RuleAPI.to_model(RuleAPI(**docs["rule"])) rule.trigger = reference.get_str_resource_ref_from_model(trigger) rule = Rule.add_or_update(rule) trigger_instance = TriggerInstance.add_or_update( TriggerInstanceAPI.to_model( TriggerInstanceAPI(**docs["trigger_instance"]))) trace_service.add_or_update_given_trace_context( trace_context={"trace_tag": "test_triggered_execution_trace"}, trigger_instances=[str(trigger_instance.id)], ) enforcer = RuleEnforcer(trigger_instance, rule) enforcer.enforce() # Wait for the action execution to complete and then confirm outcome. liveaction = LiveAction.get( context__trigger_instance__id=str(trigger_instance.id)) self.assertIsNotNone(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_FAILED) execution = self._get_action_execution(liveaction__id=str( liveaction.id), raise_exception=True) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type))) self.assertDictEqual( execution.trigger_instance, vars(TriggerInstanceAPI.from_model(trigger_instance)), ) self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule))) action = action_utils.get_action_by_ref(liveaction.action) self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action))) runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(execution.start_timestamp, liveaction.start_timestamp) # NOTE: Timestamp of liveaction and execution may be a bit different, depending on how long # it takes to persist each object in the database self.assertEqual( execution.end_timestamp.replace(microsecond=0), liveaction.end_timestamp.replace(microsecond=0), ) self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) self.assertEqual(execution.liveaction["callback"], liveaction.callback) self.assertEqual(execution.liveaction["action"], liveaction.action)
def update_trigger_instance_status(trigger_instance, status): trigger_instance.status = status return TriggerInstance.add_or_update(trigger_instance)
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB(trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now()) return TriggerInstance.add_or_update(created)
def _append_view_properties(self, rule_enforcement_apis): """ Method which appends corresponding execution (if available) and trigger instance object properties. """ trigger_instance_ids = set([]) execution_ids = [] for rule_enforcement_api in rule_enforcement_apis: if rule_enforcement_api.get('trigger_instance_id', None): trigger_instance_ids.add(str(rule_enforcement_api['trigger_instance_id'])) if rule_enforcement_api.get('execution_id', None): execution_ids.append(rule_enforcement_api['execution_id']) # 1. Retrieve corresponding execution objects # NOTE: Executions contain a lot of field and could contain a lot of data so we only # retrieve fields we need only_fields = [ 'id', 'action.ref', 'action.parameters', 'runner.name', 'runner.runner_parameters', 'parameters', 'status' ] execution_dbs = ActionExecution.query(id__in=execution_ids, only_fields=only_fields) execution_dbs_by_id = {} for execution_db in execution_dbs: execution_dbs_by_id[str(execution_db.id)] = execution_db # 2. Retrieve corresponding trigger instance objects trigger_instance_dbs = TriggerInstance.query(id__in=list(trigger_instance_ids)) trigger_instance_dbs_by_id = {} for trigger_instance_db in trigger_instance_dbs: trigger_instance_dbs_by_id[str(trigger_instance_db.id)] = trigger_instance_db # Ammend rule enforcement objects with additional data for rule_enforcement_api in rule_enforcement_apis: rule_enforcement_api['trigger_instance'] = {} rule_enforcement_api['execution'] = {} trigger_instance_id = rule_enforcement_api.get('trigger_instance_id', None) execution_id = rule_enforcement_api.get('execution_id', None) trigger_instance_db = trigger_instance_dbs_by_id.get(trigger_instance_id, None) execution_db = execution_dbs_by_id.get(execution_id, None) if trigger_instance_db: trigger_instance_api = TriggerInstanceAPI.from_model(trigger_instance_db) rule_enforcement_api['trigger_instance'] = trigger_instance_api if execution_db: execution_api = ActionExecutionAPI.from_model(execution_db) rule_enforcement_api['execution'] = execution_api return rule_enforcement_apis
def _create_save_triggerinstance(trigger): created = TriggerInstanceDB( trigger=trigger.get_reference().ref, payload={}, occurrence_time=date_utils.get_datetime_utc_now()) return TriggerInstance.add_or_update(created)