def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
query_filters = {'occurrence_time__lt': isotime.parse(timestamp)}
# TODO: Update this code to return statistics on deleted objects once we
# upgrade to newer version of MongoDB where delete_by_query actually returns
# some data
try:
TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete trigger instances: %s'
'Please contact support.' % (query_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deleting instances using query_filters %s failed.',
query_filters)
# Print stats
logger.info(
'All trigger instance models older than timestamp %s were deleted.',
timestamp)
def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
query_filters = {'occurrence_time__lt': isotime.parse(timestamp)}
# TODO: Update this code to return statistics on deleted objects once we
# upgrade to newer version of MongoDB where delete_by_query actually returns
# some data
try:
TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete trigger instances: %s'
'Please contact support.' % (query_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deleting instances using query_filters %s failed.', query_filters)
# Print stats
logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def _purge_model(instance):
try:
TriggerInstance.delete(instance)
except:
LOG.exception('Exception deleting instance %s.', instance)
else:
global DELETED_COUNT
DELETED_COUNT += 1
def _purge_model(instance):
try:
TriggerInstance.delete(instance)
except:
LOG.exception('Exception deleting instance %s.', instance)
else:
global DELETED_COUNT
DELETED_COUNT += 1
def test_no_timestamp_doesnt_delete(self):
now = date_utils.get_datetime_utc_now()
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={'hola': 'hi', 'kuraci': 'chicken'},
occurrence_time=now - timedelta(days=20))
TriggerInstance.add_or_update(instance_db)
self.assertEqual(len(TriggerInstance.get_all()), 1)
purge_trigger_instances()
self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_triggerinstance_crud(self):
triggertype = ReactorModelTest._create_save_triggertype()
trigger = ReactorModelTest._create_save_trigger(triggertype)
saved = ReactorModelTest._create_save_triggerinstance(trigger)
retrieved = TriggerInstance.get_by_id(saved.id)
self.assertIsNotNone(retrieved, 'No triggerinstance created.')
ReactorModelTest._delete([retrieved, trigger, triggertype])
try:
retrieved = TriggerInstance.get_by_id(saved.id)
except StackStormDBObjectNotFoundError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_triggerinstance_crud(self):
triggertype = ReactorModelTest._create_save_triggertype()
trigger = ReactorModelTest._create_save_trigger(triggertype)
saved = ReactorModelTest._create_save_triggerinstance(trigger)
retrieved = TriggerInstance.get_by_id(saved.id)
self.assertIsNotNone(retrieved, "No triggerinstance created.")
ReactorModelTest._delete([retrieved, trigger, triggertype])
try:
retrieved = TriggerInstance.get_by_id(saved.id)
except StackStormDBObjectNotFoundError:
retrieved = None
self.assertIsNone(retrieved, "managed to retrieve after failure.")
def test_triggerinstance_crud(self):
triggertype = ReactorModelTest._create_save_triggertype()
trigger = ReactorModelTest._create_save_trigger(triggertype)
saved = ReactorModelTest._create_save_triggerinstance(trigger)
retrieved = TriggerInstance.get_by_id(saved.id)
self.assertIsNotNone(retrieved, 'No triggerinstance created.')
ReactorModelTest._delete([retrieved, trigger, triggertype])
try:
retrieved = TriggerInstance.get_by_id(saved.id)
except ValueError:
retrieved = None
self.assertIsNone(retrieved, 'managed to retrieve after failure.')
def test_no_timestamp_doesnt_delete(self):
now = date_utils.get_datetime_utc_now()
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={'hola': 'hi', 'kuraci': 'chicken'},
occurrence_time=now - timedelta(days=20))
TriggerInstance.add_or_update(instance_db)
self.assertEqual(len(TriggerInstance.get_all()), 1)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg,
purge_trigger_instances,
logger=LOG, timestamp=None)
self.assertEqual(len(TriggerInstance.get_all()), 1)
def test_no_timestamp_doesnt_delete(self):
now = date_utils.get_datetime_utc_now()
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={'hola': 'hi', 'kuraci': 'chicken'},
occurrence_time=now - timedelta(days=20),
status=TRIGGER_INSTANCE_PROCESSED)
TriggerInstance.add_or_update(instance_db)
self.assertEqual(len(TriggerInstance.get_all()), 1)
expected_msg = 'Specify a valid timestamp'
self.assertRaisesRegexp(ValueError, expected_msg,
purge_trigger_instances,
logger=LOG, timestamp=None)
self.assertEqual(len(TriggerInstance.get_all()), 1)
def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
query_filters = {'occurrence_time__lt': isotime.parse(timestamp)}
try:
deleted_count = TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete trigger instances: %s'
'Please contact support.' % (query_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deleting instances using query_filters %s failed.', query_filters)
else:
logger.info('Deleted %s trigger instance objects' % (deleted_count))
# Print stats
logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or
'uid' or type' and 'parameters' keys.
:param trigger: Trigger reference or dictionary with trigger query filters.
:type trigger: ``str`` or ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger)
if not trigger_db:
LOG.debug('No trigger in db for %s', trigger)
if raise_on_no_trigger:
raise StackStormDBObjectNotFoundError('Trigger not found for %s' % trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
trigger_instance.status = TRIGGER_INSTANCE_PENDING
return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger, payload, occurrence_time):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict``
containing 'type' and 'parameters'.
:param trigger: Dictionary with trigger query filters.
:type trigger: ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
# TODO: This is nasty, this should take a unique reference and not a dict
if isinstance(trigger, six.string_types):
trigger_db = TriggerService.get_trigger_db_by_ref(trigger)
else:
type_ = trigger.get('type', None)
parameters = trigger.get('parameters', {})
trigger_db = TriggerService.get_trigger_db_given_type_and_params(
type=type_, parameters=parameters)
if trigger_db is None:
LOG.info('No trigger in db for %s', trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
return TriggerInstance.add_or_update(trigger_instance)
def _create_save_triggerinstance(trigger):
created = TriggerInstanceDB(
trigger=trigger.get_reference().ref,
payload={},
occurrence_time=date_utils.get_datetime_utc_now(),
status=TRIGGER_INSTANCE_PROCESSED)
return TriggerInstance.add_or_update(created)
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict``
containing 'type' and 'parameters'.
:param trigger: Dictionary with trigger query filters.
:type trigger: ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
# TODO: This is nasty, this should take a unique reference and not a dict
if isinstance(trigger, six.string_types):
trigger_db = TriggerService.get_trigger_db_by_ref(trigger)
else:
type_ = trigger.get('type', None)
parameters = trigger.get('parameters', {})
trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=type_,
parameters=parameters)
if trigger_db is None:
LOG.debug('No trigger in db for %s', trigger)
if raise_on_no_trigger:
raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
trigger_instance.status = TRIGGER_INSTANCE_PENDING
return TriggerInstance.add_or_update(trigger_instance)
def create_trigger_instance(trigger,
payload,
occurrence_time,
raise_on_no_trigger=False):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or
'uid' or type' and 'parameters' keys.
:param trigger: Trigger reference or dictionary with trigger query filters.
:type trigger: ``str`` or ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
trigger_db = get_trigger_db_by_ref_or_dict(trigger=trigger)
if not trigger_db:
LOG.debug("No trigger in db for %s", trigger)
if raise_on_no_trigger:
raise StackStormDBObjectNotFoundError("Trigger not found for %s" %
trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
trigger_instance.status = TRIGGER_INSTANCE_PENDING
return TriggerInstance.add_or_update(trigger_instance)
def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError("Specify a valid timestamp to purge.")
logger.info("Purging trigger instances older than timestamp: %s" %
timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
query_filters = {"occurrence_time__lt": isotime.parse(timestamp)}
try:
deleted_count = TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ("Bad query (%s) used to delete trigger instances: %s"
"Please contact support." % (
query_filters,
six.text_type(e),
))
raise InvalidQueryError(msg)
except:
logger.exception("Deleting instances using query_filters %s failed.",
query_filters)
else:
logger.info("Deleted %s trigger instance objects" % (deleted_count))
# Print stats
logger.info(
"All trigger instance models older than timestamp %s were deleted.",
timestamp)
def _create_trigger_instance(cls, trigger_ref, payload):
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = datetime.datetime.utcnow()
created = TriggerInstance.add_or_update(trigger_instance)
cls.triggerinstance_count += 1
return created
def _create_save_triggerinstance(trigger):
created = TriggerInstanceDB(
trigger=trigger.get_reference().ref,
payload={},
occurrence_time=date_utils.get_datetime_utc_now(),
status=TRIGGER_INSTANCE_PROCESSED,
)
return TriggerInstance.add_or_update(created)
def _create_trigger_instance(cls, trigger_ref, payload):
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = date_utils.get_datetime_utc_now()
created = TriggerInstance.add_or_update(trigger_instance)
cls.triggerinstance_count += 1
return created
def create_trigger_instance(trigger,
payload,
occurrence_time,
raise_on_no_trigger=False):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or
'uid' or type' and 'parameters' keys.
:param trigger: Trigger reference or dictionary with trigger query filters.
:type trigger: ``str`` or ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
# TODO: This is nasty, this should take a unique reference and not a dict
if isinstance(trigger, six.string_types):
trigger_db = TriggerService.get_trigger_db_by_ref(trigger)
else:
# If id / uid is available we try to look up Trigger by id. This way we can avoid bug in
# pymongo / mongoengine related to "parameters" dictionary lookups
trigger_id = trigger.get('id', None)
trigger_uid = trigger.get('uid', None)
# TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary
# passed to this method always contains id or uid
if trigger_id:
LOG.debug('Looking up TriggerDB by id: %s', trigger_id)
trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id)
elif trigger_uid:
LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid)
trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid)
else:
# Last resort - look it up by parameters
trigger_type = trigger.get('type', None)
parameters = trigger.get('parameters', {})
LOG.debug(
'Looking up TriggerDB by type and parameters: type=%s, parameters=%s',
trigger_type, parameters)
trigger_db = TriggerService.get_trigger_db_given_type_and_params(
type=trigger_type, parameters=parameters)
if trigger_db is None:
LOG.debug('No trigger in db for %s', trigger)
if raise_on_no_trigger:
raise StackStormDBObjectNotFoundError('Trigger not found for %s',
trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
trigger_instance.status = TRIGGER_INSTANCE_PENDING
return TriggerInstance.add_or_update(trigger_instance)
def get_all(self, **kw):
"""
List all triggerinstances.
Handles requests:
GET /triggerinstances/
"""
trigger_instance_apis = [TriggerInstanceAPI.from_model(trigger_instance_db)
for trigger_instance_db in TriggerInstance.get_all(**kw)]
return trigger_instance_apis
def create_execution_object(liveaction, action_db=None, runnertype_db=None, publish=True):
if not action_db:
action_db = action_utils.get_action_by_ref(liveaction.action)
if not runnertype_db:
runnertype_db = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runnertype_db))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
# TODO: This object initialization takes 20-30or so ms
execution = ActionExecutionDB(**attrs)
# TODO: Do 100% research this is fully safe and unique in distributed setups
execution.id = ObjectId()
execution.web_url = _get_web_url_for_execution(str(execution.id))
# NOTE: User input data is already validate as part of the API request,
# other data is set by us. Skipping validation here makes operation 10%-30% faster
execution = ActionExecution.add_or_update(execution, publish=publish, validate=False)
if parent and str(execution.id) not in parent.children:
values = {}
values['push__children'] = str(execution.id)
ActionExecution.update(parent, **values)
return execution
def create_trigger_instance(trigger, payload, occurrence_time, raise_on_no_trigger=False):
"""
This creates a trigger instance object given trigger and payload.
Trigger can be just a string reference (pack.name) or a ``dict`` containing 'id' or
'uid' or type' and 'parameters' keys.
:param trigger: Trigger reference or dictionary with trigger query filters.
:type trigger: ``str`` or ``dict``
:param payload: Trigger payload.
:type payload: ``dict``
"""
# TODO: This is nasty, this should take a unique reference and not a dict
if isinstance(trigger, six.string_types):
trigger_db = TriggerService.get_trigger_db_by_ref(trigger)
else:
# If id / uid is available we try to look up Trigger by id. This way we can avoid bug in
# pymongo / mongoengine related to "parameters" dictionary lookups
trigger_id = trigger.get('id', None)
trigger_uid = trigger.get('uid', None)
# TODO: Remove parameters dictionary look up when we can confirm each trigger dictionary
# passed to this method always contains id or uid
if trigger_id:
LOG.debug('Looking up TriggerDB by id: %s', trigger_id)
trigger_db = TriggerService.get_trigger_db_by_id(id=trigger_id)
elif trigger_uid:
LOG.debug('Looking up TriggerDB by uid: %s', trigger_uid)
trigger_db = TriggerService.get_trigger_db_by_uid(uid=trigger_uid)
else:
# Last resort - look it up by parameters
trigger_type = trigger.get('type', None)
parameters = trigger.get('parameters', {})
LOG.debug('Looking up TriggerDB by type and parameters: type=%s, parameters=%s',
trigger_type, parameters)
trigger_db = TriggerService.get_trigger_db_given_type_and_params(type=trigger_type,
parameters=parameters)
if trigger_db is None:
LOG.debug('No trigger in db for %s', trigger)
if raise_on_no_trigger:
raise StackStormDBObjectNotFoundError('Trigger not found for %s', trigger)
return None
trigger_ref = trigger_db.get_reference().ref
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
trigger_instance.status = TRIGGER_INSTANCE_PENDING
return TriggerInstance.add_or_update(trigger_instance)
def test_purge(self):
now = date_utils.get_datetime_utc_now()
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={
'hola': 'hi',
'kuraci': 'chicken'
},
occurrence_time=now -
timedelta(days=20),
status=TRIGGER_INSTANCE_PROCESSED)
TriggerInstance.add_or_update(instance_db)
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={
'hola': 'hi',
'kuraci': 'chicken'
},
occurrence_time=now -
timedelta(days=5),
status=TRIGGER_INSTANCE_PROCESSED)
TriggerInstance.add_or_update(instance_db)
self.assertEqual(len(TriggerInstance.get_all()), 2)
purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10))
self.assertEqual(len(TriggerInstance.get_all()), 1)
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule,
liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(
db_api=Trigger, ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(
TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=False)
# Update the web_url field in execution. Unfortunately, we need
# the execution id for constructing the URL which we only get
# after the model is written to disk.
execution.web_url = _get_web_url_for_execution(str(execution.id))
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def _create_trigger_instance(cls, trigger_ref, payload, seconds):
# Note: We use 1 second intervals between occurence time to prevent
# occasional test failures
occurrence_time = date_utils.get_datetime_utc_now()
occurrence_time = occurrence_time + datetime.timedelta(seconds=seconds)
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
created = TriggerInstance.add_or_update(trigger_instance)
cls.triggerinstance_count += 1
return created
def _create_trigger_instance(cls, trigger_ref, payload, seconds):
# Note: We use 1 second intervals between occurence time to prevent
# occasional test failures
occurrence_time = date_utils.get_datetime_utc_now()
occurrence_time = occurrence_time + datetime.timedelta(seconds=seconds)
trigger_instance = TriggerInstanceDB()
trigger_instance.trigger = trigger_ref
trigger_instance.payload = payload
trigger_instance.occurrence_time = occurrence_time
created = TriggerInstance.add_or_update(trigger_instance)
cls.triggerinstance_count += 1
return created
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
attrs['log'] = [_create_execution_log_entry(liveaction['status'])]
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=False)
# Update the web_url field in execution. Unfortunately, we need
# the execution id for constructing the URL which we only get
# after the model is written to disk.
execution.web_url = _get_web_url_for_execution(str(execution.id))
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def test_triggered_execution(self):
docs = {
'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance'])}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs['trigger_type'])))
trigger = Trigger.add_or_update(TriggerAPI.to_model(TriggerAPI(**docs['trigger'])))
rule = RuleAPI.to_model(RuleAPI(**docs['rule']))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(TriggerInstanceAPI(**docs['trigger_instance'])))
trace_service.add_or_update_given_trace_context(
trace_context={'trace_tag': 'test_triggered_execution_trace'},
trigger_instances=[str(trigger_instance.id)])
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(
liveaction__id=str(liveaction.id),
raise_exception=True
)
self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type, vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)))
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action, vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type['name'])
self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
self.assertEqual(execution.end_timestamp, liveaction.end_timestamp)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction['callback'], liveaction.callback)
self.assertEqual(execution.liveaction['action'], liveaction.action)
def get_one(self, id):
"""
List triggerinstance by id.
Handle:
GET /triggerinstances/1
"""
try:
trigger_instance_db = TriggerInstance.get_by_id(id)
except (ValueError, ValidationError):
LOG.exception('Database lookup for id="%s" resulted in exception.', id)
abort(http_client.NOT_FOUND)
return
trigger_instance_api = TriggerInstanceAPI.from_model(trigger_instance_db)
return trigger_instance_api
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule,
liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(
db_api=Trigger, ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(
TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = ActionExecution.get(
liveaction__id=liveaction.context.get('parent', ''))
if parent:
attrs['parent'] = str(parent.id)
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def create_execution_object(liveaction, publish=True):
action_db = action_utils.get_action_by_ref(liveaction.action)
runner = RunnerType.get_by_name(action_db.runner_type['name'])
attrs = {
'action': vars(ActionAPI.from_model(action_db)),
'parameters': liveaction['parameters'],
'runner': vars(RunnerTypeAPI.from_model(runner))
}
attrs.update(_decompose_liveaction(liveaction))
if 'rule' in liveaction.context:
rule = reference.get_model_from_ref(Rule, liveaction.context.get('rule', {}))
attrs['rule'] = vars(RuleAPI.from_model(rule))
if 'trigger_instance' in liveaction.context:
trigger_instance_id = liveaction.context.get('trigger_instance', {})
trigger_instance_id = trigger_instance_id.get('id', None)
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger = reference.get_model_by_resource_ref(db_api=Trigger,
ref=trigger_instance.trigger)
trigger_type = reference.get_model_by_resource_ref(db_api=TriggerType,
ref=trigger.type)
trigger_instance = reference.get_model_from_ref(
TriggerInstance, liveaction.context.get('trigger_instance', {}))
attrs['trigger_instance'] = vars(TriggerInstanceAPI.from_model(trigger_instance))
attrs['trigger'] = vars(TriggerAPI.from_model(trigger))
attrs['trigger_type'] = vars(TriggerTypeAPI.from_model(trigger_type))
parent = _get_parent_execution(liveaction)
if parent:
attrs['parent'] = str(parent.id)
execution = ActionExecutionDB(**attrs)
execution = ActionExecution.add_or_update(execution, publish=publish)
if parent:
if str(execution.id) not in parent.children:
parent.children.append(str(execution.id))
ActionExecution.add_or_update(parent)
return execution
def purge_trigger_instances(timestamp=None):
if not timestamp:
LOG.error('Specify a valid timestamp to purge.')
return
LOG.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
# XXX: Think about paginating this call.
filters = {'occurrence_time__lt': isotime.parse(timestamp)}
instances = TriggerInstance.query(**filters)
LOG.info('#### Total number of trigger instances to delete: %d' % len(instances))
# Purge execution and liveaction models now
for instance in instances:
_purge_model(instance)
# Print stats
LOG.info('#### Total trigger instances deleted: %d' % DELETED_COUNT)
def purge_trigger_instances(timestamp=None):
if not timestamp:
LOG.error('Specify a valid timestamp to purge.')
return
LOG.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
# XXX: Think about paginating this call.
filters = {'occurrence_time__lt': isotime.parse(timestamp)}
instances = TriggerInstance.query(**filters)
LOG.info('#### Total number of trigger instances to delete: %d' %
len(instances))
# Purge execution and liveaction models now
for instance in instances:
_purge_model(instance)
# Print stats
LOG.info('#### Total trigger instances deleted: %d' % DELETED_COUNT)
def test_purge(self):
now = date_utils.get_datetime_utc_now()
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={'hola': 'hi', 'kuraci': 'chicken'},
occurrence_time=now - timedelta(days=20),
status=TRIGGER_INSTANCE_PROCESSED)
TriggerInstance.add_or_update(instance_db)
instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
payload={'hola': 'hi', 'kuraci': 'chicken'},
occurrence_time=now - timedelta(days=5),
status=TRIGGER_INSTANCE_PROCESSED)
TriggerInstance.add_or_update(instance_db)
self.assertEqual(len(TriggerInstance.get_all()), 2)
purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10))
self.assertEqual(len(TriggerInstance.get_all()), 1)
def _refire_trigger_instance(trigger_instance_id, log_):
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger_dispatcher = TriggerDispatcher(log_)
trigger_dispatcher.dispatch(trigger=trigger_instance.trigger,
payload=trigger_instance.payload)
def test_migrate_triggers(self):
TriggerInstanceDB._meta["allow_inheritance"] = True
class TriggerInstanceDB_OldFieldType(TriggerInstanceDB):
payload = stormbase.EscapedDictField()
trigger_instance_dbs = TriggerInstance.query(
__raw__={"payload": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(trigger_instance_dbs), 0)
trigger_instance_dbs = TriggerInstance.query(__raw__={
"payload": {
"$type": "object",
},
})
self.assertEqual(len(trigger_instance_dbs), 0)
# 1. Insert data in old format
trigger_instance_1_db = TriggerInstanceDB_OldFieldType()
trigger_instance_1_db.payload = MOCK_PAYLOAD_1
trigger_instance_1_db.status = TRIGGER_INSTANCE_PROCESSED
trigger_instance_1_db.occurrence_time = datetime.datetime.utcnow()
trigger_instance_1_db = TriggerInstance.add_or_update(
trigger_instance_1_db, publish=False)
trigger_instance_2_db = TriggerInstanceDB_OldFieldType()
trigger_instance_2_db.payload = MOCK_PAYLOAD_2
trigger_instance_2_db.status = TRIGGER_INSTANCE_PENDING
trigger_instance_2_db.occurrence_time = datetime.datetime.utcnow()
trigger_instance_2_db = TriggerInstance.add_or_update(
trigger_instance_2_db, publish=False)
# This object is older than the default threshold so it should not be migrated
trigger_instance_3_db = TriggerInstanceDB_OldFieldType()
trigger_instance_3_db.payload = MOCK_PAYLOAD_2
trigger_instance_3_db.status = TRIGGER_INSTANCE_PROCESSED
trigger_instance_3_db.occurrence_time = datetime.datetime.utcfromtimestamp(
0)
trigger_instance_3_db = TriggerInstance.add_or_update(
trigger_instance_3_db, publish=False)
# Verify data has been inserted in old format
trigger_instance_dbs = TriggerInstance.query(
__raw__={"payload": {
"$not": {
"$type": "binData",
},
}})
self.assertEqual(len(trigger_instance_dbs), 3)
trigger_instance_dbs = TriggerInstance.query(__raw__={
"payload": {
"$type": "object",
},
})
self.assertEqual(len(trigger_instance_dbs), 3)
# Update inserted documents and remove special _cls field added by mongoengine. We need to
# do that here due to how mongoengine works with subclasses.
TriggerInstance.query(__raw__={
"payload": {
"$type": "object",
},
}).update(set___cls="TriggerInstanceDB")
# 2. Run migration
start_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=2)
end_dt = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
migration_module.migrate_triggers(start_dt=start_dt, end_dt=end_dt)
# 3. Verify data has been migrated - only 1 item should have been migrated since it's in a
# completed state
trigger_instance_dbs = TriggerInstance.query(
__raw__={"payload": {
"$not": {
"$type": "binData",
},
}})
# TODO: Also verify raw as_pymongo() bin field value
self.assertEqual(len(trigger_instance_dbs), 2)
trigger_instance_dbs = TriggerInstance.query(__raw__={
"payload": {
"$type": "object",
},
})
self.assertEqual(len(trigger_instance_dbs), 2)
trigger_instance_1_db_retrieved = TriggerInstance.get_by_id(
trigger_instance_1_db.id)
self.assertEqual(trigger_instance_1_db_retrieved.payload,
MOCK_PAYLOAD_1)
trigger_instance_2_db_retrieved = TriggerInstance.get_by_id(
trigger_instance_2_db.id)
self.assertEqual(trigger_instance_2_db_retrieved.payload,
MOCK_PAYLOAD_2)
def test_triggered_execution(self):
docs = {
"trigger_type":
copy.deepcopy(fixture.ARTIFACTS["trigger_type"]),
"trigger":
copy.deepcopy(fixture.ARTIFACTS["trigger"]),
"rule":
copy.deepcopy(fixture.ARTIFACTS["rule"]),
"trigger_instance":
copy.deepcopy(fixture.ARTIFACTS["trigger_instance"]),
}
# Trigger an action execution.
trigger_type = TriggerType.add_or_update(
TriggerTypeAPI.to_model(TriggerTypeAPI(**docs["trigger_type"])))
trigger = Trigger.add_or_update(
TriggerAPI.to_model(TriggerAPI(**docs["trigger"])))
rule = RuleAPI.to_model(RuleAPI(**docs["rule"]))
rule.trigger = reference.get_str_resource_ref_from_model(trigger)
rule = Rule.add_or_update(rule)
trigger_instance = TriggerInstance.add_or_update(
TriggerInstanceAPI.to_model(
TriggerInstanceAPI(**docs["trigger_instance"])))
trace_service.add_or_update_given_trace_context(
trace_context={"trace_tag": "test_triggered_execution_trace"},
trigger_instances=[str(trigger_instance.id)],
)
enforcer = RuleEnforcer(trigger_instance, rule)
enforcer.enforce()
# Wait for the action execution to complete and then confirm outcome.
liveaction = LiveAction.get(
context__trigger_instance__id=str(trigger_instance.id))
self.assertIsNotNone(liveaction)
liveaction = self._wait_on_status(
liveaction, action_constants.LIVEACTION_STATUS_FAILED)
execution = self._get_action_execution(liveaction__id=str(
liveaction.id),
raise_exception=True)
self.assertDictEqual(execution.trigger,
vars(TriggerAPI.from_model(trigger)))
self.assertDictEqual(execution.trigger_type,
vars(TriggerTypeAPI.from_model(trigger_type)))
self.assertDictEqual(
execution.trigger_instance,
vars(TriggerInstanceAPI.from_model(trigger_instance)),
)
self.assertDictEqual(execution.rule, vars(RuleAPI.from_model(rule)))
action = action_utils.get_action_by_ref(liveaction.action)
self.assertDictEqual(execution.action,
vars(ActionAPI.from_model(action)))
runner = RunnerType.get_by_name(action.runner_type["name"])
self.assertDictEqual(execution.runner,
vars(RunnerTypeAPI.from_model(runner)))
liveaction = LiveAction.get_by_id(str(liveaction.id))
self.assertEqual(execution.start_timestamp, liveaction.start_timestamp)
# NOTE: Timestamp of liveaction and execution may be a bit different, depending on how long
# it takes to persist each object in the database
self.assertEqual(
execution.end_timestamp.replace(microsecond=0),
liveaction.end_timestamp.replace(microsecond=0),
)
self.assertEqual(execution.result, liveaction.result)
self.assertEqual(execution.status, liveaction.status)
self.assertEqual(execution.context, liveaction.context)
self.assertEqual(execution.liveaction["callback"], liveaction.callback)
self.assertEqual(execution.liveaction["action"], liveaction.action)
def update_trigger_instance_status(trigger_instance, status):
trigger_instance.status = status
return TriggerInstance.add_or_update(trigger_instance)
def _create_save_triggerinstance(trigger):
created = TriggerInstanceDB(trigger=trigger.get_reference().ref, payload={},
occurrence_time=date_utils.get_datetime_utc_now())
return TriggerInstance.add_or_update(created)
def _append_view_properties(self, rule_enforcement_apis):
"""
Method which appends corresponding execution (if available) and trigger instance object
properties.
"""
trigger_instance_ids = set([])
execution_ids = []
for rule_enforcement_api in rule_enforcement_apis:
if rule_enforcement_api.get('trigger_instance_id', None):
trigger_instance_ids.add(str(rule_enforcement_api['trigger_instance_id']))
if rule_enforcement_api.get('execution_id', None):
execution_ids.append(rule_enforcement_api['execution_id'])
# 1. Retrieve corresponding execution objects
# NOTE: Executions contain a lot of field and could contain a lot of data so we only
# retrieve fields we need
only_fields = [
'id',
'action.ref',
'action.parameters',
'runner.name',
'runner.runner_parameters',
'parameters',
'status'
]
execution_dbs = ActionExecution.query(id__in=execution_ids,
only_fields=only_fields)
execution_dbs_by_id = {}
for execution_db in execution_dbs:
execution_dbs_by_id[str(execution_db.id)] = execution_db
# 2. Retrieve corresponding trigger instance objects
trigger_instance_dbs = TriggerInstance.query(id__in=list(trigger_instance_ids))
trigger_instance_dbs_by_id = {}
for trigger_instance_db in trigger_instance_dbs:
trigger_instance_dbs_by_id[str(trigger_instance_db.id)] = trigger_instance_db
# Ammend rule enforcement objects with additional data
for rule_enforcement_api in rule_enforcement_apis:
rule_enforcement_api['trigger_instance'] = {}
rule_enforcement_api['execution'] = {}
trigger_instance_id = rule_enforcement_api.get('trigger_instance_id', None)
execution_id = rule_enforcement_api.get('execution_id', None)
trigger_instance_db = trigger_instance_dbs_by_id.get(trigger_instance_id, None)
execution_db = execution_dbs_by_id.get(execution_id, None)
if trigger_instance_db:
trigger_instance_api = TriggerInstanceAPI.from_model(trigger_instance_db)
rule_enforcement_api['trigger_instance'] = trigger_instance_api
if execution_db:
execution_api = ActionExecutionAPI.from_model(execution_db)
rule_enforcement_api['execution'] = execution_api
return rule_enforcement_apis
def update_trigger_instance_status(trigger_instance, status):
trigger_instance.status = status
return TriggerInstance.add_or_update(trigger_instance)
def _create_save_triggerinstance(trigger):
created = TriggerInstanceDB(
trigger=trigger.get_reference().ref,
payload={},
occurrence_time=date_utils.get_datetime_utc_now())
return TriggerInstance.add_or_update(created)
def _refire_trigger_instance(trigger_instance_id, log_):
trigger_instance = TriggerInstance.get_by_id(trigger_instance_id)
trigger_dispatcher = TriggerDispatcher(log_)
trigger_dispatcher.dispatch(trigger=trigger_instance.trigger,
payload=trigger_instance.payload)
