def test_what_comes_in_goes_out(self): field = ComplexDateTimeField() date = date_utils.get_datetime_utc_now() us = field._datetime_to_microseconds_since_epoch(date) result = field._microseconds_since_epoch_to_datetime(us) self.assertEqual(date, result)
def test_round_trip_conversion(self): datetime_values = [ datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=500), datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=0), datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=999999) ] datetime_values = [ date_utils.add_utc_tz(datetime_values[0]), date_utils.add_utc_tz(datetime_values[1]), date_utils.add_utc_tz(datetime_values[2]) ] microsecond_values = [] # Calculate microsecond values for value in datetime_values: seconds = calendar.timegm(value.timetuple()) microseconds_reminder = value.time().microsecond result = int(seconds * 1000000) + microseconds_reminder microsecond_values.append(result) field = ComplexDateTimeField() # datetime to us for index, value in enumerate(datetime_values): actual_value = field._datetime_to_microseconds_since_epoch(value=value) expected_value = microsecond_values[index] expected_microseconds = value.time().microsecond self.assertEqual(actual_value, expected_value) self.assertTrue(str(actual_value).endswith(str(expected_microseconds))) # us to datetime for index, value in enumerate(microsecond_values): actual_value = field._microseconds_since_epoch_to_datetime(data=value) expected_value = datetime_values[index] self.assertEqual(actual_value, expected_value)
def test_round_trip_conversion(self): datetime_values = [ datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=500), datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=0), datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=999999) ] microsecond_values = [] # Calculate microsecond values for value in datetime_values: seconds = time.mktime(value.timetuple()) microseconds_reminder = value.time().microsecond result = int(seconds * 1000000) + microseconds_reminder microsecond_values.append(result) field = ComplexDateTimeField() # datetime to us for index, value in enumerate(datetime_values): actual_value = field._datetime_to_microseconds_since_epoch( value=value) expected_value = microsecond_values[index] expected_microseconds = value.time().microsecond self.assertEqual(actual_value, expected_value) self.assertTrue( str(actual_value).endswith(str(expected_microseconds))) # us to datetime for index, value in enumerate(microsecond_values): actual_value = field._microseconds_since_epoch_to_datetime( data=value) expected_value = datetime_values[index] self.assertEqual(actual_value, expected_value)
class ActionExecutionSchedulingQueueItemDB(stormbase.StormFoundationDB, stormbase.ChangeRevisionFieldMixin): """ A model which represents a request for execution to be scheduled. Those models are picked up by the scheduler and scheduled to be ran by an action runner. """ RESOURCE_TYPE = ResourceType.EXECUTION_REQUEST UID_FIELDS = ["id"] liveaction_id = me.StringField( required=True, help_text="Foreign key to the LiveActionDB which is to be scheduled", ) action_execution_id = me.StringField( help_text= "Foreign key to the ActionExecutionDB which is to be scheduled") original_start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text= "The timestamp when the liveaction was created and originally be scheduled to " "run.", ) scheduled_start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text="The timestamp when liveaction is scheduled to run.", ) delay = me.IntField() handling = me.BooleanField( default=False, help_text="Flag indicating if this item is currently being handled / " "processed by a scheduler service", ) meta = { "indexes": [ # NOTE: We limit index names to 65 characters total for compatibility with AWS # DocumentDB. # See https://github.com/StackStorm/st2/pull/4690 for details. { "fields": ["action_execution_id"], "name": "ac_exc_id" }, { "fields": ["liveaction_id"], "name": "lv_ac_id" }, { "fields": ["original_start_timestamp"], "name": "orig_s_ts" }, { "fields": ["scheduled_start_timestamp"], "name": "schd_s_ts" }, ] }
class ActionExecutionDB(stormbase.StormFoundationDB): trigger = stormbase.EscapedDictField() trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() action = stormbase.EscapedDictField(required=True) runner = stormbase.EscapedDictField(required=True) # Only the diff between the liveaction type and what is replicated # in the ActionExecutionDB object. liveaction = stormbase.EscapedDictField(required=True) status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') parameters = me.DictField( default={}, help_text='The key-value pairs passed as to the action runner & action.' ) result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') parent = me.StringField() children = me.ListField(field=me.StringField()) meta = { 'indexes': [{ 'fields': ['parent'] }, { 'fields': ['liveaction.id'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['action.ref'] }, { 'fields': ['status'] }] } def mask_secrets(self, value): result = copy.deepcopy(value) execution_parameters = value['parameters'] parameters = {} # pylint: disable=no-member parameters.update(value.get('action', {}).get('parameters', {})) parameters.update(value.get('runner', {}).get('runner_parameters', {})) secret_parameters = get_secret_parameters(parameters=parameters) result['parameters'] = mask_secret_parameters( parameters=execution_parameters, secret_parameters=secret_parameters) return result
class ActionExecutionSchedulingQueueItemDB(stormbase.StormFoundationDB, stormbase.ChangeRevisionFieldMixin): """ A model which represents a request for execution to be scheduled. Those models are picked up by the scheduler and scheduled to be ran by an action runner. """ RESOURCE_TYPE = ResourceType.EXECUTION_REQUEST UID_FIELDS = ['id'] liveaction_id = me.StringField( required=True, help_text='Foreign key to the LiveActionDB which is to be scheduled') action_execution_id = me.StringField( help_text= 'Foreign key to the ActionExecutionDB which is to be scheduled') original_start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text= 'The timestamp when the liveaction was created and originally be scheduled to ' 'run.') scheduled_start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when liveaction is scheduled to run.') delay = me.IntField() handling = me.BooleanField( default=False, help_text='Flag indicating if this item is currently being handled / ' 'processed by a scheduler service') meta = { 'indexes': [ { 'fields': ['action_execution_id'] }, { 'fields': ['liveaction_id'] }, { 'fields': ['original_start_timestamp'] }, { 'fields': ['scheduled_start_timestamp'] }, ] }
class RuleEnforcementDB(stormbase.StormFoundationDB, stormbase.TagsMixin): UID_FIELDS = ['id'] trigger_instance_id = me.StringField(required=True) execution_id = me.StringField(required=False) failure_reason = me.StringField(required=False) rule = me.EmbeddedDocumentField(RuleReferenceSpecDB, required=True) enforced_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the rule enforcement happened.') status = me.StringField( required=True, default=RULE_ENFORCEMENT_STATUS_SUCCEEDED, help_text='Rule enforcement status.') meta = { 'indexes': [ {'fields': ['trigger_instance_id']}, {'fields': ['execution_id']}, {'fields': ['rule.id']}, {'fields': ['rule.ref']}, {'fields': ['enforced_at']}, {'fields': ['-enforced_at']}, {'fields': ['-enforced_at', 'rule.ref']}, {'fields': ['status']}, ] + stormbase.TagsMixin.get_indices() } # NOTE: Note the following method is exposed so loggers in rbac resolvers can log objects # with a consistent get_uid interface. def get_uid(self): # TODO Construct uid from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] return ':'.join(uid)
class RuleEnforcementDB(stormbase.StormFoundationDB, stormbase.TagsMixin): UID_FIELDS = ['id'] trigger_instance_id = me.StringField(required=True) execution_id = me.StringField(required=False) failure_reason = me.StringField(required=False) rule = me.EmbeddedDocumentField(RuleReferenceSpecDB, required=True) enforced_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the rule enforcement happened.') status = me.StringField(required=True, default=RULE_ENFORCEMENT_STATUS_SUCCEEDED, help_text='Rule enforcement status.') meta = { 'indexes': [ { 'fields': ['trigger_instance_id'] }, { 'fields': ['execution_id'] }, { 'fields': ['rule.id'] }, { 'fields': ['rule.ref'] }, { 'fields': ['enforced_at'] }, { 'fields': ['-enforced_at'] }, { 'fields': ['-enforced_at', 'rule.ref'] }, { 'fields': ['status'] }, ] + stormbase.TagsMixin.get_indexes() } def __init__(self, *args, **values): super(RuleEnforcementDB, self).__init__(*args, **values) # Set status to succeeded for old / existing RuleEnforcementDB which predate status field status = getattr(self, 'status', None) failure_reason = getattr(self, 'failure_reason', None) if status in [None, RULE_ENFORCEMENT_STATUS_SUCCEEDED ] and failure_reason: self.status = RULE_ENFORCEMENT_STATUS_FAILED # NOTE: Note the following method is exposed so loggers in rbac resolvers can log objects # with a consistent get_uid interface. def get_uid(self): # TODO Construct uid from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] # pylint: disable=E1101 return ':'.join(uid)
class ActionExecutionOutputDB(stormbase.StormFoundationDB): """ Stores output of a particular execution. New document is inserted dynamically when a new chunk / line is received which means you can simulate tail behavior by periodically reading from this collection. Attribute: execution_id: ID of the execution to which this output belongs. action_ref: Parent action reference. runner_ref: Parent action runner reference. timestamp: Timestamp when this output has been produced / received. output_type: Type of the output (e.g. stdout, stderr, output) data: Actual output data. This could either be line, chunk or similar, depending on the runner. """ execution_id = me.StringField(required=True) action_ref = me.StringField(required=True) runner_ref = me.StringField(required=True) timestamp = ComplexDateTimeField(required=True, default=date_utils.get_datetime_utc_now) output_type = me.StringField(required=True, default='output') data = me.StringField() meta = { 'indexes': [ {'fields': ['execution_id']}, {'fields': ['action_ref']}, {'fields': ['runner_ref']}, {'fields': ['timestamp']}, {'fields': ['output_type']} ] }
class ApiKeyDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin): """ """ RESOURCE_TYPE = ResourceType.API_KEY UID_FIELDS = ['key_hash'] user = me.StringField(required=True) key_hash = me.StringField(required=True, unique=True) metadata = me.DictField(required=False, help_text='Arbitrary metadata associated with this token') created_at = ComplexDateTimeField(default=date_utils.get_datetime_utc_now, help_text='The creation time of this ApiKey.') enabled = me.BooleanField(required=True, default=True, help_text='A flag indicating whether the ApiKey is enabled.') meta = { 'indexes': [ {'fields': ['user']}, {'fields': ['key_hash']} ] } def __init__(self, *args, **values): super(ApiKeyDB, self).__init__(*args, **values) self.uid = self.get_uid() def mask_secrets(self, value): result = copy.deepcopy(value) # In theory the key_hash is safe to return as it is one way. On the other # hand given that this is actually a secret no real point in letting the hash # escape. Since uid contains key_hash masking that as well. result['key_hash'] = MASKED_ATTRIBUTE_VALUE result['uid'] = MASKED_ATTRIBUTE_VALUE return result
def test_get_(self, mock_get): field = ComplexDateTimeField() # No value set mock_get.return_value = None self.assertEqual(field.__get__(instance=None, owner=None), None) # Already a datetime mock_get.return_value = datetime.datetime.now() self.assertEqual(field.__get__(instance=None, owner=None), mock_get.return_value) # Microseconds dt = datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=500) us = field._datetime_to_microseconds_since_epoch(value=dt) mock_get.return_value = us self.assertEqual(field.__get__(instance=None, owner=None), dt)
class TraceDB(stormbase.StormFoundationDB): """ Trace is a collection of all TriggerInstances, Rules and ActionExecutions that represent an activity which begins with the introduction of a TriggerInstance or request of an ActionExecution and ends with the completion of an ActionExecution. Given the closed feedback look sort of nature of StackStorm this implies a Trace can comprise of multiple TriggerInstances, Rules and ActionExecutions. :param trace_tag: A user specified reference to the trace. :param trigger_instances: TriggerInstances associated with this trace. :param rules: Rules associated with this trace. :param action_executions: ActionExecutions associated with this trace. """ trace_tag = me.StringField( required=True, help_text='A user specified reference to the trace.') trigger_instances = me.ListField( field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated TriggerInstances.') rules = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated Rules.') action_executions = me.ListField( field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated ActionExecutions.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the Trace was created.') meta = { 'indexes': [ { 'fields': ['trace_tag'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['action_executions.object_id'] }, { 'fields': ['trigger_instances.object_id'] }, { 'fields': ['rules.object_id'] }, { 'fields': ['-start_timestamp', 'trace_tag'] }, ] }
class LiveActionDB(stormbase.StormFoundationDB): """ The databse entity that represents a Stack Action/Automation in the system. Attributes: status: the most recently observed status of the execution. One of "starting", "running", "completed", "error". result: an embedded document structure that holds the output and exit status code from the action. """ # TODO: Can status be an enum at the Mongo layer? status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') action = me.StringField( required=True, help_text='Reference to the action that has to be executed.') parameters = me.DictField( default={}, help_text= 'The key-value pairs passed as to the action runner & execution.') result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') callback = me.DictField( default={}, help_text= 'Callback information for the on completion of action execution.') runner_info = me.DictField( default={}, help_text='Reference to the runner that executed this liveaction.') notify = me.EmbeddedDocumentField(NotificationSchema) meta = {'indexes': ['-start_timestamp', 'action']}
class ActionExecutionDB(stormbase.StormFoundationDB): trigger = stormbase.EscapedDictField() trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() action = stormbase.EscapedDictField(required=True) runner = stormbase.EscapedDictField(required=True) # Only the diff between the liveaction type and what is replicated # in the ActionExecutionDB object. liveaction = stormbase.EscapedDictField(required=True) status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=datetime.datetime.utcnow, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') parameters = me.DictField( default={}, help_text= 'The key-value pairs passed as to the action runner & execution.') result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') parent = me.StringField() children = me.ListField(field=me.StringField()) meta = { 'indexes': [{ 'fields': ['parent'] }, { 'fields': ['liveaction.id'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['action.ref'] }, { 'fields': ['status'] }] }
class TraceComponentDB(me.EmbeddedDocument): """ """ object_id = me.StringField() updated_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the TraceComponent was included.') def __str__(self): return 'TraceComponentDB@(object_id:{}, updated_at:{})'.format( self.object_id, self.updated_at)
class TraceComponentDB(me.EmbeddedDocument): """""" object_id = me.StringField() ref = me.StringField(default="") updated_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text="The timestamp when the TraceComponent was included.", ) caused_by = me.DictField(help_text="Causal component.") def __str__(self): return "TraceComponentDB@(object_id:{}, updated_at:{})".format( self.object_id, self.updated_at)
class MarkerDB(stormbase.StormFoundationDB): """ Abstract model for storing marker (or cursor) in db. This is typically used when doing iteration. :param marker: Cursor string. :type marker: ``str`` :param updated_at: Timestamp when marker was updated. :type updated_at: ``datetime.datetime`` """ marker = me.StringField(required=True) updated_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') meta = {'abstract': True}
class ApiKeyDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin): """ An entity representing an API key object. Each API key object is scoped to the user and inherits permissions from that user. """ RESOURCE_TYPE = ResourceType.API_KEY UID_FIELDS = ["key_hash"] user = me.StringField(required=True) key_hash = me.StringField(required=True, unique=True) metadata = me.DictField( required=False, help_text="Arbitrary metadata associated with this token") created_at = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text="The creation time of this ApiKey.", ) enabled = me.BooleanField( required=True, default=True, help_text="A flag indicating whether the ApiKey is enabled.", ) meta = {"indexes": [{"fields": ["user"]}, {"fields": ["key_hash"]}]} def __init__(self, *args, **values): super(ApiKeyDB, self).__init__(*args, **values) self.uid = self.get_uid() def mask_secrets(self, value): result = copy.deepcopy(value) # In theory the key_hash is safe to return as it is one way. On the other # hand given that this is actually a secret no real point in letting the hash # escape. Since uid contains key_hash masking that as well. result["key_hash"] = MASKED_ATTRIBUTE_VALUE result["uid"] = MASKED_ATTRIBUTE_VALUE return result
class LiveActionDB(stormbase.StormFoundationDB): """ The databse entity that represents a Stack Action/Automation in the system. Attributes: status: the most recently observed status of the execution. One of "starting", "running", "completed", "error". result: an embedded document structure that holds the output and exit status code from the action. """ # TODO: Can status be an enum at the Mongo layer? status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') action = me.StringField( required=True, help_text='Reference to the action that has to be executed.') parameters = stormbase.EscapedDynamicField( default={}, help_text= 'The key-value pairs passed as to the action runner & execution.') result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') callback = me.DictField( default={}, help_text= 'Callback information for the on completion of action execution.') runner_info = me.DictField( default={}, help_text= 'Information about the runner which executed this live action (hostname, pid).' ) notify = me.EmbeddedDocumentField(NotificationSchema) meta = { 'indexes': [ { 'fields': ['-start_timestamp', 'action'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['end_timestamp'] }, { 'fields': ['action'] }, { 'fields': ['status'] }, ] } def mask_secrets(self, value): from st2common.util import action_db result = copy.deepcopy(value) execution_parameters = value['parameters'] # TODO: This results into two DB looks, we should cache action and runner type object # for each liveaction... # # ,-'"-. # . f .--. \ # .\._,\._',' j_ # 7______""-'__`, parameters = action_db.get_action_parameters_specs( action_ref=self.action) secret_parameters = get_secret_parameters(parameters=parameters) result['parameters'] = mask_secret_parameters( parameters=execution_parameters, secret_parameters=secret_parameters) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict['parameters']
class LiveActionDB(stormbase.StormFoundationDB): workflow_execution = me.StringField() task_execution = me.StringField() # TODO: Can status be an enum at the Mongo layer? status = me.StringField(required=True, help_text="The current status of the liveaction.") start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text="The timestamp when the liveaction was created.", ) end_timestamp = ComplexDateTimeField( help_text="The timestamp when the liveaction has finished.") action = me.StringField( required=True, help_text="Reference to the action that has to be executed.") action_is_workflow = me.BooleanField( default=False, help_text= "A flag indicating whether the referenced action is a workflow.", ) parameters = stormbase.EscapedDynamicField( default={}, help_text= "The key-value pairs passed as to the action runner & execution.", ) result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result.") context = me.DictField( default={}, help_text="Contextual information on the action execution.") callback = me.DictField( default={}, help_text= "Callback information for the on completion of action execution.", ) runner_info = me.DictField( default={}, help_text= "Information about the runner which executed this live action (hostname, pid).", ) notify = me.EmbeddedDocumentField(NotificationSchema) delay = me.IntField( min_value=0, help_text= "How long (in milliseconds) to delay the execution before scheduling.", ) meta = { "indexes": [ { "fields": ["-start_timestamp", "action"] }, { "fields": ["start_timestamp"] }, { "fields": ["end_timestamp"] }, { "fields": ["action"] }, { "fields": ["status"] }, { "fields": ["context.trigger_instance.id"] }, { "fields": ["workflow_execution"] }, { "fields": ["task_execution"] }, ], } def mask_secrets(self, value): from st2common.util import action_db result = copy.deepcopy(value) execution_parameters = value["parameters"] # TODO: This results into two DB looks, we should cache action and runner type object # for each liveaction... # # ,-'"-. # . f .--. \ # .\._,\._',' j_ # 7______""-'__`, parameters = action_db.get_action_parameters_specs( action_ref=self.action) secret_parameters = get_secret_parameters(parameters=parameters) result["parameters"] = mask_secret_parameters( parameters=execution_parameters, secret_parameters=secret_parameters) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict["parameters"]
class ActionExecutionDB(stormbase.StormFoundationDB): RESOURCE_TYPE = ResourceType.EXECUTION UID_FIELDS = ['id'] trigger = stormbase.EscapedDictField() trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() action = stormbase.EscapedDictField(required=True) runner = stormbase.EscapedDictField(required=True) # Only the diff between the liveaction type and what is replicated # in the ActionExecutionDB object. liveaction = stormbase.EscapedDictField(required=True) status = me.StringField( required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') parameters = stormbase.EscapedDynamicField( default={}, help_text='The key-value pairs passed as to the action runner & action.') result = stormbase.EscapedDynamicField( default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) meta = { 'indexes': [ {'fields': ['rule.ref']}, {'fields': ['action.ref']}, {'fields': ['liveaction.id']}, {'fields': ['start_timestamp']}, {'fields': ['end_timestamp']}, {'fields': ['status']}, {'fields': ['parent']}, {'fields': ['-start_timestamp', 'action.ref', 'status']} ] } def get_uid(self): # TODO Construct od from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] return ':'.join(uid) def mask_secrets(self, value): result = copy.deepcopy(value) execution_parameters = value['parameters'] parameters = {} # pylint: disable=no-member parameters.update(value.get('action', {}).get('parameters', {})) parameters.update(value.get('runner', {}).get('runner_parameters', {})) secret_parameters = get_secret_parameters(parameters=parameters) result['parameters'] = mask_secret_parameters(parameters=execution_parameters, secret_parameters=secret_parameters) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict['parameters']
class TraceDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin): """ Trace is a collection of all TriggerInstances, Rules and ActionExecutions that represent an activity which begins with the introduction of a TriggerInstance or request of an ActionExecution and ends with the completion of an ActionExecution. Given the closed feedback look sort of nature of StackStorm this implies a Trace can comprise of multiple TriggerInstances, Rules and ActionExecutions. :param trace_tag: A user specified reference to the trace. :param trigger_instances: TriggerInstances associated with this trace. :param rules: Rules associated with this trace. :param action_executions: ActionExecutions associated with this trace. """ RESOURCE_TYPE = ResourceType.TRACE trace_tag = me.StringField( required=True, help_text='A user specified reference to the trace.') trigger_instances = me.ListField( field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated TriggerInstances.') rules = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated Rules.') action_executions = me.ListField( field=me.EmbeddedDocumentField(TraceComponentDB), required=False, help_text='Associated ActionExecutions.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the Trace was created.') meta = { 'indexes': [ { 'fields': ['trace_tag'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['action_executions.object_id'] }, { 'fields': ['trigger_instances.object_id'] }, { 'fields': ['rules.object_id'] }, { 'fields': ['-start_timestamp', 'trace_tag'] }, ] } def __init__(self, *args, **values): super(TraceDB, self).__init__(*args, **values) self.uid = self.get_uid() def get_uid(self): parts = [] parts.append(self.RESOURCE_TYPE) componenets_hash = hashlib.md5() componenets_hash.update(str(self.trace_tag).encode()) componenets_hash.update(str(self.trigger_instances).encode()) componenets_hash.update(str(self.rules).encode()) componenets_hash.update(str(self.action_executions).encode()) componenets_hash.update(str(self.start_timestamp).encode()) parts.append(componenets_hash.hexdigest()) uid = self.UID_SEPARATOR.join(parts) return uid
class LiveActionDB(stormbase.StormFoundationDB): workflow_execution = me.StringField() task_execution = me.StringField() # TODO: Can status be an enum at the Mongo layer? status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') action = me.StringField( required=True, help_text='Reference to the action that has to be executed.') action_is_workflow = me.BooleanField( default=False, help_text= 'A flag indicating whether the referenced action is a workflow.') parameters = stormbase.EscapedDynamicField( default={}, help_text= 'The key-value pairs passed as to the action runner & execution.') result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') callback = me.DictField( default={}, help_text= 'Callback information for the on completion of action execution.') runner_info = me.DictField( default={}, help_text= 'Information about the runner which executed this live action (hostname, pid).' ) notify = me.EmbeddedDocumentField(NotificationSchema) meta = { 'indexes': [{ 'fields': ['-start_timestamp', 'action'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['end_timestamp'] }, { 'fields': ['action'] }, { 'fields': ['status'] }, { 'fields': ['context.trigger_instance.id'] }, { 'fields': ['workflow_execution'] }, { 'fields': ['task_execution'] }] } def mask_secrets(self, value): from st2common.util import action_db result = copy.deepcopy(value) execution_parameters = value['parameters'] # TODO: This results into two DB looks, we should cache action and runner type object # for each liveaction... # # ,-'"-. # . f .--. \ # .\._,\._',' j_ # 7______""-'__`, parameters = action_db.get_action_parameters_specs( action_ref=self.action) secret_parameters = get_secret_parameters(parameters=parameters) result['parameters'] = mask_secret_parameters( parameters=execution_parameters, secret_parameters=secret_parameters) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict['parameters']
class ActionExecutionDB(stormbase.StormFoundationDB): RESOURCE_TYPE = ResourceType.EXECUTION UID_FIELDS = ["id"] trigger = stormbase.EscapedDictField() trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() action = stormbase.EscapedDictField(required=True) runner = stormbase.EscapedDictField(required=True) # Only the diff between the liveaction type and what is replicated # in the ActionExecutionDB object. liveaction = stormbase.EscapedDictField(required=True) workflow_execution = me.StringField() task_execution = me.StringField() status = me.StringField(required=True, help_text="The current status of the liveaction.") start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text="The timestamp when the liveaction was created.", ) end_timestamp = ComplexDateTimeField( help_text="The timestamp when the liveaction has finished.") parameters = stormbase.EscapedDynamicField( default={}, help_text= "The key-value pairs passed as to the action runner & action.", ) result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result.") result_size = me.IntField(default=0, help_text="Serialized result size in bytes") context = me.DictField( default={}, help_text="Contextual information on the action execution.") parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) delay = me.IntField(min_value=0) # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. web_url = me.StringField(required=False) meta = { "indexes": [ { "fields": ["rule.ref"] }, { "fields": ["action.ref"] }, { "fields": ["liveaction.id"] }, { "fields": ["start_timestamp"] }, { "fields": ["end_timestamp"] }, { "fields": ["status"] }, { "fields": ["parent"] }, { "fields": ["rule.name"] }, { "fields": ["runner.name"] }, { "fields": ["trigger.name"] }, { "fields": ["trigger_type.name"] }, { "fields": ["trigger_instance.id"] }, { "fields": ["context.user"] }, { "fields": ["-start_timestamp", "action.ref", "status"] }, { "fields": ["workflow_execution"] }, { "fields": ["task_execution"] }, ] } def get_uid(self): # TODO Construct id from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] # pylint: disable=no-member return ":".join(uid) def mask_secrets(self, value): result = copy.deepcopy(value) liveaction = result["liveaction"] parameters = {} # pylint: disable=no-member parameters.update(value.get("action", {}).get("parameters", {})) parameters.update(value.get("runner", {}).get("runner_parameters", {})) secret_parameters = get_secret_parameters(parameters=parameters) result["parameters"] = mask_secret_parameters( parameters=result.get("parameters", {}), secret_parameters=secret_parameters) if "parameters" in liveaction: liveaction["parameters"] = mask_secret_parameters( parameters=liveaction["parameters"], secret_parameters=secret_parameters) if liveaction.get("action", "") == "st2.inquiry.respond": # Special case to mask parameters for `st2.inquiry.respond` action # In this case, this execution is just a plain python action, not # an inquiry, so we don't natively have a handle on the response # schema. # # To prevent leakage, we can just mask all response fields. # # Note: The 'string' type in secret_parameters doesn't matter, # it's just a placeholder to tell mask_secret_parameters() # that this parameter is indeed a secret parameter and to # mask it. result["parameters"]["response"] = mask_secret_parameters( parameters=liveaction["parameters"]["response"], secret_parameters={ p: "string" for p in liveaction["parameters"]["response"] }, ) # TODO(mierdin): This logic should be moved to the dedicated Inquiry # data model once it exists. result["result"] = ActionExecutionDB.result.parse_field_value( result["result"]) if self.runner.get("name") == "inquirer": schema = result["result"].get("schema", {}) response = result["result"].get("response", {}) # We can only mask response secrets if response and schema exist and are # not empty if response and schema: result["result"]["response"] = mask_inquiry_response( response, schema) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict["parameters"]
class ActionExecutionDB(stormbase.StormFoundationDB): RESOURCE_TYPE = ResourceType.EXECUTION UID_FIELDS = ['id'] trigger = stormbase.EscapedDictField() trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() action = stormbase.EscapedDictField(required=True) runner = stormbase.EscapedDictField(required=True) # Only the diff between the liveaction type and what is replicated # in the ActionExecutionDB object. liveaction = stormbase.EscapedDictField(required=True) workflow_execution = me.StringField() task_execution = me.StringField() status = me.StringField(required=True, help_text='The current status of the liveaction.') start_timestamp = ComplexDateTimeField( default=date_utils.get_datetime_utc_now, help_text='The timestamp when the liveaction was created.') end_timestamp = ComplexDateTimeField( help_text='The timestamp when the liveaction has finished.') parameters = stormbase.EscapedDynamicField( default={}, help_text='The key-value pairs passed as to the action runner & action.' ) result = stormbase.EscapedDynamicField(default={}, help_text='Action defined result.') context = me.DictField( default={}, help_text='Contextual information on the action execution.') parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. web_url = me.StringField(required=False) meta = { 'indexes': [{ 'fields': ['rule.ref'] }, { 'fields': ['action.ref'] }, { 'fields': ['liveaction.id'] }, { 'fields': ['start_timestamp'] }, { 'fields': ['end_timestamp'] }, { 'fields': ['status'] }, { 'fields': ['parent'] }, { 'fields': ['rule.name'] }, { 'fields': ['runner.name'] }, { 'fields': ['trigger.name'] }, { 'fields': ['trigger_type.name'] }, { 'fields': ['trigger_instance.id'] }, { 'fields': ['context.user'] }, { 'fields': ['-start_timestamp', 'action.ref', 'status'] }, { 'fields': ['workflow_execution'] }, { 'fields': ['task_execution'] }] } def get_uid(self): # TODO Construct od from non id field: uid = [self.RESOURCE_TYPE, str(self.id)] return ':'.join(uid) def mask_secrets(self, value): result = copy.deepcopy(value) liveaction = result['liveaction'] parameters = {} # pylint: disable=no-member parameters.update(value.get('action', {}).get('parameters', {})) parameters.update(value.get('runner', {}).get('runner_parameters', {})) secret_parameters = get_secret_parameters(parameters=parameters) result['parameters'] = mask_secret_parameters( parameters=result['parameters'], secret_parameters=secret_parameters) if 'parameters' in liveaction: liveaction['parameters'] = mask_secret_parameters( parameters=liveaction['parameters'], secret_parameters=secret_parameters) if liveaction.get('action', '') == 'st2.inquiry.respond': # Special case to mask parameters for `st2.inquiry.respond` action # In this case, this execution is just a plain python action, not # an inquiry, so we don't natively have a handle on the response # schema. # # To prevent leakage, we can just mask all response fields. # # Note: The 'string' type in secret_parameters doesn't matter, # it's just a placeholder to tell mask_secret_parameters() # that this parameter is indeed a secret parameter and to # mask it. result['parameters']['response'] = mask_secret_parameters( parameters=liveaction['parameters']['response'], secret_parameters={ p: 'string' for p in liveaction['parameters']['response'] }) # TODO(mierdin): This logic should be moved to the dedicated Inquiry # data model once it exists. if self.runner.get('name') == "inquirer": schema = result['result'].get('schema', {}) response = result['result'].get('response', {}) # We can only mask response secrets if response and schema exist and are # not empty if response and schema: result['result']['response'] = mask_inquiry_response( response, schema) return result def get_masked_parameters(self): """ Retrieve parameters with the secrets masked. :rtype: ``dict`` """ serializable_dict = self.to_serializable_dict(mask_secrets=True) return serializable_dict['parameters']