class SnuggleIndex(CritsDocument, Document): """Snugglefish Index Document Object""" meta = { "collection": 'snugglefish_indexes', "crits_type": 'snugglefish_index', "latest_schema_version": 1, "schema_doc": { 'name': "Name of this index", 'name': "Directory where this index lives", 'query': "Query for this index", 'created': "Date this index was created", 'last_update': "Date this index was last updated", 'last_id': "Last object ID fetched for this index", 'total': "Total number of objects available for this index", 'count': "Total number of objects fetched for this index" }, } name = StringField(required=True) directory = StringField(required=True) query = StringField(required=True) created = CritsDateTimeField(required=True) last_update = CritsDateTimeField() last_id = ObjectIdField() total = IntField(default=0) count = IntField(default=0) def migrate(self): pass
class Taxii(CritsDocument, Document): """TAXII Document Object""" meta = { # mongoengine adds fields _cls and _types and uses them to filter database # responses unless you disallow inheritance. In other words, we # can't see any of our old data unless we add _cls and _types # attributes to them or turn off inheritance. #So we'll turn inheritance off. # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data) "allow_inheritance": False, "collection": 'taxii', "crits_type": 'TAXII', "latest_schema_version": 2, #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields "minify_defaults": ['runtime', 'end', 'feed'], "schema_doc": { 'runtime': 'The last time we made a TAXII request.', 'end': 'End date of this taxii document.', 'feed': 'Feed of this taxii document.' }, } runtime = CritsDateTimeField(required=True) end = CritsDateTimeField(required=True) def migrate(self): pass @classmethod def get_last(cls, feed): #returns the last time for the particular feed return cls.objects(unsupported_attrs={ 'feed': feed }).order_by('-end').first()
class EmbeddedActivity(EmbeddedDocument, CritsDocumentFormatter): """ Indicator activity class. """ analyst = StringField() end_date = CritsDateTimeField(default=datetime.datetime.now) date = CritsDateTimeField(default=datetime.datetime.now) description = StringField() start_date = CritsDateTimeField(default=datetime.datetime.now)
class EmbeddedHighlight(EmbeddedDocument, CritsDocumentFormatter): """ Raw Data highlight comment class. """ date = CritsDateTimeField(default=datetime.datetime.now) line_date = CritsDateTimeField() analyst = StringField() line = IntField() line_data = StringField() comment = StringField()
class EmbeddedAction(EmbeddedDocument, CritsDocumentFormatter): """ Indicator action class. """ action_type = StringField() active = StringField() analyst = StringField() begin_date = CritsDateTimeField(default=datetime.datetime.now) date = CritsDateTimeField(default=datetime.datetime.now) end_date = CritsDateTimeField(default=datetime.datetime.now) performed_date = CritsDateTimeField(default=datetime.datetime.now) reason = StringField()
class TaxiiContent(CritsDocument, Document): """TAXII Content Block Document Object""" meta = { # mongoengine adds fields _cls and _types and uses them to filter database # responses unless you disallow inheritance. In other words, we # can't see any of our old data unless we add _cls and _types # attributes to them or turn off inheritance. #So we'll turn inheritance off. # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data) "allow_inheritance": False, "collection": 'taxii.content', "crits_type": 'TAXIIContent', "latest_schema_version": 1, #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields "minify_defaults": [ 'taxii_msg_id', 'hostname', 'feed', 'timestamp', 'poll_time', 'timerange', 'analyst', 'content', 'errors', 'import_failed' ], "schema_doc": { 'taxii_msg_id': 'The ID of the TAXII message from which this content came', 'hostname': 'The hostname of the TAXII server', 'feed': 'The name of the TAXII feed/collection', 'timestamp': 'When the content was submitted to the TAXII server', 'poll_time': 'A timestamp representing when this data was polled', 'timerange': 'The timerange of the TAXII poll', 'analyst': 'The analyst who polled the data', 'content': 'The content being stored', 'errors': 'Any errors that prevented import of the content', 'import_failed': 'Boolean indicating that an attempt to import failed' }, } taxii_msg_id = StringField(required=True) hostname = StringField(required=True) feed = StringField(required=True) timestamp = CritsDateTimeField(required=True) poll_time = CritsDateTimeField(required=True) timerange = StringField(required=True) analyst = StringField(required=True) content = StringField(required=True) errors = ListField(StringField(required=True)) import_failed = BooleanField(required=True, default=False) def migrate(self): pass
class EmbeddedParentField(EmbeddedDocument, CritsDocumentFormatter): """ Embedded Parent Field """ date = CritsDateTimeField() analyst = StringField()
class AuditLog(CritsDocument, CritsSchemaDocument, Document): """ Audit Log Class """ meta = { "collection": settings.COL_AUDIT_LOG, "auto_create_index": False, "allow_inheritance": False, "crits_type": "AuditLog", "latest_schema_version": 1, "schema_doc": { 'value': 'Value of the audit log entry', 'user': '******', 'date': 'Date of the entry', 'type': 'Type of the audit entry', 'method': 'Method of the audit entry' } } value = StringField() user = StringField() date = CritsDateTimeField(default=datetime.datetime.now) target_type = StringField(db_field='type') target_id = ObjectIdField() method = StringField()
class EmbeddedBackdoor(EmbeddedDocument, CritsDocumentFormatter): """Sample backdoors object""" name = StringField() version = StringField() analyst = StringField() date = CritsDateTimeField(default=datetime.datetime.now)
class EmbeddedActorIdentifier(EmbeddedDocument, CritsDocumentFormatter): """ Embedded Actor Identifier class. """ analyst = StringField(required=True) date = CritsDateTimeField(default=datetime.datetime.now) identifier_id = StringField(required=True) confidence = StringField(default="unknown")
class EmbeddedInline(EmbeddedDocument, CritsDocumentFormatter): """ Raw Data Inline comment class. """ date = CritsDateTimeField(default=datetime.datetime.now) analyst = StringField() line = IntField() comment = StringField() counter = IntField()
class Notification(CritsDocument, CritsSchemaDocument, Document): """ Notification Class. """ meta = { "collection": settings.COL_NOTIFICATIONS, "auto_create_index": False, "crits_type": "Notification", "latest_schema_version": 1, "schema_doc": { 'notification': 'The notification body', 'notification_type': 'The type of notification, e.g. alert, error', 'header': 'The notification header, optional', 'link_url': 'A link URL for the header, optional', 'status': 'New/Processed - used to determine whether or not to notify', 'obj_type': 'The type of the object this notification is for', 'obj_id': 'The MongoDB ObjectId for the object this notification is for', 'created': 'ISODate when this notification was made', 'users': 'List [] of users for this notification', 'analyst': 'The analyst, if any, that made this notification', } } # This is not a date field! # It exists to provide default values for created and edit_date date = datetime.datetime.now() analyst = StringField() notification = StringField() notification_type = StringField() header = StringField() created = CritsDateTimeField(default=date, db_field="date") obj_id = ObjectIdField() obj_type = StringField() status = StringField(default="new") link_url = StringField() users = ListField(StringField()) def set_status(self, status): """ Set the status of the notification. :param status: The status ("new", "processed"). :type status: str """ if status in ("new", "processed"): self.status = status
class ActorIdentifier(CritsDocument, CritsSchemaDocument, CritsSourceDocument, Document): """ Actor Identifier class. """ meta = { "collection": settings.COL_ACTOR_IDENTIFIERS, "crits_type": 'ActorIdentifier', "latest_schema_version": 1, "schema_doc": { 'name': 'The name of this Action', 'active': 'Enabled in the UI (on/off)' }, "jtable_opts": { 'details_url': '', 'details_url_key': '', 'default_sort': "created DESC", 'searchurl': 'crits.actors.views.actor_identifiers_listing', 'fields': [ "name", "created", "source", "identifier_type", "id"], 'jtopts_fields': [ "name", "identifier_type", "created", "source"], 'hidden_fields': [], 'linked_fields': ["name", "source"], 'details_link': '', 'no_sort': [''] } } active = StringField(default="on") created = CritsDateTimeField(default=datetime.datetime.now) identifier_type = StringField(required=True) # name is misleading, it's actually the value. We use name so we can # leverage a lot of the work done for Item editing in the control panel and # changing active state. name = StringField(required=True) def set_identifier_type(self, identifier_type): identifier_type = identifier_type.strip() it = ActorThreatIdentifier.objects(name=identifier_type).first() if it: self.identifier_type = identifier_type
class Email(CritsBaseAttributes, CritsSourceDocument, Document): """ Email Class. """ meta = { # mongoengine adds fields _cls and _types and uses them to filter database # responses unless you disallow inheritance. In other words, we # can't see any of our old data unless we add _cls and _types # attributes to them or turn off inheritance. #So we'll turn inheritance off. # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data) "collection": settings.COL_EMAIL, "crits_type": 'Email', "latest_schema_version": 2, "schema_doc": { 'boundary': 'Email boundary', 'campaign': 'List [] of campaigns attributed to this email', 'cc': 'List [] of CC recipients', 'date': 'String of date header field', 'from': 'From header field', 'helo': 'HELO', 'isodate': 'ISODate conversion of date header field', 'message_id': 'Message-ID header field', 'modified': 'When this object was last modified', 'objects': 'List of objects in this email', 'originating_ip': 'Originating-IP header field', 'raw_body': 'Email raw body', 'raw_header': 'Email raw headers', 'relationships': 'List of relationships with this email', 'reply_to': 'Reply-To header field', 'sender': 'Sender header field', 'shared_with': 'Dictionary of sources that this email may be shared with and whether it has been shared already', 'source': 'List [] of sources that provided information on this email', 'subject': 'Email subject', 'to': 'To header field', 'x_originating_ip': 'X-Originating-IP header field', 'x_mailer': 'X-Mailer header field', }, "jtable_opts": { 'details_url': 'crits.emails.views.email_detail', 'details_url_key': 'id', 'default_sort': "isodate DESC", 'searchurl': 'crits.emails.views.emails_listing', 'fields': [ "from_address", "subject", "isodate", "source", "campaign", "id", "to", "status", "cc" ], 'jtopts_fields': [ "details", "from", "recip", "subject", "isodate", "source", "campaign", "status", "favorite", "id" ], 'hidden_fields': [], 'linked_fields': ["source", "campaign", "from", "subject"], 'details_link': 'details', 'no_sort': ['recip', 'details'] } } boundary = StringField() cc = ListField(StringField()) date = StringField(required=True) from_address = StringField(db_field="from") helo = StringField() # isodate is an interally-set attribute and on save will be overwritten # with the isodate version of the email's date attribute. isodate = CritsDateTimeField() message_id = StringField() originating_ip = StringField() raw_body = StringField() raw_header = RawHeadersField(db_field="raw_headers") reply_to = StringField() sender = StringField() subject = StringField() to = ListField(StringField()) x_originating_ip = StringField() x_mailer = StringField() def migrate(self): """ Migrate to latest schema version. """ migrate_email(self) def _custom_save(self, force_insert=False, validate=True, clean=False, write_concern=None, cascade=None, cascade_kwargs=None, _refs=None, username=None, **kwargs): """ Override our core custom save. This will ensure if there is a "date" string available for the email that we generate a corresponding "isodate" field which is more useful for database sorting/searching. """ if hasattr(self, 'date'): if self.date: if isinstance(self.date, datetime.datetime): self.isodate = self.date self.date = convert_datetimes_to_string(self.date) else: self.isodate = date_parser(self.date, fuzzy=True) else: if self.isodate: if isinstance(self.isodate, datetime.datetime): self.date = convert_datetimes_to_string(self.isodate) else: self.isodate = None return super(self.__class__, self)._custom_save(force_insert, validate, clean, write_concern, cascade, cascade_kwargs, _refs, username)
class Email(CritsBaseAttributes, CritsSourceDocument, Document): """ Email Class. """ meta = { # mongoengine adds fields _cls and _types and uses them to filter database # responses unless you disallow inheritance. In other words, we # can't see any of our old data unless we add _cls and _types # attributes to them or turn off inheritance. #So we'll turn inheritance off. # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data) "collection": settings.COL_EMAIL, "crits_type": 'Email', "latest_schema_version": 1, "schema_doc": { 'boundary': 'Email boundary', 'campaign': 'List [] of campaigns attributed to this email', 'cc': 'List [] of CC recipients', 'date': 'String of date header field', 'from': 'From header field', 'helo': 'HELO', 'isodate': 'ISODate conversion of date header field', 'message_id': 'Message-ID header field', 'modified': 'When this object was last modified', 'objects': 'List of objects in this email', 'originating_ip': 'Originating-IP header field', 'raw_body': 'Email raw body', 'raw_header': 'Email raw headers', 'relationships': 'List of relationships with this email', 'reply_to': 'Reply-To header field', 'sender': 'Sender header field', 'shared_with': 'Dictionary of sources that this email may be shared with and whether it has been shared already', 'source': 'List [] of sources that provided information on this email', 'subject': 'Email subject', 'to': 'To header field', 'x_originating_ip': 'X-Originating-IP header field', 'x_mailer': 'X-Mailer header field', }, "jtable_opts": { 'details_url': 'crits.emails.views.email_detail', 'details_url_key': 'id', 'default_sort': "isodate DESC", 'searchurl': 'crits.emails.views.emails_listing', 'fields': [ "from_address", "subject", "isodate", "source", "campaign", "id", "to", "status", "cc" ], 'jtopts_fields': [ "details", "from", "recip", "subject", "isodate", "source", "campaign", "status", "favorite", "id"], 'hidden_fields': [], 'linked_fields': [ "source", "campaign", "from", "subject" ], 'details_link': 'details', 'no_sort': ['recip', 'details'] } } boundary = StringField() cc = ListField(StringField()) date = StringField(required=True) from_address = StringField(db_field="from") helo = StringField() # isodate is an interally-set attribute and on save will be overwritten # with the isodate version of the email's date attribute. isodate = CritsDateTimeField() message_id = StringField() originating_ip = StringField() raw_body = StringField() raw_header = RawHeadersField(db_field="raw_headers") reply_to = StringField() sender = StringField() subject = StringField() to = ListField(StringField()) x_originating_ip = StringField() x_mailer = StringField() def migrate(self): """ Migrate to latest schema version. """ migrate_email(self) def _custom_save(self, force_insert=False, validate=True, clean=False, write_concern=None, cascade=None, cascade_kwargs=None, _refs=None, username=None, **kwargs): """ Override our core custom save. This will ensure if there is a "date" string available for the email that we generate a corresponding "isodate" field which is more useful for database sorting/searching. """ if hasattr(self, 'date'): if self.date: if isinstance(self.date, datetime.datetime): self.isodate = self.date self.date = convert_datetimes_to_string(self.date) else: self.isodate = date_parser(self.date, fuzzy=True) else: if self.isodate: if isinstance(self.isodate, datetime.datetime): self.date = convert_datetimes_to_string(self.isodate) else: self.isodate = None return super(self.__class__, self)._custom_save(force_insert, validate, clean, write_concern, cascade, cascade_kwargs, _refs, username) def to_cybox_observable(self, exclude=None): """ Convert an email to a CybOX Observables. Pass parameter exclude to specify fields that should not be included in the returned object. Returns a tuple of (CybOX object, releasability list). To get the cybox object as xml or json, call to_xml() or to_json(), respectively, on the resulting CybOX object. """ if exclude == None: exclude = [] observables = [] obj = EmailMessage() # Assume there is going to be at least one header obj.header = EmailHeader() if 'message_id' not in exclude: obj.header.message_id = String(self.message_id) if 'subject' not in exclude: obj.header.subject = String(self.subject) if 'sender' not in exclude: obj.header.sender = Address(self.sender, Address.CAT_EMAIL) if 'reply_to' not in exclude: obj.header.reply_to = Address(self.reply_to, Address.CAT_EMAIL) if 'x_originating_ip' not in exclude: obj.header.x_originating_ip = Address(self.x_originating_ip, Address.CAT_IPV4) if 'x_mailer' not in exclude: obj.header.x_mailer = String(self.x_mailer) if 'boundary' not in exclude: obj.header.boundary = String(self.boundary) if 'raw_body' not in exclude: obj.raw_body = self.raw_body if 'raw_header' not in exclude: obj.raw_header = self.raw_header #copy fields where the names differ between objects if 'helo' not in exclude and 'email_server' not in exclude: obj.email_server = String(self.helo) if ('from_' not in exclude and 'from' not in exclude and 'from_address' not in exclude): obj.header.from_ = EmailAddress(self.from_address) if 'date' not in exclude and 'isodate' not in exclude: obj.header.date = DateTime(self.isodate) observables.append(Observable(obj)) return (observables, self.releasability) @classmethod def from_cybox(cls, cybox_obs, source): """ Convert a Cybox DefinedObject to a MongoEngine Email object. """ cybox_obj = cybox_obs.object_.properties email = cls(source=source) if cybox_obj.header: email.from_address = str(cybox_obj.header.from_) if cybox_obj.header.to: email.to = [str(recpt) for recpt in cybox_obj.header.to.to_list()] for field in ['message_id', 'sender', 'reply_to', 'x_originating_ip', 'subject', 'date', 'x_mailer', 'boundary']: setattr(email, field, str(getattr(cybox_obj.header, field))) email.helo = str(cybox_obj.email_server) if cybox_obj.raw_body: email.raw_body = str(cybox_obj.raw_body) if cybox_obj.raw_header: email.raw_header = str(cybox_obj.raw_header) return email
class Comment(CritsDocument, CritsSchemaDocument, CritsSourceDocument, Document): """ Comment Class. """ meta = { "collection": settings.COL_COMMENTS, "crits_type": "Comment", "latest_schema_version": 1, "schema_doc": { 'comment': 'The comment body', 'obj_type': 'The type of the object this comment is for', 'obj_id': 'The MongoDB ObjectId for the object this comment is for', 'created': 'ISODate when this comment was made', 'users': 'List [] of users mentioned in the comment', 'tags': 'List [] of hashtags in the comment', 'url_key': 'The key used to redirect to the object for this comment', 'analyst': 'The analyst, if any, that made this comment', 'parent': { 'date': 'The date of the parent comment if this is a reply', 'analyst': 'Analyst who made the comment this is a reply to' }, 'source': ('List [] of source information about who provided this' ' comment') } } # This is not a date field! # It exists to provide default values for created and edit_date date = datetime.datetime.now() analyst = StringField() comment = StringField() created = CritsDateTimeField(default=date, db_field="date") edit_date = CritsDateTimeField(default=date) obj_id = ObjectIdField() obj_type = StringField() #TODO: seems like this might be a good candidate for # a reference field? parent = EmbeddedDocumentField(EmbeddedParentField) tags = ListField(StringField()) url_key = StringField() users = ListField(StringField()) def get_parent(self): """ Get the parent CRITs object. :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes`. """ col_obj = class_from_type(self.obj_type) doc = col_obj.objects(id=self.obj_id).first() return doc def comment_to_html(self): """ Convert the comment from str to HTML. """ if len(self.comment) > 0: self.comment = parse_comment(self.comment)['html'] def parse_comment(self): """ Parse the comment str for users and tags. """ if len(self.comment) > 0: pc = parse_comment(self.comment) self.users = pc['users'] self.tags = pc['tags'] def edit_comment(self, new_comment): """ Edit comment contents. Reparse for users and tags, and set the edit date. :param new_comment: The new comment. :type new_comment: str """ self.comment = new_comment self.parse_comment() self.edit_date = datetime.datetime.now() def set_parent_comment(self, date, analyst): """ Set the parent comment if this is a reply. :param date: The date of the parent comment. :type date: datetime.datetime :param analyst: The user replying to the comment. :type analyst: str """ p = EmbeddedParentField() p.date = date p.analyst = analyst self.parent = p def set_parent_object(self, type_, id_): """ Set the top-level object this comment is for. :param type_: The CRITs type of the object this comment is for. :type type_: str :param id_: The ObjectId of the object this comment is for. :type id_: str """ if type_: self.obj_type = type_ if isinstance(id_, ObjectId): self.obj_id = id_ else: self.obj_id = ObjectId(id_) def set_url_key(self, url_key): """ Set the url_key to link back to the parent object. :param url_key: The URL key to use. :type url_key: str """ if isinstance(url_key, basestring): self.url_key = url_key
class TaxiiContent(CritsSchemaDocument, CritsDocument, Document): """TAXII Content Block Document Object""" meta = { # mongoengine adds fields _cls and _types and uses them to filter database # responses unless you disallow inheritance. In other words, we # can't see any of our old data unless we add _cls and _types # attributes to them or turn off inheritance. #So we'll turn inheritance off. # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data) "allow_inheritance": False, "collection": 'taxii.content', "crits_type": 'TAXIIContent', "latest_schema_version": 2, #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields "minify_defaults": [ 'taxii_msg_id', 'hostname', 'use_hdr_src', 'feed', 'block_label', 'poll_time', 'timerange', 'analyst', 'content', 'errors', 'import_failed' ], "schema_doc": { 'taxii_msg_id': 'A reference to the data (ID of the TAXII message)', 'hostname': 'The source of the data (or TAXII server hostname)', 'use_hdr_src': 'Indicates if STIX Header Info Source is preferred', 'feed': 'Name of the TAXII feed/collection, or ZIP file name', 'block_label': 'STIX filename, or when block submitted to TAXII server', 'poll_time': 'When the data was polled or uploaded', 'timerange': 'Timerange of the TAXII poll, or indication of upload', 'analyst': 'The analyst who retrieved or provided the data', 'content': 'The content being stored (STIX)', 'errors': 'Errors that occurred while parsing or importing content', 'import_failed': 'Boolean indicating that an attempt to import failed' }, } taxii_msg_id = StringField(required=True) hostname = StringField(required=True) use_hdr_src = BooleanField(required=True, default=False) feed = StringField(required=True) block_label = StringField(required=True) poll_time = CritsDateTimeField(required=True) timerange = StringField(required=True) analyst = StringField(required=True) content = StringField(required=True) errors = ListField(StringField(required=True)) import_failed = BooleanField(required=True, default=False) def populate(self, data, analyst, message_id, hostname, feed, block_label, begin=None, end=None, poll_time=None, use_hdr_src=False, errors=[]): """ Populate the class attributes :param data: The STIX content :type data: string :param analyst: The analyst who retrieved or provided the data :type analyst: string :param message_id: A reference to the data (ID of the TAXII message) :type message_id: string :param hostname: The source of the data (or TAXII server hostname) :type hostname: string :param feed: Name of the TAXII feed/collection, or ZIP file name :type feed: string :param block_label: Filename, or when block submitted to TAXII server :type block_label: string :param begin: Exclusive begin component of the timerange that was polled :type begin: :class:`datetime.datetime` :param end: Inclusive end component of the timerange that was polled :type end: :class:`datetime.datetime` :param poll_time: When the data was polled or uploaded :type poll_time: :class:`datetime.datetime` :param use_hdr_src: Indicates if STIX Header Info Source is preferred :type use_hdr_src: boolean :param errors: Errors that occurred while parsing or importing content :type errors: list """ if data or errors: self.taxii_msg_id = message_id self.hostname = hostname self.use_hdr_src = use_hdr_src self.feed = feed self.block_label = block_label self.poll_time = poll_time or datetime.now() if end: # TAXII poll will always have end timestamp end = end.strftime('%Y-%m-%d %H:%M:%S') begin = begin.strftime( '%Y-%m-%d %H:%M:%S') if begin else 'None' self.timerange = '%s to %s' % (begin, end) else: # Must be a STIX file upload self.timerange = 'STIX File Upload' self.analyst = analyst self.content = data or "" self.errors = errors self.import_failed = False def migrate(self): migrate_taxii_content(self)