コード例 #1
0
class SnuggleIndex(CritsDocument, Document):
    """Snugglefish Index Document Object"""
    meta = {
        "collection": 'snugglefish_indexes',
        "crits_type": 'snugglefish_index',
        "latest_schema_version": 1,
        "schema_doc": {
            'name': "Name of this index",
            'name': "Directory where this index lives",
            'query': "Query for this index",
            'created': "Date this index was created",
            'last_update': "Date this index was last updated",
            'last_id': "Last object ID fetched for this index",
            'total': "Total number of objects available for this index",
            'count': "Total number of objects fetched for this index"
        },
    }

    name = StringField(required=True)
    directory = StringField(required=True)
    query = StringField(required=True)
    created = CritsDateTimeField(required=True)
    last_update = CritsDateTimeField()
    last_id = ObjectIdField()
    total = IntField(default=0)
    count = IntField(default=0)

    def migrate(self):
        pass
コード例 #2
0
class Taxii(CritsDocument, Document):
    """TAXII Document Object"""
    meta = {
        # mongoengine adds fields _cls and _types and uses them to filter database
        # responses unless you disallow inheritance. In other words, we
        # can't see any of our old data unless we add _cls and _types
        # attributes to them or turn off inheritance.
        #So we'll turn inheritance off.
        # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data)
        "allow_inheritance": False,
        "collection": 'taxii',
        "crits_type": 'TAXII',
        "latest_schema_version": 2,
        #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields
        "minify_defaults": ['runtime', 'end', 'feed'],
        "schema_doc": {
            'runtime': 'The last time we made a TAXII request.',
            'end': 'End date of this taxii document.',
            'feed': 'Feed of this taxii document.'
        },
    }

    runtime = CritsDateTimeField(required=True)
    end = CritsDateTimeField(required=True)

    def migrate(self):
        pass

    @classmethod
    def get_last(cls, feed):
        #returns the last time for the particular feed
        return cls.objects(unsupported_attrs={
            'feed': feed
        }).order_by('-end').first()
コード例 #3
0
ファイル: indicator.py プロジェクト: plouzek/crits-1
class EmbeddedActivity(EmbeddedDocument, CritsDocumentFormatter):
    """
    Indicator activity class.
    """

    analyst = StringField()
    end_date = CritsDateTimeField(default=datetime.datetime.now)
    date = CritsDateTimeField(default=datetime.datetime.now)
    description = StringField()
    start_date = CritsDateTimeField(default=datetime.datetime.now)
コード例 #4
0
class EmbeddedHighlight(EmbeddedDocument, CritsDocumentFormatter):
    """
    Raw Data highlight comment class.
    """

    date = CritsDateTimeField(default=datetime.datetime.now)
    line_date = CritsDateTimeField()
    analyst = StringField()
    line = IntField()
    line_data = StringField()
    comment = StringField()
コード例 #5
0
ファイル: indicator.py プロジェクト: plouzek/crits-1
class EmbeddedAction(EmbeddedDocument, CritsDocumentFormatter):
    """
    Indicator action class.
    """

    action_type = StringField()
    active = StringField()
    analyst = StringField()
    begin_date = CritsDateTimeField(default=datetime.datetime.now)
    date = CritsDateTimeField(default=datetime.datetime.now)
    end_date = CritsDateTimeField(default=datetime.datetime.now)
    performed_date = CritsDateTimeField(default=datetime.datetime.now)
    reason = StringField()
コード例 #6
0
class TaxiiContent(CritsDocument, Document):
    """TAXII Content Block Document Object"""
    meta = {
        # mongoengine adds fields _cls and _types and uses them to filter database
        # responses unless you disallow inheritance. In other words, we
        # can't see any of our old data unless we add _cls and _types
        # attributes to them or turn off inheritance.
        #So we'll turn inheritance off.
        # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data)
        "allow_inheritance":
        False,
        "collection":
        'taxii.content',
        "crits_type":
        'TAXIIContent',
        "latest_schema_version":
        1,
        #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields
        "minify_defaults": [
            'taxii_msg_id', 'hostname', 'feed', 'timestamp', 'poll_time',
            'timerange', 'analyst', 'content', 'errors', 'import_failed'
        ],
        "schema_doc": {
            'taxii_msg_id':
            'The ID of the TAXII message from which this content came',
            'hostname': 'The hostname of the TAXII server',
            'feed': 'The name of the TAXII feed/collection',
            'timestamp': 'When the content was submitted to the TAXII server',
            'poll_time': 'A timestamp representing when this data was polled',
            'timerange': 'The timerange of the TAXII poll',
            'analyst': 'The analyst who polled the data',
            'content': 'The content being stored',
            'errors': 'Any errors that prevented import of the content',
            'import_failed':
            'Boolean indicating that an attempt to import failed'
        },
    }

    taxii_msg_id = StringField(required=True)
    hostname = StringField(required=True)
    feed = StringField(required=True)
    timestamp = CritsDateTimeField(required=True)
    poll_time = CritsDateTimeField(required=True)
    timerange = StringField(required=True)
    analyst = StringField(required=True)
    content = StringField(required=True)
    errors = ListField(StringField(required=True))
    import_failed = BooleanField(required=True, default=False)

    def migrate(self):
        pass
コード例 #7
0
class EmbeddedParentField(EmbeddedDocument, CritsDocumentFormatter):
    """
    Embedded Parent Field
    """

    date = CritsDateTimeField()
    analyst = StringField()
コード例 #8
0
class AuditLog(CritsDocument, CritsSchemaDocument, Document):
    """
    Audit Log Class
    """
    meta = {
        "collection": settings.COL_AUDIT_LOG,
        "auto_create_index": False,
        "allow_inheritance": False,
        "crits_type": "AuditLog",
        "latest_schema_version": 1,
        "schema_doc": {
            'value': 'Value of the audit log entry',
            'user': '******',
            'date': 'Date of the entry',
            'type': 'Type of the audit entry',
            'method': 'Method of the audit entry'
        }
    }

    value = StringField()
    user = StringField()
    date = CritsDateTimeField(default=datetime.datetime.now)
    target_type = StringField(db_field='type')
    target_id = ObjectIdField()
    method = StringField()
コード例 #9
0
ファイル: sample.py プロジェクト: stefanvangastel/crits
class EmbeddedBackdoor(EmbeddedDocument, CritsDocumentFormatter):
    """Sample backdoors object"""

    name = StringField()
    version = StringField()
    analyst = StringField()
    date = CritsDateTimeField(default=datetime.datetime.now)
コード例 #10
0
ファイル: actor.py プロジェクト: xtracerx/crits
class EmbeddedActorIdentifier(EmbeddedDocument, CritsDocumentFormatter):
    """
    Embedded Actor Identifier class.
    """

    analyst = StringField(required=True)
    date = CritsDateTimeField(default=datetime.datetime.now)
    identifier_id = StringField(required=True)
    confidence = StringField(default="unknown")
コード例 #11
0
class EmbeddedInline(EmbeddedDocument, CritsDocumentFormatter):
    """
    Raw Data Inline comment class.
    """

    date = CritsDateTimeField(default=datetime.datetime.now)
    analyst = StringField()
    line = IntField()
    comment = StringField()
    counter = IntField()
コード例 #12
0
ファイル: notification.py プロジェクト: zhangjunqiang/crits
class Notification(CritsDocument, CritsSchemaDocument, Document):
    """
    Notification Class.
    """

    meta = {
        "collection": settings.COL_NOTIFICATIONS,
        "auto_create_index": False,
        "crits_type": "Notification",
        "latest_schema_version": 1,
        "schema_doc": {
            'notification': 'The notification body',
            'notification_type': 'The type of notification, e.g. alert, error',
            'header': 'The notification header, optional',
            'link_url': 'A link URL for the header, optional',
            'status':
            'New/Processed - used to determine whether or not to notify',
            'obj_type': 'The type of the object this notification is for',
            'obj_id':
            'The MongoDB ObjectId for the object this notification is for',
            'created': 'ISODate when this notification was made',
            'users': 'List [] of users for this notification',
            'analyst': 'The analyst, if any, that made this notification',
        }
    }
    # This is not a date field!
    # It exists to provide default values for created and edit_date
    date = datetime.datetime.now()

    analyst = StringField()
    notification = StringField()
    notification_type = StringField()
    header = StringField()
    created = CritsDateTimeField(default=date, db_field="date")
    obj_id = ObjectIdField()
    obj_type = StringField()
    status = StringField(default="new")
    link_url = StringField()
    users = ListField(StringField())

    def set_status(self, status):
        """
        Set the status of the notification.

        :param status: The status ("new", "processed").
        :type status: str
        """

        if status in ("new", "processed"):
            self.status = status
コード例 #13
0
class ActorIdentifier(CritsDocument, CritsSchemaDocument, CritsSourceDocument,
                      Document):
    """
    Actor Identifier class.
    """

    meta = {
        "collection": settings.COL_ACTOR_IDENTIFIERS,
        "crits_type": 'ActorIdentifier',
        "latest_schema_version": 1,
        "schema_doc": {
            'name': 'The name of this Action',
            'active': 'Enabled in the UI (on/off)'
        },
        "jtable_opts": {
                         'details_url': '',
                         'details_url_key': '',
                         'default_sort': "created DESC",
                         'searchurl': 'crits.actors.views.actor_identifiers_listing',
                         'fields': [ "name", "created", "source",
                                    "identifier_type", "id"],
                         'jtopts_fields': [ "name",
                                            "identifier_type",
                                            "created",
                                            "source"],
                         'hidden_fields': [],
                         'linked_fields': ["name", "source"],
                         'details_link': '',
                         'no_sort': ['']
                       }
    }

    active = StringField(default="on")
    created = CritsDateTimeField(default=datetime.datetime.now)
    identifier_type = StringField(required=True)
    # name is misleading, it's actually the value. We use name so we can
    # leverage a lot of the work done for Item editing in the control panel and
    # changing active state.
    name = StringField(required=True)

    def set_identifier_type(self, identifier_type):
        identifier_type = identifier_type.strip()
        it = ActorThreatIdentifier.objects(name=identifier_type).first()
        if it:
            self.identifier_type = identifier_type
コード例 #14
0
class Email(CritsBaseAttributes, CritsSourceDocument, Document):
    """
    Email Class.
    """

    meta = {
        # mongoengine adds fields _cls and _types and uses them to filter database
        # responses unless you disallow inheritance. In other words, we
        # can't see any of our old data unless we add _cls and _types
        # attributes to them or turn off inheritance.
        #So we'll turn inheritance off.
        # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data)
        "collection": settings.COL_EMAIL,
        "crits_type": 'Email',
        "latest_schema_version": 2,
        "schema_doc": {
            'boundary': 'Email boundary',
            'campaign': 'List [] of campaigns attributed to this email',
            'cc': 'List [] of CC recipients',
            'date': 'String of date header field',
            'from': 'From header field',
            'helo': 'HELO',
            'isodate': 'ISODate conversion of date header field',
            'message_id': 'Message-ID header field',
            'modified': 'When this object was last modified',
            'objects': 'List of objects in this email',
            'originating_ip': 'Originating-IP header field',
            'raw_body': 'Email raw body',
            'raw_header': 'Email raw headers',
            'relationships': 'List of relationships with this email',
            'reply_to': 'Reply-To header field',
            'sender': 'Sender header field',
            'shared_with':
            'Dictionary of sources that this email may be shared with and whether it has been shared already',
            'source':
            'List [] of sources that provided information on this email',
            'subject': 'Email subject',
            'to': 'To header field',
            'x_originating_ip': 'X-Originating-IP header field',
            'x_mailer': 'X-Mailer header field',
        },
        "jtable_opts": {
            'details_url':
            'crits.emails.views.email_detail',
            'details_url_key':
            'id',
            'default_sort':
            "isodate DESC",
            'searchurl':
            'crits.emails.views.emails_listing',
            'fields': [
                "from_address", "subject", "isodate", "source", "campaign",
                "id", "to", "status", "cc"
            ],
            'jtopts_fields': [
                "details", "from", "recip", "subject", "isodate", "source",
                "campaign", "status", "favorite", "id"
            ],
            'hidden_fields': [],
            'linked_fields': ["source", "campaign", "from", "subject"],
            'details_link':
            'details',
            'no_sort': ['recip', 'details']
        }
    }

    boundary = StringField()
    cc = ListField(StringField())
    date = StringField(required=True)
    from_address = StringField(db_field="from")
    helo = StringField()
    # isodate is an interally-set attribute and on save will be overwritten
    # with the isodate version of the email's date attribute.
    isodate = CritsDateTimeField()
    message_id = StringField()
    originating_ip = StringField()
    raw_body = StringField()
    raw_header = RawHeadersField(db_field="raw_headers")
    reply_to = StringField()
    sender = StringField()
    subject = StringField()
    to = ListField(StringField())
    x_originating_ip = StringField()
    x_mailer = StringField()

    def migrate(self):
        """
        Migrate to latest schema version.
        """

        migrate_email(self)

    def _custom_save(self,
                     force_insert=False,
                     validate=True,
                     clean=False,
                     write_concern=None,
                     cascade=None,
                     cascade_kwargs=None,
                     _refs=None,
                     username=None,
                     **kwargs):
        """
        Override our core custom save. This will ensure if there is a "date"
        string available for the email that we generate a corresponding
        "isodate" field which is more useful for database sorting/searching.
        """

        if hasattr(self, 'date'):
            if self.date:
                if isinstance(self.date, datetime.datetime):
                    self.isodate = self.date
                    self.date = convert_datetimes_to_string(self.date)
                else:
                    self.isodate = date_parser(self.date, fuzzy=True)
            else:
                if self.isodate:
                    if isinstance(self.isodate, datetime.datetime):
                        self.date = convert_datetimes_to_string(self.isodate)
        else:
            self.isodate = None

        return super(self.__class__,
                     self)._custom_save(force_insert, validate, clean,
                                        write_concern, cascade, cascade_kwargs,
                                        _refs, username)
コード例 #15
0
class Email(CritsBaseAttributes, CritsSourceDocument, Document):
    """
    Email Class.
    """

    meta = {
        # mongoengine adds fields _cls and _types and uses them to filter database
        # responses unless you disallow inheritance. In other words, we
        # can't see any of our old data unless we add _cls and _types
        # attributes to them or turn off inheritance.
        #So we'll turn inheritance off.
        # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data)
        "collection": settings.COL_EMAIL,
        "crits_type": 'Email',
        "latest_schema_version": 1,
        "schema_doc": {
            'boundary': 'Email boundary',
            'campaign': 'List [] of campaigns attributed to this email',
            'cc': 'List [] of CC recipients',
            'date': 'String of date header field',
            'from': 'From header field',
            'helo': 'HELO',
            'isodate': 'ISODate conversion of date header field',
            'message_id': 'Message-ID header field',
            'modified': 'When this object was last modified',
            'objects': 'List of objects in this email',
            'originating_ip': 'Originating-IP header field',
            'raw_body': 'Email raw body',
            'raw_header': 'Email raw headers',
            'relationships': 'List of relationships with this email',
            'reply_to': 'Reply-To header field',
            'sender': 'Sender header field',
            'shared_with': 'Dictionary of sources that this email may be shared with and whether it has been shared already',
            'source': 'List [] of sources that provided information on this email',
            'subject': 'Email subject',
            'to': 'To header field',
            'x_originating_ip': 'X-Originating-IP header field',
            'x_mailer': 'X-Mailer header field',
        },
        "jtable_opts": {
                         'details_url': 'crits.emails.views.email_detail',
                         'details_url_key': 'id',
                         'default_sort': "isodate DESC",
                         'searchurl': 'crits.emails.views.emails_listing',
                         'fields': [ "from_address", "subject", "isodate",
                                     "source", "campaign", "id", "to",
                                     "status", "cc" ],
                          'jtopts_fields': [ "details",
                                             "from",
                                             "recip",
                                             "subject",
                                             "isodate",
                                             "source",
                                             "campaign",
                                             "status",
                                             "favorite",
                                             "id"],
                         'hidden_fields': [],
                         'linked_fields': [ "source", "campaign",
                                            "from", "subject" ],
                         'details_link': 'details',
                         'no_sort': ['recip', 'details']
                       }
    }

    boundary = StringField()
    cc = ListField(StringField())
    date = StringField(required=True)
    from_address = StringField(db_field="from")
    helo = StringField()
    # isodate is an interally-set attribute and on save will be overwritten
    # with the isodate version of the email's date attribute.
    isodate = CritsDateTimeField()
    message_id = StringField()
    originating_ip = StringField()
    raw_body = StringField()
    raw_header = RawHeadersField(db_field="raw_headers")
    reply_to = StringField()
    sender = StringField()
    subject = StringField()
    to = ListField(StringField())
    x_originating_ip = StringField()
    x_mailer = StringField()

    def migrate(self):
        """
        Migrate to latest schema version.
        """

        migrate_email(self)

    def _custom_save(self, force_insert=False, validate=True, clean=False,
        write_concern=None,  cascade=None, cascade_kwargs=None,
        _refs=None, username=None, **kwargs):
        """
        Override our core custom save. This will ensure if there is a "date"
        string available for the email that we generate a corresponding
        "isodate" field which is more useful for database sorting/searching.
        """

        if hasattr(self, 'date'):
            if self.date:
                if isinstance(self.date, datetime.datetime):
                    self.isodate = self.date
                    self.date = convert_datetimes_to_string(self.date)
                else:
                    self.isodate = date_parser(self.date, fuzzy=True)
            else:
                if self.isodate:
                    if isinstance(self.isodate, datetime.datetime):
                        self.date = convert_datetimes_to_string(self.isodate)
        else:
            self.isodate = None

        return super(self.__class__, self)._custom_save(force_insert, validate,
            clean, write_concern, cascade, cascade_kwargs, _refs, username)

    def to_cybox_observable(self, exclude=None):
        """
        Convert an email to a CybOX Observables.

        Pass parameter exclude to specify fields that should not be
        included in the returned object.

        Returns a tuple of (CybOX object, releasability list).

        To get the cybox object as xml or json, call to_xml() or
        to_json(), respectively, on the resulting CybOX object.
        """

        if exclude == None:
            exclude = []

        observables = []

        obj = EmailMessage()
        # Assume there is going to be at least one header
        obj.header = EmailHeader()

        if 'message_id' not in exclude:
            obj.header.message_id = String(self.message_id)

        if 'subject' not in exclude:
            obj.header.subject = String(self.subject)

        if 'sender' not in exclude:
            obj.header.sender = Address(self.sender, Address.CAT_EMAIL)

        if 'reply_to' not in exclude:
            obj.header.reply_to = Address(self.reply_to, Address.CAT_EMAIL)

        if 'x_originating_ip' not in exclude:
            obj.header.x_originating_ip = Address(self.x_originating_ip,
                                                  Address.CAT_IPV4)

        if 'x_mailer' not in exclude:
            obj.header.x_mailer = String(self.x_mailer)

        if 'boundary' not in exclude:
            obj.header.boundary = String(self.boundary)

        if 'raw_body' not in exclude:
            obj.raw_body = self.raw_body

        if 'raw_header' not in exclude:
            obj.raw_header = self.raw_header

        #copy fields where the names differ between objects
        if 'helo' not in exclude and 'email_server' not in exclude:
            obj.email_server = String(self.helo)
        if ('from_' not in exclude and 'from' not in exclude and
            'from_address' not in exclude):
            obj.header.from_ = EmailAddress(self.from_address)
        if 'date' not in exclude and 'isodate' not in exclude:
            obj.header.date = DateTime(self.isodate)

        observables.append(Observable(obj))
        return (observables, self.releasability)

    @classmethod
    def from_cybox(cls, cybox_obs, source):
        """
        Convert a Cybox DefinedObject to a MongoEngine Email object.
        """

        cybox_obj = cybox_obs.object_.properties
        email = cls(source=source)

        if cybox_obj.header:
            email.from_address = str(cybox_obj.header.from_)
            if cybox_obj.header.to:
                email.to = [str(recpt) for recpt in cybox_obj.header.to.to_list()]
            for field in ['message_id', 'sender', 'reply_to', 'x_originating_ip',
                          'subject', 'date', 'x_mailer', 'boundary']:
                setattr(email, field, str(getattr(cybox_obj.header, field)))

        email.helo = str(cybox_obj.email_server)
        if cybox_obj.raw_body:
            email.raw_body = str(cybox_obj.raw_body)
        if cybox_obj.raw_header:
            email.raw_header = str(cybox_obj.raw_header)

        return email
コード例 #16
0
class Comment(CritsDocument, CritsSchemaDocument, CritsSourceDocument, Document):
    """
    Comment Class.
    """

    meta = {
        "collection": settings.COL_COMMENTS,
        "crits_type": "Comment",
        "latest_schema_version": 1,
        "schema_doc": {
            'comment': 'The comment body',
            'obj_type': 'The type of the object this comment is for',
            'obj_id': 'The MongoDB ObjectId for the object this comment is for',
            'created': 'ISODate when this comment was made',
            'users': 'List [] of users mentioned in the comment',
            'tags': 'List [] of hashtags in the comment',
            'url_key': 'The key used to redirect to the object for this comment',
            'analyst': 'The analyst, if any, that made this comment',
            'parent': {
                'date': 'The date of the parent comment if this is a reply',
                'analyst': 'Analyst who made the comment this is a reply to'
            },
            'source': ('List [] of source information about who provided this'
                       ' comment')
        }
    }
    # This is not a date field!
    # It exists to provide default values for created and edit_date
    date = datetime.datetime.now()

    analyst = StringField()
    comment = StringField()
    created = CritsDateTimeField(default=date, db_field="date")
    edit_date = CritsDateTimeField(default=date)
    obj_id = ObjectIdField()
    obj_type = StringField()
    #TODO: seems like this might be a good candidate for
    #   a reference field?
    parent = EmbeddedDocumentField(EmbeddedParentField)
    tags = ListField(StringField())
    url_key = StringField()
    users = ListField(StringField())

    def get_parent(self):
        """
        Get the parent CRITs object.

        :returns: class which inherits from
                  :class:`crits.core.crits_mongoengine.CritsBaseAttributes`.
        """

        col_obj = class_from_type(self.obj_type)
        doc = col_obj.objects(id=self.obj_id).first()
        return doc

    def comment_to_html(self):
        """
        Convert the comment from str to HTML.
        """

        if len(self.comment) > 0:
            self.comment = parse_comment(self.comment)['html']

    def parse_comment(self):
        """
        Parse the comment str for users and tags.
        """

        if len(self.comment) > 0:
            pc = parse_comment(self.comment)
            self.users = pc['users']
            self.tags = pc['tags']

    def edit_comment(self, new_comment):
        """
        Edit comment contents. Reparse for users and tags, and set the edit
        date.

        :param new_comment: The new comment.
        :type new_comment: str
        """

        self.comment = new_comment
        self.parse_comment()
        self.edit_date = datetime.datetime.now()

    def set_parent_comment(self, date, analyst):
        """
        Set the parent comment if this is a reply.

        :param date: The date of the parent comment.
        :type date: datetime.datetime
        :param analyst: The user replying to the comment.
        :type analyst: str
        """

        p = EmbeddedParentField()
        p.date = date
        p.analyst = analyst
        self.parent = p

    def set_parent_object(self, type_, id_):
        """
        Set the top-level object this comment is for.

        :param type_: The CRITs type of the object this comment is for.
        :type type_: str
        :param id_: The ObjectId of the object this comment is for.
        :type id_: str
        """

        if type_:
            self.obj_type = type_
        if isinstance(id_, ObjectId):
            self.obj_id = id_
        else:
            self.obj_id = ObjectId(id_)

    def set_url_key(self, url_key):
        """
        Set the url_key to link back to the parent object.

        :param url_key: The URL key to use.
        :type url_key: str
        """

        if isinstance(url_key, basestring):
                self.url_key = url_key
コード例 #17
0
ファイル: taxii.py プロジェクト: brlogan/crits_services
class TaxiiContent(CritsSchemaDocument, CritsDocument, Document):
    """TAXII Content Block Document Object"""
    meta = {
        # mongoengine adds fields _cls and _types and uses them to filter database
        # responses unless you disallow inheritance. In other words, we
        # can't see any of our old data unless we add _cls and _types
        # attributes to them or turn off inheritance.
        #So we'll turn inheritance off.
        # (See http://mongoengine-odm.readthedocs.org/en/latest/guide/defining-documents.html#working-with-existing-data)
        "allow_inheritance":
        False,
        "collection":
        'taxii.content',
        "crits_type":
        'TAXIIContent',
        "latest_schema_version":
        2,
        #NOTE: minify_defaults fields should match the MongoEngine field names, NOT the database fields
        "minify_defaults": [
            'taxii_msg_id', 'hostname', 'use_hdr_src', 'feed', 'block_label',
            'poll_time', 'timerange', 'analyst', 'content', 'errors',
            'import_failed'
        ],
        "schema_doc": {
            'taxii_msg_id':
            'A reference to the data (ID of the TAXII message)',
            'hostname': 'The source of the data (or TAXII server hostname)',
            'use_hdr_src': 'Indicates if STIX Header Info Source is preferred',
            'feed': 'Name of the TAXII feed/collection, or ZIP file name',
            'block_label':
            'STIX filename, or when block submitted to TAXII server',
            'poll_time': 'When the data was polled or uploaded',
            'timerange':
            'Timerange of the TAXII poll, or indication of upload',
            'analyst': 'The analyst who retrieved or provided the data',
            'content': 'The content being stored (STIX)',
            'errors':
            'Errors that occurred while parsing or importing content',
            'import_failed':
            'Boolean indicating that an attempt to import failed'
        },
    }

    taxii_msg_id = StringField(required=True)
    hostname = StringField(required=True)
    use_hdr_src = BooleanField(required=True, default=False)
    feed = StringField(required=True)
    block_label = StringField(required=True)
    poll_time = CritsDateTimeField(required=True)
    timerange = StringField(required=True)
    analyst = StringField(required=True)
    content = StringField(required=True)
    errors = ListField(StringField(required=True))
    import_failed = BooleanField(required=True, default=False)

    def populate(self,
                 data,
                 analyst,
                 message_id,
                 hostname,
                 feed,
                 block_label,
                 begin=None,
                 end=None,
                 poll_time=None,
                 use_hdr_src=False,
                 errors=[]):
        """
        Populate the class attributes

        :param data: The STIX content
        :type data: string
        :param analyst: The analyst who retrieved or provided the data
        :type analyst: string
        :param message_id: A reference to the data (ID of the TAXII message)
        :type message_id: string
        :param hostname: The source of the data (or TAXII server hostname)
        :type hostname: string
        :param feed: Name of the TAXII feed/collection, or ZIP file name
        :type feed: string
        :param block_label: Filename, or when block submitted to TAXII server
        :type block_label: string
        :param begin: Exclusive begin component of the timerange that was polled
        :type begin: :class:`datetime.datetime`
        :param end: Inclusive end component of the timerange that was polled
        :type end: :class:`datetime.datetime`
        :param poll_time: When the data was polled or uploaded
        :type poll_time: :class:`datetime.datetime`
        :param use_hdr_src: Indicates if STIX Header Info Source is preferred
        :type use_hdr_src: boolean
        :param errors: Errors that occurred while parsing or importing content
        :type errors: list
        """

        if data or errors:
            self.taxii_msg_id = message_id
            self.hostname = hostname
            self.use_hdr_src = use_hdr_src
            self.feed = feed
            self.block_label = block_label
            self.poll_time = poll_time or datetime.now()
            if end:  # TAXII poll will always have end timestamp
                end = end.strftime('%Y-%m-%d %H:%M:%S')
                begin = begin.strftime(
                    '%Y-%m-%d %H:%M:%S') if begin else 'None'
                self.timerange = '%s to %s' % (begin, end)
            else:  # Must be a STIX file upload
                self.timerange = 'STIX File Upload'
            self.analyst = analyst
            self.content = data or ""
            self.errors = errors
            self.import_failed = False

    def migrate(self):
        migrate_taxii_content(self)