class Sample(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument, Document): """Sample object""" meta = { "collection": settings.COL_SAMPLES, "crits_type": 'Sample', "latest_schema_version": 4, "shard_key": ('md5',), "schema_doc": { 'filename': 'The name of the last file that was uploaded with this'\ 'MD5', 'filenames': 'A list of filenames this binary has gone by.', 'filetype': 'The filetype of the file', 'mimetype': 'The mimetype of the file', 'size': 'The size of the file', 'md5': 'The MD5 of the file', 'sha1': 'The SHA1 of the file', 'sha256': 'The SHA256 of the file', 'ssdeep': 'The ssdeep of the file', 'campaign': 'List [] of campaigns using this file', 'source': 'List [] of sources that provided this file', 'created': 'ISODate of when this file was uploaded', 'modified': 'ISODate of when the file metadata was last modified', 'filedata': 'The ObjectId of the file in GridFS' }, "jtable_opts": { 'details_url': 'crits.samples.views.detail', 'details_url_key': 'md5', 'default_sort': "created DESC", 'searchurl': 'crits.samples.views.samples_listing', 'fields': [ "filename", "size", "filetype", "created", "modified", "campaign", "source", "md5", "id", "status"], 'jtopts_fields': [ "details", "filename", "size", "filetype", "created", "campaign", "source", "md5", "status", "favorite", "id"], 'hidden_fields': ["md5"], 'linked_fields': ["filename", "source", "campaign", "filetype"], 'details_link': 'details', 'no_sort': ['details', 'id'] }, } filedata = getFileField(collection_name=settings.COL_SAMPLES) filename = StringField(required=True) filenames = ListField(StringField()) filetype = StringField() md5 = StringField(required=True) mimetype = StringField() sha1 = StringField() sha256 = StringField() size = IntField(default=0) ssdeep = StringField() def migrate(self): migrate_sample(self) def add_file_data(self, file_data): self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): import pydeep import magic from hashlib import md5, sha1, sha256 try: self.filetype = magic.from_buffer(data) except: self.filetype = "Unavailable" try: mimetype = magic.from_buffer(data, mime=True) if mimetype: self.mimetype = mimetype.split(";")[0] if not mimetype: self.mimetype = "unknown" except: self.mimetype = "Unavailable" self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() self.sha1 = sha1(data).hexdigest() self.sha256 = sha256(data).hexdigest() try: self.ssdeep = pydeep.hash_bytes(data) except: self.ssdeep = None def is_pe(self): """ Is this a PE file. """ return self.filedata.grid_id != None and self.filedata.read(2) == "MZ" def is_pdf(self): """ Is this a PDF. """ return self.filedata.grid_id != None and "%PDF-" in self.filedata.read( 1024) def discover_binary(self): """ Queries GridFS for a matching binary to this sample document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def set_filenames(self, filenames): """ Set the Sample filenames to a specified list. :param filenames: The filenames to set. :type filenames: list """ if isinstance(filenames, list): self.filenames = filenames
class Sample(CritsBaseAttributes, CritsSourceDocument, Document): """Sample object""" meta = { "collection": settings.COL_SAMPLES, "crits_type": 'Sample', "latest_schema_version": 4, "shard_key": ('md5',), "schema_doc": { 'filename': 'The name of the last file that was uploaded with this'\ 'MD5', 'filenames': 'A list of filenames this binary has gone by.', 'filetype': 'The filetype of the file', 'mimetype': 'The mimetype of the file', 'size': 'The size of the file', 'md5': 'The MD5 of the file', 'sha1': 'The SHA1 of the file', 'sha256': 'The SHA256 of the file', 'ssdeep': 'The ssdeep of the file', 'campaign': 'List [] of campaigns using this file', 'source': 'List [] of sources that provided this file', 'created': 'ISODate of when this file was uploaded', 'modified': 'ISODate of when the file metadata was last modified', 'filedata': 'The ObjectId of the file in GridFS' }, "jtable_opts": { 'details_url': 'crits.samples.views.detail', 'details_url_key': 'md5', 'default_sort': "created DESC", 'searchurl': 'crits.samples.views.samples_listing', 'fields': [ "filename", "size", "filetype", "created", "modified", "campaign", "source", "md5", "id", "status"], 'jtopts_fields': [ "details", "filename", "size", "filetype", "created", "campaign", "source", "md5", "status", "favorite", "id"], 'hidden_fields': ["md5"], 'linked_fields': ["filename", "source", "campaign", "filetype"], 'details_link': 'details', 'no_sort': ['details', 'id'] }, } filedata = getFileField(collection_name=settings.COL_SAMPLES) filename = StringField(required=True) filenames = ListField(StringField()) filetype = StringField() md5 = StringField(required=True) mimetype = StringField() sha1 = StringField() sha256 = StringField() size = IntField(default=0) ssdeep = StringField() def migrate(self): migrate_sample(self) def add_file_data(self, file_data): self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): import pydeep import magic from hashlib import md5, sha1, sha256 try: self.filetype = magic.from_buffer(data) except: self.filetype = "Unavailable" try: mimetype = magic.from_buffer(data, mime=True) if mimetype: self.mimetype = mimetype.split(";")[0] if not mimetype: self.mimetype = "unknown" except: self.mimetype = "Unavailable" self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() self.sha1 = sha1(data).hexdigest() self.sha256 = sha256(data).hexdigest() try: self.ssdeep = pydeep.hash_bytes(data) except: self.ssdeep = None def is_pe(self): """ Is this a PE file. """ return self.filedata.grid_id != None and self.filedata.read(2) == "MZ" def is_pdf(self): """ Is this a PDF. """ return self.filedata.grid_id != None and "%PDF-" in self.filedata.read( 1024) def to_cybox_observable(self, exclude=None, bin_fmt="raw"): if exclude == None: exclude = [] observables = [] f = File() for attr in ['md5', 'sha1', 'sha256']: if attr not in exclude: val = getattr(self, attr, None) if val: setattr(f, attr, val) if self.ssdeep and 'ssdeep' not in exclude: f.add_hash(Hash(self.ssdeep, Hash.TYPE_SSDEEP)) if 'size' not in exclude and 'size_in_bytes' not in exclude: f.size_in_bytes = UnsignedLong(self.size) if 'filename' not in exclude and 'file_name' not in exclude: f.file_name = self.filename # create an Artifact object for the binary if it exists if 'filedata' not in exclude and bin_fmt: data = self.filedata.read() if data: # if sample data available a = Artifact(data, Artifact.TYPE_FILE) # create artifact w/data if bin_fmt == "zlib": a.packaging.append(ZlibCompression()) a.packaging.append(Base64Encoding()) elif bin_fmt == "base64": a.packaging.append(Base64Encoding()) f.add_related(a, "Child_Of") # relate artifact to file if 'filetype' not in exclude and 'file_format' not in exclude: #NOTE: this doesn't work because the CybOX File object does not # have any support built in for setting the filetype to a # CybOX-binding friendly object (e.g., calling .to_dict() on # the resulting CybOX object fails on this field. f.file_format = self.filetype observables.append(Observable(f)) return (observables, self.releasability) @classmethod def from_cybox(cls, cybox_obs): """ Convert a Cybox DefinedObject to a MongoEngine Sample object. :param cybox_obs: The cybox object to create the Sample from. :type cybox_obs: :class:`cybox.core.Observable`` :returns: :class:`crits.samples.sample.Sample` """ cybox_object = cybox_obs.object_.properties if cybox_object.md5: db_obj = Sample.objects(md5=cybox_object.md5).first() if db_obj: # if a sample with md5 already exists return db_obj # don't modify, just return sample = cls() # else, start creating new sample record sample.filename = str(cybox_object.file_name) sample.size = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0 for hash_ in cybox_object.hashes: if hash_.type_.value.upper() in [ Hash.TYPE_MD5, Hash.TYPE_SHA1, Hash.TYPE_SHA256, Hash.TYPE_SSDEEP ]: setattr(sample, hash_.type_.value.lower(), str(hash_.simple_hash_value).strip().lower()) for obj in cybox_object.parent.related_objects: # attempt to find data in cybox if isinstance( obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE: sample.add_file_data(obj.properties.data) break return sample def discover_binary(self): """ Queries GridFS for a matching binary to this sample document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def set_filenames(self, filenames): """ Set the Sample filenames to a specified list. :param filenames: The filenames to set. :type filenames: list """ if isinstance(filenames, list): self.filenames = filenames
class Certificate(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument, Document): """ Certificate Class. """ meta = { "collection": settings.COL_CERTIFICATES, "crits_type": 'Certificate', "latest_schema_version": 2, "schema_doc": { 'filename': 'The filename of the certificate', 'filetype': 'The filetype of the certificate', 'md5': 'The MD5 of the certificate file', 'size': 'The filesize of the certificate', 'source': 'List [] of source information about who provided the certificate' }, "jtable_opts": { 'details_url': 'crits.certificates.views.certificate_details', 'details_url_key': 'md5', 'default_sort': "modified DESC", 'searchurl': 'crits.certificates.views.certificates_listing', 'fields': [ "filename", "description", "filetype", "size", "modified", "source", "campaign", "id", "md5", "status" ], 'jtopts_fields': [ "details", "filename", "description", "filetype", "size", "modified", "source", "campaign", "status", "md5", "favorite", "id" ], 'hidden_fields': ["md5"], 'linked_fields': ["source", "campaign"], 'details_link': 'details', 'no_sort': ['details'] }, } filedata = getFileField(collection_name=settings.COL_CERTIFICATES) filename = StringField(required=True) filetype = StringField(required=True) size = IntField(default=0) md5 = StringField() def migrate(self): """ Migrate the Certificate tot he latest schema version. """ migrate_certificate(self) def add_file_data(self, file_data): """ Add the Certificate to GridFS. :param file_data: The Certificate. :type file_data: str """ self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): """ Add the Certificate to GridFS. :param file_obj: The Certificate. :type file_data: file handle """ data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): """ Set the filetype, size, and MD5 of the Certificate. :param data: The Certificate. :type data: str """ import magic from hashlib import md5 self.filetype = magic.from_buffer(data) self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() def discover_binary(self): """ Queries GridFS for a matching binary to this Certificate document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed()
class Screenshot(CritsBaseDocument, CritsSourceDocument, CritsSchemaDocument, CritsDocument, Document): """ Screenshot Class. """ meta = { "collection": settings.COL_SCREENSHOTS, "crits_type": 'Screenshot', "latest_schema_version": 1, "schema_doc": { 'filename': 'The name of the screenshot', 'thumb': 'Thumbnail of screenshot', 'screenshot': 'The screenshot', 'width': 'The width in pixels', 'height': 'The height in pixels', 'description': 'Description of the screenshot', 'tags': 'List of tags about this screenshot', 'source': 'List [] of source information about who provided the screenshot' }, "jtable_opts": { 'details_url': 'crits.screenshots.views.render_screenshot', 'details_url_key': 'id', 'default_sort': "created DESC", 'searchurl': 'crits.screenshots.views.screenshots_listing', 'fields': ["thumb", "description", "created", "source", "id", "md5", "tags"], 'jtopts_fields': [ "details", "thumb", "description", "created", "source", "md5", "tags", "favorite" ], 'hidden_fields': ["md5"], 'linked_fields': ["source", "tags"], 'details_link': 'details', 'no_sort': ['details'] }, } analyst = StringField() description = StringField() filename = StringField() height = IntField() md5 = StringField() screenshot = getFileField(collection_name=settings.COL_SCREENSHOTS, required=True) tags = ListField(StringField()) thumb = getFileField(collection_name=settings.COL_SCREENSHOTS, required=True) width = IntField() def migrate(self): """ Migrate the Screenshot to the latest schema version. """ pass def add_screenshot(self, screenshot=None, tags=None): """ Add the screenshot to the class. This will write the screenshot to GridFS, set the filename, width, height, and generate the thumbnail. :param screenshot: The screenshot to add. :type screenshot: file handle :param tags: A tag or list of tags for this screenshot :type param: str, list """ if not screenshot: return self.filename = screenshot.name im = Image.open(screenshot) self.width, self.height = im.size fs = StringIO.StringIO() im.save(fs, "PNG") fs.seek(0) self.screenshot = fs.read() self.generate_thumbnail(im) if isinstance(tags, basestring): tlist = tags.split(',') self.tags = [t.strip() for t in tlist if len(t.strip())] elif isinstance(tags, list): self.tags = tags def add_tags(self, tags): """ Add tags to a screenshot. :param tags: The tags to add. :type tags: str, list """ if isinstance(tags, basestring): tag_list = [t.strip() for t in tags.split(',') if len(t.strip())] if isinstance(tags, list): tag_list = [t.strip() for t in tags if len(t.strip())] for t in tag_list: if t not in self.tags: self.tags.append(t) def generate_thumbnail(self, im=None): """ Generate a thumbnail out of a screenshot. Will write the thumbnail to GridFS. :param im: The PIL Image. :type im: :class:`PIL.Image` """ if not im: return size = (128, 128) im.thumbnail(size, Image.ANTIALIAS) im.save(self.thumb, "PNG") fs = StringIO.StringIO() im.save(fs, "PNG") fs.seek(0) self.thumb = fs.read() def sanitize(self, username=None, sources=None, rels=None): """ Sanitize the source list down to only those a user has access to see. This was sniped from core/crits_mongoengine. :param username: The user requesting this data. :type username: str :param sources: A list of sources the user has access to. :type sources: list """ if username and hasattr(self, 'source'): length = len(self.source) if not sources: sources = user_sources(username) # use slice to modify in place in case any code is referencing # the source already will reflect the changes as well self.source[:] = [s for s in self.source if s.name in sources] # a bit of a hack but we add a poorly formatted source to the # source list which has an instances length equal to the amount # of sources that were sanitized out of the user's list. # not tested but this has the added benefit of throwing a # ValidationError if someone were to try and save() this. new_length = len(self.source) if length > new_length: i_length = length - new_length s = EmbeddedSource() s.name = "Other" s.instances = [0] * i_length self.source.append(s)
class PCAP(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument, Document): """ PCAP class. """ meta = { "collection": settings.COL_PCAPS, "auto_create_index": False, "crits_type": 'PCAP', "latest_schema_version": 3, "shard_key": ('md5',), "schema_doc": { 'filename': 'The filename of the PCAP', 'md5': 'The MD5 of the PCAP file', 'length': 'The filesize of the PCAP', 'uploadDate': 'The ISODate when the PCAP was uploaded', 'contentType': 'The filetype of the PCAP', 'source': 'List [] of source information about who provided the PCAP' }, "jtable_opts": { 'details_url': 'crits-pcaps-views-pcap_details', 'details_url_key': 'md5', 'default_sort': "modified DESC", 'searchurl': 'crits-pcaps-views-pcaps_listing', 'fields': [ "filename", "description", "length", "modified", "source", "campaign", "id", "md5", "status"], 'jtopts_fields': [ "details", "filename", "description", "length", "modified", "source", "campaign", "status", "md5", "favorite", "id"], 'hidden_fields': ['md5'], 'linked_fields': ["source", "campaign"], 'details_link': 'details', 'no_sort': ['details'] } } contentType = StringField() filedata = getFileField(collection_name=settings.COL_PCAPS) filename = StringField(required=True) length = IntField(default=0) md5 = StringField() def migrate(self): """ Migrate to the latest schema version. """ migrate_pcap(self) def add_file_data(self, file_data): """ Add filedata to this PCAP. :param file_data: The filedata to add. :type file_data: str """ self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): """ Add filedata to this PCAP. :param file_data: The filedata to add. :type file_data: file handle """ data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): """ Generate metadata from the file data. Will add content-type, length, and MD5. :param data: The data to generate metadata from. :type data: str """ import magic from hashlib import md5 self.contentType = magic.from_buffer(data) self.length = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() def discover_binary(self): """ Queries GridFS for a matching binary to this pcap document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed()
class Sample(CritsBaseAttributes, CritsSourceDocument, Document): """Sample object""" meta = { "collection": settings.COL_SAMPLES, "crits_type": 'Sample', "latest_schema_version": 2, "shard_key": ('md5',), "schema_doc": { 'filename': 'The name of the last file that was uploaded with this'\ 'MD5', 'filetype': 'The filetype of the file', 'mimetype': 'The mimetype of the file', 'size': 'The size of the file', 'md5': 'The MD5 of the file', 'sha1': 'The SHA1 of the file', 'sha256': 'The SHA256 of the file', 'ssdeep': 'The ssdeep of the file', 'exploit': [ { 'cve': 'The CVE of the exploit used by this file' } ], 'backdoor': { 'name': 'The name of the backdoor used by this file', 'version': 'The version of the backdoor used by this file', 'analyst': 'The analyst who added this backdoor', 'date': 'The date this backdoor was added' }, 'campaign': 'List [] of campaigns using this file', 'analysis': 'List [] of analysis results from tools for this file', 'source': 'List [] of sources that provided this file', 'created': 'ISODate of when this file was uploaded', 'modified': 'ISODate of when the file metadata was last modified', 'filedata': 'The ObjectId of the file in GridFS' }, "jtable_opts": { 'details_url': 'crits.samples.views.detail', 'details_url_key': 'md5', 'default_sort': "created DESC", 'searchurl': 'crits.samples.views.samples_listing', 'fields': [ "filename", "size", "filetype", "created", "modified", "exploit", "campaign", "source", "md5", "id", "status"], 'jtopts_fields': [ "details", "filename", "size", "filetype", "created", "exploit", "campaign", "source", "md5", "status", "favorite", "id"], 'hidden_fields': ["md5"], 'linked_fields': ["filename", "source", "campaign", "filetype","exploit"], 'details_link': 'details', 'no_sort': ['details', 'id'] }, } backdoor = EmbeddedDocumentField(EmbeddedBackdoor) exploit = ListField(EmbeddedDocumentField(EmbeddedExploit)) filedata = getFileField(collection_name=settings.COL_SAMPLES) filename = StringField(required=True) filetype = StringField() md5 = StringField(required=True) mimetype = StringField() sha1 = StringField() sha256 = StringField() size = IntField(default=0) ssdeep = StringField() def migrate(self): migrate_sample(self) def add_file_data(self, file_data): self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): import pydeep import magic from hashlib import md5, sha1, sha256 try: self.filetype = magic.from_buffer(data) except: self.filetype = "Unavailable" try: mimetype = magic.from_buffer(data, mime=True) if mimetype: self.mimetype = mimetype.split(";")[0] if not mimetype: self.mimetype = "unknown" except: self.mimetype = "Unavailable" self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() self.sha1 = sha1(data).hexdigest() self.sha256 = sha256(data).hexdigest() try: self.ssdeep = pydeep.hash_bytes(data) except: self.ssdeep = None def to_cybox(self, exclude=None): if exclude == None: exclude = [] observables = [] f = File() for attr in ['md5', 'sha1', 'sha256']: if attr not in exclude: val = getattr(self, attr, None) if val: setattr(f, attr, val) if self.ssdeep and 'ssdeep' not in exclude: f.add_hash(Hash(self.ssdeep, Hash.TYPE_SSDEEP)) if 'size' not in exclude and 'size_in_bytes' not in exclude: f.size_in_bytes = UnsignedLong(self.size) if 'filename' not in exclude and 'file_name' not in exclude: f.file_name = self.filename # create an Artifact object for the binary if it exists if 'filedata' not in exclude: data = self.filedata.read() if data: data = base64.b64encode(data) a = Artifact(data=data, type_=Artifact.TYPE_FILE) observables.append(Observable(a)) #if 'filetype' not in exclude and 'file_format' not in exclude: #NOTE: this doesn't work because the CybOX File object does not # have any support built in for setting the filetype to a # CybOX-binding friendly object (e.g., calling .to_dict() on # the resulting CybOX object fails on this field. #f.file_format = self.filetype observables.append(Observable(f)) return (observables, self.releasability) @classmethod def from_cybox(cls, cybox_object, source, filedata=None): """ Convert a Cybox DefinedObject to a MongoEngine Sample object. """ sample = cls(source=source) #TODO: we need to find all *required* fields and check for them # and handle optional fields accordingly. if cybox_object.size_in_bytes: sample.size = cybox_object.size_in_bytes.value else: sample.size = 0 sample.filename = str(cybox_object.file_name) for hash_ in cybox_object.hashes: if hash_.type_.value.upper() in [ Hash.TYPE_MD5, Hash.TYPE_SHA1, Hash.TYPE_SHA256, Hash.TYPE_SSDEEP ]: setattr(sample, hash_.type_.value.lower(), str(hash_.simple_hash_value).strip().lower()) if filedata: if isinstance(filedata, file): sample.filedata = filedata.read() else: sample.filedata = filedata return sample def discover_binary(self): """ Queries GridFS for a matching binary to this sample document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def set_backdoor(self, name, version, analyst): if self.backdoor: bd = Backdoor.objects(name=self.backdoor.name).first() bd.decrement_count() bd.save(username=analyst) eb = EmbeddedBackdoor() eb.name = name eb.version = version eb.analyst = analyst self.backdoor = eb bd = Backdoor.objects(name=name).first() bd.increment_count() bd.save(username=analyst) def add_exploit(self, cve): found = False for e in self.exploit: if e.cve == cve: found = True if not found: ee = EmbeddedExploit() ee.cve = cve self.exploit.append(ee) def delete_exploit(self, cve): c = 0 for e in self.exploit: if e.cve == cve: del self.exploit[c] c += 0
class Certificate(CritsBaseAttributes, CritsSourceDocument, Document): """ Certificate Class. """ meta = { "collection": settings.COL_CERTIFICATES, "crits_type": 'Certificate', "latest_schema_version": 1, "schema_doc": { 'filename': 'The filename of the certificate', 'filetype': 'The filetype of the certificate', 'md5': 'The MD5 of the certificate file', 'size': 'The filesize of the certificate', 'description': 'Description of what the certificate contains', 'source': 'List [] of source information about who provided the certificate' }, "jtable_opts": { 'details_url': 'crits.certificates.views.certificate_details', 'details_url_key': 'md5', 'default_sort': "modified DESC", 'searchurl': 'crits.certificates.views.certificates_listing', 'fields': [ "filename", "description", "filetype", "size", "modified", "source", "campaign", "id", "md5", "status" ], 'jtopts_fields': [ "details", "filename", "description", "filetype", "size", "modified", "source", "campaign", "status", "md5", "favorite", "id" ], 'hidden_fields': ["md5"], 'linked_fields': ["source", "campaign"], 'details_link': 'details', 'no_sort': ['details'] }, } description = StringField() filedata = getFileField(collection_name=settings.COL_CERTIFICATES) filename = StringField(required=True) filetype = StringField(required=True) size = IntField(default=0) md5 = StringField() def migrate(self): """ Migrate the Certificate tot he latest schema version. """ pass def add_file_data(self, file_data): """ Add the Certificate to GridFS. :param file_data: The Certificate. :type file_data: str """ self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): """ Add the Certificate to GridFS. :param file_obj: The Certificate. :type file_data: file handle """ data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): """ Set the filetype, size, and MD5 of the Certificate. :param data: The Certificate. :type data: str """ import magic from hashlib import md5 self.filetype = magic.from_buffer(data) self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() def discover_binary(self): """ Queries GridFS for a matching binary to this Certificate document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def to_cybox_observable(self): """ Convert a Certificate to a CybOX Observables. Returns a tuple of (CybOX object, releasability list). To get the cybox object as xml or json, call to_xml() or to_json(), respectively, on the resulting CybOX object. """ custom_prop = Property( ) # make a custom property so CRITs import can identify Certificate exports custom_prop.name = "crits_type" custom_prop.description = "Indicates the CRITs type of the object this CybOX object represents" custom_prop._value = "Certificate" obj = File() # represent cert information as file obj.md5 = self.md5 obj.file_name = self.filename obj.file_format = self.filetype obj.size_in_bytes = self.size obj.custom_properties = CustomProperties() obj.custom_properties.append(custom_prop) obs = Observable(obj) obs.description = self.description data = self.filedata.read() if data: # if cert data available a = Artifact(data, Artifact.TYPE_FILE) # create artifact w/data a.packaging.append(Base64Encoding()) obj.add_related(a, "Child_Of") # relate artifact to file return ([obs], self.releasability) @classmethod def from_cybox(cls, cybox_obs, source): """ Convert a Cybox DefinedObject to a MongoEngine Indicator object. :param cybox_obs: The cybox object to create the indicator from. :type cybox_obs: :class:`cybox.core.Observable`` :param source: The source list for the Indicator. :type source: list :returns: :class:`crits.indicators.indicator.Indicator` """ cybox_object = cybox_obs.object_.properties if cybox_object.md5: db_obj = Certificate.objects(md5=cybox_object.md5).first() if db_obj: return db_obj cert = cls(source=source) cert.md5 = cybox_object.md5 cert.filename = cybox_object.file_name cert.filetype = cybox_object.file_format cert.size = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0 cert.description = cybox_obs.description for obj in cybox_object.parent.related_objects: # attempt to find data in cybox if isinstance(obj.properties, Artifact): cert.add_file_data(base64.b64decode(obj.properties.data)) return cert
class PCAP(CritsBaseAttributes, CritsSourceDocument, Document): """ PCAP class. """ meta = { "collection": settings.COL_PCAPS, "crits_type": 'PCAP', "latest_schema_version": 3, "shard_key": ('md5', ), "schema_doc": { 'filename': 'The filename of the PCAP', 'md5': 'The MD5 of the PCAP file', 'length': 'The filesize of the PCAP', 'uploadDate': 'The ISODate when the PCAP was uploaded', 'contentType': 'The filetype of the PCAP', 'source': 'List [] of source information about who provided the PCAP' }, "jtable_opts": { 'details_url': 'crits.pcaps.views.pcap_details', 'details_url_key': 'md5', 'default_sort': "modified DESC", 'searchurl': 'crits.pcaps.views.pcaps_listing', 'fields': [ "filename", "description", "length", "modified", "source", "campaign", "id", "md5", "status" ], 'jtopts_fields': [ "details", "filename", "description", "length", "modified", "source", "campaign", "status", "md5", "favorite", "id" ], 'hidden_fields': ['md5'], 'linked_fields': ["source", "campaign"], 'details_link': 'details', 'no_sort': ['details'] } } contentType = StringField() filedata = getFileField(collection_name=settings.COL_PCAPS) filename = StringField(required=True) length = IntField(default=0) md5 = StringField() def migrate(self): """ Migrate to the latest schema version. """ migrate_pcap(self) def add_file_data(self, file_data): """ Add filedata to this PCAP. :param file_data: The filedata to add. :type file_data: str """ self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): """ Add filedata to this PCAP. :param file_data: The filedata to add. :type file_data: file handle """ data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): """ Generate metadata from the file data. Will add content-type, length, and MD5. :param data: The data to generate metadata from. :type data: str """ import magic from hashlib import md5 self.contentType = magic.from_buffer(data) self.length = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() def discover_binary(self): """ Queries GridFS for a matching binary to this pcap document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def to_cybox_observable(self): """ Convert a PCAP to a CybOX Observables. Returns a tuple of (CybOX object, releasability list). To get the cybox object as xml or json, call to_xml() or to_json(), respectively, on the resulting CybOX object. """ obj = File() obj.md5 = self.md5 obj.file_name = self.filename obj.file_format = self.contentType obj.size_in_bytes = self.length obs = Observable(obj) obs.description = self.description art = Artifact(self.filedata.read(), Artifact.TYPE_NETWORK) art.packaging.append(Base64Encoding()) obj.add_related(art, "Child_Of") # relate artifact to file return ([obs], self.releasability) @classmethod def from_cybox(cls, cybox_obs): """ Convert a Cybox Artifact to a CRITs PCAP object. :param cybox_obs: The cybox object to create the PCAP from. :type cybox_obs: :class:`cybox.core.Observable` :returns: :class:`crits.pcaps.pcap.PCAP` """ cybox_object = cybox_obs.object_.properties if cybox_object.md5: db_obj = PCAP.objects(md5=cybox_object.md5).first() if db_obj: return db_obj pcap = cls() pcap.description = str(cybox_obs.description) pcap.md5 = cybox_object.md5 pcap.filename = str(cybox_object.file_name) pcap.contentType = cybox_object.file_format pcap.length = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0 for obj in cybox_object.parent.related_objects: # attempt to find data in cybox if isinstance(obj.properties, Artifact ) and obj.properties.type_ == Artifact.TYPE_NETWORK: pcap.add_file_data(obj.properties.data) break return pcap
class Sample(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument, Document): """Sample object""" meta = { "collection": settings.COL_SAMPLES, "auto_create_index": False, "crits_type": 'Sample', "latest_schema_version": 5, "shard_key": ('md5',), "schema_doc": { 'filename': 'The name of the last file that was uploaded with this'\ 'MD5', 'filenames': 'A list of filenames this binary has gone by.', 'filetype': 'The filetype of the file', 'mimetype': 'The mimetype of the file', 'size': 'The size of the file', 'md5': 'The MD5 of the file', 'sha1': 'The SHA1 of the file', 'sha256': 'The SHA256 of the file', 'ssdeep': 'The ssdeep of the file', 'impfuzzy': 'The impfuzzy of the executable file', 'campaign': 'List [] of campaigns using this file', 'source': 'List [] of sources that provided this file', 'created': 'ISODate of when this file was uploaded', 'modified': 'ISODate of when the file metadata was last modified', 'filedata': 'The ObjectId of the file in GridFS' }, "jtable_opts": { 'details_url': 'crits-samples-views-detail', 'details_url_key': 'md5', 'default_sort': "created DESC", 'searchurl': 'crits-samples-views-samples_listing', 'fields': [ "filename", "size", "filetype", "created", "modified", "campaign", "source", "md5", "id", "status"], 'jtopts_fields': [ "details", "filename", "size", "filetype", "created", "campaign", "source", "md5", "status", "favorite", "id"], 'hidden_fields': ["md5"], 'linked_fields': ["filename", "source", "campaign", "filetype"], 'details_link': 'details', 'no_sort': ['details'] }, } filedata = getFileField(collection_name=settings.COL_SAMPLES) filename = StringField(required=True) filenames = ListField(StringField()) filetype = StringField() md5 = StringField(required=True) mimetype = StringField() sha1 = StringField() sha256 = StringField() size = IntField(default=0) ssdeep = StringField() impfuzzy = StringField() def migrate(self): migrate_sample(self) def add_file_data(self, file_data): self._generate_file_metadata(file_data) self.filedata = file_data def add_file_obj(self, file_obj): data = file_obj.read() self._generate_file_metadata(data) self.filedata = data def _generate_file_metadata(self, data): import pydeep import magic from hashlib import md5, sha1, sha256 try: import pyimpfuzzy except ImportError: pass try: self.filetype = magic.from_buffer(data) if len(self.filetype) > 1000: self.filetype = self.filetype[0:1000] + '<TRUNCATED>' except: self.filetype = "Unavailable" try: mimetype = magic.from_buffer(data, mime=True) if mimetype: self.mimetype = mimetype.split(";")[0] if not mimetype: self.mimetype = "unknown" except: self.mimetype = "Unavailable" self.size = len(data) # this is a shard key. you can't modify it once it's set. # MongoEngine will still mark the field as modified even if you set it # to the same value. if not self.md5: self.md5 = md5(data).hexdigest() self.sha1 = sha1(data).hexdigest() self.sha256 = sha256(data).hexdigest() try: self.ssdeep = pydeep.hash_bytes(data) except: self.ssdeep = None try: self.impfuzzy = pyimpfuzzy.get_impfuzzy_data(data) except: self.impfuzzy = None def is_office(self): """ Is this a Office file. """ ret = self.filedata.grid_id != None and self.filedata.read(8) == "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" if self.filedata.grid_id: self.filedata.seek(0) return ret def is_pe(self): """ Is this a PE file. """ ret = self.filedata.grid_id != None and self.filedata.read(2) == "MZ" if self.filedata.grid_id: self.filedata.seek(0) return ret def is_pdf(self): """ Is this a PDF. """ ret = self.filedata.grid_id != None and "%PDF-" in self.filedata.read(1024) if self.filedata.grid_id: self.filedata.seek(0) return ret def discover_binary(self): """ Queries GridFS for a matching binary to this sample document. """ from crits.core.mongo_tools import mongo_connector fm = mongo_connector("%s.files" % self._meta['collection']) objectid = fm.find_one({'md5': self.md5}, {'_id': 1}) if objectid: self.filedata.grid_id = objectid['_id'] self.filedata._mark_as_changed() def set_filenames(self, filenames): """ Set the Sample filenames to a specified list. :param filenames: The filenames to set. :type filenames: list """ if isinstance(filenames, list): self.filenames = filenames def _json_yaml_convert(self, exclude=[]): """ Helper to convert to a dict before converting to JSON. :param exclude: list of fields to exclude. :type exclude: list :returns: json """ d = self.to_dict(exclude) if 'filedata' not in exclude: (d['filedata'], ext) = format_file(self.filedata.read(), 'base64') return json.dumps(d, default=json_handler)