class Sample(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""Sample object"""
meta = {
"collection": settings.COL_SAMPLES,
"crits_type": 'Sample',
"latest_schema_version": 4,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The name of the last file that was uploaded with this'\
'MD5',
'filenames': 'A list of filenames this binary has gone by.',
'filetype': 'The filetype of the file',
'mimetype': 'The mimetype of the file',
'size': 'The size of the file',
'md5': 'The MD5 of the file',
'sha1': 'The SHA1 of the file',
'sha256': 'The SHA256 of the file',
'ssdeep': 'The ssdeep of the file',
'campaign': 'List [] of campaigns using this file',
'source': 'List [] of sources that provided this file',
'created': 'ISODate of when this file was uploaded',
'modified': 'ISODate of when the file metadata was last modified',
'filedata': 'The ObjectId of the file in GridFS'
},
"jtable_opts": {
'details_url': 'crits.samples.views.detail',
'details_url_key': 'md5',
'default_sort': "created DESC",
'searchurl': 'crits.samples.views.samples_listing',
'fields': [ "filename", "size", "filetype",
"created", "modified", "campaign",
"source", "md5", "id", "status"],
'jtopts_fields': [ "details",
"filename",
"size",
"filetype",
"created",
"campaign",
"source",
"md5",
"status",
"favorite",
"id"],
'hidden_fields': ["md5"],
'linked_fields': ["filename", "source", "campaign",
"filetype"],
'details_link': 'details',
'no_sort': ['details', 'id']
},
}
filedata = getFileField(collection_name=settings.COL_SAMPLES)
filename = StringField(required=True)
filenames = ListField(StringField())
filetype = StringField()
md5 = StringField(required=True)
mimetype = StringField()
sha1 = StringField()
sha256 = StringField()
size = IntField(default=0)
ssdeep = StringField()
def migrate(self):
migrate_sample(self)
def add_file_data(self, file_data):
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
import pydeep
import magic
from hashlib import md5, sha1, sha256
try:
self.filetype = magic.from_buffer(data)
except:
self.filetype = "Unavailable"
try:
mimetype = magic.from_buffer(data, mime=True)
if mimetype:
self.mimetype = mimetype.split(";")[0]
if not mimetype:
self.mimetype = "unknown"
except:
self.mimetype = "Unavailable"
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
self.sha1 = sha1(data).hexdigest()
self.sha256 = sha256(data).hexdigest()
try:
self.ssdeep = pydeep.hash_bytes(data)
except:
self.ssdeep = None
def is_pe(self):
"""
Is this a PE file.
"""
return self.filedata.grid_id != None and self.filedata.read(2) == "MZ"
def is_pdf(self):
"""
Is this a PDF.
"""
return self.filedata.grid_id != None and "%PDF-" in self.filedata.read(
1024)
def discover_binary(self):
"""
Queries GridFS for a matching binary to this sample document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def set_filenames(self, filenames):
"""
Set the Sample filenames to a specified list.
:param filenames: The filenames to set.
:type filenames: list
"""
if isinstance(filenames, list):
self.filenames = filenames
class Sample(CritsBaseAttributes, CritsSourceDocument, Document):
"""Sample object"""
meta = {
"collection": settings.COL_SAMPLES,
"crits_type": 'Sample',
"latest_schema_version": 4,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The name of the last file that was uploaded with this'\
'MD5',
'filenames': 'A list of filenames this binary has gone by.',
'filetype': 'The filetype of the file',
'mimetype': 'The mimetype of the file',
'size': 'The size of the file',
'md5': 'The MD5 of the file',
'sha1': 'The SHA1 of the file',
'sha256': 'The SHA256 of the file',
'ssdeep': 'The ssdeep of the file',
'campaign': 'List [] of campaigns using this file',
'source': 'List [] of sources that provided this file',
'created': 'ISODate of when this file was uploaded',
'modified': 'ISODate of when the file metadata was last modified',
'filedata': 'The ObjectId of the file in GridFS'
},
"jtable_opts": {
'details_url': 'crits.samples.views.detail',
'details_url_key': 'md5',
'default_sort': "created DESC",
'searchurl': 'crits.samples.views.samples_listing',
'fields': [ "filename", "size", "filetype",
"created", "modified", "campaign",
"source", "md5", "id", "status"],
'jtopts_fields': [ "details",
"filename",
"size",
"filetype",
"created",
"campaign",
"source",
"md5",
"status",
"favorite",
"id"],
'hidden_fields': ["md5"],
'linked_fields': ["filename", "source", "campaign",
"filetype"],
'details_link': 'details',
'no_sort': ['details', 'id']
},
}
filedata = getFileField(collection_name=settings.COL_SAMPLES)
filename = StringField(required=True)
filenames = ListField(StringField())
filetype = StringField()
md5 = StringField(required=True)
mimetype = StringField()
sha1 = StringField()
sha256 = StringField()
size = IntField(default=0)
ssdeep = StringField()
def migrate(self):
migrate_sample(self)
def add_file_data(self, file_data):
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
import pydeep
import magic
from hashlib import md5, sha1, sha256
try:
self.filetype = magic.from_buffer(data)
except:
self.filetype = "Unavailable"
try:
mimetype = magic.from_buffer(data, mime=True)
if mimetype:
self.mimetype = mimetype.split(";")[0]
if not mimetype:
self.mimetype = "unknown"
except:
self.mimetype = "Unavailable"
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
self.sha1 = sha1(data).hexdigest()
self.sha256 = sha256(data).hexdigest()
try:
self.ssdeep = pydeep.hash_bytes(data)
except:
self.ssdeep = None
def is_pe(self):
"""
Is this a PE file.
"""
return self.filedata.grid_id != None and self.filedata.read(2) == "MZ"
def is_pdf(self):
"""
Is this a PDF.
"""
return self.filedata.grid_id != None and "%PDF-" in self.filedata.read(
1024)
def to_cybox_observable(self, exclude=None, bin_fmt="raw"):
if exclude == None:
exclude = []
observables = []
f = File()
for attr in ['md5', 'sha1', 'sha256']:
if attr not in exclude:
val = getattr(self, attr, None)
if val:
setattr(f, attr, val)
if self.ssdeep and 'ssdeep' not in exclude:
f.add_hash(Hash(self.ssdeep, Hash.TYPE_SSDEEP))
if 'size' not in exclude and 'size_in_bytes' not in exclude:
f.size_in_bytes = UnsignedLong(self.size)
if 'filename' not in exclude and 'file_name' not in exclude:
f.file_name = self.filename
# create an Artifact object for the binary if it exists
if 'filedata' not in exclude and bin_fmt:
data = self.filedata.read()
if data: # if sample data available
a = Artifact(data,
Artifact.TYPE_FILE) # create artifact w/data
if bin_fmt == "zlib":
a.packaging.append(ZlibCompression())
a.packaging.append(Base64Encoding())
elif bin_fmt == "base64":
a.packaging.append(Base64Encoding())
f.add_related(a, "Child_Of") # relate artifact to file
if 'filetype' not in exclude and 'file_format' not in exclude:
#NOTE: this doesn't work because the CybOX File object does not
# have any support built in for setting the filetype to a
# CybOX-binding friendly object (e.g., calling .to_dict() on
# the resulting CybOX object fails on this field.
f.file_format = self.filetype
observables.append(Observable(f))
return (observables, self.releasability)
@classmethod
def from_cybox(cls, cybox_obs):
"""
Convert a Cybox DefinedObject to a MongoEngine Sample object.
:param cybox_obs: The cybox object to create the Sample from.
:type cybox_obs: :class:`cybox.core.Observable``
:returns: :class:`crits.samples.sample.Sample`
"""
cybox_object = cybox_obs.object_.properties
if cybox_object.md5:
db_obj = Sample.objects(md5=cybox_object.md5).first()
if db_obj: # if a sample with md5 already exists
return db_obj # don't modify, just return
sample = cls() # else, start creating new sample record
sample.filename = str(cybox_object.file_name)
sample.size = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0
for hash_ in cybox_object.hashes:
if hash_.type_.value.upper() in [
Hash.TYPE_MD5, Hash.TYPE_SHA1, Hash.TYPE_SHA256,
Hash.TYPE_SSDEEP
]:
setattr(sample, hash_.type_.value.lower(),
str(hash_.simple_hash_value).strip().lower())
for obj in cybox_object.parent.related_objects: # attempt to find data in cybox
if isinstance(
obj.properties,
Artifact) and obj.properties.type_ == Artifact.TYPE_FILE:
sample.add_file_data(obj.properties.data)
break
return sample
def discover_binary(self):
"""
Queries GridFS for a matching binary to this sample document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def set_filenames(self, filenames):
"""
Set the Sample filenames to a specified list.
:param filenames: The filenames to set.
:type filenames: list
"""
if isinstance(filenames, list):
self.filenames = filenames
class Certificate(CritsBaseAttributes, CritsSourceDocument,
CritsActionsDocument, Document):
"""
Certificate Class.
"""
meta = {
"collection": settings.COL_CERTIFICATES,
"crits_type": 'Certificate',
"latest_schema_version": 2,
"schema_doc": {
'filename':
'The filename of the certificate',
'filetype':
'The filetype of the certificate',
'md5':
'The MD5 of the certificate file',
'size':
'The filesize of the certificate',
'source':
'List [] of source information about who provided the certificate'
},
"jtable_opts": {
'details_url':
'crits.certificates.views.certificate_details',
'details_url_key':
'md5',
'default_sort':
"modified DESC",
'searchurl':
'crits.certificates.views.certificates_listing',
'fields': [
"filename", "description", "filetype", "size", "modified",
"source", "campaign", "id", "md5", "status"
],
'jtopts_fields': [
"details", "filename", "description", "filetype", "size",
"modified", "source", "campaign", "status", "md5", "favorite",
"id"
],
'hidden_fields': ["md5"],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
},
}
filedata = getFileField(collection_name=settings.COL_CERTIFICATES)
filename = StringField(required=True)
filetype = StringField(required=True)
size = IntField(default=0)
md5 = StringField()
def migrate(self):
"""
Migrate the Certificate tot he latest schema version.
"""
migrate_certificate(self)
def add_file_data(self, file_data):
"""
Add the Certificate to GridFS.
:param file_data: The Certificate.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
"""
Add the Certificate to GridFS.
:param file_obj: The Certificate.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
"""
Set the filetype, size, and MD5 of the Certificate.
:param data: The Certificate.
:type data: str
"""
import magic
from hashlib import md5
self.filetype = magic.from_buffer(data)
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
def discover_binary(self):
"""
Queries GridFS for a matching binary to this Certificate document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
class Screenshot(CritsBaseDocument, CritsSourceDocument, CritsSchemaDocument,
CritsDocument, Document):
"""
Screenshot Class.
"""
meta = {
"collection": settings.COL_SCREENSHOTS,
"crits_type": 'Screenshot',
"latest_schema_version": 1,
"schema_doc": {
'filename':
'The name of the screenshot',
'thumb':
'Thumbnail of screenshot',
'screenshot':
'The screenshot',
'width':
'The width in pixels',
'height':
'The height in pixels',
'description':
'Description of the screenshot',
'tags':
'List of tags about this screenshot',
'source':
'List [] of source information about who provided the screenshot'
},
"jtable_opts": {
'details_url':
'crits.screenshots.views.render_screenshot',
'details_url_key':
'id',
'default_sort':
"created DESC",
'searchurl':
'crits.screenshots.views.screenshots_listing',
'fields':
["thumb", "description", "created", "source", "id", "md5", "tags"],
'jtopts_fields': [
"details", "thumb", "description", "created", "source", "md5",
"tags", "favorite"
],
'hidden_fields': ["md5"],
'linked_fields': ["source", "tags"],
'details_link':
'details',
'no_sort': ['details']
},
}
analyst = StringField()
description = StringField()
filename = StringField()
height = IntField()
md5 = StringField()
screenshot = getFileField(collection_name=settings.COL_SCREENSHOTS,
required=True)
tags = ListField(StringField())
thumb = getFileField(collection_name=settings.COL_SCREENSHOTS,
required=True)
width = IntField()
def migrate(self):
"""
Migrate the Screenshot to the latest schema version.
"""
pass
def add_screenshot(self, screenshot=None, tags=None):
"""
Add the screenshot to the class. This will write the screenshot to
GridFS, set the filename, width, height, and generate the thumbnail.
:param screenshot: The screenshot to add.
:type screenshot: file handle
:param tags: A tag or list of tags for this screenshot
:type param: str, list
"""
if not screenshot:
return
self.filename = screenshot.name
im = Image.open(screenshot)
self.width, self.height = im.size
fs = StringIO.StringIO()
im.save(fs, "PNG")
fs.seek(0)
self.screenshot = fs.read()
self.generate_thumbnail(im)
if isinstance(tags, basestring):
tlist = tags.split(',')
self.tags = [t.strip() for t in tlist if len(t.strip())]
elif isinstance(tags, list):
self.tags = tags
def add_tags(self, tags):
"""
Add tags to a screenshot.
:param tags: The tags to add.
:type tags: str, list
"""
if isinstance(tags, basestring):
tag_list = [t.strip() for t in tags.split(',') if len(t.strip())]
if isinstance(tags, list):
tag_list = [t.strip() for t in tags if len(t.strip())]
for t in tag_list:
if t not in self.tags:
self.tags.append(t)
def generate_thumbnail(self, im=None):
"""
Generate a thumbnail out of a screenshot. Will write the thumbnail to
GridFS.
:param im: The PIL Image.
:type im: :class:`PIL.Image`
"""
if not im:
return
size = (128, 128)
im.thumbnail(size, Image.ANTIALIAS)
im.save(self.thumb, "PNG")
fs = StringIO.StringIO()
im.save(fs, "PNG")
fs.seek(0)
self.thumb = fs.read()
def sanitize(self, username=None, sources=None, rels=None):
"""
Sanitize the source list down to only those a user has access to see.
This was sniped from core/crits_mongoengine.
:param username: The user requesting this data.
:type username: str
:param sources: A list of sources the user has access to.
:type sources: list
"""
if username and hasattr(self, 'source'):
length = len(self.source)
if not sources:
sources = user_sources(username)
# use slice to modify in place in case any code is referencing
# the source already will reflect the changes as well
self.source[:] = [s for s in self.source if s.name in sources]
# a bit of a hack but we add a poorly formatted source to the
# source list which has an instances length equal to the amount
# of sources that were sanitized out of the user's list.
# not tested but this has the added benefit of throwing a
# ValidationError if someone were to try and save() this.
new_length = len(self.source)
if length > new_length:
i_length = length - new_length
s = EmbeddedSource()
s.name = "Other"
s.instances = [0] * i_length
self.source.append(s)
class PCAP(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""
PCAP class.
"""
meta = {
"collection": settings.COL_PCAPS,
"auto_create_index": False,
"crits_type": 'PCAP',
"latest_schema_version": 3,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The filename of the PCAP',
'md5': 'The MD5 of the PCAP file',
'length': 'The filesize of the PCAP',
'uploadDate': 'The ISODate when the PCAP was uploaded',
'contentType': 'The filetype of the PCAP',
'source': 'List [] of source information about who provided the PCAP'
},
"jtable_opts": {
'details_url': 'crits-pcaps-views-pcap_details',
'details_url_key': 'md5',
'default_sort': "modified DESC",
'searchurl': 'crits-pcaps-views-pcaps_listing',
'fields': [ "filename", "description", "length",
"modified", "source", "campaign", "id",
"md5", "status"],
'jtopts_fields': [ "details",
"filename",
"description",
"length",
"modified",
"source",
"campaign",
"status",
"md5",
"favorite",
"id"],
'hidden_fields': ['md5'],
'linked_fields': ["source", "campaign"],
'details_link': 'details',
'no_sort': ['details']
}
}
contentType = StringField()
filedata = getFileField(collection_name=settings.COL_PCAPS)
filename = StringField(required=True)
length = IntField(default=0)
md5 = StringField()
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_pcap(self)
def add_file_data(self, file_data):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
"""
Generate metadata from the file data. Will add content-type, length, and
MD5.
:param data: The data to generate metadata from.
:type data: str
"""
import magic
from hashlib import md5
self.contentType = magic.from_buffer(data)
self.length = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
def discover_binary(self):
"""
Queries GridFS for a matching binary to this pcap document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
class Sample(CritsBaseAttributes, CritsSourceDocument, Document):
"""Sample object"""
meta = {
"collection": settings.COL_SAMPLES,
"crits_type": 'Sample',
"latest_schema_version": 2,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The name of the last file that was uploaded with this'\
'MD5',
'filetype': 'The filetype of the file',
'mimetype': 'The mimetype of the file',
'size': 'The size of the file',
'md5': 'The MD5 of the file',
'sha1': 'The SHA1 of the file',
'sha256': 'The SHA256 of the file',
'ssdeep': 'The ssdeep of the file',
'exploit': [
{
'cve': 'The CVE of the exploit used by this file'
}
],
'backdoor': {
'name': 'The name of the backdoor used by this file',
'version': 'The version of the backdoor used by this file',
'analyst': 'The analyst who added this backdoor',
'date': 'The date this backdoor was added'
},
'campaign': 'List [] of campaigns using this file',
'analysis': 'List [] of analysis results from tools for this file',
'source': 'List [] of sources that provided this file',
'created': 'ISODate of when this file was uploaded',
'modified': 'ISODate of when the file metadata was last modified',
'filedata': 'The ObjectId of the file in GridFS'
},
"jtable_opts": {
'details_url': 'crits.samples.views.detail',
'details_url_key': 'md5',
'default_sort': "created DESC",
'searchurl': 'crits.samples.views.samples_listing',
'fields': [ "filename", "size", "filetype",
"created", "modified", "exploit",
"campaign", "source", "md5", "id",
"status"],
'jtopts_fields': [ "details",
"filename",
"size",
"filetype",
"created",
"exploit",
"campaign",
"source",
"md5",
"status",
"favorite",
"id"],
'hidden_fields': ["md5"],
'linked_fields': ["filename", "source", "campaign",
"filetype","exploit"],
'details_link': 'details',
'no_sort': ['details', 'id']
},
}
backdoor = EmbeddedDocumentField(EmbeddedBackdoor)
exploit = ListField(EmbeddedDocumentField(EmbeddedExploit))
filedata = getFileField(collection_name=settings.COL_SAMPLES)
filename = StringField(required=True)
filetype = StringField()
md5 = StringField(required=True)
mimetype = StringField()
sha1 = StringField()
sha256 = StringField()
size = IntField(default=0)
ssdeep = StringField()
def migrate(self):
migrate_sample(self)
def add_file_data(self, file_data):
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
import pydeep
import magic
from hashlib import md5, sha1, sha256
try:
self.filetype = magic.from_buffer(data)
except:
self.filetype = "Unavailable"
try:
mimetype = magic.from_buffer(data, mime=True)
if mimetype:
self.mimetype = mimetype.split(";")[0]
if not mimetype:
self.mimetype = "unknown"
except:
self.mimetype = "Unavailable"
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
self.sha1 = sha1(data).hexdigest()
self.sha256 = sha256(data).hexdigest()
try:
self.ssdeep = pydeep.hash_bytes(data)
except:
self.ssdeep = None
def to_cybox(self, exclude=None):
if exclude == None:
exclude = []
observables = []
f = File()
for attr in ['md5', 'sha1', 'sha256']:
if attr not in exclude:
val = getattr(self, attr, None)
if val:
setattr(f, attr, val)
if self.ssdeep and 'ssdeep' not in exclude:
f.add_hash(Hash(self.ssdeep, Hash.TYPE_SSDEEP))
if 'size' not in exclude and 'size_in_bytes' not in exclude:
f.size_in_bytes = UnsignedLong(self.size)
if 'filename' not in exclude and 'file_name' not in exclude:
f.file_name = self.filename
# create an Artifact object for the binary if it exists
if 'filedata' not in exclude:
data = self.filedata.read()
if data:
data = base64.b64encode(data)
a = Artifact(data=data, type_=Artifact.TYPE_FILE)
observables.append(Observable(a))
#if 'filetype' not in exclude and 'file_format' not in exclude:
#NOTE: this doesn't work because the CybOX File object does not
# have any support built in for setting the filetype to a
# CybOX-binding friendly object (e.g., calling .to_dict() on
# the resulting CybOX object fails on this field.
#f.file_format = self.filetype
observables.append(Observable(f))
return (observables, self.releasability)
@classmethod
def from_cybox(cls, cybox_object, source, filedata=None):
"""
Convert a Cybox DefinedObject to a MongoEngine Sample object.
"""
sample = cls(source=source)
#TODO: we need to find all *required* fields and check for them
# and handle optional fields accordingly.
if cybox_object.size_in_bytes:
sample.size = cybox_object.size_in_bytes.value
else:
sample.size = 0
sample.filename = str(cybox_object.file_name)
for hash_ in cybox_object.hashes:
if hash_.type_.value.upper() in [
Hash.TYPE_MD5, Hash.TYPE_SHA1, Hash.TYPE_SHA256,
Hash.TYPE_SSDEEP
]:
setattr(sample, hash_.type_.value.lower(),
str(hash_.simple_hash_value).strip().lower())
if filedata:
if isinstance(filedata, file):
sample.filedata = filedata.read()
else:
sample.filedata = filedata
return sample
def discover_binary(self):
"""
Queries GridFS for a matching binary to this sample document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def set_backdoor(self, name, version, analyst):
if self.backdoor:
bd = Backdoor.objects(name=self.backdoor.name).first()
bd.decrement_count()
bd.save(username=analyst)
eb = EmbeddedBackdoor()
eb.name = name
eb.version = version
eb.analyst = analyst
self.backdoor = eb
bd = Backdoor.objects(name=name).first()
bd.increment_count()
bd.save(username=analyst)
def add_exploit(self, cve):
found = False
for e in self.exploit:
if e.cve == cve:
found = True
if not found:
ee = EmbeddedExploit()
ee.cve = cve
self.exploit.append(ee)
def delete_exploit(self, cve):
c = 0
for e in self.exploit:
if e.cve == cve:
del self.exploit[c]
c += 0
class Certificate(CritsBaseAttributes, CritsSourceDocument, Document):
"""
Certificate Class.
"""
meta = {
"collection": settings.COL_CERTIFICATES,
"crits_type": 'Certificate',
"latest_schema_version": 1,
"schema_doc": {
'filename':
'The filename of the certificate',
'filetype':
'The filetype of the certificate',
'md5':
'The MD5 of the certificate file',
'size':
'The filesize of the certificate',
'description':
'Description of what the certificate contains',
'source':
'List [] of source information about who provided the certificate'
},
"jtable_opts": {
'details_url':
'crits.certificates.views.certificate_details',
'details_url_key':
'md5',
'default_sort':
"modified DESC",
'searchurl':
'crits.certificates.views.certificates_listing',
'fields': [
"filename", "description", "filetype", "size", "modified",
"source", "campaign", "id", "md5", "status"
],
'jtopts_fields': [
"details", "filename", "description", "filetype", "size",
"modified", "source", "campaign", "status", "md5", "favorite",
"id"
],
'hidden_fields': ["md5"],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
},
}
description = StringField()
filedata = getFileField(collection_name=settings.COL_CERTIFICATES)
filename = StringField(required=True)
filetype = StringField(required=True)
size = IntField(default=0)
md5 = StringField()
def migrate(self):
"""
Migrate the Certificate tot he latest schema version.
"""
pass
def add_file_data(self, file_data):
"""
Add the Certificate to GridFS.
:param file_data: The Certificate.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
"""
Add the Certificate to GridFS.
:param file_obj: The Certificate.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
"""
Set the filetype, size, and MD5 of the Certificate.
:param data: The Certificate.
:type data: str
"""
import magic
from hashlib import md5
self.filetype = magic.from_buffer(data)
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
def discover_binary(self):
"""
Queries GridFS for a matching binary to this Certificate document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def to_cybox_observable(self):
"""
Convert a Certificate to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
custom_prop = Property(
) # make a custom property so CRITs import can identify Certificate exports
custom_prop.name = "crits_type"
custom_prop.description = "Indicates the CRITs type of the object this CybOX object represents"
custom_prop._value = "Certificate"
obj = File() # represent cert information as file
obj.md5 = self.md5
obj.file_name = self.filename
obj.file_format = self.filetype
obj.size_in_bytes = self.size
obj.custom_properties = CustomProperties()
obj.custom_properties.append(custom_prop)
obs = Observable(obj)
obs.description = self.description
data = self.filedata.read()
if data: # if cert data available
a = Artifact(data, Artifact.TYPE_FILE) # create artifact w/data
a.packaging.append(Base64Encoding())
obj.add_related(a, "Child_Of") # relate artifact to file
return ([obs], self.releasability)
@classmethod
def from_cybox(cls, cybox_obs, source):
"""
Convert a Cybox DefinedObject to a MongoEngine Indicator object.
:param cybox_obs: The cybox object to create the indicator from.
:type cybox_obs: :class:`cybox.core.Observable``
:param source: The source list for the Indicator.
:type source: list
:returns: :class:`crits.indicators.indicator.Indicator`
"""
cybox_object = cybox_obs.object_.properties
if cybox_object.md5:
db_obj = Certificate.objects(md5=cybox_object.md5).first()
if db_obj:
return db_obj
cert = cls(source=source)
cert.md5 = cybox_object.md5
cert.filename = cybox_object.file_name
cert.filetype = cybox_object.file_format
cert.size = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0
cert.description = cybox_obs.description
for obj in cybox_object.parent.related_objects: # attempt to find data in cybox
if isinstance(obj.properties, Artifact):
cert.add_file_data(base64.b64decode(obj.properties.data))
return cert
class PCAP(CritsBaseAttributes, CritsSourceDocument, Document):
"""
PCAP class.
"""
meta = {
"collection": settings.COL_PCAPS,
"crits_type": 'PCAP',
"latest_schema_version": 3,
"shard_key": ('md5', ),
"schema_doc": {
'filename': 'The filename of the PCAP',
'md5': 'The MD5 of the PCAP file',
'length': 'The filesize of the PCAP',
'uploadDate': 'The ISODate when the PCAP was uploaded',
'contentType': 'The filetype of the PCAP',
'source':
'List [] of source information about who provided the PCAP'
},
"jtable_opts": {
'details_url':
'crits.pcaps.views.pcap_details',
'details_url_key':
'md5',
'default_sort':
"modified DESC",
'searchurl':
'crits.pcaps.views.pcaps_listing',
'fields': [
"filename", "description", "length", "modified", "source",
"campaign", "id", "md5", "status"
],
'jtopts_fields': [
"details", "filename", "description", "length", "modified",
"source", "campaign", "status", "md5", "favorite", "id"
],
'hidden_fields': ['md5'],
'linked_fields': ["source", "campaign"],
'details_link':
'details',
'no_sort': ['details']
}
}
contentType = StringField()
filedata = getFileField(collection_name=settings.COL_PCAPS)
filename = StringField(required=True)
length = IntField(default=0)
md5 = StringField()
def migrate(self):
"""
Migrate to the latest schema version.
"""
migrate_pcap(self)
def add_file_data(self, file_data):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: str
"""
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
"""
Add filedata to this PCAP.
:param file_data: The filedata to add.
:type file_data: file handle
"""
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
"""
Generate metadata from the file data. Will add content-type, length, and
MD5.
:param data: The data to generate metadata from.
:type data: str
"""
import magic
from hashlib import md5
self.contentType = magic.from_buffer(data)
self.length = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
def discover_binary(self):
"""
Queries GridFS for a matching binary to this pcap document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def to_cybox_observable(self):
"""
Convert a PCAP to a CybOX Observables.
Returns a tuple of (CybOX object, releasability list).
To get the cybox object as xml or json, call to_xml() or
to_json(), respectively, on the resulting CybOX object.
"""
obj = File()
obj.md5 = self.md5
obj.file_name = self.filename
obj.file_format = self.contentType
obj.size_in_bytes = self.length
obs = Observable(obj)
obs.description = self.description
art = Artifact(self.filedata.read(), Artifact.TYPE_NETWORK)
art.packaging.append(Base64Encoding())
obj.add_related(art, "Child_Of") # relate artifact to file
return ([obs], self.releasability)
@classmethod
def from_cybox(cls, cybox_obs):
"""
Convert a Cybox Artifact to a CRITs PCAP object.
:param cybox_obs: The cybox object to create the PCAP from.
:type cybox_obs: :class:`cybox.core.Observable`
:returns: :class:`crits.pcaps.pcap.PCAP`
"""
cybox_object = cybox_obs.object_.properties
if cybox_object.md5:
db_obj = PCAP.objects(md5=cybox_object.md5).first()
if db_obj:
return db_obj
pcap = cls()
pcap.description = str(cybox_obs.description)
pcap.md5 = cybox_object.md5
pcap.filename = str(cybox_object.file_name)
pcap.contentType = cybox_object.file_format
pcap.length = cybox_object.size_in_bytes.value if cybox_object.size_in_bytes else 0
for obj in cybox_object.parent.related_objects: # attempt to find data in cybox
if isinstance(obj.properties, Artifact
) and obj.properties.type_ == Artifact.TYPE_NETWORK:
pcap.add_file_data(obj.properties.data)
break
return pcap
class Sample(CritsBaseAttributes, CritsSourceDocument, CritsActionsDocument,
Document):
"""Sample object"""
meta = {
"collection": settings.COL_SAMPLES,
"auto_create_index": False,
"crits_type": 'Sample',
"latest_schema_version": 5,
"shard_key": ('md5',),
"schema_doc": {
'filename': 'The name of the last file that was uploaded with this'\
'MD5',
'filenames': 'A list of filenames this binary has gone by.',
'filetype': 'The filetype of the file',
'mimetype': 'The mimetype of the file',
'size': 'The size of the file',
'md5': 'The MD5 of the file',
'sha1': 'The SHA1 of the file',
'sha256': 'The SHA256 of the file',
'ssdeep': 'The ssdeep of the file',
'impfuzzy': 'The impfuzzy of the executable file',
'campaign': 'List [] of campaigns using this file',
'source': 'List [] of sources that provided this file',
'created': 'ISODate of when this file was uploaded',
'modified': 'ISODate of when the file metadata was last modified',
'filedata': 'The ObjectId of the file in GridFS'
},
"jtable_opts": {
'details_url': 'crits-samples-views-detail',
'details_url_key': 'md5',
'default_sort': "created DESC",
'searchurl': 'crits-samples-views-samples_listing',
'fields': [ "filename", "size", "filetype",
"created", "modified", "campaign",
"source", "md5", "id", "status"],
'jtopts_fields': [ "details",
"filename",
"size",
"filetype",
"created",
"campaign",
"source",
"md5",
"status",
"favorite",
"id"],
'hidden_fields': ["md5"],
'linked_fields': ["filename", "source", "campaign",
"filetype"],
'details_link': 'details',
'no_sort': ['details']
},
}
filedata = getFileField(collection_name=settings.COL_SAMPLES)
filename = StringField(required=True)
filenames = ListField(StringField())
filetype = StringField()
md5 = StringField(required=True)
mimetype = StringField()
sha1 = StringField()
sha256 = StringField()
size = IntField(default=0)
ssdeep = StringField()
impfuzzy = StringField()
def migrate(self):
migrate_sample(self)
def add_file_data(self, file_data):
self._generate_file_metadata(file_data)
self.filedata = file_data
def add_file_obj(self, file_obj):
data = file_obj.read()
self._generate_file_metadata(data)
self.filedata = data
def _generate_file_metadata(self, data):
import pydeep
import magic
from hashlib import md5, sha1, sha256
try:
import pyimpfuzzy
except ImportError:
pass
try:
self.filetype = magic.from_buffer(data)
if len(self.filetype) > 1000:
self.filetype = self.filetype[0:1000] + '<TRUNCATED>'
except:
self.filetype = "Unavailable"
try:
mimetype = magic.from_buffer(data, mime=True)
if mimetype:
self.mimetype = mimetype.split(";")[0]
if not mimetype:
self.mimetype = "unknown"
except:
self.mimetype = "Unavailable"
self.size = len(data)
# this is a shard key. you can't modify it once it's set.
# MongoEngine will still mark the field as modified even if you set it
# to the same value.
if not self.md5:
self.md5 = md5(data).hexdigest()
self.sha1 = sha1(data).hexdigest()
self.sha256 = sha256(data).hexdigest()
try:
self.ssdeep = pydeep.hash_bytes(data)
except:
self.ssdeep = None
try:
self.impfuzzy = pyimpfuzzy.get_impfuzzy_data(data)
except:
self.impfuzzy = None
def is_office(self):
"""
Is this a Office file.
"""
ret = self.filedata.grid_id != None and self.filedata.read(8) == "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def is_pe(self):
"""
Is this a PE file.
"""
ret = self.filedata.grid_id != None and self.filedata.read(2) == "MZ"
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def is_pdf(self):
"""
Is this a PDF.
"""
ret = self.filedata.grid_id != None and "%PDF-" in self.filedata.read(1024)
if self.filedata.grid_id:
self.filedata.seek(0)
return ret
def discover_binary(self):
"""
Queries GridFS for a matching binary to this sample document.
"""
from crits.core.mongo_tools import mongo_connector
fm = mongo_connector("%s.files" % self._meta['collection'])
objectid = fm.find_one({'md5': self.md5}, {'_id': 1})
if objectid:
self.filedata.grid_id = objectid['_id']
self.filedata._mark_as_changed()
def set_filenames(self, filenames):
"""
Set the Sample filenames to a specified list.
:param filenames: The filenames to set.
:type filenames: list
"""
if isinstance(filenames, list):
self.filenames = filenames
def _json_yaml_convert(self, exclude=[]):
"""
Helper to convert to a dict before converting to JSON.
:param exclude: list of fields to exclude.
:type exclude: list
:returns: json
"""
d = self.to_dict(exclude)
if 'filedata' not in exclude:
(d['filedata'], ext) = format_file(self.filedata.read(), 'base64')
return json.dumps(d, default=json_handler)
