def run(self, obj, config):
if isinstance(obj, Event):
data = obj.description
elif isinstance(obj, RawData):
data = obj.data
elif isinstance(obj, Sample):
samp_data = obj.filedata.read()
data = make_ascii_strings(data=samp_data)
if not data:
self._debug("Could not find sample data to parse.")
return
else:
self._debug("This type is not supported by this service.")
return
ips = extract_ips(data)
for ip in ips:
tdict = {'Type': IndicatorTypes.IPV4_ADDRESS}
id_ = Indicator.objects(value=ip).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential IP Address', ip, tdict)
domains = extract_domains(data)
for domain in domains:
tdict = {'Type': IndicatorTypes.DOMAIN}
id_ = Indicator.objects(value=domain).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Domains', domain, tdict)
emails = extract_emails(data)
for email in emails:
tdict = {'Type': IndicatorTypes.EMAIL_ADDRESS}
id_ = Indicator.objects(value=email).only('id').first()
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Emails', email, tdict)
hashes = extract_hashes(data)
for hash_ in hashes:
type_ = hash_[0]
val = hash_[1]
tdict = {'Type': type_}
if type_ == IndicatorTypes.MD5:
id_ = Sample.objects(md5=val).only('id').first()
elif type_ == IndicatorTypes.SHA1:
id_ = Sample.objects(sha1=val).only('id').first()
elif type_ == IndicatorTypes.SHA256:
id_ = Sample.objects(sha256=val).only('id').first()
elif type_ == IndicatorTypes.SSDEEP:
id_ = Sample.objects(ssdeep=val).only('id').first()
else:
id_ = None
if id_:
tdict['exists'] = str(id_.id)
self._add_result('Potential Samples', val, tdict)
def run(self, obj, config):
threshold = config.get("threshold", 50)
target_ssdeep = obj.ssdeep
target_md5 = obj.md5
target_mimetype = obj.mimetype
if not target_ssdeep:
logger.error = "Could not get the target ssdeep value for sample"
self._error("Could not get the target ssdeep value for sample")
return
# setup the sample space to compare against
# first use the mimetype as a comparator if available
query_filter = {}
if target_mimetype:
query_filter['mimetype'] = target_mimetype
# then use only samples with a multiple of chunksize
chunk_size = int(target_ssdeep.split(":")[0])
query_filter["$or"] = []
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size * 2}})
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size}})
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % (chunk_size / 2)}})
result_filter = {'md5': 1, 'ssdeep': 1}
candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter)
match_list = []
for candidate in candidate_space:
if "ssdeep" in candidate:
score = pydeep.compare(target_ssdeep, candidate["ssdeep"])
if score >= threshold and candidate["md5"] != target_md5:
match_list.append({'md5': candidate["md5"], 'score': score})
# finally sort the results
match_list.sort(key=lambda sample: sample["score"], reverse=True)
for match in match_list:
self._add_result("ssdeep_match", match["md5"], {'md5': match["md5"], 'score': match["score"]})
def snugglefish_search(indexes, search, user):
"""Execute search of selected index with the given string."""
# Return a dictionary where the key is the index name and the
# value a dictionary with status and a list of potential matches.
ret = {}
# If there are no sources, return early.
sources = user_sources(user)
if not sources:
return ret
for idx in indexes:
ret[idx] = {'success': True, 'reason': '', 'files': []}
sngindex = SnuggleIndex.objects(name=idx).first()
if not sngindex:
ret[idx]['reason'] = "Index not found in database."
ret[idx]['success'] = False
continue
snuggle = pysnugglefish.init(str(sngindex.directory + "/" + idx))
try:
tmp = snuggle.search(search)
for res in tmp:
if Sample.objects(md5=res,
source__name__in=sources).count() > 0:
ret[idx]['files'].append(res)
except Exception, e:
ret[idx]['reason'] = "Error: %s" % e
ret[idx]['success'] = False
def run(self, argv):
parser = OptionParser()
parser.add_option("-f", "--filter", action="store", dest="filter",
type="string", help="filter for samples to discover binaries")
(opts, args) = parser.parse_args(argv)
if opts.filter:
query = ast.literal_eval(opts.filter)
else:
query = {}
errorpath = "/tmp/gridfs_migrate_errors.txt"
try:
err = open(errorpath, "w")
except:
print "Could not open file handle to write to: %s" % errorpath
sys.exit(1)
error_count = 0
samples = Sample.objects(__raw__=query)
count = len(samples)
print "Migrating %s samples found with query %s...\n" % (count, query)
i = 1
for s in samples:
md5 = s.md5
try:
print >> sys.stdout, "\r\tWorking on sample %d of %d" % (i, count),
sys.stdout.flush()
s.discover_binary()
s.save()
except Exception, e:
error_count += 1
err.write("Error saving sample for discover binary: %s - %s" % (md5, e))
i += 1
def _check_triage(self):
sample = Sample.objects(md5=self.test_md5).first()
results = False
if sample and len(sample.analysis) > 0 and sample.filedata:
results = True
print "[?] sample analysis executed == %s" % results
return results
def snugglefish_search(indexes, search, user):
"""Execute search of selected index with the given string."""
# Return a dictionary where the key is the index name and the
# value a dictionary with status and a list of potential matches.
ret = {}
# If there are no sources, return early.
sources = user_sources(user)
if not sources:
return ret
for idx in indexes:
ret[idx] = {
'success': True,
'reason': '',
'files': []
}
sngindex = SnuggleIndex.objects(name=idx).first()
if not sngindex:
ret[idx]['reason'] = "Index not found in database."
ret[idx]['success'] = False
continue
snuggle = pysnugglefish.init(str(sngindex.directory + "/" + idx))
try:
tmp = snuggle.search(search)
for res in tmp:
if Sample.objects(md5=res, source__name__in=sources).count() > 0:
ret[idx]['files'].append(res)
except Exception, e:
ret[idx]['reason'] = "Error: %s" % e
ret[idx]['success'] = False
def run(self, obj, config):
threshold = config.get("threshold", 50)
target_ssdeep = obj.ssdeep
target_md5 = obj.md5
target_mimetype = obj.mimetype
if not target_ssdeep:
logger.error = "Could not get the target ssdeep value for sample"
self._error("Could not get the target ssdeep value for sample")
return
# setup the sample space to compare against
# first use the mimetype as a comparator if available
query_filter = {}
if target_mimetype:
query_filter['mimetype'] = target_mimetype
# then use only samples with a multiple of chunksize
chunk_size = int(target_ssdeep.split(":")[0])
query_filter["$or"] = []
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size * 2}})
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size}})
query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % (chunk_size / 2)}})
result_filter = {'md5': 1, 'ssdeep': 1, 'description':1}
candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter)
match_list = []
for candidate in candidate_space:
if "ssdeep" in candidate:
score = pydeep.compare(target_ssdeep, candidate["ssdeep"])
if score >= threshold and candidate["md5"] != target_md5:
match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score})
# finally sort the results
match_list.sort(key=lambda sample: sample["score"], reverse=True)
for match in match_list:
self._add_result("ssdeep_match (MD5)", match["md5"], {'description': match["description"], 'score': match["score"]})
def _check_triage(self):
sample = Sample.objects(md5=self.test_md5).first()
results = False
if sample and sample.filedata:
if len(AnalysisResult.objects(object_id=str(sample.id))) > 0:
results = True
print "[?] sample analysis executed == %s" % results
return results
def _check_triage(self):
sample = Sample.objects(md5=self.test_md5).first()
results = False
if sample and sample.filedata:
if len(AnalysisResult.objects(object_id=str(sample.id))) > 0:
results = True
print "[?] sample analysis executed == %s" % results
return results
def start_pyew_shell(request, id_, token):
# Make sure we can find pyew
svc = CRITsService.objects(name='Pyew').first()
if not svc:
text = "\nPyew not found"
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
sc = svc.config
pyew = str(sc['pyew'])
if not os.path.exists(pyew):
text = "\nPyew not found"
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
# Find CRITs user by token
query = {'unsupported_attrs.pyew_token': token}
user = CRITsUser.objects(__raw__=query).first()
if not user:
text = "\nCould not validate user"
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
# Remove this one-time use token
ua = user.unsupported_attrs
delattr(ua, 'pyew_token')
user.unsupported_attrs = ua
try:
user.save()
except:
pass
# Make sure we have a sample to work with that this user has access to
sample = Sample.objects(id=id_,
source__name__in=user.get_sources_list()).first()
if not sample:
text = "\nNo Sample found"
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
sample_data = sample.filedata.read()
if not sample_data:
text = "\nCould not get Sample from GridFS: %s" % id_
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
# write Sample to disk
# temp_sample is the sample to read
try:
temp_sample = tempfile.NamedTemporaryFile(delete=False)
sample_name = temp_sample.name
temp_sample.write(sample_data)
temp_sample.close()
except Exception, e:
text = "\nError writing file to disk: %s" % e
request.ws_stream.send_message(base64.b64encode(text), binary=False)
sys.exit(1)
def _check_grid(self):
sample = Sample.objects(md5=self.test_md5).first()
result = False
if sample:
if sample.filedata:
data = sample.filedata.read()
result = data == self.test_data
print "[?] check grid == %s" % result
return result
def start_pyew_shell(request, id_, token):
# Make sure we can find pyew
sc = manager.get_config('Pyew')
pyew = str(sc['pyew'])
if not os.path.exists(pyew):
text = "\nPyew not found"
request.ws_stream.send_message(base64.b64encode(text),
binary=False)
sys.exit(1)
# Find CRITs user by token
query = {'unsupported_attrs.pyew_token': token}
user = CRITsUser.objects(__raw__=query).first()
if not user:
text = "\nCould not validate user"
request.ws_stream.send_message(base64.b64encode(text),
binary=False)
sys.exit(1)
# Remove this one-time use token
ua = user.unsupported_attrs
delattr(ua, 'pyew_token')
user.unsupported_attrs = ua
try:
user.save()
except:
pass
# Make sure we have a sample to work with that this user has access to
sample = Sample.objects(id=id_, source__name__in=user.sources).first()
if not sample:
text = "\nNo Sample found"
request.ws_stream.send_message(base64.b64encode(text),
binary=False)
sys.exit(1)
sample_data = sample.filedata.read()
if not sample_data:
text = "\nCould not get Sample from GridFS: %s" % id_
request.ws_stream.send_message(base64.b64encode(text),
binary=False)
sys.exit(1)
# write Sample to disk
# temp_sample is the sample to read
try:
temp_sample = tempfile.NamedTemporaryFile(delete=False)
sample_name = temp_sample.name
temp_sample.write(sample_data)
temp_sample.close()
except Exception, e:
text = "\nError writing file to disk: %s" % e
request.ws_stream.send_message(base64.b64encode(text),
binary=False)
sys.exit(1)
def _del_sample(self):
sample = Sample.objects(md5=self.test_md5).first()
if sample:
print "[-] deleting from grid"
if sample.filedata:
sample.filedata.delete()
print "[-] deleting sample"
sample.delete()
else:
print "[-] could not find sample to delete"
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
def run(self, argv):
parser = OptionParser()
parser.add_option("-m", "--md5", action="store", dest="md5",
type="string", help="filetype filter")
(opts, args) = parser.parse_args(argv)
try:
if opts.md5:
sample = Sample.objects(md5=opts.md5).first()
except Exception as e:
print "Bad things - '%s'" % e
if sample:
print sample.id
def process_saved_artifacts(self):
"""
Process anything in saved_artifacts that didn't have a match.
"""
for md5_, value in self.saved_artifacts.iteritems():
(saved_obj, data) = value
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if data:
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
def process_saved_artifacts(self):
"""
Process anything in saved_artifacts that didn't have a match.
"""
for md5_, value in self.saved_artifacts.iteritems():
(saved_obj, data) = value
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if data:
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
def _fetch_meta(self, query_filter, result_filter):
"""
Fetch sample metadata.
:param query_filter: The filter to use to find the sample.
:type query_filter: dict
:param result_filter: Limit the result to these fields.
:type result_filter: tuple
:returns: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
self.ensure_current_task()
results = Sample.objects(__raw__=query_filter).only(*result_filter)
return results
def _fetch_meta(self, query_filter, result_filter):
"""
Fetch sample metadata.
:param query_filter: The filter to use to find the sample.
:type query_filter: dict
:param result_filter: Limit the result to these fields.
:type result_filter: tuple
:returns: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
self.ensure_current_task()
results = Sample.objects(__raw__=query_filter).only(*result_filter)
return results
def run(self, obj, config):
self._info("Impfuzzy: run()")
threshold = config.get("threshold", 50)
target_impfuzzy = None
try:
target_impfuzzy = pyimpfuzzy.get_impfuzzy_data(obj.filedata.read())
except Exception:
pass
target_md5 = obj.md5
if not target_impfuzzy:
logger.error = "impfuzzy: Could not generate impfuzzy value for sample: %s" % str(obj.id)
self._error("Could not generate impfuzzy value for sample")
return
# setup the sample space to compare against
# first use the mimetype as a comparator if available
if obj.impfuzzy:
obj.impfuzzy = target_impfuzzy
obj.save()
self._info("impfuzzy: Filled-in in the impfuzzy")
else:
self._info("impfuzzy attribute already present, not overwriting")
self._add_result('impfuzzy_hash', target_impfuzzy,{'impfuzzy': target_impfuzzy})
target_mimetype = obj.mimetype
query_filter = {}
if target_mimetype:
query_filter['mimetype'] = target_mimetype
# then use only samples with a multiple of chunksize
chunk_size = int(target_impfuzzy.split(":")[0])
query_filter["$or"] = []
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size * 2}})
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size}})
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % (chunk_size // 2)}})
result_filter = {'md5': 1, 'impfuzzy': 1, 'description':1}
candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter)
# self.info("candidate: %s" % repr(candidate_space))
match_list = []
for candidate in candidate_space:
if "impfuzzy" in candidate:
score = pyimpfuzzy.hash_compare(target_impfuzzy, candidate["impfuzzy"])
if score >= threshold and candidate["md5"] != target_md5:
# Grab the md5 and the description for later
match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score})
# finally sort the results
match_list.sort(key=lambda sample: sample["score"], reverse=True)
for match in match_list:
#Show the MD5 and the Description
self._add_result("impfuzzy_match", match["md5"], {'description': match["description"], 'md5': match["md5"], 'score': match["score"]})
self._info("impfuzzy run() done")
def run(self, argv):
parser = OptionParser()
parser.add_option("-m",
"--md5",
action="store",
dest="md5",
type="string",
help="filetype filter")
(opts, args) = parser.parse_args(argv)
try:
if opts.md5:
sample = Sample.objects(md5=opts.md5).first()
except Exception as e:
print "Bad things - '%s'" % e
if sample:
print sample.id
def run(self, obj, config):
threshold = config.get("threshold", 50)
target_impfuzzy = None
try:
target_impfuzzy = pyimpfuzzy.get_impfuzzy_data(obj.filedata.read())
except Exception:
pass
target_md5 = obj.md5
if not target_impfuzzy:
logger.error = "impfuzzy: Could not generate impfuzzy value for sample: %s" % str(obj.id)
self._error("Could not generate impfuzzy value for sample")
return
# setup the sample space to compare against
# first use the mimetype as a comparator if available
if obj.impfuzzy:
obj.impfuzzy = target_impfuzzy
obj.save()
self._info("impfuzzy: Filled-in in the impfuzzy")
else:
self._info("impfuzzy attribute already present, not overwriting")
self._add_result('impfuzzy_hash', target_impfuzzy,{'impfuzzy': target_impfuzzy})
target_mimetype = obj.mimetype
query_filter = {}
if target_mimetype:
query_filter['mimetype'] = target_mimetype
# then use only samples with a multiple of chunksize
chunk_size = int(target_impfuzzy.split(":")[0])
query_filter["$or"] = []
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size * 2}})
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size}})
query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % (chunk_size // 2)}})
result_filter = {'md5': 1, 'impfuzzy': 1, 'description':1}
candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter)
# self.info("candidate: %s" % repr(candidate_space))
match_list = []
for candidate in candidate_space:
if "impfuzzy" in candidate:
score = pyimpfuzzy.hash_compare(target_impfuzzy, candidate["impfuzzy"])
if score >= threshold and candidate["md5"] != target_md5:
# Grab the md5 and the description for later
match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score})
# finally sort the results
match_list.sort(key=lambda sample: sample["score"], reverse=True)
for match in match_list:
#Show the MD5 and the Description
self._add_result("impfuzzy_match (MD5)", match["md5"], {'description': match["description"], 'score': match["score"]})
def run(self, argv):
parser = OptionParser()
parser.add_option('-b',
'--bucket',
action='store',
dest='bucket',
type='string',
help='bucket list name')
parser.add_option("-o",
"--output-file",
action="store",
dest="outfile",
type="string",
help="output archive file (no extension)")
(opts, args) = parser.parse_args(argv)
samples = Sample.objects(bucket_list=opts.bucket)
if opts.bucket and opts.outfile:
filename = "%s.tar.bz2" % opts.outfile
try:
tar = tarfile.open(filename, "w:bz2")
except Exception as e:
print("Error when attempting to open %s for writing: %s" %
(filename, e))
sys.exit(1)
count = len(samples)
if count <= 0:
print("No matching bucket name found!")
sys.exit(1)
for sample in samples:
m = sample.md5
f = sample.filename
s = sample.filedata.read()
info = tarfile.TarInfo(name="%s" % f)
info.mtime = time.time()
if s is not None:
info.size = len(s)
else:
info.size = 0
try:
tar.addfile(info, BytesIO(s))
except Exception as e:
print("Error attempting to add %s to the tarfile: %s" % (f, e))
pass
tar.close()
print("Generated %s containing %s files." % (filename, count))
def test_yara_rule(id_, rule):
sample = Sample.objects(id=id_).first()
data = sample.filedata.read()
success = False
message = ""
if not sample or not data:
message = "No sample found!"
else:
try:
rules = yara.compile(source=rule)
matches = rules.match(data=data)
yara_results = []
mcount = 0
for match in matches:
strings = {}
for s in match.strings:
s_name = s[1]
s_offset = s[0]
try:
s_data = s[2].decode('ascii')
except UnicodeError:
s_data = "Hex: " + binascii.hexlify(s[2])
s_key = "{0}-{1}".format(s_name, s_data)
if s_key in strings:
strings[s_key]['offset'].append(s_offset)
else:
strings[s_key] = {
'rule': str(match),
'offset': [s_offset],
'name': s_name,
'data': s_data,
}
string_list = []
for key in strings:
string_list.append(strings[key])
yara_results.append(string_list)
mcount += 1
success = True
if mcount == 0:
yara_results.append("No matches!")
message = pprint.pformat(yara_results)
except SyntaxError as e:
message = "Syntax error in YARA rule: %s" % str(e)
return {"success": success, "message": message}
def test_yara_rule(id_, rule):
sample = Sample.objects(id=id_).first()
data = sample.filedata.read()
success = False
message = ""
if not sample or not data:
message = "No sample found!"
else:
try:
rules = yara.compile(source=rule)
matches = rules.match(data=data)
yara_results = []
mcount = 0
for match in matches:
strings = {}
for s in match.strings:
s_name = s[1]
s_offset = s[0]
try:
s_data = s[2].decode('ascii')
except UnicodeError:
s_data = "Hex: " + binascii.hexlify(s[2])
s_key = "{0}-{1}".format(s_name, s_data)
if s_key in strings:
strings[s_key]['offset'].append(s_offset)
else:
strings[s_key] = {
'rule': str(match),
'offset': [s_offset],
'name': s_name,
'data': s_data,
}
string_list = []
for key in strings:
string_list.append(strings[key])
yara_results.append(string_list)
mcount += 1
success = True
if mcount == 0:
yara_results.append("No matches!")
message = pprint.pformat(yara_results)
except SyntaxError as e:
message = "Syntax error in YARA rule: %s" % str(e)
return {"success": success, "message": message}
def create_sample_context(self, identifier, username):
# .only() is currently broken in MongoEngine :(
#fields = ('size', 'filetype', 'filename', 'md5',
# 'mimetype', 'filedata')
#sample = Sample.objects(id=identifier).only(*fields).first()
sample = Sample.objects(id=identifier).first()
if not sample:
raise ValueError("Sample not found in database")
data = sample.filedata.read()
if not data:
raise ValueError("Sample not found in GridFS")
sample_md5 = sample.md5
self._check_length(data, getattr(sample, 'size', 0))
return SampleContext(username, data, sample_md5, sample.to_dict())
def create_sample_context(self, identifier, username):
# .only() is currently broken in MongoEngine :(
#fields = ('size', 'filetype', 'filename', 'md5',
# 'mimetype', 'filedata')
#sample = Sample.objects(id=identifier).only(*fields).first()
sample = Sample.objects(id=identifier).first()
if not sample:
raise ValueError("Sample not found in database")
data = sample.filedata.read()
if not data:
raise ValueError("Sample not found in GridFS")
sample_md5 = sample.md5
self._check_length(data, getattr(sample, 'size', 0))
return SampleContext(username, data, sample_md5, sample.to_dict())
def run(self, argv):
parser = OptionParser()
parser.add_option("-f",
"--filter",
action="store",
dest="filter",
type="string",
help="filter for samples to discover binaries")
(opts, args) = parser.parse_args(argv)
if opts.filter:
query = ast.literal_eval(opts.filter)
else:
query = {}
errorpath = "/tmp/gridfs_migrate_errors.txt"
try:
err = open(errorpath, "w")
except:
print "Could not open file handle to write to: %s" % errorpath
sys.exit(1)
error_count = 0
samples = Sample.objects(__raw__=query)
count = len(samples)
print "Migrating %s samples found with query %s...\n" % (count, query)
i = 1
for s in samples:
md5 = s.md5
try:
print >> sys.stdout, "\r\tWorking on sample %d of %d" % (
i, count),
sys.stdout.flush()
s.discover_binary()
s.save()
except Exception, e:
error_count += 1
err.write("Error saving sample for discover binary: %s - %s" %
(md5, e))
i += 1
def run(self, argv):
parser = OptionParser()
parser.add_option('-b', '--bucket', action='store', dest='bucket',
type='string', help='bucket list name')
parser.add_option("-o", "--output-file", action="store", dest="outfile",
type="string", help="output archive file (no extension)")
(opts, args) = parser.parse_args(argv)
samples = Sample.objects(bucket_list=opts.bucket)
if opts.bucket and opts.outfile:
filename = "%s.tar.bz2" % opts.outfile
try:
tar = tarfile.open(filename, "w:bz2")
except Exception as e:
print ("Error when attempting to open %s for writing: %s" % (filename, e))
sys.exit(1)
count = len(samples)
if count <= 0:
print ("No matching bucket name found!")
sys.exit(1)
for sample in samples:
m = sample.md5
f = sample.filename
s = sample.filedata.read()
info = tarfile.TarInfo(name="%s" % f)
info.mtime = time.time()
if s is not None:
info.size = len(s)
else:
info.size = 0
try:
tar.addfile(info, BytesIO(s))
except Exception as e:
print ("Error attempting to add %s to the tarfile: %s" % (f, e))
pass
tar.close()
print ("Generated %s containing %s files." % (filename, count))
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def upload_child(request, parent_md5):
"""
Upload a new child sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param parent_md5: The MD5 of the parent sample.
:type parent_md5: str
:returns: :class:`django.http.HttpResponse`
"""
new_samples = []
if request.method == "POST":
form = EmailAttachForm(request.user.username, request.POST,
request.FILES)
if form.is_valid():
if request.FILES or 'filename' in request.POST and 'md5' in request.POST:
# Child samples inherit all of the sources of the parent.
parent = Sample.objects(md5=parent_md5).first()
if not parent:
return render_to_response(
'error.html', {'error': "Unable to find parent."},
RequestContext(request))
source = parent.source
campaign_name = request.POST['campaign']
confidence = request.POST['confidence']
parent.campaign.append(
EmbeddedCampaign(name=campaign_name,
confidence=confidence,
analyst=request.user.username))
campaigns = parent.campaign
try:
if request.FILES:
new_samples = handle_uploaded_file(
request.FILES["filedata"],
source,
None,
form.cleaned_data["file_format"],
form.cleaned_data["password"],
user=request.user.username,
campaign=campaigns,
parent_md5=parent_md5,
bucket_list=form.cleaned_data[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME])
else:
filename = request.POST['filename'].strip()
md5 = request.POST['md5'].strip().lower()
if not filename or not md5:
error = "Need a file, or a filename and an md5."
return render_to_response('error.html',
{'error': error},
RequestContext(request))
else:
new_samples = handle_uploaded_file(
None,
source,
None,
form.cleaned_data["file_format"],
form.cleaned_data["password"],
user=request.user.username,
campaign=campaigns,
parent_md5=parent_md5,
filename=filename,
bucket_list=form.cleaned_data[
form_consts.Common.
BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME],
md5=md5)
except ZipFileError, zfe:
return render_to_response('error.html',
{'error': zfe.value},
RequestContext(request))
else:
return render_to_response(
'error.html',
{'error': "Need a file, or a filename and an md5."},
RequestContext(request))
else:
return render_to_response('error.html', {'error': 'form error'},
RequestContext(request))
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def upload_file(request, related_md5=None):
"""
Upload a new sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param related_md5: The MD5 of a related sample.
:type related_md5: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
email_errmsg = None
if form.is_valid():
response = {'success': False,
'message': 'Unknown error; unable to upload file.'}
inherited_source = None
backdoor = form.cleaned_data['backdoor']
campaign = form.cleaned_data['campaign']
confidence = form.cleaned_data['confidence']
source = form.cleaned_data['source']
method = form.cleaned_data['method']
reference = form.cleaned_data['reference']
analyst = request.user.username
if related_md5:
reload_page = True
else:
reload_page = False
related_md5 = form.cleaned_data['related_md5']
if related_md5:
related_sample = Sample.objects(md5=related_md5).first()
if not related_sample:
response['message'] = "Upload Failed. Unable to locate related sample."
return render_to_response("file_upload_response.html",
{'response': json.dumps(response)},
RequestContext(request))
# If selected, new sample inherits the campaigns of the related sample.
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_sample.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst))
campaign = related_sample.campaign
# If selected, new sample inherits the sources of the related sample
if form.cleaned_data['inherit_sources']:
inherited_source = related_sample.source
backdoor_name = None
backdoor_version = None
if backdoor:
backdoor = backdoor.split('|||')
if len(backdoor) == 2:
(backdoor_name, backdoor_version) = backdoor[0], backdoor[1]
try:
if request.FILES:
result = handle_uploaded_file(
request.FILES['filedata'],
source,
method,
reference,
form.cleaned_data['file_format'],
form.cleaned_data['password'],
analyst,
campaign,
confidence,
related_md5,
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version)
else:
result = handle_uploaded_file(
None,
source,
method,
reference,
form.cleaned_data['file_format'],
None,
analyst,
campaign,
confidence,
related_md5 = related_md5,
filename=request.POST['filename'].strip(),
md5=request.POST['md5'].strip().lower(),
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
is_return_only_md5=False,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version)
except ZipFileError, zfe:
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'message': zfe.value})},
RequestContext(request))
else:
if len(result) > 1:
filedata = request.FILES['filedata']
message = ('<a href="%s">View Uploaded Samples.</a>'
% reverse('crits.samples.views.view_upload_list',
args=[filedata.name, result]))
response = {'success': True,
'message': message }
md5_response = result
elif len(result) == 1:
md5_response = None
if not request.FILES:
response['success'] = result[0].get('success', False)
if(response['success'] == False):
response['message'] = result[0].get('message', response.get('message'))
else:
md5_response = [result[0].get('object').md5]
else:
md5_response = [result[0]]
response['success'] = True
if md5_response != None:
response['message'] = ('File uploaded successfully. <a href="%s">View Sample.</a>'
% reverse('crits.samples.views.detail',
args=md5_response))
if response['success']:
if request.POST.get('email'):
for s in md5_response:
email_errmsg = mail_sample(s, [request.user.email])
if email_errmsg is not None:
msg = "<br>Error emailing sample %s: %s\n" % (s, email_errmsg)
response['message'] = response['message'] + msg
if reload_page:
response['redirect_url'] = reverse('crits.samples.views.detail', args=[related_md5])
return render_to_response("file_upload_response.html",
{'response': json.dumps(response)},
RequestContext(request))
else:
if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear
form.fields['related_md5'].widget = forms.HiddenInput()
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'form': form.as_table()})},
RequestContext(request))
except Exception, e:
print "Error with query: %s" % e
return
sngindex = self.__create_index(opts.name, opts.query,
opts.directory)
if not query:
try:
query = ast.literal_eval(sngindex.query)
except Exception, e:
print "Error with query: %s" % e
return
# XXX: Get count worth of samples using query...
samples = Sample.objects(__raw__=query).order_by('+id').only(
'md5')[sngindex.count:sngindex.count + count]
if not samples:
print "No objects found."
return
for sample in samples:
print sample.md5
sngindex.last_id = samples[len(samples) - 1].id
sngindex.save()
elif opts.action == 'update':
sngindex = SnuggleIndex.objects(name=opts.name).first()
if not sngindex:
print "Index does not exist."
return
except Exception, e:
print "Error with query: %s" % e
return
sngindex = self.__create_index(opts.name, opts.query,
opts.directory)
if not query:
try:
query = ast.literal_eval(sngindex.query)
except Exception, e:
print "Error with query: %s" % e
return
# XXX: Get count worth of samples using query...
samples = Sample.objects(__raw__=query).order_by('+id').only('md5')[sngindex.count:sngindex.count + count]
if not samples:
print "No objects found."
return
for sample in samples:
print sample.md5
sngindex.last_id = samples[len(samples) - 1].id
sngindex.save()
elif opts.action == 'update':
sngindex = SnuggleIndex.objects(name=opts.name).first()
if not sngindex:
print "Index does not exist."
return
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info("CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal.")
self._add_result("Download Canceled", "Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result("Download Canceled", "User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info("Requesting binary with md5 '{0}' from VirusTotal.".format(obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error("Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options.".format(size, sizeLimit))
self._add_result("Download Aborted", "Match found, but binary is larger than maximum size limit.")
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info("No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect.")
self._add_result("Not Found", "Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result("Download Canceled", "CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info("Replace = True. Deleting any previous binary with md5 {0}".format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error("VirusTotal: Error deleting existing binary ({0})".format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(filename = obj.md5,
data = data,
source = "VirusTotal",
reference = "Binary downloaded from VT based on MD5",
user = "******",
method = "VirusTotal Download Service",
md5_digest = obj.md5 )
except Exception as e:
logger.error("VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user = "******")
self._add_result("Download Successful", "Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info(
"CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal."
)
self._add_result("Download Canceled",
"Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result(
"Download Canceled",
"User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({
'http': settings.HTTP_PROXY,
'https': settings.HTTP_PROXY
})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info(
"Requesting binary with md5 '{0}' from VirusTotal.".format(
obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error(
"Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options."
.format(size, sizeLimit))
self._add_result(
"Download Aborted",
"Match found, but binary is larger than maximum size limit."
)
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info(
"No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect."
)
self._add_result(
"Not Found",
"Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result(
"Download Canceled",
"CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
return
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info(
"Replace = True. Deleting any previous binary with md5 {0}"
.format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error(
"VirusTotal: Error deleting existing binary ({0})".
format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(
filename=obj.md5,
data=data,
source="VirusTotal",
source_reference="Binary downloaded from VT based on MD5",
user=self.current_task.user,
source_method="VirusTotal Download Service",
md5_digest=obj.md5)
except Exception as e:
logger.error(
"VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user="******")
self._add_result(
"Download Successful",
"Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def upload_file(request, related_md5=None):
"""
Upload a new sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param related_md5: The MD5 of a related sample.
:type related_md5: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
email_errmsg = None
if form.is_valid():
response = {
'success': False,
'message': 'Unknown error; unable to upload file.'
}
inherited_source = None
backdoor = form.cleaned_data['backdoor']
campaign = form.cleaned_data['campaign']
confidence = form.cleaned_data['confidence']
source = form.cleaned_data['source_name']
source_method = form.cleaned_data['source_method']
source_reference = form.cleaned_data['source_reference']
source_tlp = form.cleaned_data['source_tlp']
user = request.user
description = form.cleaned_data['description']
related_id = form.cleaned_data.get('related_id', None)
related_type = form.cleaned_data.get('related_type', None)
relationship_type = form.cleaned_data.get('relationship_type',
None)
if related_md5:
reload_page = True
else:
reload_page = False
related_md5 = form.cleaned_data['related_md5']
if related_md5:
related_sample = Sample.objects(md5=related_md5).first()
if not related_sample:
response['message'] = (
"Upload Failed. Unable to locate related sample. %s" %
related_md5)
return render(request, "file_upload_response.html",
{'response': json.dumps(response)})
# If selected, new sample inherits the campaigns of the related sample.
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_sample.campaign.append(
EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user))
campaign = related_sample.campaign
# If selected, new sample inherits the sources of the related sample
if form.cleaned_data['inherit_sources']:
inherited_source = related_sample.source
elif related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
response['success'] = False
response['message'] = (
"Upload Failed. Unable to locate related Item")
return render(
request,
"file_upload_response.html",
{'response': json.dumps(response)},
)
else:
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_obj.campaign.append(
EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user))
campaign = related_obj.campaign
if form.cleaned_data['inherit_sources']:
inherited_source = related_obj.source
backdoor_name = None
backdoor_version = None
if backdoor:
backdoor = backdoor.split('|||')
if len(backdoor) == 2:
(backdoor_name,
backdoor_version) = backdoor[0], backdoor[1]
try:
if request.FILES:
result = handle_uploaded_file(
request.FILES['filedata'],
source,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
file_format=form.cleaned_data['file_format'],
password=form.cleaned_data['password'],
user=user,
campaign=campaign,
confidence=confidence,
related_md5=related_md5,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type,
bucket_list=form.cleaned_data[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version,
description=description)
else:
result = handle_uploaded_file(
None,
source,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
file_format=form.cleaned_data['file_format'],
password=None,
user=user,
campaign=campaign,
confidence=confidence,
related_md5=related_md5,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type,
filename=request.POST['filename'].strip(),
md5=request.POST['md5'].strip().lower(),
sha1=request.POST['sha1'].strip().lower(),
sha256=request.POST['sha256'].strip().lower(),
bucket_list=form.cleaned_data[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
is_return_only_md5=False,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version,
description=description)
except ZipFileError, zfe:
return render(
request, 'file_upload_response.html', {
'response':
json.dumps({
'success': False,
'message': zfe.value
})
})
else:
# zip file upload, etc; result is a list of strings (1 hash per file)
if len(result) > 0 and not isinstance(result[0], dict):
filedata = request.FILES['filedata']
message = ('<a href="%s">View Uploaded Samples.</a>' %
reverse('crits-samples-views-view_upload_list',
args=[filedata.name, result]))
response = {'success': True, 'message': message}
md5_response = result
# regular file upload; result is a list with a single dict
else:
response['success'] = result[0].get('success', False)
response['message'] = result[0].get(
'message', response.get('message'))
try:
md5_response = [result[0].get('object').md5]
except:
md5_response = None
if response['success']:
if request.POST.get('email') and md5_response:
for s in md5_response:
email_errmsg = mail_sample(s, [request.user.email])
if email_errmsg is not None:
msg = "<br>Error emailing sample %s: %s\n" % (
s, email_errmsg)
response['message'] = response['message'] + msg
if reload_page:
response['redirect_url'] = reverse(
'crits-samples-views-detail', args=[related_md5])
return render(request, "file_upload_response.html",
{'response': json.dumps(response)})
else:
if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear
form.fields['related_md5'].widget = forms.HiddenInput()
return render(
request, 'file_upload_response.html', {
'response':
json.dumps({
'success': False,
'form': form.as_table()
})
})
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
print "Copying GridFS data matching md5s to new collection..."
filename = opts.gridfs
elif opts.griddelete:
print "Deleting matching md5s from GridFS..."
filename = opts.griddelete
count = 0
no_data = 0
try:
with open(filename) as o:
for line in o:
md5 = line.strip()
data = get_file(md5, opts.collection)
if data:
if opts.gridfs:
put_file(md5, data)
s = Sample.objects(md5=md5).first()
if s:
s.discover_binary()
try:
s.save()
except:
error_count += 1
err.write("Error saving sample for discover binary: %s" % md5)
elif opts.griddelete:
delete_file(md5, opts.collection)
count += 1
else:
no_data += 1
if opts.gridfs:
print "Copied %s md5s to new collection." % count
if error_count:
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def to_stix(self, username=None):
"""
Converts a CRITs event to a STIX document.
The resulting document includes all related emails, samples, and
indicators converted to CybOX Observable objects.
Returns the STIX document and releasability constraints.
(NOTE: the following statement is untrue until the
releasability checking is finished, which includes setting
releasability on all CRITs objects.)
Raises UnreleasableEventError if the releasability on the
relationships and the event do not share any common releasability
sources.
"""
from crits.emails.email import Email
from crits.samples.sample import Sample
from crits.indicators.indicator import Indicator
from cybox.common import Time, ToolInformationList, ToolInformation
from cybox.core import Observables
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
from stix.common import InformationSource
from stix.common.identity import Identity
stix_indicators = []
stix_observables = []
final_objects = []
# create a list of sources to send as part of the results.
# list should be limited to the sources this user is allowed to use.
# this list should be used along with the list of objects to set the
# appropriate source's 'released' key to True for each object.
final_sources = []
user_source_list = user_sources(username)
for f in self.releasability:
if f.name in user_source_list:
final_sources.append(f.name)
final_sources = set(final_sources)
# TODO: eventually we can use class_from_id instead of the if block
# but only once we support all CRITs types.
for r in self.relationships:
obj = None
if r.rel_type == Email._meta['crits_type']:
obj = Email.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
stix_observables.append(ind[0])
elif r.rel_type == Sample._meta['crits_type']:
obj = Sample.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
for i in ind:
stix_observables.append(i)
elif r.rel_type == Indicator._meta['crits_type']:
#NOTE: Currently this will raise an exception if there
# are multiple indicators with the same value.
# Should be fixed automatically once we transition
# indicators to be related based on ObjectId rather
# than value.
obj = Indicator.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_stix_indicator()
stix_indicators.append(ind)
else:
continue
#Create a releasability list that is the intersection of
# each related item's releasability with the event's
# releasability. If the resulting set is empty, raise exception
#TODO: Set releasability on all objects so that we actually
# get results here instead of always raising an exception.
if obj:
releas_sources = set([rel.name for rel in releas])
final_sources = final_sources.intersection(releas_sources)
#TODO: uncomment the following lines when objects have
# releasability set.
#if not final_sources:
# raise UnreleasableEventError(r.value)
# add to the final_objects list to send as part of the results
final_objects.append(obj)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(
time=Time(produced_time= datetime.datetime.now()),
identity = Identity(name=settings.COMPANY_NAME),
tools = tool_list
)
description = StructuredText(value=self.description)
header = STIXHeader(information_source=i_s,
description=description,
package_intent=self.event_type,
title=self.title)
return (STIXPackage(indicators=stix_indicators,
observables=Observables(stix_observables),
stix_header=header,
id_=self.event_id),
final_sources,
final_objects)
def _get_meta_count(self, meta_type, meta_val):
query_field = "analysis.results.{0}".format(meta_type)
query = {query_field: meta_val}
total_count = Sample.objects(__raw__=query).only('md5').count()
return total_count
def upload_file(request, related_md5=None):
"""
Upload a new sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param related_md5: The MD5 of a related sample.
:type related_md5: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
email_errmsg = None
if form.is_valid():
response = {
'success': False,
'message': 'Unknown error; unable to upload file.'
}
inherited_source = None
backdoor = form.cleaned_data['backdoor']
campaign = form.cleaned_data['campaign']
confidence = form.cleaned_data['confidence']
source = form.cleaned_data['source']
method = form.cleaned_data['method']
reference = form.cleaned_data['reference']
analyst = request.user.username
if related_md5:
reload_page = True
else:
reload_page = False
related_md5 = form.cleaned_data['related_md5']
if related_md5:
related_sample = Sample.objects(md5=related_md5).first()
if not related_sample:
response[
'message'] = "Upload Failed. Unable to locate related sample."
return render_to_response(
"file_upload_response.html",
{'response': json.dumps(response)},
RequestContext(request))
# If selected, new sample inherits the campaigns of the related sample.
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_sample.campaign.append(
EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst))
campaign = related_sample.campaign
# If selected, new sample inherits the sources of the related sample
if form.cleaned_data['inherit_sources']:
inherited_source = related_sample.source
backdoor_name = None
backdoor_version = None
if backdoor:
backdoor = backdoor.split('|||')
if len(backdoor) == 2:
(backdoor_name,
backdoor_version) = backdoor[0], backdoor[1]
try:
if request.FILES:
result = handle_uploaded_file(
request.FILES['filedata'],
source,
method=method,
reference=reference,
file_format=form.cleaned_data['file_format'],
password=form.cleaned_data['password'],
user=analyst,
campaign=campaign,
confidence=confidence,
related_md5=related_md5,
bucket_list=form.cleaned_data[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version)
else:
result = handle_uploaded_file(
None,
source,
method=method,
reference=reference,
file_format=form.cleaned_data['file_format'],
password=None,
user=analyst,
campaign=campaign,
confidence=confidence,
related_md5=related_md5,
filename=request.POST['filename'].strip(),
md5=request.POST['md5'].strip().lower(),
sha1=request.POST['sha1'].strip().lower(),
sha256=request.POST['sha256'].strip().lower(),
bucket_list=form.cleaned_data[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[
form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
is_return_only_md5=False,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version)
except ZipFileError, zfe:
return render_to_response(
'file_upload_response.html', {
'response':
json.dumps({
'success': False,
'message': zfe.value
})
}, RequestContext(request))
else:
if len(result) > 1:
filedata = request.FILES['filedata']
message = ('<a href="%s">View Uploaded Samples.</a>' %
reverse('crits.samples.views.view_upload_list',
args=[filedata.name, result]))
response = {'success': True, 'message': message}
md5_response = result
elif len(result) == 1:
md5_response = None
if not request.FILES:
response['success'] = result[0].get('success', False)
if (response['success'] == False):
response['message'] = result[0].get(
'message', response.get('message'))
else:
md5_response = [result[0].get('object').md5]
else:
md5_response = [result[0]]
response['success'] = True
if md5_response != None:
response['message'] = (
'File uploaded successfully. <a href="%s">View Sample.</a>'
% reverse('crits.samples.views.detail',
args=md5_response))
if response['success']:
if request.POST.get('email'):
for s in md5_response:
email_errmsg = mail_sample(s, [request.user.email])
if email_errmsg is not None:
msg = "<br>Error emailing sample %s: %s\n" % (
s, email_errmsg)
response['message'] = response['message'] + msg
if reload_page:
response['redirect_url'] = reverse(
'crits.samples.views.detail', args=[related_md5])
return render_to_response("file_upload_response.html",
{'response': json.dumps(response)},
RequestContext(request))
else:
if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear
form.fields['related_md5'].widget = forms.HiddenInput()
return render_to_response(
'file_upload_response.html', {
'response':
json.dumps({
'success': False,
'form': form.as_table()
})
}, RequestContext(request))
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def upload_child(request, parent_md5):
"""
Upload a new child sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param parent_md5: The MD5 of the parent sample.
:type parent_md5: str
:returns: :class:`django.http.HttpResponse`
"""
new_samples = []
if request.method == "POST":
form = EmailAttachForm(request.user.username, request.POST, request.FILES)
if form.is_valid():
if request.FILES or 'filename' in request.POST and 'md5' in request.POST:
# Child samples inherit all of the sources of the parent.
parent = Sample.objects(md5=parent_md5).first()
if not parent:
return render_to_response('error.html',
{'error': "Unable to find parent."},
RequestContext(request))
source = parent.source
campaign_name = request.POST['campaign']
confidence = request.POST['confidence']
parent.campaign.append(EmbeddedCampaign(name=campaign_name, confidence=confidence, analyst=request.user.username))
campaigns = parent.campaign
try:
if request.FILES:
new_samples = handle_uploaded_file(request.FILES["filedata"],
source,
None,
form.cleaned_data["file_format"],
form.cleaned_data["password"],
user=request.user.username,
campaign=campaigns,
parent_md5=parent_md5,
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME])
else:
filename = request.POST['filename'].strip()
md5= request.POST['md5'].strip().lower()
if not filename or not md5:
error = "Need a file, or a filename and an md5."
return render_to_response('error.html',
{'error': error},
RequestContext(request))
else:
new_samples = handle_uploaded_file(None,
source,
None,
form.cleaned_data["file_format"],
form.cleaned_data["password"],
user=request.user.username,
campaign=campaigns,
parent_md5=parent_md5,
filename=filename,
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME],
md5=md5)
except ZipFileError, zfe:
return render_to_response('error.html',
{'error': zfe.value},
RequestContext(request))
else:
return render_to_response('error.html',
{'error': "Need a file, or a filename and an md5."},
RequestContext(request))
else:
return render_to_response('error.html',
{'error': 'form error'},
RequestContext(request))
def upload_file(request, related_md5=None):
"""
Upload a new sample.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param related_md5: The MD5 of a related sample.
:type related_md5: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
email_errmsg = None
if form.is_valid():
response = {'success': False,
'message': 'Unknown error; unable to upload file.'}
inherited_source = None
backdoor = form.cleaned_data['backdoor']
campaign = form.cleaned_data['campaign']
confidence = form.cleaned_data['confidence']
source = form.cleaned_data['source_name']
source_method = form.cleaned_data['source_method']
source_reference = form.cleaned_data['source_reference']
source_tlp = form.cleaned_data['source_tlp']
user = request.user
description = form.cleaned_data['description']
related_id = form.cleaned_data.get('related_id', None)
related_type = form.cleaned_data.get('related_type', None)
relationship_type = form.cleaned_data.get('relationship_type', None)
if related_md5:
reload_page = True
else:
reload_page = False
related_md5 = form.cleaned_data['related_md5']
if related_md5:
related_sample = Sample.objects(md5=related_md5).first()
if not related_sample:
response['message'] = ("Upload Failed. Unable to locate related sample. %s" % related_md5)
return render(request, "file_upload_response.html", {'response': json.dumps(response)})
# If selected, new sample inherits the campaigns of the related sample.
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_sample.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user))
campaign = related_sample.campaign
# If selected, new sample inherits the sources of the related sample
if form.cleaned_data['inherit_sources']:
inherited_source = related_sample.source
elif related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
response['success'] = False
response['message'] = ("Upload Failed. Unable to locate related Item")
return render(request, "file_upload_response.html",{'response': json.dumps(response)}, )
else:
if form.cleaned_data['inherit_campaigns']:
if campaign:
related_obj.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user))
campaign = related_obj.campaign
if form.cleaned_data['inherit_sources']:
inherited_source = related_obj.source
backdoor_name = None
backdoor_version = None
if backdoor:
backdoor = backdoor.split('|||')
if len(backdoor) == 2:
(backdoor_name, backdoor_version) = backdoor[0], backdoor[1]
try:
if request.FILES:
result = handle_uploaded_file(
request.FILES['filedata'],
source,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
file_format=form.cleaned_data['file_format'],
password=form.cleaned_data['password'],
user=user,
campaign=campaign,
confidence=confidence,
related_md5=related_md5,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type,
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version,
description=description)
else:
result = handle_uploaded_file(
None,
source,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
file_format=form.cleaned_data['file_format'],
password=None,
user=user,
campaign=campaign,
confidence=confidence,
related_md5 = related_md5,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type,
filename=request.POST['filename'].strip(),
md5=request.POST['md5'].strip().lower(),
sha1=request.POST['sha1'].strip().lower(),
sha256=request.POST['sha256'].strip().lower(),
bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME],
inherited_source=inherited_source,
is_return_only_md5=False,
backdoor_name=backdoor_name,
backdoor_version=backdoor_version,
description=description)
except ZipFileError, zfe:
return render(request, 'file_upload_response.html', {'response': json.dumps({'success': False,
'message': zfe.value})})
else:
# zip file upload, etc; result is a list of strings (1 hash per file)
if len(result) > 0 and not isinstance(result[0], dict):
filedata = request.FILES['filedata']
message = ('<a href="%s">View Uploaded Samples.</a>'
% reverse('crits-samples-views-view_upload_list',
args=[filedata.name, result]))
response = {'success': True,
'message': message }
md5_response = result
# regular file upload; result is a list with a single dict
else:
response['success'] = result[0].get('success', False)
response['message'] = result[0].get('message',
response.get('message'))
try:
md5_response = [result[0].get('object').md5]
except:
md5_response = None
if response['success']:
if request.POST.get('email') and md5_response:
for s in md5_response:
email_errmsg = mail_sample(s, [request.user.email])
if email_errmsg is not None:
msg = "<br>Error emailing sample %s: %s\n" % (s, email_errmsg)
response['message'] = response['message'] + msg
if reload_page:
response['redirect_url'] = reverse('crits-samples-views-detail', args=[related_md5])
return render(request, "file_upload_response.html", {'response': json.dumps(response)})
else:
if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear
form.fields['related_md5'].widget = forms.HiddenInput()
return render(request, 'file_upload_response.html', {'response': json.dumps({'success': False,
'form': form.as_table()})})
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(
Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
else:
if not filename or not md5:
error = "Need a file, or a filename and an md5"
return {'success': False, 'error': error}
else:
sample_md5 = handle_uploaded_file(None,
source,
reference,
file_format,
password,
analyst,
campaign=campaign,
confidence=confidence,
bucket_list=bucket_list,
ticket=ticket,
filename=filename.strip(),
md5=md5.strip())
except ZipFileError, zfe:
return {'success': False, 'error': zfe.value}
samples = Sample.objects(md5__in=sample_md5, source__name__in=sources)
if samples:
for s in samples:
event.add_relationship(rel_item=s,
rel_type='Related_To',
analyst=analyst,
get_rels=False)
s.save(username=analyst)
event.save(username=analyst)
return {'success': True}
