def setUp(self):
prep_db()
self.factory = RequestFactory()
self.user = CRITsUser.objects(username=TUSER_NAME).first()
self.user.sources.append(TSRC)
self.user.save()
# Add a test email
handlers.handle_eml(EML_DATA, TSRC, None, self.user, "Test")
def setUp(self):
prep_db()
self.factory = RequestFactory()
self.user = CRITsUser.objects(username=TUSER_NAME).first()
self.user.sources.append(TSRC)
self.user.save()
# Add a test email
handlers.handle_eml(EML_DATA, TSRC, None, self.user.username, "Test")
def obj_create(self, bundle, **kwargs):
"""
Handles creating Emails through the API.
:param bundle: Bundle containing the information to create the Campaign.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If a type_ is not provided or creation fails.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get('upload_type', None)
if not type_:
raise BadRequest('You must specify the upload type.')
elif type_ not in ('eml', 'msg', 'raw', 'yaml', 'fields'):
raise BadRequest('Unknown or unsupported upload type.')
# Remove this so it doesn't get included with the fields upload
del bundle.data['upload_type']
result = None
# Extract common information
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
campaign = bundle.data.get('campaign', None)
confidence = bundle.data.get('confidence', None)
if type_ == 'eml':
file_ = bundle.data.get('filedata', None)
if not file_:
raise BadRequest('No file uploaded.')
filedata = file_.read()
result = handle_eml(filedata, source, reference, analyst, 'Upload',
campaign, confidence)
if type_ == 'msg':
raw_email = bundle.data.get('filedata', None)
password = bundle.data.get('password', None)
result = handle_msg(raw_email, source, reference, analyst,
'Upload', password, campaign, confidence)
if type_ == 'raw':
raw_email = bundle.data.get('filedata', None)
result = handle_pasted_eml(raw_email, source, reference, analyst,
'Upload', campaign, confidence)
if type_ == 'yaml':
yaml_data = bundle.data.get('filedata', None)
email_id = bundle.data.get('email_id', None)
save_unsupported = bundle.data.get('save_unsupported', False)
result = handle_yaml(yaml_data, source, reference, analyst,
'Upload', email_id, save_unsupported,
campaign, confidence)
if type_ == 'fields':
fields = bundle.data
result = handle_email_fields(fields, analyst, 'Upload')
if not result:
raise BadRequest('No upload type found.')
if not result['status']:
raise BadRequest(result['reason'])
else:
return bundle
def email_eml_add(request):
"""
Upload an email using EML.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
eml_form = EmailEMLForm(request.user, request.POST, request.FILES)
json_reply = {
'form': eml_form.as_table(),
'success': False,
'message': ""
}
if request.method != "POST":
json_reply['message'] = "Must submit via POST."
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
if not eml_form.is_valid():
json_reply['message'] = "Form is invalid."
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
data = ''
for chunk in request.FILES['filedata']:
data += chunk
method = "EML Upload"
if eml_form.cleaned_data['source_method']:
method = method + " - " + eml_form.cleaned_data['source_method']
obj = handle_eml(data, eml_form.cleaned_data['source'],
eml_form.cleaned_data['source_reference'],
request.user.username,
method,
campaign=eml_form.cleaned_data['campaign'],
confidence=eml_form.cleaned_data['campaign_confidence'],
bucket_list=eml_form.cleaned_data['bucket_list'],
ticket=eml_form.cleaned_data['ticket'],
related_id=eml_form.cleaned_data['related_id'],
related_type=eml_form.cleaned_data['related_type'],
relationship_type=eml_form.cleaned_data['relationship_type'])
if not obj['status']:
json_reply['message'] = obj['reason']
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
json_reply['success'] = True
json_reply['message'] = 'Email uploaded successfully. <a href="%s">View email.</a>' % reverse('crits.emails.views.email_detail', args=[obj['object'].id])
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
def email_eml_add(request):
"""
Upload an email using EML.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
eml_form = EmailEMLForm(request.user, request.POST, request.FILES)
json_reply = {"form": eml_form.as_table(), "success": False, "message": ""}
if request.method != "POST":
json_reply["message"] = "Must submit via POST."
return render_to_response(
"file_upload_response.html", {"response": json.dumps(json_reply)}, RequestContext(request)
)
if not eml_form.is_valid():
json_reply["message"] = "Form is invalid."
return render_to_response(
"file_upload_response.html", {"response": json.dumps(json_reply)}, RequestContext(request)
)
data = ""
for chunk in request.FILES["filedata"]:
data += chunk
method = "EML Upload"
if eml_form.cleaned_data["source_method"]:
method = method + " - " + eml_form.cleaned_data["source_method"]
obj = handle_eml(
data,
eml_form.cleaned_data["source"],
eml_form.cleaned_data["source_reference"],
request.user.username,
method,
campaign=eml_form.cleaned_data["campaign"],
confidence=eml_form.cleaned_data["campaign_confidence"],
)
if not obj["status"]:
json_reply["message"] = obj["reason"]
return render_to_response(
"file_upload_response.html", {"response": json.dumps(json_reply)}, RequestContext(request)
)
json_reply["success"] = True
json_reply["message"] = 'Email uploaded successfully. <a href="%s">View email.</a>' % reverse(
"crits.emails.views.email_detail", args=[obj["object"].id]
)
return render_to_response(
"file_upload_response.html", {"response": json.dumps(json_reply)}, RequestContext(request)
)
def email_eml_add(request):
"""
Upload an email using EML.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
eml_form = EmailEMLForm(request.user, request.POST, request.FILES)
user = request.user
json_reply = {'form': eml_form.as_table(), 'success': False}
if request.method != "POST":
message = "Must submit via POST."
else:
if not eml_form.is_valid():
message = "Form is invalid."
elif not user.has_access_to(EmailACL.WRITE):
message = "User does not have permission to add email."
else:
form_data = eml_form.cleaned_data
data = ''
for chunk in request.FILES['filedata']:
data += chunk
method = "EML Upload"
if form_data['source_method']:
method = method + " - " + form_data['source_method']
result = handle_eml(
data, form_data['source_name'], form_data['source_reference'],
method, form_data['source_tlp'], user, form_data['campaign'],
form_data['campaign_confidence'], form_data['bucket_list'],
form_data['ticket'], form_data['related_id'],
form_data['related_type'], form_data['relationship_type'])
if result['status']:
redirect = reverse('crits-emails-views-email_detail',
args=[result['object'].id])
json_reply['success'] = True
message = 'Email uploaded successfully'
if result.get('reason'):
message += ', but %s' % result['reason']
message += ('. <a href="%s">View email.</a>' % redirect)
else:
message = result['reason']
json_reply['message'] = message
return render(request, 'file_upload_response.html',
{'response': json.dumps(json_reply)})
def obj_create(self, bundle, **kwargs):
"""
Handles creating Emails through the API.
:param bundle: Bundle containing the information to create the Campaign.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get('upload_type', None)
content = {'return_code': 1,
'type': 'Email',
'message': ''}
if not type_:
content['message'] = 'You must specify the upload type.'
self.crits_response(content)
elif type_ not in ('eml', 'msg', 'raw', 'yaml', 'fields'):
content['message'] = 'Unknown or unsupported upload type.'
self.crits_response(content)
# Remove this so it doesn't get included with the fields upload
del bundle.data['upload_type']
result = None
# Extract common information
source = bundle.data.get('source', None)
method = bundle.data.get('method', '')
reference = bundle.data.get('reference', None)
campaign = bundle.data.get('campaign', None)
confidence = bundle.data.get('confidence', None)
if method:
method = " - " + method
if type_ == 'eml':
file_ = bundle.data.get('filedata', None)
if not file_:
content['message'] = 'No file uploaded.'
self.crits_response(content)
filedata = file_.read()
result = handle_eml(filedata, source, reference,
analyst, 'EML Upload' + method, campaign,
confidence)
if type_ == 'msg':
raw_email = bundle.data.get('filedata', None)
password = bundle.data.get('password', None)
result = handle_msg(raw_email,
source,
reference,
analyst,
'Outlook MSG Upload' + method,
password,
campaign,
confidence)
if type_ == 'raw':
raw_email = bundle.data.get('filedata', None)
result = handle_pasted_eml(raw_email,
source,
reference,
analyst,
'Raw Upload' + method,
campaign,
confidence)
if type_ == 'yaml':
yaml_data = bundle.data.get('filedata', None)
email_id = bundle.data.get('email_id', None)
save_unsupported = bundle.data.get('save_unsupported', False)
result = handle_yaml(yaml_data,
source,
reference,
analyst,
'YAML Upload' + method,
email_id,
save_unsupported,
campaign,
confidence)
if type_ == 'fields':
fields = bundle.data
# Strip these so they don't get put in unsupported_attrs.
del fields['username']
del fields['api_key']
result = handle_email_fields(fields,
analyst,
'Fields Upload')
if result.get('message'):
content['message'] = result.get('message')
if result.get('reason'):
content['message'] += result.get('reason')
if result.get('obj_id'):
content['id'] = result.get('obj_id', '')
elif result.get('object'):
content['id'] = str(result.get('object').id)
if content.get('id'):
url = reverse('api_dispatch_detail',
kwargs={'resource_name': 'emails',
'api_name': 'v1',
'pk': content.get('id')})
content['url'] = url
if result['status']:
content['return_code'] = 0
self.crits_response(content)
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
user = get_user_info(analyst)
if not sc:
return {'success': False, 'message': 'Could not find ChopShop service.'}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {'success': False, 'message': "ChopShop shop path does not exist."}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {'success': False, 'message': "Could not get PCAP from GridFS: %s" % pcap_md5}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {'success': False,
'message': 'Lost race condition in chopui. Try again.'}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
related_id=pcap.id, related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
if user.has_access_to(SampleACL.WRITE):
md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
else:
md5_digest = None
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def testEmailAdd(self):
result = handlers.handle_eml(EML_DATA, TSRC, None, self.user.username, "Test")
self.assertEqual(result['status'], True)
self.assertEqual(result['data']['x_mailer'],"YahooMailWebService/0.8.121.416")
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
if not sc:
return {
'success': False,
'message': 'Could not find ChopShop service.'
}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {
'success': False,
'message': "ChopShop shop path does not exist."
}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {
'success': False,
'message': "Can not process SMTP and raw simultaneously."
}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {
'success': False,
'message': "Could not get PCAP from GridFS: %s" % pcap_md5
}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {
'success': False,
'message': 'Lost race condition in chopui. Try again.'
}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if handle_file(name,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
md5_digest=md5_digest,
related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
reverse('crits.samples.views.detail', args=[md5_digest
]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob,
source,
None,
analyst,
'ChopShop FileCarver',
related_id=pcap.id,
related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
reverse('crits.emails.views.email_detail', args=[
ret['object'].id
]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse(
'crits.samples.views.detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
md5_digest = handle_file(id_,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
related_type='PCAP')
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
id_, reverse('crits.samples.views.detail',
args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def email_eml_add(request):
"""
Upload an email using EML.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
eml_form = EmailEMLForm(request.user, request.POST, request.FILES)
user = request.user
json_reply = {
'form': eml_form.as_table(),
'success': False
}
if request.method != "POST":
message = "Must submit via POST."
else:
if not eml_form.is_valid():
message = "Form is invalid."
elif not user.has_access_to(EmailACL.WRITE):
message = "User does not have permission to add email."
else:
form_data = eml_form.cleaned_data
data = ''
for chunk in request.FILES['filedata']:
data += chunk
method = "EML Upload"
if form_data['source_method']:
method = method + " - " + form_data['source_method']
result = handle_eml(data,
form_data['source_name'],
form_data['source_reference'],
method,
form_data['source_tlp'],
request.user,
form_data['campaign'],
form_data['campaign_confidence'],
form_data['bucket_list'],
form_data['ticket'],
form_data['related_id'],
form_data['related_type'],
form_data['relationship_type'])
if result['status']:
redirect = reverse('crits.emails.views.email_detail',
args=[result['object'].id])
json_reply['success'] = True
message = 'Email uploaded successfully'
if result.get('reason'):
message += ', but %s' % result['reason']
message += ('. <a href="%s">View email.</a>' % redirect)
else:
message = result['reason']
json_reply['message'] = message
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
def testEmailAdd(self):
result = handlers.handle_eml(EML_DATA, TSRC, None, self.user, "Test")
self.assertEqual(result['status'], True)
self.assertEqual(result['data']['x_mailer'],
"YahooMailWebService/0.8.121.416")
def obj_create(self, bundle, **kwargs):
"""
Handles creating Emails through the API.
:param bundle: Bundle containing the information to create the Campaign.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If a type_ is not provided or creation fails.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get('upload_type', None)
if not type_:
raise BadRequest('You must specify the upload type.')
elif type_ not in ('eml', 'msg', 'raw', 'yaml', 'fields'):
raise BadRequest('Unknown or unsupported upload type.')
# Remove this so it doesn't get included with the fields upload
del bundle.data['upload_type']
result = None
# Extract common information
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
campaign = bundle.data.get('campaign', None)
confidence = bundle.data.get('confidence', None)
if type_ == 'eml':
file_ = bundle.data.get('filedata', None)
if not file_:
raise BadRequest('No file uploaded.')
filedata = file_.read()
result = handle_eml(filedata, source, reference,
analyst, 'Upload', campaign,
confidence)
if type_ == 'msg':
raw_email = bundle.data.get('filedata', None)
password = bundle.data.get('password', None)
result = handle_msg(raw_email,
source,
reference,
analyst,
'Upload',
password,
campaign,
confidence)
if type_ == 'raw':
raw_email = bundle.data.get('filedata', None)
result = handle_pasted_eml(raw_email,
source,
reference,
analyst,
'Upload',
campaign,
confidence)
if type_ == 'yaml':
yaml_data = bundle.data.get('filedata', None)
email_id = bundle.data.get('email_id', None)
save_unsupported = bundle.data.get('save_unsupported', False)
result = handle_yaml(yaml_data,
source,
reference,
analyst,
'Upload',
email_id,
save_unsupported,
campaign,
confidence)
if type_ == 'fields':
fields = bundle.data
result = handle_email_fields(fields,
analyst,
'Upload')
if not result:
raise BadRequest('No upload type found.')
if not result['status']:
raise BadRequest(result['reason'])
else:
return bundle
