def obj_create(self, bundle, **kwargs):
"""
Handles creating RawData through the API.
:param bundle: Bundle containing the information to create the RawData.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If filedata is not provided or creation fails.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get('upload_type', None)
if not type_:
raise BadRequest('Must provide an upload type.')
if type_ not in ('metadata', 'file'):
raise BadRequest('Not a valid upload type.')
if type_ == 'metadata':
data = bundle.data.get('data', None)
elif type_ == 'file':
file_ = bundle.data.get('filedata', None)
if not file_:
raise BadRequest("Upload type of 'file' but no file uploaded.")
data = file_.read()
source = bundle.data.get('source', None)
description = bundle.data.get('description', '')
title = bundle.data.get('title', None)
data_type = bundle.data.get('data_type', None)
tool_name = bundle.data.get('tool_name', '')
tool_version = bundle.data.get('tool_version', '')
tool_details = bundle.data.get('tool_details', '')
link_id = bundle.data.get('link_id', None)
copy_rels = bundle.data.get('copy_relationships', False)
method = 'Upload'
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
status = handle_raw_data_file(data,
source,
analyst,
description,
title,
data_type,
tool_name,
tool_version,
tool_details,
link_id,
method=method,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket)
if status['success']:
return bundle
else:
raise BadRequest(status['message'])
def obj_create(self, bundle, **kwargs):
"""
Handles creating RawData through the API.
:param bundle: Bundle containing the information to create the RawData.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If filedata is not provided or creation fails.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get('upload_type', None)
if not type_:
raise BadRequest('Must provide an upload type.')
if type_ not in ('metadata', 'file'):
raise BadRequest('Not a valid upload type.')
if type_ == 'metadata':
data = bundle.data.get('data', None)
elif type_ == 'file':
file_ = bundle.data.get('filedata', None)
if not file_:
raise BadRequest("Upload type of 'file' but no file uploaded.")
data = file_.read()
source = bundle.data.get('source', None)
description = bundle.data.get('description', '')
title = bundle.data.get('title', None)
data_type = bundle.data.get('data_type', None)
tool_name = bundle.data.get('tool_name', '')
tool_version = bundle.data.get('tool_version', '')
tool_details = bundle.data.get('tool_details', '')
link_id = bundle.data.get('link_id', None)
copy_rels = bundle.data.get('copy_relationships', False)
method = 'Upload'
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if not title:
raise BadRequest("Must provide a title.")
if not data_type:
raise BadRequest("Must provide a data type.")
status = handle_raw_data_file(data, source, analyst,
description, title, data_type,
tool_name, tool_version, tool_details,
link_id,
method=method,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket)
if status['success']:
return bundle
else:
raise BadRequest(status['message'])
def run(self, obj, config):
self.config = config
self.obj = obj
self._debug("pdf2txt started\n")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
args = [pdf2txt_path, filename, "-"]
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output = proc.communicate()[0]
self._debug(output)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(
output,
self.obj.source,
self.current_task.username,
title="pdftotext",
data_type='text',
tool_name='pdftotext',
tool_version='0.1',
tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" %
(str(obj.id), str(raw_obj.id), repr(res['success'])))
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.username)
obj.save(username=self.current_task.username)
raw_obj.save(username=self.current_task.username)
self._warning("resy: %s" % (str(resy)))
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def run(self, obj, config):
self.config = config
self.obj = obj
self._debug("pdf2txt started\n")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
args = [pdf2txt_path, filename, "-"]
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output = proc.communicate()[0]
self._debug(output)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(output, self.obj.source, self.current_task.username,
title="pdftotext", data_type='text',
tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) )
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.username)
obj.save(username=self.current_task.username)
raw_obj.save(username=self.current_task.username)
self._warning("resy: %s" % (str(resy)) )
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if not obs.object_ or not obs.object_.properties:
self.failed.append(("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value),
[self.source],
username=analyst)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False)
self.parse_res(imp_type, obs, res)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
res = handle_email_fields(data,
analyst,
"STIX")
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append((obs.id_,
"Contains",
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
type(item).__name__,
item.parent.id_)) # note for display in UI
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if obs.observable_composition:
object_list = obs.observable_composition.observables
else:
object_list = [obs]
for obs_comp in object_list:
if not obs_comp.object_ or not obs_comp.object_.properties:
self.failed.append(
("No valid object_properties was found!",
type(obs_comp).__name__,
obs_comp.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs_comp.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask',
'ipv6-subnet'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype, [self.source],
analyst=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value), [self.source],
username=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value
== "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item,
File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if item.extracted_features:
self.parse_filenames(item.extracted_features,
res['object'].id)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data[
'source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to]
if item.header.cc:
data['cc'] = [str(r) for r in item.header.cc]
res = handle_email_fields(data,
analyst,
"STIX",
id=self.package.id_)
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append(
(obs.id_, "Contains", rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def upload_raw_data(request, link_id=None):
"""
Upload new RawData to CRITs.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param link_id: The LinkId of RawData if this is a new version upload.
:type link_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
if 'filedata' in request.FILES:
form = UploadRawDataFileForm(request.user,
request.POST,
request.FILES)
filedata = request.FILES['filedata']
data = filedata.read() # XXX: Should be using chunks here.
has_file = True
else:
form = UploadRawDataForm(request.user,request.POST)
data = request.POST.get('data', None)
has_file = False
if form.is_valid():
source = form.cleaned_data.get('source_name')
user = request.user
description = form.cleaned_data.get('description', '')
title = form.cleaned_data.get('title', None)
tool_name = form.cleaned_data.get('tool_name', '')
tool_version = form.cleaned_data.get('tool_version', '')
tool_details = form.cleaned_data.get('tool_details', '')
data_type = form.cleaned_data.get('data_type', None)
copy_rels = request.POST.get('copy_relationships', False)
link_id = link_id
bucket_list = form.cleaned_data.get('bucket_list')
ticket = form.cleaned_data.get('ticket')
method = form.cleaned_data.get('source_method', '') or 'Upload'
reference = form.cleaned_data.get('source_reference', '')
tlp = form.cleaned_data.get('source_tlp', '')
related_id = form.cleaned_data.get('related_id', '')
related_type = form.cleaned_data.get('related_type', '')
relationship_type = form.cleaned_data.get('relationship_type')
status = handle_raw_data_file(data, source, user,
description, title, data_type,
tool_name, tool_version, tool_details,
link_id,
method=method,
reference=reference,
tlp=tlp,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type)
if status['success']:
jdump = json.dumps({
'message': 'raw_data uploaded successfully! <a href="%s">View raw_data</a>'
% reverse('crits.raw_data.views.raw_data_details',
args=[status['_id']]), 'success': True})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
jdump = json.dumps({'success': False,
'message': status['message']})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
jdump = json.dumps({'success': False,
'form': form.as_table()})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
return render_to_response('error.html',
{'error': "Expected POST."},
RequestContext(request))
def run(self, obj, config):
obj.filedata.seek(0)
data8 = obj.filedata.read(8)
obj.filedata.seek(0)
self.config = config
self.obj = obj
self._debug("pdf2txt started")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
antiword_path = self.config.get("antiword_path", "/usr/bin/antiword")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
new_env = dict(os.environ) # Copy current environment
args = []
obj.filedata.seek(0)
if obj.is_pdf():
self._debug("PDF")
args = [pdf2txt_path, filename, "-"]
elif data8.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
self._debug("Word")
#new_env['LANG'] = 'en_US'
#env=dict(os.environ, LANG="en_US")
args = [antiword_path, '-r', '-s', '-t', filename]
else:
self._error("Not a valid PDF or Word document")
return False
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args, env=new_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output, serr = proc.communicate()
if serr:
self._warning(serr)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(output, self.obj.source, self.current_task.username,
title="pdftotext", data_type='Text',
tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) )
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.username)
obj.save(username=self.current_task.username)
raw_obj.save(username=self.current_task.username)
self._warning("resy: %s" % (str(resy)) )
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def upload_raw_data(request, link_id=None):
"""
Upload new RawData to CRITs.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param link_id: The LinkId of RawData if this is a new version upload.
:type link_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
if 'filedata' in request.FILES:
form = UploadRawDataFileForm(request.user, request.POST,
request.FILES)
filedata = request.FILES['filedata']
data = filedata.read() # XXX: Should be using chunks here.
has_file = True
else:
form = UploadRawDataForm(request.user, request.POST)
data = request.POST.get('data', None)
has_file = False
if form.is_valid():
source = form.cleaned_data.get('source_name')
user = request.user
description = form.cleaned_data.get('description', '')
title = form.cleaned_data.get('title', None)
tool_name = form.cleaned_data.get('tool_name', '')
tool_version = form.cleaned_data.get('tool_version', '')
tool_details = form.cleaned_data.get('tool_details', '')
data_type = form.cleaned_data.get('data_type', None)
copy_rels = request.POST.get('copy_relationships', False)
link_id = link_id
bucket_list = form.cleaned_data.get('bucket_list')
ticket = form.cleaned_data.get('ticket')
method = form.cleaned_data.get('source_method', '') or 'Upload'
reference = form.cleaned_data.get('source_reference', '')
tlp = form.cleaned_data.get('source_tlp', '')
related_id = form.cleaned_data.get('related_id', '')
related_type = form.cleaned_data.get('related_type', '')
relationship_type = form.cleaned_data.get('relationship_type')
status = handle_raw_data_file(data,
source,
user,
description,
title,
data_type,
tool_name,
tool_version,
tool_details,
link_id,
method=method,
reference=reference,
tlp=tlp,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type)
if status['success']:
jdump = json.dumps({
'message':
'raw_data uploaded successfully! <a href="%s">View raw_data</a>'
% reverse('crits.raw_data.views.raw_data_details',
args=[status['_id']]),
'success':
True
})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
jdump = json.dumps({
'success': False,
'message': status['message']
})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
jdump = json.dumps({'success': False, 'form': form.as_table()})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response('file_upload_response.html',
{'response': jdump},
RequestContext(request))
else:
return render_to_response('error.html', {'error': "Expected POST."},
RequestContext(request))
def run_archive_viewer(self, obj):
"""
Get data using the archive viewer.
"""
safe = [
'pyi_carchive',
'pyi_rth_win32comgenpy',
'_pyi_bootstrap',
'_pyi_egg_install.py'
]
# This doesn't work. Everything is showing as an invalid CArchive file.
with self._write_to_file() as tmp_file:
try:
arch = get_archive(tmp_file)
if type(arch.toc) == type({}):
toc = arch.toc
else:
toc = arch.toc.data
for t in toc:
d = {'Position': t[0],
'Length': t[1],
'Uncompressed': t[2],
'IsCompressed': t[3],
'Type': t[4],
'RawData': ""
}
if t[4] == 's' and t[5] not in safe:
try:
block = self.get_data(t[5], arch).encode('utf-8',
"ignore")
except:
self._info("%s: Block not valid utf-8. Trying utf-16." % t[5])
try:
block = self.get_data(t[5], arch).encode('utf-16',
"ignore")
except:
self._info("%s: Block not valid utf-16. Trying utf-32." % t[5])
try:
block = self.get_data(t[5], arch).encode('utf-32',
"ignore")
except:
self._info("%s: Block not valid utf-32. Trying latin-1." % t[5])
try:
block = self.get_data(t[5], arch).encode('latin-1',
'ignore')
except:
self._info("%s: Block not valid latin-1. Done trying." % t[5])
block = None
if block is not None:
bmd5 = md5(block).hexdigest()
bsha1 = sha1(block).hexdigest()
bsha256 = sha256(block).hexdigest()
block = block.replace('http', 'hxxp')
description = '"%s" pulled from Sample\n\n' % t[5]
description += 'MD5: %s\n' % bmd5
description += 'SHA1: %s\n' % bsha1
description += 'SHA256: %s\n' % bsha256
title = t[5]
data_type = "Python"
tool_name = "pyinstaller_service"
result = handle_raw_data_file(
block,
obj.source,
user=self.current_task.user,
description=description,
title=title,
data_type=data_type,
tool_name=tool_name,
)
if result['success']:
self._info("RawData added for %s" % t[5])
res = obj.add_relationship(
rel_item=result['object'],
rel_type=RelationshipTypes.CONTAINED_WITHIN,
rel_confidence="high",
analyst=self.current_task.user
)
if res['success']:
obj.save(username=self.current_task.user.username)
result['object'].save(username=self.current_task.user.username)
url = reverse('crits-core-views.details',
args=('RawData',
result['_id']))
url = '<a href="%s">View Raw Data</a>' % url
d['RawData'] = url
self._info("Relationship added for %s" % t[5])
else:
self._info("Error adding relationship: %s" % res['message'])
else:
self._info(
"RawData addition failed for %s:%s" % (t[5],
result['message'])
)
self._add_result("Info", t[5], d)
except Exception, e:
self._info("Error: %s" % str(e))
class MacroExtractService(Service):
"""
Attempts to extract VBA Macros from MS Office files.
"""
name = "macro_extract"
version = '0.1.0'
supported_types = ['Sample']
description = "Extracs VBA Macros from MS Office documents."
@staticmethod
def get_config(existing_config):
# This service no longer uses config options, so blow away any existing
# configs.
return {}
@staticmethod
def valid_for(obj):
return
def run(self, obj, config):
username = self.current_task.username
filename = obj.filename
filedata = obj.filedata.read()
try:
vbaparser = VBA_Parser(filename, data=filedata)
except Exception, e:
self._error("Cannot parse file: %s" % str(e))
return
if vbaparser.detect_vba_macros():
for (filename, stream_path, vba_filename,
vba_code) in vbaparser.extract_macros():
d = {
'OLE stream': stream_path,
'VBA filename': vba_filename,
'Length': len(vba_code)
}
result = handle_raw_data_file(
vba_code,
obj.source,
user=username,
description="VBA Macro source code for %s" % vba_filename,
title=vba_filename,
data_type="Text",
tool_name=self.name,
tool_version=self.version,
tool_details=self.description)
if result['success']:
obj.add_relationship(
result['object'],
RelationshipTypes.RELATED_TO,
analyst=username,
rel_reason="Extracted from related Sample")
obj.save()
d['RawData TLO ID'] = result['_id']
self._add_result('Macros', filename, d)
results = vbaparser.analyze_macros(show_decoded_strings=True)
self._add_result('Counts', 'Suspicious keywords',
{'Count': vbaparser.nb_suspicious})
self._add_result('Counts', 'AutoExec keywords',
{'Count': vbaparser.nb_autoexec})
self._add_result('Counts', 'IOCs', {'Count': vbaparser.nb_iocs})
self._add_result('Counts', 'Hex obfuscated strings',
{'Count': vbaparser.nb_hexstrings})
self._add_result('Counts', 'Base64 obfuscated strings',
{'Count': vbaparser.nb_base64strings})
self._add_result('Counts', 'Dridex obfuscated strings',
{'Count': vbaparser.nb_dridexstrings})
self._add_result('Counts', 'VBA obfuscated strings',
{'Count': vbaparser.nb_vbastrings})
for kw_type, keyword, description in results:
try:
d = {
'type': kw_type,
'description': description.decode('utf-8')
}
self._add_result('Keywords', keyword.decode('utf-8'), d)
except:
pass
else:
self._info('No VBA Macros found')
vbaparser.close()
return
def upload_raw_data(request, link_id=None):
"""
Upload new RawData to CRITs.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param link_id: The LinkId of RawData if this is a new version upload.
:type link_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST":
if "filedata" in request.FILES:
form = UploadRawDataFileForm(request.user, request.POST, request.FILES)
filedata = request.FILES["filedata"]
data = filedata.read() # XXX: Should be using chunks here.
has_file = True
else:
form = UploadRawDataForm(request.user, request.POST)
data = request.POST.get("data", None)
has_file = False
if form.is_valid():
source = form.cleaned_data.get("source")
user = request.user.username
description = form.cleaned_data.get("description", "")
title = form.cleaned_data.get("title", None)
tool_name = form.cleaned_data.get("tool_name", "")
tool_version = form.cleaned_data.get("tool_version", "")
tool_details = form.cleaned_data.get("tool_details", "")
data_type = form.cleaned_data.get("data_type", None)
copy_rels = request.POST.get("copy_relationships", False)
link_id = link_id
bucket_list = form.cleaned_data.get("bucket_list")
ticket = form.cleaned_data.get("ticket")
method = form.cleaned_data.get("method", "") or "Upload"
reference = form.cleaned_data.get("reference", "")
related_id = form.cleaned_data.get("related_id", "")
related_type = form.cleaned_data.get("related_type", "")
relationship_type = form.cleaned_data.get("relationship_type")
status = handle_raw_data_file(
data,
source,
user,
description,
title,
data_type,
tool_name,
tool_version,
tool_details,
link_id,
method=method,
reference=reference,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type,
)
if status["success"]:
jdump = json.dumps(
{
"message": 'raw_data uploaded successfully! <a href="%s">View raw_data</a>'
% reverse("crits.raw_data.views.raw_data_details", args=[status["_id"]]),
"success": True,
}
)
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request))
else:
jdump = json.dumps({"success": False, "message": status["message"]})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request))
else:
jdump = json.dumps({"success": False, "form": form.as_table()})
if not has_file:
return HttpResponse(jdump, content_type="application/json")
return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request))
else:
return render_to_response("error.html", {"error": "Expected POST."}, RequestContext(request))
def run(self, obj, config):
obj.filedata.seek(0)
data8 = obj.filedata.read(8)
obj.filedata.seek(0)
self.config = config
self.obj = obj
self._debug("pdf2txt started")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
antiword_path = self.config.get("antiword_path", "/usr/bin/antiword")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
new_env = dict(os.environ) # Copy current environment
args = []
obj.filedata.seek(0)
if obj.is_pdf():
self._debug("PDF")
args = [pdf2txt_path, filename, "-"]
elif data8.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
self._debug("Word")
#new_env['LANG'] = 'en_US'
#env=dict(os.environ, LANG="en_US")
args = [antiword_path, '-r', '-s', '-t', filename]
else:
self._error("Not a valid PDF or Word document")
return False
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args,
env=new_env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output, serr = proc.communicate()
if serr:
self._warning(serr)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(
output,
self.obj.source,
self.current_task.username,
title="pdftotext",
data_type='Text',
tool_name='pdftotext',
tool_version='0.1',
tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" %
(str(obj.id), str(raw_obj.id), repr(res['success'])))
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.username)
obj.save(username=self.current_task.username)
raw_obj.save(username=self.current_task.username)
self._warning("resy: %s" % (str(resy)))
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype, [self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {
'success': False,
'reason': 'No IP Type'
}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and
(isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val), [self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact)
and rel_obj.properties.type_
== Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get(
'date', 'None'), data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data, analyst, "STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(
data[key], self.source, analyst,
description, title, d_type, "STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj, res,
ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(
cbx_obj.id_, []).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to',
'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(
data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append(
(oid, RelationshipTypes.RELATED_TO, oid2,
"High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append(
(oid, RelationshipTypes.CONTAINS, rel_id,
"High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, "%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {'success': False, 'reason': 'No IP Type'}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and (isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val),
[self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data,
analyst,
"STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(data[key],
self.source,
analyst,
description,
title,
d_type,
"STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj,
res, ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
x, y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to', 'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append((oid,
RelationshipTypes.RELATED_TO,
oid2, "High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append((oid,
RelationshipTypes.CONTAINS,
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
"%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def run_archive_viewer(self, obj):
"""
Get data using the archive viewer.
"""
safe = [
'pyi_carchive', 'pyi_rth_win32comgenpy', '_pyi_bootstrap',
'_pyi_egg_install.py'
]
# This doesn't work. Everything is showing as an invalid CArchive file.
with self._write_to_file() as tmp_file:
try:
arch = get_archive(tmp_file)
if type(arch.toc) == type({}):
toc = arch.toc
else:
toc = arch.toc.data
for t in toc:
d = {
'Position': t[0],
'Length': t[1],
'Uncompressed': t[2],
'IsCompressed': t[3],
'Type': t[4],
'RawData': ""
}
if t[4] == 's' and t[5] not in safe:
try:
block = self.get_data(t[5], arch).encode(
'utf-8', "ignore")
except:
self._info(
"%s: Block not valid utf-8. Trying utf-16." %
t[5])
try:
block = self.get_data(t[5], arch).encode(
'utf-16', "ignore")
except:
self._info(
"%s: Block not valid utf-16. Trying utf-32." %
t[5])
try:
block = self.get_data(t[5], arch).encode(
'utf-32', "ignore")
except:
self._info(
"%s: Block not valid utf-32. Trying latin-1." %
t[5])
try:
block = self.get_data(t[5], arch).encode(
'latin-1', 'ignore')
except:
self._info(
"%s: Block not valid latin-1. Done trying." %
t[5])
block = None
if block is not None:
bmd5 = md5(block).hexdigest()
bsha1 = sha1(block).hexdigest()
bsha256 = sha256(block).hexdigest()
block = block.replace('http', 'hxxp')
description = '"%s" pulled from Sample\n\n' % t[5]
description += 'MD5: %s\n' % bmd5
description += 'SHA1: %s\n' % bsha1
description += 'SHA256: %s\n' % bsha256
title = t[5]
data_type = "Python"
tool_name = "pyinstaller_service"
result = handle_raw_data_file(
block,
obj.source,
user=self.current_task.user,
description=description,
title=title,
data_type=data_type,
tool_name=tool_name,
)
if result['success']:
self._info("RawData added for %s" % t[5])
res = obj.add_relationship(
rel_item=result['object'],
rel_type=RelationshipTypes.
CONTAINED_WITHIN,
rel_confidence="high",
analyst=self.current_task.user)
if res['success']:
obj.save(username=self.current_task.user.
username)
result['object'].save(
username=self.current_task.user.
username)
url = reverse('crits-core-views.details',
args=('RawData',
result['_id']))
url = '<a href="%s">View Raw Data</a>' % url
d['RawData'] = url
self._info("Relationship added for %s" %
t[5])
else:
self._info(
"Error adding relationship: %s" %
res['message'])
else:
self._info(
"RawData addition failed for %s:%s" %
(t[5], result['message']))
self._add_result("Info", t[5], d)
except Exception, e:
self._info("Error: %s" % str(e))
def run(self, obj, config):
self.config = config
self.obj = obj
user = self.current_task.user
tlp_value = self.config.get("tlp_value", "tlp_value")
url = obj['value']
if not (obj._meta['crits_type'] == 'Indicator'
and obj['ind_type'] == 'URI'):
self._error('This object type cannot use service Url analysis.')
return False
#verify url http or https
if url.startswith('https://') or url.startswith('http://'):
#put url in file
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"
)
driver = webdriver.PhantomJS(desired_capabilities=dcap,
service_args=[
'--ignore-ssl-errors=true',
'--ssl-protocol=any',
'--web-security=false'
])
driver.set_window_size(1024, 768)
driver.set_page_load_timeout(30)
driver.get(url)
time.sleep(3)
#driver.save_screenshot('testing1.png')
screen = driver.get_screenshot_as_png()
ofile = io.BytesIO()
im = Image.open(StringIO.StringIO(screen))
im.save(ofile, 'PNG', optimize=True)
ofile.seek(0)
res = add_screenshot(description='Render of a website URL',
tags=None,
method=self.name,
source=obj.source,
reference=None,
analyst=self.current_task.user.username,
screenshot=ofile,
screenshot_ids=None,
oid=obj.id,
tlp=tlp_value,
otype="Indicator")
if res.get('message') and res.get('success') == True:
self._warning("res-message: %s id:%s" %
(res.get('message'), res.get('id')))
self._add_result('ScreenShot URL', res.get('id'),
{'Message': res.get('message')})
#parse HAR
har = driver.get_log('har')
if type(har) is list and har:
if type(har[0]) is dict and 'message' in har[0]:
#change unicode to json
try:
har[0]['message'] = json.loads(har[0]['message'])
except:
self._warning('Har log error to parse json')
if type(har[0]['message']
) is dict and 'log' in har[0]['message'] and type(
har[0]['message']['log']
) is dict and 'pages' in har[0]['message']['log']:
if type(har[0]['message']['log']['pages']
) is list and har[0]['message']['log'][
'pages'] and type(har[0]['message']['log']
['pages'][0]) is dict:
title = 'Result of '
if 'id' in har[0]['message']['log']['pages'][0]:
title += har[0]['message']['log']['pages'][0][
'id']
if 'title' in har[0]['message']['log']['pages'][0]:
self._add_result(
title, 'Title', {
'value':
har[0]['message']['log']['pages'][0]
['title']
})
#parse each request and response
if 'entries' in har[0]['message']['log'] and type(
har[0]['message']['log']['entries']
) is list and har[0]['message']['log']['entries']:
count = 1
type_r = ['cookies', 'queryString', 'headers']
type_rs = ['content', 'timings', 'cache']
for elem_rr in har[0]['message']['log']['entries']:
for k, v in elem_rr.iteritems():
if type(v) is not dict:
self._add_result(
title +
' -- Informations Request & Response num:'
+ str(count), k, {'value': v})
for k, v in elem_rr.iteritems():
if type(v) is dict:
for kx, vx in v.iteritems():
self._add_result(
title +
' -- Informations Request & Response num:'
+ str(count) + ' -- ' + str(k),
kx, {'value': vx})
count += 1
#save page source in rawdata
if not user.has_access_to(RawDataACL.WRITE):
self._info(driver.page_source.encode('utf8'))
else:
#can write
result = handle_raw_data_file(
driver.page_source.encode('utf8'),
obj.source,
user=self.current_task.user,
description="Code page for URL: %s" % url,
title=url,
data_type="Text",
tool_name=self.name,
tool_version=self.version,
tool_details=self.description)
if result['success']:
obj.add_relationship(
result['object'],
RelationshipTypes.CONTAINED_WITHIN,
analyst=self.current_task.user.username,
rel_reason="Extracted from URI")
obj.save()
self._add_result(
'Code Page', url, {
'RawData TLO ID':
result['_id'],
'md5 file':
md5(driver.page_source.encode('utf8')).hexdigest(),
'simhash':
str(simhash(driver.page_source.encode('utf8')))
})
driver.close()
driver.service.process.terminate()
time.sleep(1)
#get certificat information - ref: https://stackoverflow.com/questions/30862099/how-can-i-get-certificate-issuer-information-in-python
#because selenium not functionnality
if url.startswith('https://'):
try:
host = urlparse(url).hostname
port = urlparse(url).port
if (port is None):
port = 443
s = socks.socksocket()
if settings.HTTP_PROXY:
type_proxy = socks.PROXY_TYPE_SOCKS5
if settings.HTTP_PROXY.startswith('http://'):
type_proxy = socks.PROXY_TYPE_HTTP
s.setproxy(type_proxy,
urlparse(settings.HTTP_PROXY).hostname,
port=urlparse(settings.HTTP_PROXY).port)
s.connect((host, port))
ss = ssl.wrap_socket(s)
pem_data = ssl.DER_cert_to_PEM_cert(ss.getpeercert(True))
ss.close()
s.close()
cert = M2Crypto.X509.load_cert_string(pem_data)
#put ssl information
self._add_result(
'SSL informations', 'Subject',
{'value': str(cert.get_subject().as_text())})
self._add_result(
'SSL informations', 'Issuer',
{'value': str(cert.get_issuer().as_text())})
self._add_result('SSL informations', 'Version',
{'value': str(cert.get_version())})
self._add_result('SSL informations', 'Date before',
{'value': str(cert.get_not_before())})
self._add_result('SSL informations', 'Date after',
{'value': str(cert.get_not_after())})
self._add_result('SSL informations', 'Serial Number',
{'value': str(cert.get_serial_number())})
self._add_result('SSL informations', 'Verify',
{'value': str(cert.verify())})
self._add_result('SSL informations', 'Fingerprint MD5',
{'value': str(cert.get_fingerprint())})
for i in range(0, cert.get_ext_count()):
self._add_result(
'SSL informations Extension',
str(cert.get_ext_at(i).get_name()),
{'value': str(cert.get_ext_at(i).get_value())})
#https://www.heikkitoivonen.net/m2crypto/api/M2Crypto.X509-module.html
except:
self._error('Error: get certificate informations.')
self._info(str(cert))
driver.service.process.kill()
driver.quit()
self._info('END')
def obj_create(self, bundle, **kwargs):
"""
Handles creating RawData through the API.
:param bundle: Bundle containing the information to create the RawData.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
type_ = bundle.data.get("upload_type", None)
content = {"return_code": 1, "type": "RawData"}
if not type_:
content["message"] = "Must provide an upload type."
self.crits_response(content)
if type_ not in ("metadata", "file"):
content["message"] = "Not a valid upload type."
self.crits_response(content)
if type_ == "metadata":
data = bundle.data.get("data", None)
elif type_ == "file":
file_ = bundle.data.get("filedata", None)
if not file_:
content["message"] = "Upload type of 'file' but no file uploaded."
self.crits_response(content)
data = file_.read()
source = bundle.data.get("source", None)
description = bundle.data.get("description", "")
title = bundle.data.get("title", None)
data_type = bundle.data.get("data_type", None)
tool_name = bundle.data.get("tool_name", "")
tool_version = bundle.data.get("tool_version", "")
tool_details = bundle.data.get("tool_details", "")
link_id = bundle.data.get("link_id", None)
copy_rels = bundle.data.get("copy_relationships", False)
method = bundle.data.get("method", None) or "Upload"
reference = bundle.data.get("reference", None)
bucket_list = bundle.data.get("bucket_list", None)
ticket = bundle.data.get("ticket", None)
if not title:
content["message"] = "Must provide a title."
self.crits_response(content)
if not data_type:
content["message"] = "Must provide a data type."
self.crits_response(content)
result = handle_raw_data_file(
data,
source,
analyst,
description,
title,
data_type,
tool_name,
tool_version,
tool_details,
link_id,
method=method,
reference=reference,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket,
)
if result.get("message"):
content["message"] = result.get("message")
if result.get("_id"):
url = reverse(
"api_dispatch_detail",
kwargs={"resource_name": "raw_data", "api_name": "v1", "pk": str(result.get("_id"))},
)
content["url"] = url
content["id"] = str(result.get("_id"))
if result["success"]:
content["return_code"] = 0
self.crits_response(content)
def obj_create(self, bundle, **kwargs):
"""
Handles creating RawData through the API.
:param bundle: Bundle containing the information to create the RawData.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
user = bundle.request.user
type_ = bundle.data.get('upload_type', None)
content = {'return_code': 1, 'type': 'RawData'}
if not type_:
content['message'] = 'Must provide an upload type.'
self.crits_response(content)
if type_ not in ('metadata', 'file'):
content['message'] = 'Not a valid upload type.'
self.crits_response(content)
if type_ == 'metadata':
data = bundle.data.get('data', None)
elif type_ == 'file':
file_ = bundle.data.get('filedata', None)
if not file_:
content[
'message'] = "Upload type of 'file' but no file uploaded."
self.crits_response(content)
data = file_.read()
source = bundle.data.get('source', None)
description = bundle.data.get('description', '')
title = bundle.data.get('title', None)
data_type = bundle.data.get('data_type', None)
tool_name = bundle.data.get('tool_name', '')
tool_version = bundle.data.get('tool_version', '')
tool_details = bundle.data.get('tool_details', '')
link_id = bundle.data.get('link_id', None)
copy_rels = bundle.data.get('copy_relationships', False)
method = bundle.data.get('method', None) or 'Upload'
reference = bundle.data.get('reference', None)
tlp = bundle.data.get('tlp', 'amber')
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if not title:
content['message'] = "Must provide a title."
self.crits_response(content)
if not data_type:
content['message'] = "Must provide a data type."
self.crits_response(content)
if not user.has_access_to(RawDataACL.WRITE):
content[
'message'] = 'User does not have permission to create Object.'
self.crits_response(content)
result = handle_raw_data_file(data,
source,
user,
description,
title,
data_type,
tool_name,
tool_version,
tool_details,
link_id,
method=method,
reference=reference,
tlp=tlp,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket)
if result.get('message'):
content['message'] = result.get('message')
if result.get('_id'):
url = reverse('api_dispatch_detail',
kwargs={
'resource_name': 'raw_data',
'api_name': 'v1',
'pk': str(result.get('_id'))
})
content['url'] = url
content['id'] = str(result.get('_id'))
if result['success']:
content['return_code'] = 0
self.crits_response(content)
def obj_create(self, bundle, **kwargs):
"""
Handles creating RawData through the API.
:param bundle: Bundle containing the information to create the RawData.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
user = bundle.request.user
type_ = bundle.data.get('upload_type', None)
content = {'return_code': 1,
'type': 'RawData'}
if not type_:
content['message'] = 'Must provide an upload type.'
self.crits_response(content)
if type_ not in ('metadata', 'file'):
content['message'] = 'Not a valid upload type.'
self.crits_response(content)
if type_ == 'metadata':
data = bundle.data.get('data', None)
elif type_ == 'file':
file_ = bundle.data.get('filedata', None)
if not file_:
content['message'] = "Upload type of 'file' but no file uploaded."
self.crits_response(content)
data = file_.read()
source = bundle.data.get('source', None)
description = bundle.data.get('description', '')
title = bundle.data.get('title', None)
data_type = bundle.data.get('data_type', None)
tool_name = bundle.data.get('tool_name', '')
tool_version = bundle.data.get('tool_version', '')
tool_details = bundle.data.get('tool_details', '')
link_id = bundle.data.get('link_id', None)
copy_rels = bundle.data.get('copy_relationships', False)
method = bundle.data.get('method', None) or 'Upload'
reference = bundle.data.get('reference', None)
tlp = bundle.data.get('tlp', 'amber')
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if not title:
content['message'] = "Must provide a title."
self.crits_response(content)
if not data_type:
content['message'] = "Must provide a data type."
self.crits_response(content)
if not user.has_access_to(RawDataACL.WRITE):
content['message'] = 'User does not have permission to create Object.'
self.crits_response(content)
result = handle_raw_data_file(data, source, user,
description, title, data_type,
tool_name, tool_version, tool_details,
link_id,
method=method,
reference=reference,
tlp=tlp,
copy_rels=copy_rels,
bucket_list=bucket_list,
ticket=ticket)
if result.get('message'):
content['message'] = result.get('message')
if result.get('_id'):
url = reverse('api_dispatch_detail',
kwargs={'resource_name': 'raw_data',
'api_name': 'v1',
'pk': str(result.get('_id'))})
content['url'] = url
content['id'] = str(result.get('_id'))
if result['success']:
content['return_code'] = 0
self.crits_response(content)
