def obj_create(self, bundle, **kwargs): """ Handles creating RawData through the API. :param bundle: Bundle containing the information to create the RawData. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If filedata is not provided or creation fails. """ analyst = bundle.request.user.username type_ = bundle.data.get('upload_type', None) if not type_: raise BadRequest('Must provide an upload type.') if type_ not in ('metadata', 'file'): raise BadRequest('Not a valid upload type.') if type_ == 'metadata': data = bundle.data.get('data', None) elif type_ == 'file': file_ = bundle.data.get('filedata', None) if not file_: raise BadRequest("Upload type of 'file' but no file uploaded.") data = file_.read() source = bundle.data.get('source', None) description = bundle.data.get('description', '') title = bundle.data.get('title', None) data_type = bundle.data.get('data_type', None) tool_name = bundle.data.get('tool_name', '') tool_version = bundle.data.get('tool_version', '') tool_details = bundle.data.get('tool_details', '') link_id = bundle.data.get('link_id', None) copy_rels = bundle.data.get('copy_relationships', False) method = 'Upload' bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) status = handle_raw_data_file(data, source, analyst, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket) if status['success']: return bundle else: raise BadRequest(status['message'])
def obj_create(self, bundle, **kwargs): """ Handles creating RawData through the API. :param bundle: Bundle containing the information to create the RawData. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If filedata is not provided or creation fails. """ analyst = bundle.request.user.username type_ = bundle.data.get('upload_type', None) if not type_: raise BadRequest('Must provide an upload type.') if type_ not in ('metadata', 'file'): raise BadRequest('Not a valid upload type.') if type_ == 'metadata': data = bundle.data.get('data', None) elif type_ == 'file': file_ = bundle.data.get('filedata', None) if not file_: raise BadRequest("Upload type of 'file' but no file uploaded.") data = file_.read() source = bundle.data.get('source', None) description = bundle.data.get('description', '') title = bundle.data.get('title', None) data_type = bundle.data.get('data_type', None) tool_name = bundle.data.get('tool_name', '') tool_version = bundle.data.get('tool_version', '') tool_details = bundle.data.get('tool_details', '') link_id = bundle.data.get('link_id', None) copy_rels = bundle.data.get('copy_relationships', False) method = 'Upload' bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if not title: raise BadRequest("Must provide a title.") if not data_type: raise BadRequest("Must provide a data type.") status = handle_raw_data_file(data, source, analyst, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket) if status['success']: return bundle else: raise BadRequest(status['message'])
def run(self, obj, config): self.config = config self.obj = obj self._debug("pdf2txt started\n") pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext") # The _write_to_file() context manager will delete this file at the # end of the "with" block. with self._write_to_file() as tmp_file: (working_dir, filename) = os.path.split(tmp_file) args = [pdf2txt_path, filename, "-"] # pdftotext does not generate a lot of output, so we should not have to # worry about this hanging because the buffer is full proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir) # Note that we are redirecting STDERR to STDOUT, so we can ignore # the second element of the tuple returned by communicate(). output = proc.communicate()[0] self._debug(output) if proc.returncode: msg = ("pdftotext could not process the file.") self._warning(msg) return raw_hash = md5(output).hexdigest() res = handle_raw_data_file( output, self.obj.source, self.current_task.username, title="pdftotext", data_type='text', tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org', method=self.name, copy_rels=True) raw_obj = class_from_id("RawData", res["_id"]) self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']))) # update relationship if a related top-level object is supplied rel_type = RelationshipTypes.RELATED_TO if obj.id != raw_obj.id: #don't form relationship to itself resy = obj.add_relationship(rel_item=raw_obj, rel_type=rel_type, rel_date=datetime.now(), analyst=self.current_task.username) obj.save(username=self.current_task.username) raw_obj.save(username=self.current_task.username) self._warning("resy: %s" % (str(resy))) self._add_result("rawdata_added", raw_hash, {'md5': raw_hash}) return
def run(self, obj, config): self.config = config self.obj = obj self._debug("pdf2txt started\n") pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext") # The _write_to_file() context manager will delete this file at the # end of the "with" block. with self._write_to_file() as tmp_file: (working_dir, filename) = os.path.split(tmp_file) args = [pdf2txt_path, filename, "-"] # pdftotext does not generate a lot of output, so we should not have to # worry about this hanging because the buffer is full proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=working_dir) # Note that we are redirecting STDERR to STDOUT, so we can ignore # the second element of the tuple returned by communicate(). output = proc.communicate()[0] self._debug(output) if proc.returncode: msg = ("pdftotext could not process the file.") self._warning(msg) return raw_hash = md5(output).hexdigest() res = handle_raw_data_file(output, self.obj.source, self.current_task.username, title="pdftotext", data_type='text', tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org', method=self.name, copy_rels=True) raw_obj = class_from_id("RawData", res["_id"]) self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) ) # update relationship if a related top-level object is supplied rel_type = RelationshipTypes.RELATED_TO if obj.id != raw_obj.id: #don't form relationship to itself resy = obj.add_relationship(rel_item=raw_obj, rel_type=rel_type, rel_date=datetime.now(), analyst=self.current_task.username) obj.save(username=self.current_task.username) raw_obj.save(username=self.current_task.username) self._warning("resy: %s" % (str(resy)) ) self._add_result("rawdata_added", raw_hash, {'md5': raw_hash}) return
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if not obs.object_ or not obs.object_.properties: self.failed.append(("No valid object_properties was found!", type(obs).__name__, obs.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, is_add_indicator=True) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False) self.parse_res(imp_type, obs, res) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] res = handle_email_fields(data, analyst, "STIX") # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append((obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if obs.observable_composition: object_list = obs.observable_composition.observables else: object_list = [obs] for obs_comp in object_list: if not obs_comp.object_ or not obs_comp.object_.properties: self.failed.append( ("No valid object_properties was found!", type(obs_comp).__name__, obs_comp.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs_comp.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask', 'ipv6-subnet'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, id=self.package.id_) self.parse_res(imp_type, obs, res) if item.extracted_features: self.parse_filenames(item.extracted_features, res['object'].id) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data[ 'source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to] if item.header.cc: data['cc'] = [str(r) for r in item.header.cc] res = handle_email_fields(data, analyst, "STIX", id=self.package.id_) # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append( (obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def upload_raw_data(request, link_id=None): """ Upload new RawData to CRITs. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param link_id: The LinkId of RawData if this is a new version upload. :type link_id: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': if 'filedata' in request.FILES: form = UploadRawDataFileForm(request.user, request.POST, request.FILES) filedata = request.FILES['filedata'] data = filedata.read() # XXX: Should be using chunks here. has_file = True else: form = UploadRawDataForm(request.user,request.POST) data = request.POST.get('data', None) has_file = False if form.is_valid(): source = form.cleaned_data.get('source_name') user = request.user description = form.cleaned_data.get('description', '') title = form.cleaned_data.get('title', None) tool_name = form.cleaned_data.get('tool_name', '') tool_version = form.cleaned_data.get('tool_version', '') tool_details = form.cleaned_data.get('tool_details', '') data_type = form.cleaned_data.get('data_type', None) copy_rels = request.POST.get('copy_relationships', False) link_id = link_id bucket_list = form.cleaned_data.get('bucket_list') ticket = form.cleaned_data.get('ticket') method = form.cleaned_data.get('source_method', '') or 'Upload' reference = form.cleaned_data.get('source_reference', '') tlp = form.cleaned_data.get('source_tlp', '') related_id = form.cleaned_data.get('related_id', '') related_type = form.cleaned_data.get('related_type', '') relationship_type = form.cleaned_data.get('relationship_type') status = handle_raw_data_file(data, source, user, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, tlp=tlp, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket, related_id=related_id, related_type=related_type, relationship_type=relationship_type) if status['success']: jdump = json.dumps({ 'message': 'raw_data uploaded successfully! <a href="%s">View raw_data</a>' % reverse('crits.raw_data.views.raw_data_details', args=[status['_id']]), 'success': True}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: jdump = json.dumps({'success': False, 'message': status['message']}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: jdump = json.dumps({'success': False, 'form': form.as_table()}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: return render_to_response('error.html', {'error': "Expected POST."}, RequestContext(request))
def run(self, obj, config): obj.filedata.seek(0) data8 = obj.filedata.read(8) obj.filedata.seek(0) self.config = config self.obj = obj self._debug("pdf2txt started") pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext") antiword_path = self.config.get("antiword_path", "/usr/bin/antiword") # The _write_to_file() context manager will delete this file at the # end of the "with" block. with self._write_to_file() as tmp_file: (working_dir, filename) = os.path.split(tmp_file) new_env = dict(os.environ) # Copy current environment args = [] obj.filedata.seek(0) if obj.is_pdf(): self._debug("PDF") args = [pdf2txt_path, filename, "-"] elif data8.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): self._debug("Word") #new_env['LANG'] = 'en_US' #env=dict(os.environ, LANG="en_US") args = [antiword_path, '-r', '-s', '-t', filename] else: self._error("Not a valid PDF or Word document") return False # pdftotext does not generate a lot of output, so we should not have to # worry about this hanging because the buffer is full proc = subprocess.Popen(args, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir) # Note that we are redirecting STDERR to STDOUT, so we can ignore # the second element of the tuple returned by communicate(). output, serr = proc.communicate() if serr: self._warning(serr) if proc.returncode: msg = ("pdftotext could not process the file.") self._warning(msg) return raw_hash = md5(output).hexdigest() res = handle_raw_data_file(output, self.obj.source, self.current_task.username, title="pdftotext", data_type='Text', tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org', method=self.name, copy_rels=True) raw_obj = class_from_id("RawData", res["_id"]) self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) ) # update relationship if a related top-level object is supplied rel_type = RelationshipTypes.RELATED_TO if obj.id != raw_obj.id: #don't form relationship to itself resy = obj.add_relationship(rel_item=raw_obj, rel_type=rel_type, rel_date=datetime.now(), analyst=self.current_task.username) obj.save(username=self.current_task.username) raw_obj.save(username=self.current_task.username) self._warning("resy: %s" % (str(resy)) ) self._add_result("rawdata_added", raw_hash, {'md5': raw_hash}) return
def upload_raw_data(request, link_id=None): """ Upload new RawData to CRITs. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param link_id: The LinkId of RawData if this is a new version upload. :type link_id: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': if 'filedata' in request.FILES: form = UploadRawDataFileForm(request.user, request.POST, request.FILES) filedata = request.FILES['filedata'] data = filedata.read() # XXX: Should be using chunks here. has_file = True else: form = UploadRawDataForm(request.user, request.POST) data = request.POST.get('data', None) has_file = False if form.is_valid(): source = form.cleaned_data.get('source_name') user = request.user description = form.cleaned_data.get('description', '') title = form.cleaned_data.get('title', None) tool_name = form.cleaned_data.get('tool_name', '') tool_version = form.cleaned_data.get('tool_version', '') tool_details = form.cleaned_data.get('tool_details', '') data_type = form.cleaned_data.get('data_type', None) copy_rels = request.POST.get('copy_relationships', False) link_id = link_id bucket_list = form.cleaned_data.get('bucket_list') ticket = form.cleaned_data.get('ticket') method = form.cleaned_data.get('source_method', '') or 'Upload' reference = form.cleaned_data.get('source_reference', '') tlp = form.cleaned_data.get('source_tlp', '') related_id = form.cleaned_data.get('related_id', '') related_type = form.cleaned_data.get('related_type', '') relationship_type = form.cleaned_data.get('relationship_type') status = handle_raw_data_file(data, source, user, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, tlp=tlp, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket, related_id=related_id, related_type=related_type, relationship_type=relationship_type) if status['success']: jdump = json.dumps({ 'message': 'raw_data uploaded successfully! <a href="%s">View raw_data</a>' % reverse('crits.raw_data.views.raw_data_details', args=[status['_id']]), 'success': True }) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: jdump = json.dumps({ 'success': False, 'message': status['message'] }) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: jdump = json.dumps({'success': False, 'form': form.as_table()}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response('file_upload_response.html', {'response': jdump}, RequestContext(request)) else: return render_to_response('error.html', {'error': "Expected POST."}, RequestContext(request))
def run_archive_viewer(self, obj): """ Get data using the archive viewer. """ safe = [ 'pyi_carchive', 'pyi_rth_win32comgenpy', '_pyi_bootstrap', '_pyi_egg_install.py' ] # This doesn't work. Everything is showing as an invalid CArchive file. with self._write_to_file() as tmp_file: try: arch = get_archive(tmp_file) if type(arch.toc) == type({}): toc = arch.toc else: toc = arch.toc.data for t in toc: d = {'Position': t[0], 'Length': t[1], 'Uncompressed': t[2], 'IsCompressed': t[3], 'Type': t[4], 'RawData': "" } if t[4] == 's' and t[5] not in safe: try: block = self.get_data(t[5], arch).encode('utf-8', "ignore") except: self._info("%s: Block not valid utf-8. Trying utf-16." % t[5]) try: block = self.get_data(t[5], arch).encode('utf-16', "ignore") except: self._info("%s: Block not valid utf-16. Trying utf-32." % t[5]) try: block = self.get_data(t[5], arch).encode('utf-32', "ignore") except: self._info("%s: Block not valid utf-32. Trying latin-1." % t[5]) try: block = self.get_data(t[5], arch).encode('latin-1', 'ignore') except: self._info("%s: Block not valid latin-1. Done trying." % t[5]) block = None if block is not None: bmd5 = md5(block).hexdigest() bsha1 = sha1(block).hexdigest() bsha256 = sha256(block).hexdigest() block = block.replace('http', 'hxxp') description = '"%s" pulled from Sample\n\n' % t[5] description += 'MD5: %s\n' % bmd5 description += 'SHA1: %s\n' % bsha1 description += 'SHA256: %s\n' % bsha256 title = t[5] data_type = "Python" tool_name = "pyinstaller_service" result = handle_raw_data_file( block, obj.source, user=self.current_task.user, description=description, title=title, data_type=data_type, tool_name=tool_name, ) if result['success']: self._info("RawData added for %s" % t[5]) res = obj.add_relationship( rel_item=result['object'], rel_type=RelationshipTypes.CONTAINED_WITHIN, rel_confidence="high", analyst=self.current_task.user ) if res['success']: obj.save(username=self.current_task.user.username) result['object'].save(username=self.current_task.user.username) url = reverse('crits-core-views.details', args=('RawData', result['_id'])) url = '<a href="%s">View Raw Data</a>' % url d['RawData'] = url self._info("Relationship added for %s" % t[5]) else: self._info("Error adding relationship: %s" % res['message']) else: self._info( "RawData addition failed for %s:%s" % (t[5], result['message']) ) self._add_result("Info", t[5], d) except Exception, e: self._info("Error: %s" % str(e))
class MacroExtractService(Service): """ Attempts to extract VBA Macros from MS Office files. """ name = "macro_extract" version = '0.1.0' supported_types = ['Sample'] description = "Extracs VBA Macros from MS Office documents." @staticmethod def get_config(existing_config): # This service no longer uses config options, so blow away any existing # configs. return {} @staticmethod def valid_for(obj): return def run(self, obj, config): username = self.current_task.username filename = obj.filename filedata = obj.filedata.read() try: vbaparser = VBA_Parser(filename, data=filedata) except Exception, e: self._error("Cannot parse file: %s" % str(e)) return if vbaparser.detect_vba_macros(): for (filename, stream_path, vba_filename, vba_code) in vbaparser.extract_macros(): d = { 'OLE stream': stream_path, 'VBA filename': vba_filename, 'Length': len(vba_code) } result = handle_raw_data_file( vba_code, obj.source, user=username, description="VBA Macro source code for %s" % vba_filename, title=vba_filename, data_type="Text", tool_name=self.name, tool_version=self.version, tool_details=self.description) if result['success']: obj.add_relationship( result['object'], RelationshipTypes.RELATED_TO, analyst=username, rel_reason="Extracted from related Sample") obj.save() d['RawData TLO ID'] = result['_id'] self._add_result('Macros', filename, d) results = vbaparser.analyze_macros(show_decoded_strings=True) self._add_result('Counts', 'Suspicious keywords', {'Count': vbaparser.nb_suspicious}) self._add_result('Counts', 'AutoExec keywords', {'Count': vbaparser.nb_autoexec}) self._add_result('Counts', 'IOCs', {'Count': vbaparser.nb_iocs}) self._add_result('Counts', 'Hex obfuscated strings', {'Count': vbaparser.nb_hexstrings}) self._add_result('Counts', 'Base64 obfuscated strings', {'Count': vbaparser.nb_base64strings}) self._add_result('Counts', 'Dridex obfuscated strings', {'Count': vbaparser.nb_dridexstrings}) self._add_result('Counts', 'VBA obfuscated strings', {'Count': vbaparser.nb_vbastrings}) for kw_type, keyword, description in results: try: d = { 'type': kw_type, 'description': description.decode('utf-8') } self._add_result('Keywords', keyword.decode('utf-8'), d) except: pass else: self._info('No VBA Macros found') vbaparser.close() return
def upload_raw_data(request, link_id=None): """ Upload new RawData to CRITs. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param link_id: The LinkId of RawData if this is a new version upload. :type link_id: str :returns: :class:`django.http.HttpResponse` """ if request.method == "POST": if "filedata" in request.FILES: form = UploadRawDataFileForm(request.user, request.POST, request.FILES) filedata = request.FILES["filedata"] data = filedata.read() # XXX: Should be using chunks here. has_file = True else: form = UploadRawDataForm(request.user, request.POST) data = request.POST.get("data", None) has_file = False if form.is_valid(): source = form.cleaned_data.get("source") user = request.user.username description = form.cleaned_data.get("description", "") title = form.cleaned_data.get("title", None) tool_name = form.cleaned_data.get("tool_name", "") tool_version = form.cleaned_data.get("tool_version", "") tool_details = form.cleaned_data.get("tool_details", "") data_type = form.cleaned_data.get("data_type", None) copy_rels = request.POST.get("copy_relationships", False) link_id = link_id bucket_list = form.cleaned_data.get("bucket_list") ticket = form.cleaned_data.get("ticket") method = form.cleaned_data.get("method", "") or "Upload" reference = form.cleaned_data.get("reference", "") related_id = form.cleaned_data.get("related_id", "") related_type = form.cleaned_data.get("related_type", "") relationship_type = form.cleaned_data.get("relationship_type") status = handle_raw_data_file( data, source, user, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket, related_id=related_id, related_type=related_type, relationship_type=relationship_type, ) if status["success"]: jdump = json.dumps( { "message": 'raw_data uploaded successfully! <a href="%s">View raw_data</a>' % reverse("crits.raw_data.views.raw_data_details", args=[status["_id"]]), "success": True, } ) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request)) else: jdump = json.dumps({"success": False, "message": status["message"]}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request)) else: jdump = json.dumps({"success": False, "form": form.as_table()}) if not has_file: return HttpResponse(jdump, content_type="application/json") return render_to_response("file_upload_response.html", {"response": jdump}, RequestContext(request)) else: return render_to_response("error.html", {"error": "Expected POST."}, RequestContext(request))
def run(self, obj, config): obj.filedata.seek(0) data8 = obj.filedata.read(8) obj.filedata.seek(0) self.config = config self.obj = obj self._debug("pdf2txt started") pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext") antiword_path = self.config.get("antiword_path", "/usr/bin/antiword") # The _write_to_file() context manager will delete this file at the # end of the "with" block. with self._write_to_file() as tmp_file: (working_dir, filename) = os.path.split(tmp_file) new_env = dict(os.environ) # Copy current environment args = [] obj.filedata.seek(0) if obj.is_pdf(): self._debug("PDF") args = [pdf2txt_path, filename, "-"] elif data8.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): self._debug("Word") #new_env['LANG'] = 'en_US' #env=dict(os.environ, LANG="en_US") args = [antiword_path, '-r', '-s', '-t', filename] else: self._error("Not a valid PDF or Word document") return False # pdftotext does not generate a lot of output, so we should not have to # worry about this hanging because the buffer is full proc = subprocess.Popen(args, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir) # Note that we are redirecting STDERR to STDOUT, so we can ignore # the second element of the tuple returned by communicate(). output, serr = proc.communicate() if serr: self._warning(serr) if proc.returncode: msg = ("pdftotext could not process the file.") self._warning(msg) return raw_hash = md5(output).hexdigest() res = handle_raw_data_file( output, self.obj.source, self.current_task.username, title="pdftotext", data_type='Text', tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org', method=self.name, copy_rels=True) raw_obj = class_from_id("RawData", res["_id"]) self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']))) # update relationship if a related top-level object is supplied rel_type = RelationshipTypes.RELATED_TO if obj.id != raw_obj.id: #don't form relationship to itself resy = obj.add_relationship(rel_item=raw_obj, rel_type=rel_type, rel_date=datetime.now(), analyst=self.current_task.username) obj.save(username=self.current_task.username) raw_obj.save(username=self.current_task.username) self._warning("resy: %s" % (str(resy))) self._add_result("rawdata_added", raw_hash, {'md5': raw_hash}) return
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = { 'success': False, 'reason': 'No IP Type' } self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get( 'date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file( data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault( cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind( data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append( (oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append( (oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = {'success': False, 'reason': 'No IP Type'} self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get('date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file(data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind(data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append((oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append((oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI
def run_archive_viewer(self, obj): """ Get data using the archive viewer. """ safe = [ 'pyi_carchive', 'pyi_rth_win32comgenpy', '_pyi_bootstrap', '_pyi_egg_install.py' ] # This doesn't work. Everything is showing as an invalid CArchive file. with self._write_to_file() as tmp_file: try: arch = get_archive(tmp_file) if type(arch.toc) == type({}): toc = arch.toc else: toc = arch.toc.data for t in toc: d = { 'Position': t[0], 'Length': t[1], 'Uncompressed': t[2], 'IsCompressed': t[3], 'Type': t[4], 'RawData': "" } if t[4] == 's' and t[5] not in safe: try: block = self.get_data(t[5], arch).encode( 'utf-8', "ignore") except: self._info( "%s: Block not valid utf-8. Trying utf-16." % t[5]) try: block = self.get_data(t[5], arch).encode( 'utf-16', "ignore") except: self._info( "%s: Block not valid utf-16. Trying utf-32." % t[5]) try: block = self.get_data(t[5], arch).encode( 'utf-32', "ignore") except: self._info( "%s: Block not valid utf-32. Trying latin-1." % t[5]) try: block = self.get_data(t[5], arch).encode( 'latin-1', 'ignore') except: self._info( "%s: Block not valid latin-1. Done trying." % t[5]) block = None if block is not None: bmd5 = md5(block).hexdigest() bsha1 = sha1(block).hexdigest() bsha256 = sha256(block).hexdigest() block = block.replace('http', 'hxxp') description = '"%s" pulled from Sample\n\n' % t[5] description += 'MD5: %s\n' % bmd5 description += 'SHA1: %s\n' % bsha1 description += 'SHA256: %s\n' % bsha256 title = t[5] data_type = "Python" tool_name = "pyinstaller_service" result = handle_raw_data_file( block, obj.source, user=self.current_task.user, description=description, title=title, data_type=data_type, tool_name=tool_name, ) if result['success']: self._info("RawData added for %s" % t[5]) res = obj.add_relationship( rel_item=result['object'], rel_type=RelationshipTypes. CONTAINED_WITHIN, rel_confidence="high", analyst=self.current_task.user) if res['success']: obj.save(username=self.current_task.user. username) result['object'].save( username=self.current_task.user. username) url = reverse('crits-core-views.details', args=('RawData', result['_id'])) url = '<a href="%s">View Raw Data</a>' % url d['RawData'] = url self._info("Relationship added for %s" % t[5]) else: self._info( "Error adding relationship: %s" % res['message']) else: self._info( "RawData addition failed for %s:%s" % (t[5], result['message'])) self._add_result("Info", t[5], d) except Exception, e: self._info("Error: %s" % str(e))
def run(self, obj, config): self.config = config self.obj = obj user = self.current_task.user tlp_value = self.config.get("tlp_value", "tlp_value") url = obj['value'] if not (obj._meta['crits_type'] == 'Indicator' and obj['ind_type'] == 'URI'): self._error('This object type cannot use service Url analysis.') return False #verify url http or https if url.startswith('https://') or url.startswith('http://'): #put url in file dcap = dict(DesiredCapabilities.PHANTOMJS) dcap["phantomjs.page.settings.userAgent"] = ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" ) driver = webdriver.PhantomJS(desired_capabilities=dcap, service_args=[ '--ignore-ssl-errors=true', '--ssl-protocol=any', '--web-security=false' ]) driver.set_window_size(1024, 768) driver.set_page_load_timeout(30) driver.get(url) time.sleep(3) #driver.save_screenshot('testing1.png') screen = driver.get_screenshot_as_png() ofile = io.BytesIO() im = Image.open(StringIO.StringIO(screen)) im.save(ofile, 'PNG', optimize=True) ofile.seek(0) res = add_screenshot(description='Render of a website URL', tags=None, method=self.name, source=obj.source, reference=None, analyst=self.current_task.user.username, screenshot=ofile, screenshot_ids=None, oid=obj.id, tlp=tlp_value, otype="Indicator") if res.get('message') and res.get('success') == True: self._warning("res-message: %s id:%s" % (res.get('message'), res.get('id'))) self._add_result('ScreenShot URL', res.get('id'), {'Message': res.get('message')}) #parse HAR har = driver.get_log('har') if type(har) is list and har: if type(har[0]) is dict and 'message' in har[0]: #change unicode to json try: har[0]['message'] = json.loads(har[0]['message']) except: self._warning('Har log error to parse json') if type(har[0]['message'] ) is dict and 'log' in har[0]['message'] and type( har[0]['message']['log'] ) is dict and 'pages' in har[0]['message']['log']: if type(har[0]['message']['log']['pages'] ) is list and har[0]['message']['log'][ 'pages'] and type(har[0]['message']['log'] ['pages'][0]) is dict: title = 'Result of ' if 'id' in har[0]['message']['log']['pages'][0]: title += har[0]['message']['log']['pages'][0][ 'id'] if 'title' in har[0]['message']['log']['pages'][0]: self._add_result( title, 'Title', { 'value': har[0]['message']['log']['pages'][0] ['title'] }) #parse each request and response if 'entries' in har[0]['message']['log'] and type( har[0]['message']['log']['entries'] ) is list and har[0]['message']['log']['entries']: count = 1 type_r = ['cookies', 'queryString', 'headers'] type_rs = ['content', 'timings', 'cache'] for elem_rr in har[0]['message']['log']['entries']: for k, v in elem_rr.iteritems(): if type(v) is not dict: self._add_result( title + ' -- Informations Request & Response num:' + str(count), k, {'value': v}) for k, v in elem_rr.iteritems(): if type(v) is dict: for kx, vx in v.iteritems(): self._add_result( title + ' -- Informations Request & Response num:' + str(count) + ' -- ' + str(k), kx, {'value': vx}) count += 1 #save page source in rawdata if not user.has_access_to(RawDataACL.WRITE): self._info(driver.page_source.encode('utf8')) else: #can write result = handle_raw_data_file( driver.page_source.encode('utf8'), obj.source, user=self.current_task.user, description="Code page for URL: %s" % url, title=url, data_type="Text", tool_name=self.name, tool_version=self.version, tool_details=self.description) if result['success']: obj.add_relationship( result['object'], RelationshipTypes.CONTAINED_WITHIN, analyst=self.current_task.user.username, rel_reason="Extracted from URI") obj.save() self._add_result( 'Code Page', url, { 'RawData TLO ID': result['_id'], 'md5 file': md5(driver.page_source.encode('utf8')).hexdigest(), 'simhash': str(simhash(driver.page_source.encode('utf8'))) }) driver.close() driver.service.process.terminate() time.sleep(1) #get certificat information - ref: https://stackoverflow.com/questions/30862099/how-can-i-get-certificate-issuer-information-in-python #because selenium not functionnality if url.startswith('https://'): try: host = urlparse(url).hostname port = urlparse(url).port if (port is None): port = 443 s = socks.socksocket() if settings.HTTP_PROXY: type_proxy = socks.PROXY_TYPE_SOCKS5 if settings.HTTP_PROXY.startswith('http://'): type_proxy = socks.PROXY_TYPE_HTTP s.setproxy(type_proxy, urlparse(settings.HTTP_PROXY).hostname, port=urlparse(settings.HTTP_PROXY).port) s.connect((host, port)) ss = ssl.wrap_socket(s) pem_data = ssl.DER_cert_to_PEM_cert(ss.getpeercert(True)) ss.close() s.close() cert = M2Crypto.X509.load_cert_string(pem_data) #put ssl information self._add_result( 'SSL informations', 'Subject', {'value': str(cert.get_subject().as_text())}) self._add_result( 'SSL informations', 'Issuer', {'value': str(cert.get_issuer().as_text())}) self._add_result('SSL informations', 'Version', {'value': str(cert.get_version())}) self._add_result('SSL informations', 'Date before', {'value': str(cert.get_not_before())}) self._add_result('SSL informations', 'Date after', {'value': str(cert.get_not_after())}) self._add_result('SSL informations', 'Serial Number', {'value': str(cert.get_serial_number())}) self._add_result('SSL informations', 'Verify', {'value': str(cert.verify())}) self._add_result('SSL informations', 'Fingerprint MD5', {'value': str(cert.get_fingerprint())}) for i in range(0, cert.get_ext_count()): self._add_result( 'SSL informations Extension', str(cert.get_ext_at(i).get_name()), {'value': str(cert.get_ext_at(i).get_value())}) #https://www.heikkitoivonen.net/m2crypto/api/M2Crypto.X509-module.html except: self._error('Error: get certificate informations.') self._info(str(cert)) driver.service.process.kill() driver.quit() self._info('END')
def obj_create(self, bundle, **kwargs): """ Handles creating RawData through the API. :param bundle: Bundle containing the information to create the RawData. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ analyst = bundle.request.user.username type_ = bundle.data.get("upload_type", None) content = {"return_code": 1, "type": "RawData"} if not type_: content["message"] = "Must provide an upload type." self.crits_response(content) if type_ not in ("metadata", "file"): content["message"] = "Not a valid upload type." self.crits_response(content) if type_ == "metadata": data = bundle.data.get("data", None) elif type_ == "file": file_ = bundle.data.get("filedata", None) if not file_: content["message"] = "Upload type of 'file' but no file uploaded." self.crits_response(content) data = file_.read() source = bundle.data.get("source", None) description = bundle.data.get("description", "") title = bundle.data.get("title", None) data_type = bundle.data.get("data_type", None) tool_name = bundle.data.get("tool_name", "") tool_version = bundle.data.get("tool_version", "") tool_details = bundle.data.get("tool_details", "") link_id = bundle.data.get("link_id", None) copy_rels = bundle.data.get("copy_relationships", False) method = bundle.data.get("method", None) or "Upload" reference = bundle.data.get("reference", None) bucket_list = bundle.data.get("bucket_list", None) ticket = bundle.data.get("ticket", None) if not title: content["message"] = "Must provide a title." self.crits_response(content) if not data_type: content["message"] = "Must provide a data type." self.crits_response(content) result = handle_raw_data_file( data, source, analyst, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket, ) if result.get("message"): content["message"] = result.get("message") if result.get("_id"): url = reverse( "api_dispatch_detail", kwargs={"resource_name": "raw_data", "api_name": "v1", "pk": str(result.get("_id"))}, ) content["url"] = url content["id"] = str(result.get("_id")) if result["success"]: content["return_code"] = 0 self.crits_response(content)
def obj_create(self, bundle, **kwargs): """ Handles creating RawData through the API. :param bundle: Bundle containing the information to create the RawData. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ user = bundle.request.user type_ = bundle.data.get('upload_type', None) content = {'return_code': 1, 'type': 'RawData'} if not type_: content['message'] = 'Must provide an upload type.' self.crits_response(content) if type_ not in ('metadata', 'file'): content['message'] = 'Not a valid upload type.' self.crits_response(content) if type_ == 'metadata': data = bundle.data.get('data', None) elif type_ == 'file': file_ = bundle.data.get('filedata', None) if not file_: content[ 'message'] = "Upload type of 'file' but no file uploaded." self.crits_response(content) data = file_.read() source = bundle.data.get('source', None) description = bundle.data.get('description', '') title = bundle.data.get('title', None) data_type = bundle.data.get('data_type', None) tool_name = bundle.data.get('tool_name', '') tool_version = bundle.data.get('tool_version', '') tool_details = bundle.data.get('tool_details', '') link_id = bundle.data.get('link_id', None) copy_rels = bundle.data.get('copy_relationships', False) method = bundle.data.get('method', None) or 'Upload' reference = bundle.data.get('reference', None) tlp = bundle.data.get('tlp', 'amber') bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if not title: content['message'] = "Must provide a title." self.crits_response(content) if not data_type: content['message'] = "Must provide a data type." self.crits_response(content) if not user.has_access_to(RawDataACL.WRITE): content[ 'message'] = 'User does not have permission to create Object.' self.crits_response(content) result = handle_raw_data_file(data, source, user, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, tlp=tlp, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket) if result.get('message'): content['message'] = result.get('message') if result.get('_id'): url = reverse('api_dispatch_detail', kwargs={ 'resource_name': 'raw_data', 'api_name': 'v1', 'pk': str(result.get('_id')) }) content['url'] = url content['id'] = str(result.get('_id')) if result['success']: content['return_code'] = 0 self.crits_response(content)
def obj_create(self, bundle, **kwargs): """ Handles creating RawData through the API. :param bundle: Bundle containing the information to create the RawData. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ user = bundle.request.user type_ = bundle.data.get('upload_type', None) content = {'return_code': 1, 'type': 'RawData'} if not type_: content['message'] = 'Must provide an upload type.' self.crits_response(content) if type_ not in ('metadata', 'file'): content['message'] = 'Not a valid upload type.' self.crits_response(content) if type_ == 'metadata': data = bundle.data.get('data', None) elif type_ == 'file': file_ = bundle.data.get('filedata', None) if not file_: content['message'] = "Upload type of 'file' but no file uploaded." self.crits_response(content) data = file_.read() source = bundle.data.get('source', None) description = bundle.data.get('description', '') title = bundle.data.get('title', None) data_type = bundle.data.get('data_type', None) tool_name = bundle.data.get('tool_name', '') tool_version = bundle.data.get('tool_version', '') tool_details = bundle.data.get('tool_details', '') link_id = bundle.data.get('link_id', None) copy_rels = bundle.data.get('copy_relationships', False) method = bundle.data.get('method', None) or 'Upload' reference = bundle.data.get('reference', None) tlp = bundle.data.get('tlp', 'amber') bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if not title: content['message'] = "Must provide a title." self.crits_response(content) if not data_type: content['message'] = "Must provide a data type." self.crits_response(content) if not user.has_access_to(RawDataACL.WRITE): content['message'] = 'User does not have permission to create Object.' self.crits_response(content) result = handle_raw_data_file(data, source, user, description, title, data_type, tool_name, tool_version, tool_details, link_id, method=method, reference=reference, tlp=tlp, copy_rels=copy_rels, bucket_list=bucket_list, ticket=ticket) if result.get('message'): content['message'] = result.get('message') if result.get('_id'): url = reverse('api_dispatch_detail', kwargs={'resource_name': 'raw_data', 'api_name': 'v1', 'pk': str(result.get('_id'))}) content['url'] = url content['id'] = str(result.get('_id')) if result['success']: content['return_code'] = 0 self.crits_response(content)