def run(self, obj, config): if isinstance(obj, Event): data = obj.description elif isinstance(obj, RawData): data = obj.data elif isinstance(obj, Sample): samp_data = obj.filedata.read() data = make_ascii_strings(data=samp_data) if not data: self._debug("Could not find sample data to parse.") return else: self._debug("This type is not supported by this service.") return ips = extract_ips(data) for ip in ips: tdict = {'Type': IndicatorTypes.IPV4_ADDRESS} id_ = Indicator.objects(value=ip).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential IP Address', ip, tdict) domains = extract_domains(data) for domain in domains: tdict = {'Type': IndicatorTypes.DOMAIN} id_ = Indicator.objects(value=domain).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Domains', domain, tdict) emails = extract_emails(data) for email in emails: tdict = {'Type': IndicatorTypes.EMAIL_ADDRESS} id_ = Indicator.objects(value=email).only('id').first() if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Emails', email, tdict) hashes = extract_hashes(data) for hash_ in hashes: type_ = hash_[0] val = hash_[1] tdict = {'Type': type_} if type_ == IndicatorTypes.MD5: id_ = Sample.objects(md5=val).only('id').first() elif type_ == IndicatorTypes.SHA1: id_ = Sample.objects(sha1=val).only('id').first() elif type_ == IndicatorTypes.SHA256: id_ = Sample.objects(sha256=val).only('id').first() elif type_ == IndicatorTypes.SSDEEP: id_ = Sample.objects(ssdeep=val).only('id').first() else: id_ = None if id_: tdict['exists'] = str(id_.id) self._add_result('Potential Samples', val, tdict)
def run(self, obj, config): threshold = config.get("threshold", 50) target_ssdeep = obj.ssdeep target_md5 = obj.md5 target_mimetype = obj.mimetype if not target_ssdeep: logger.error = "Could not get the target ssdeep value for sample" self._error("Could not get the target ssdeep value for sample") return # setup the sample space to compare against # first use the mimetype as a comparator if available query_filter = {} if target_mimetype: query_filter['mimetype'] = target_mimetype # then use only samples with a multiple of chunksize chunk_size = int(target_ssdeep.split(":")[0]) query_filter["$or"] = [] query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size * 2}}) query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size}}) query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % (chunk_size / 2)}}) result_filter = {'md5': 1, 'ssdeep': 1} candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter) match_list = [] for candidate in candidate_space: if "ssdeep" in candidate: score = pydeep.compare(target_ssdeep, candidate["ssdeep"]) if score >= threshold and candidate["md5"] != target_md5: match_list.append({'md5': candidate["md5"], 'score': score}) # finally sort the results match_list.sort(key=lambda sample: sample["score"], reverse=True) for match in match_list: self._add_result("ssdeep_match", match["md5"], {'md5': match["md5"], 'score': match["score"]})
def snugglefish_search(indexes, search, user): """Execute search of selected index with the given string.""" # Return a dictionary where the key is the index name and the # value a dictionary with status and a list of potential matches. ret = {} # If there are no sources, return early. sources = user_sources(user) if not sources: return ret for idx in indexes: ret[idx] = {'success': True, 'reason': '', 'files': []} sngindex = SnuggleIndex.objects(name=idx).first() if not sngindex: ret[idx]['reason'] = "Index not found in database." ret[idx]['success'] = False continue snuggle = pysnugglefish.init(str(sngindex.directory + "/" + idx)) try: tmp = snuggle.search(search) for res in tmp: if Sample.objects(md5=res, source__name__in=sources).count() > 0: ret[idx]['files'].append(res) except Exception, e: ret[idx]['reason'] = "Error: %s" % e ret[idx]['success'] = False
def run(self, argv): parser = OptionParser() parser.add_option("-f", "--filter", action="store", dest="filter", type="string", help="filter for samples to discover binaries") (opts, args) = parser.parse_args(argv) if opts.filter: query = ast.literal_eval(opts.filter) else: query = {} errorpath = "/tmp/gridfs_migrate_errors.txt" try: err = open(errorpath, "w") except: print "Could not open file handle to write to: %s" % errorpath sys.exit(1) error_count = 0 samples = Sample.objects(__raw__=query) count = len(samples) print "Migrating %s samples found with query %s...\n" % (count, query) i = 1 for s in samples: md5 = s.md5 try: print >> sys.stdout, "\r\tWorking on sample %d of %d" % (i, count), sys.stdout.flush() s.discover_binary() s.save() except Exception, e: error_count += 1 err.write("Error saving sample for discover binary: %s - %s" % (md5, e)) i += 1
def _check_triage(self): sample = Sample.objects(md5=self.test_md5).first() results = False if sample and len(sample.analysis) > 0 and sample.filedata: results = True print "[?] sample analysis executed == %s" % results return results
def snugglefish_search(indexes, search, user): """Execute search of selected index with the given string.""" # Return a dictionary where the key is the index name and the # value a dictionary with status and a list of potential matches. ret = {} # If there are no sources, return early. sources = user_sources(user) if not sources: return ret for idx in indexes: ret[idx] = { 'success': True, 'reason': '', 'files': [] } sngindex = SnuggleIndex.objects(name=idx).first() if not sngindex: ret[idx]['reason'] = "Index not found in database." ret[idx]['success'] = False continue snuggle = pysnugglefish.init(str(sngindex.directory + "/" + idx)) try: tmp = snuggle.search(search) for res in tmp: if Sample.objects(md5=res, source__name__in=sources).count() > 0: ret[idx]['files'].append(res) except Exception, e: ret[idx]['reason'] = "Error: %s" % e ret[idx]['success'] = False
def run(self, obj, config): threshold = config.get("threshold", 50) target_ssdeep = obj.ssdeep target_md5 = obj.md5 target_mimetype = obj.mimetype if not target_ssdeep: logger.error = "Could not get the target ssdeep value for sample" self._error("Could not get the target ssdeep value for sample") return # setup the sample space to compare against # first use the mimetype as a comparator if available query_filter = {} if target_mimetype: query_filter['mimetype'] = target_mimetype # then use only samples with a multiple of chunksize chunk_size = int(target_ssdeep.split(":")[0]) query_filter["$or"] = [] query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size * 2}}) query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % chunk_size}}) query_filter["$or"].append({"ssdeep": {"$regex": "^%d:" % (chunk_size / 2)}}) result_filter = {'md5': 1, 'ssdeep': 1, 'description':1} candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter) match_list = [] for candidate in candidate_space: if "ssdeep" in candidate: score = pydeep.compare(target_ssdeep, candidate["ssdeep"]) if score >= threshold and candidate["md5"] != target_md5: match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score}) # finally sort the results match_list.sort(key=lambda sample: sample["score"], reverse=True) for match in match_list: self._add_result("ssdeep_match (MD5)", match["md5"], {'description': match["description"], 'score': match["score"]})
def _check_triage(self): sample = Sample.objects(md5=self.test_md5).first() results = False if sample and sample.filedata: if len(AnalysisResult.objects(object_id=str(sample.id))) > 0: results = True print "[?] sample analysis executed == %s" % results return results
def start_pyew_shell(request, id_, token): # Make sure we can find pyew svc = CRITsService.objects(name='Pyew').first() if not svc: text = "\nPyew not found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) sc = svc.config pyew = str(sc['pyew']) if not os.path.exists(pyew): text = "\nPyew not found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Find CRITs user by token query = {'unsupported_attrs.pyew_token': token} user = CRITsUser.objects(__raw__=query).first() if not user: text = "\nCould not validate user" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Remove this one-time use token ua = user.unsupported_attrs delattr(ua, 'pyew_token') user.unsupported_attrs = ua try: user.save() except: pass # Make sure we have a sample to work with that this user has access to sample = Sample.objects(id=id_, source__name__in=user.get_sources_list()).first() if not sample: text = "\nNo Sample found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) sample_data = sample.filedata.read() if not sample_data: text = "\nCould not get Sample from GridFS: %s" % id_ request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # write Sample to disk # temp_sample is the sample to read try: temp_sample = tempfile.NamedTemporaryFile(delete=False) sample_name = temp_sample.name temp_sample.write(sample_data) temp_sample.close() except Exception, e: text = "\nError writing file to disk: %s" % e request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1)
def _check_grid(self): sample = Sample.objects(md5=self.test_md5).first() result = False if sample: if sample.filedata: data = sample.filedata.read() result = data == self.test_data print "[?] check grid == %s" % result return result
def start_pyew_shell(request, id_, token): # Make sure we can find pyew sc = manager.get_config('Pyew') pyew = str(sc['pyew']) if not os.path.exists(pyew): text = "\nPyew not found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Find CRITs user by token query = {'unsupported_attrs.pyew_token': token} user = CRITsUser.objects(__raw__=query).first() if not user: text = "\nCould not validate user" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Remove this one-time use token ua = user.unsupported_attrs delattr(ua, 'pyew_token') user.unsupported_attrs = ua try: user.save() except: pass # Make sure we have a sample to work with that this user has access to sample = Sample.objects(id=id_, source__name__in=user.sources).first() if not sample: text = "\nNo Sample found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) sample_data = sample.filedata.read() if not sample_data: text = "\nCould not get Sample from GridFS: %s" % id_ request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # write Sample to disk # temp_sample is the sample to read try: temp_sample = tempfile.NamedTemporaryFile(delete=False) sample_name = temp_sample.name temp_sample.write(sample_data) temp_sample.close() except Exception, e: text = "\nError writing file to disk: %s" % e request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1)
def _del_sample(self): sample = Sample.objects(md5=self.test_md5).first() if sample: print "[-] deleting from grid" if sample.filedata: sample.filedata.delete() print "[-] deleting sample" sample.delete() else: print "[-] could not find sample to delete"
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Target': return Target.objects(email_address=value).first() else: return None
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [ a for a in obj.analysis if a.service_name != service_name ] obj.save()
def run(self, argv): parser = OptionParser() parser.add_option("-m", "--md5", action="store", dest="md5", type="string", help="filetype filter") (opts, args) = parser.parse_args(argv) try: if opts.md5: sample = Sample.objects(md5=opts.md5).first() except Exception as e: print "Bad things - '%s'" % e if sample: print sample.id
def _fetch_meta(self, query_filter, result_filter): """ Fetch sample metadata. :param query_filter: The filter to use to find the sample. :type query_filter: dict :param result_filter: Limit the result to these fields. :type result_filter: tuple :returns: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ self.ensure_current_task() results = Sample.objects(__raw__=query_filter).only(*result_filter) return results
def run(self, obj, config): self._info("Impfuzzy: run()") threshold = config.get("threshold", 50) target_impfuzzy = None try: target_impfuzzy = pyimpfuzzy.get_impfuzzy_data(obj.filedata.read()) except Exception: pass target_md5 = obj.md5 if not target_impfuzzy: logger.error = "impfuzzy: Could not generate impfuzzy value for sample: %s" % str(obj.id) self._error("Could not generate impfuzzy value for sample") return # setup the sample space to compare against # first use the mimetype as a comparator if available if obj.impfuzzy: obj.impfuzzy = target_impfuzzy obj.save() self._info("impfuzzy: Filled-in in the impfuzzy") else: self._info("impfuzzy attribute already present, not overwriting") self._add_result('impfuzzy_hash', target_impfuzzy,{'impfuzzy': target_impfuzzy}) target_mimetype = obj.mimetype query_filter = {} if target_mimetype: query_filter['mimetype'] = target_mimetype # then use only samples with a multiple of chunksize chunk_size = int(target_impfuzzy.split(":")[0]) query_filter["$or"] = [] query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size * 2}}) query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size}}) query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % (chunk_size // 2)}}) result_filter = {'md5': 1, 'impfuzzy': 1, 'description':1} candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter) # self.info("candidate: %s" % repr(candidate_space)) match_list = [] for candidate in candidate_space: if "impfuzzy" in candidate: score = pyimpfuzzy.hash_compare(target_impfuzzy, candidate["impfuzzy"]) if score >= threshold and candidate["md5"] != target_md5: # Grab the md5 and the description for later match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score}) # finally sort the results match_list.sort(key=lambda sample: sample["score"], reverse=True) for match in match_list: #Show the MD5 and the Description self._add_result("impfuzzy_match", match["md5"], {'description': match["description"], 'md5': match["md5"], 'score': match["score"]}) self._info("impfuzzy run() done")
def run(self, argv): parser = OptionParser() parser.add_option('-b', '--bucket', action='store', dest='bucket', type='string', help='bucket list name') parser.add_option("-o", "--output-file", action="store", dest="outfile", type="string", help="output archive file (no extension)") (opts, args) = parser.parse_args(argv) samples = Sample.objects(bucket_list=opts.bucket) if opts.bucket and opts.outfile: filename = "%s.tar.bz2" % opts.outfile try: tar = tarfile.open(filename, "w:bz2") except Exception as e: print("Error when attempting to open %s for writing: %s" % (filename, e)) sys.exit(1) count = len(samples) if count <= 0: print("No matching bucket name found!") sys.exit(1) for sample in samples: m = sample.md5 f = sample.filename s = sample.filedata.read() info = tarfile.TarInfo(name="%s" % f) info.mtime = time.time() if s is not None: info.size = len(s) else: info.size = 0 try: tar.addfile(info, BytesIO(s)) except Exception as e: print("Error attempting to add %s to the tarfile: %s" % (f, e)) pass tar.close() print("Generated %s containing %s files." % (filename, count))
def run(self, obj, config): threshold = config.get("threshold", 50) target_impfuzzy = None try: target_impfuzzy = pyimpfuzzy.get_impfuzzy_data(obj.filedata.read()) except Exception: pass target_md5 = obj.md5 if not target_impfuzzy: logger.error = "impfuzzy: Could not generate impfuzzy value for sample: %s" % str(obj.id) self._error("Could not generate impfuzzy value for sample") return # setup the sample space to compare against # first use the mimetype as a comparator if available if obj.impfuzzy: obj.impfuzzy = target_impfuzzy obj.save() self._info("impfuzzy: Filled-in in the impfuzzy") else: self._info("impfuzzy attribute already present, not overwriting") self._add_result('impfuzzy_hash', target_impfuzzy,{'impfuzzy': target_impfuzzy}) target_mimetype = obj.mimetype query_filter = {} if target_mimetype: query_filter['mimetype'] = target_mimetype # then use only samples with a multiple of chunksize chunk_size = int(target_impfuzzy.split(":")[0]) query_filter["$or"] = [] query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size * 2}}) query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % chunk_size}}) query_filter["$or"].append({"impfuzzy": {"$regex": "^%d:" % (chunk_size // 2)}}) result_filter = {'md5': 1, 'impfuzzy': 1, 'description':1} candidate_space = Sample.objects(__raw__=query_filter).only(*result_filter) # self.info("candidate: %s" % repr(candidate_space)) match_list = [] for candidate in candidate_space: if "impfuzzy" in candidate: score = pyimpfuzzy.hash_compare(target_impfuzzy, candidate["impfuzzy"]) if score >= threshold and candidate["md5"] != target_md5: # Grab the md5 and the description for later match_list.append({'md5': candidate["md5"], 'description': candidate["description"], 'score': score}) # finally sort the results match_list.sort(key=lambda sample: sample["score"], reverse=True) for match in match_list: #Show the MD5 and the Description self._add_result("impfuzzy_match (MD5)", match["md5"], {'description': match["description"], 'score': match["score"]})
def test_yara_rule(id_, rule): sample = Sample.objects(id=id_).first() data = sample.filedata.read() success = False message = "" if not sample or not data: message = "No sample found!" else: try: rules = yara.compile(source=rule) matches = rules.match(data=data) yara_results = [] mcount = 0 for match in matches: strings = {} for s in match.strings: s_name = s[1] s_offset = s[0] try: s_data = s[2].decode('ascii') except UnicodeError: s_data = "Hex: " + binascii.hexlify(s[2]) s_key = "{0}-{1}".format(s_name, s_data) if s_key in strings: strings[s_key]['offset'].append(s_offset) else: strings[s_key] = { 'rule': str(match), 'offset': [s_offset], 'name': s_name, 'data': s_data, } string_list = [] for key in strings: string_list.append(strings[key]) yara_results.append(string_list) mcount += 1 success = True if mcount == 0: yara_results.append("No matches!") message = pprint.pformat(yara_results) except SyntaxError as e: message = "Syntax error in YARA rule: %s" % str(e) return {"success": success, "message": message}
def create_sample_context(self, identifier, username): # .only() is currently broken in MongoEngine :( #fields = ('size', 'filetype', 'filename', 'md5', # 'mimetype', 'filedata') #sample = Sample.objects(id=identifier).only(*fields).first() sample = Sample.objects(id=identifier).first() if not sample: raise ValueError("Sample not found in database") data = sample.filedata.read() if not data: raise ValueError("Sample not found in GridFS") sample_md5 = sample.md5 self._check_length(data, getattr(sample, 'size', 0)) return SampleContext(username, data, sample_md5, sample.to_dict())
def run(self, argv): parser = OptionParser() parser.add_option("-f", "--filter", action="store", dest="filter", type="string", help="filter for samples to discover binaries") (opts, args) = parser.parse_args(argv) if opts.filter: query = ast.literal_eval(opts.filter) else: query = {} errorpath = "/tmp/gridfs_migrate_errors.txt" try: err = open(errorpath, "w") except: print "Could not open file handle to write to: %s" % errorpath sys.exit(1) error_count = 0 samples = Sample.objects(__raw__=query) count = len(samples) print "Migrating %s samples found with query %s...\n" % (count, query) i = 1 for s in samples: md5 = s.md5 try: print >> sys.stdout, "\r\tWorking on sample %d of %d" % ( i, count), sys.stdout.flush() s.discover_binary() s.save() except Exception, e: error_count += 1 err.write("Error saving sample for discover binary: %s - %s" % (md5, e)) i += 1
def process_saved_artifacts(self): """ Process anything in saved_artifacts that didn't have a match. """ for md5_, value in self.saved_artifacts.iteritems(): (saved_obj, data) = value if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if data: sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5))
def _delete_all_analysis_results(self, md5_digest, service_name): """ Delete all analysis results for this service. """ obj = Sample.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = PCAP.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Certificate.objects(md5=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = RawData.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Event.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Indicator.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = Domain.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save() obj = IP.objects(id=md5_digest).first() if obj: obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name] obj.save()
def run(self, argv): parser = OptionParser() parser.add_option('-b', '--bucket', action='store', dest='bucket', type='string', help='bucket list name') parser.add_option("-o", "--output-file", action="store", dest="outfile", type="string", help="output archive file (no extension)") (opts, args) = parser.parse_args(argv) samples = Sample.objects(bucket_list=opts.bucket) if opts.bucket and opts.outfile: filename = "%s.tar.bz2" % opts.outfile try: tar = tarfile.open(filename, "w:bz2") except Exception as e: print ("Error when attempting to open %s for writing: %s" % (filename, e)) sys.exit(1) count = len(samples) if count <= 0: print ("No matching bucket name found!") sys.exit(1) for sample in samples: m = sample.md5 f = sample.filename s = sample.filedata.read() info = tarfile.TarInfo(name="%s" % f) info.mtime = time.time() if s is not None: info.size = len(s) else: info.size = 0 try: tar.addfile(info, BytesIO(s)) except Exception as e: print ("Error attempting to add %s to the tarfile: %s" % (f, e)) pass tar.close() print ("Generated %s containing %s files." % (filename, count))
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source_name'] source_method = form.cleaned_data['source_method'] source_reference = form.cleaned_data['source_reference'] source_tlp = form.cleaned_data['source_tlp'] user = request.user description = form.cleaned_data['description'] related_id = form.cleaned_data.get('related_id', None) related_type = form.cleaned_data.get('related_type', None) relationship_type = form.cleaned_data.get('relationship_type', None) if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response['message'] = ( "Upload Failed. Unable to locate related sample. %s" % related_md5) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source elif related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: response['success'] = False response['message'] = ( "Upload Failed. Unable to locate related Item") return render( request, "file_upload_response.html", {'response': json.dumps(response)}, ) else: if form.cleaned_data['inherit_campaigns']: if campaign: related_obj.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_obj.campaign if form.cleaned_data['inherit_sources']: inherited_source = related_obj.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=form.cleaned_data['password'], user=user, campaign=campaign, confidence=confidence, related_md5=related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) else: result = handle_uploaded_file( None, source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=None, user=user, campaign=campaign, confidence=confidence, related_md5=related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), sha1=request.POST['sha1'].strip().lower(), sha256=request.POST['sha256'].strip().lower(), bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) except ZipFileError, zfe: return render( request, 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'message': zfe.value }) }) else: # zip file upload, etc; result is a list of strings (1 hash per file) if len(result) > 0 and not isinstance(result[0], dict): filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits-samples-views-view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message} md5_response = result # regular file upload; result is a list with a single dict else: response['success'] = result[0].get('success', False) response['message'] = result[0].get( 'message', response.get('message')) try: md5_response = [result[0].get('object').md5] except: md5_response = None if response['success']: if request.POST.get('email') and md5_response: for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % ( s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse( 'crits-samples-views-detail', args=[related_md5]) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render( request, 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'form': form.as_table() }) })
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'Action': return Action.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'Signature': return Signature.objects(id=_id).first() elif type_ == 'SignatureType': return SignatureType.objects(id=_id).first() elif type_ == 'SignatureDependency': return SignatureDependency.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature from crits.targets.target import Target # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in [ 'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator', 'Screenshot' ] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Actor': return Actor.objects(name=value).first() if type_ == 'Backdoor': return Backdoor.objects(id=value).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(name=value).first() elif type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(message_id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Exploit': return Exploit.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Signature': return Signature.objects(md5=value).first() elif type_ == 'Target': target = Target.objects(email_address=value).first() if target: return target else: return Target.objects(email_address__iexact=value).first() else: return None
def run(self, obj, config): key = config.get('vt_api_key', '') url = config.get('vt_download_url', '') sizeLimit = config.get('size_limit', '') replace = config.get('replace_sample', False) do_triage = config.get('run_triage', False) user = self.current_task.user sample = Sample.objects(md5=obj.md5).first() if not sample: sample = Sample() sample.md5 = md5_digest self._info("Checking if binary already exists in CRITs.") sample.discover_binary() if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace self._info("CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal.") self._add_result("Download Canceled", "Binary already exists in CRITs.") return if not user.has_access_to(SampleACL.WRITE): self._info("User does not have permission to add Samples to CRITs") self._add_result("Download Canceled", "User does not have permission to add Samples to CRITs") return parameters = urllib.urlencode({"hash": obj.md5, "apikey": key}) if settings.HTTP_PROXY: proxy = urllib2.ProxyHandler({'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY}) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) try: req = url + "?" + parameters self._info("Requesting binary with md5 '{0}' from VirusTotal.".format(obj.md5)) request = urllib2.Request(req) response = urllib2.urlopen(request) size = response.info().getheaders("Content-Length")[0] self._info("Binary size: {0} bytes".format(size)) if int(size) > sizeLimit: # Check if within size limit self._error("Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options.".format(size, sizeLimit)) self._add_result("Download Aborted", "Match found, but binary is larger than maximum size limit.") return data = response.read() except urllib2.HTTPError as e: if e.code == 404: self._info("No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect.") self._add_result("Not Found", "Binary was not found in the VirusTotal database") elif e.code == 403: self._error("Download forbidden. {0}".format(e)) self._add_result("Download Canceled", "CRITs was forbidden from downloading the binary.") else: self._error("An HTTP Error occurred: {0}".format(e)) except Exception as e: logger.error("VirusTotal: Failed connection ({0})".format(e)) self._error("Failed to get data from VirusTotal: {0}".format(e)) return if data: # Retrieved some data from VT if replace == True: try: self._info("Replace = True. Deleting any previous binary with md5 {0}".format(obj.md5)) sample.filedata.delete() except Exception as e: logger.error("VirusTotal: Error deleting existing binary ({0})".format(e)) self._error("Failed to delete existing binary") self._info("Adding new binary to CRITs.") try: handle_file(filename = obj.md5, data = data, source = "VirusTotal", reference = "Binary downloaded from VT based on MD5", user = "******", method = "VirusTotal Download Service", md5_digest = obj.md5 ) except Exception as e: logger.error("VirusTotal: Sample creation failed ({0})".format(e)) self._error("Failed to create new Sample: {0}".format(e)) return if do_triage: self._info("Running sample triage for data-reliant services.") sample.reload() run_triage(sample, user = "******") self._add_result("Download Successful", "Binary was successfully downloaded from VirusTotal") else: self._error("No data returned by VirusTotal.")
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # Quick fail if not _id or not type_: return None # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import Action from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.signatures.signature import Signature, SignatureType, SignatureDependency from crits.targets.target import Target # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode("utf8")): return None if type_ == "Actor": return Actor.objects(id=_id).first() elif type_ == "Backdoor": return Backdoor.objects(id=_id).first() elif type_ == "ActorThreatIdentifier": return ActorThreatIdentifier.objects(id=_id).first() elif type_ == "Campaign": return Campaign.objects(id=_id).first() elif type_ == "Certificate": return Certificate.objects(id=_id).first() elif type_ == "Comment": return Comment.objects(id=_id).first() elif type_ == "Domain": return Domain.objects(id=_id).first() elif type_ == "Email": return Email.objects(id=_id).first() elif type_ == "Event": return Event.objects(id=_id).first() elif type_ == "Exploit": return Exploit.objects(id=_id).first() elif type_ == "Indicator": return Indicator.objects(id=_id).first() elif type_ == "Action": return Action.objects(id=_id).first() elif type_ == "IP": return IP.objects(id=_id).first() elif type_ == "PCAP": return PCAP.objects(id=_id).first() elif type_ == "RawData": return RawData.objects(id=_id).first() elif type_ == "RawDataType": return RawDataType.objects(id=_id).first() elif type_ == "Sample": return Sample.objects(id=_id).first() elif type_ == "Signature": return Signature.objects(id=_id).first() elif type_ == "SignatureType": return SignatureType.objects(id=_id).first() elif type_ == "SignatureDependency": return SignatureDependency.objects(id=_id).first() elif type_ == "SourceAccess": return SourceAccess.objects(id=_id).first() elif type_ == "Screenshot": return Screenshot.objects(id=_id).first() elif type_ == "Target": return Target.objects(id=_id).first() elif type_ == "UserRole": return UserRole.objects(id=_id).first() else: return None
except Exception, e: print "Error with query: %s" % e return sngindex = self.__create_index(opts.name, opts.query, opts.directory) if not query: try: query = ast.literal_eval(sngindex.query) except Exception, e: print "Error with query: %s" % e return # XXX: Get count worth of samples using query... samples = Sample.objects(__raw__=query).order_by('+id').only( 'md5')[sngindex.count:sngindex.count + count] if not samples: print "No objects found." return for sample in samples: print sample.md5 sngindex.last_id = samples[len(samples) - 1].id sngindex.save() elif opts.action == 'update': sngindex = SnuggleIndex.objects(name=opts.name).first() if not sngindex: print "Index does not exist." return
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Actor': return Actor.objects(id=_id).first() elif type_ == 'Backdoor': return Backdoor.objects(id=_id).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(id=_id).first() elif type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def __parse_object(self, obs_obj): """ Parse an observable object. :param obs_obj: The observable object to parse. :type obs_obj: CybOX object type. """ properties = obs_obj.properties type_ = properties._XSI_TYPE #would isinstance be preferable? #elif isinstance(defined_obj, # cybox.objects.email_message_object.EmailMessage): #XXX: Need to check the database for an existing Sample or Indicator # and handle accordingly, or risk blowing it away!!!! if type_ == 'FileObjectType': sample = Sample.from_cybox(properties, [self.source]) md5_ = sample.md5 # do we already have this sample? db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" return elif saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Found matching Artifact in SA" sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving File to SA" self.saved_artifacts[md5_] = (properties, None) elif type_ == 'EmailMessageObjectType': # we assume all emails coming in from TAXII are new emails. # there is no way to guarantee we found a dupe in the db. email = Email.from_cybox(properties, [self.source]) email.save(username=self.source_instance.analyst) self.emails.append(('Email', str(email.id))) elif type_ in ['URIObjectType', 'AddressObjectType']: indicator = Indicator.from_cybox(properties, [self.source]) ind_type = indicator.ind_type value = indicator.value db_indicator = Indicator.objects(Q(ind_type=ind_type) & Q(value=value)).first() if db_indicator: # flat out replacing cybox indicator object with one from db. # we add the source to track we got a copy from TAXII. indicator = db_indicator indicator.add_source(self.source) indicator.save(username=self.source_instance.analyst) self.indicators.append(('Indicator', str(indicator.id))) elif type_ == 'ArtifactObjectType': # XXX: Check properties.type_ to see if it is TYPE_FILE, # TYPE_MEMORY, from CybOX definitions. This isn't implemented # yet in Greg's code. Just parse the file blindly for now. #if properties.type_ == 'File': # sample = Sample.from_cybox(properties, [self.source]) #else: # print "XXX: got unknown artifact type %s" % properties.type_ data = base64.b64decode(properties.data) md5_ = md5(data).hexdigest() #print "Found Artifact" if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Only Artifact found in SA" return elif saved_obj._XSI_TYPE == 'FileObjectType': #print "Found matching File in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving Artifact to SA" self.saved_artifacts[md5_] = (properties, data)
else: if not filename or not md5: error = "Need a file, or a filename and an md5" return {'success': False, 'error': error} else: sample_md5 = handle_uploaded_file(None, source, reference, file_format, password, analyst, campaign=campaign, confidence=confidence, bucket_list=bucket_list, ticket=ticket, filename=filename.strip(), md5=md5.strip()) except ZipFileError, zfe: return {'success': False, 'error': zfe.value} samples = Sample.objects(md5__in=sample_md5, source__name__in=sources) if samples: for s in samples: event.add_relationship(rel_item=s, rel_type='Related_To', analyst=analyst, get_rels=False) s.save(username=analyst) event.save(username=analyst) return {'success': True}
except Exception, e: print "Error with query: %s" % e return sngindex = self.__create_index(opts.name, opts.query, opts.directory) if not query: try: query = ast.literal_eval(sngindex.query) except Exception, e: print "Error with query: %s" % e return # XXX: Get count worth of samples using query... samples = Sample.objects(__raw__=query).order_by('+id').only('md5')[sngindex.count:sngindex.count + count] if not samples: print "No objects found." return for sample in samples: print sample.md5 sngindex.last_id = samples[len(samples) - 1].id sngindex.save() elif opts.action == 'update': sngindex = SnuggleIndex.objects(name=opts.name).first() if not sngindex: print "Index does not exist." return
def run(self, obj, config): key = config.get('vt_api_key', '') url = config.get('vt_download_url', '') sizeLimit = config.get('size_limit', '') replace = config.get('replace_sample', False) do_triage = config.get('run_triage', False) user = self.current_task.user sample = Sample.objects(md5=obj.md5).first() if not sample: sample = Sample() sample.md5 = md5_digest self._info("Checking if binary already exists in CRITs.") sample.discover_binary() if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace self._info( "CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal." ) self._add_result("Download Canceled", "Binary already exists in CRITs.") return if not user.has_access_to(SampleACL.WRITE): self._info("User does not have permission to add Samples to CRITs") self._add_result( "Download Canceled", "User does not have permission to add Samples to CRITs") return parameters = urllib.urlencode({"hash": obj.md5, "apikey": key}) if settings.HTTP_PROXY: proxy = urllib2.ProxyHandler({ 'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY }) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) try: req = url + "?" + parameters self._info( "Requesting binary with md5 '{0}' from VirusTotal.".format( obj.md5)) request = urllib2.Request(req) response = urllib2.urlopen(request) size = response.info().getheaders("Content-Length")[0] self._info("Binary size: {0} bytes".format(size)) if int(size) > sizeLimit: # Check if within size limit self._error( "Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options." .format(size, sizeLimit)) self._add_result( "Download Aborted", "Match found, but binary is larger than maximum size limit." ) return data = response.read() except urllib2.HTTPError as e: if e.code == 404: self._info( "No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect." ) self._add_result( "Not Found", "Binary was not found in the VirusTotal database") elif e.code == 403: self._error("Download forbidden. {0}".format(e)) self._add_result( "Download Canceled", "CRITs was forbidden from downloading the binary.") else: self._error("An HTTP Error occurred: {0}".format(e)) return except Exception as e: logger.error("VirusTotal: Failed connection ({0})".format(e)) self._error("Failed to get data from VirusTotal: {0}".format(e)) return if data: # Retrieved some data from VT if replace == True: try: self._info( "Replace = True. Deleting any previous binary with md5 {0}" .format(obj.md5)) sample.filedata.delete() except Exception as e: logger.error( "VirusTotal: Error deleting existing binary ({0})". format(e)) self._error("Failed to delete existing binary") self._info("Adding new binary to CRITs.") try: handle_file( filename=obj.md5, data=data, source="VirusTotal", source_reference="Binary downloaded from VT based on MD5", user=self.current_task.user, source_method="VirusTotal Download Service", md5_digest=obj.md5) except Exception as e: logger.error( "VirusTotal: Sample creation failed ({0})".format(e)) self._error("Failed to create new Sample: {0}".format(e)) return if do_triage: self._info("Running sample triage for data-reliant services.") sample.reload() run_triage(sample, user="******") self._add_result( "Download Successful", "Binary was successfully downloaded from VirusTotal") else: self._error("No data returned by VirusTotal.")
print "Copying GridFS data matching md5s to new collection..." filename = opts.gridfs elif opts.griddelete: print "Deleting matching md5s from GridFS..." filename = opts.griddelete count = 0 no_data = 0 try: with open(filename) as o: for line in o: md5 = line.strip() data = get_file(md5, opts.collection) if data: if opts.gridfs: put_file(md5, data) s = Sample.objects(md5=md5).first() if s: s.discover_binary() try: s.save() except: error_count += 1 err.write("Error saving sample for discover binary: %s" % md5) elif opts.griddelete: delete_file(md5, opts.collection) count += 1 else: no_data += 1 if opts.gridfs: print "Copied %s md5s to new collection." % count if error_count:
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = {'success': False, 'message': 'Unknown error; unable to upload file.'} inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source'] method = form.cleaned_data['method'] reference = form.cleaned_data['reference'] analyst = request.user.username if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response['message'] = "Upload Failed. Unable to locate related sample." return render_to_response("file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, method, reference, form.cleaned_data['file_format'], form.cleaned_data['password'], analyst, campaign, confidence, related_md5, bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version) else: result = handle_uploaded_file( None, source, method, reference, form.cleaned_data['file_format'], None, analyst, campaign, confidence, related_md5 = related_md5, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version) except ZipFileError, zfe: return render_to_response('file_upload_response.html', {'response': json.dumps({'success': False, 'message': zfe.value})}, RequestContext(request)) else: if len(result) > 1: filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits.samples.views.view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message } md5_response = result elif len(result) == 1: md5_response = None if not request.FILES: response['success'] = result[0].get('success', False) if(response['success'] == False): response['message'] = result[0].get('message', response.get('message')) else: md5_response = [result[0].get('object').md5] else: md5_response = [result[0]] response['success'] = True if md5_response != None: response['message'] = ('File uploaded successfully. <a href="%s">View Sample.</a>' % reverse('crits.samples.views.detail', args=md5_response)) if response['success']: if request.POST.get('email'): for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % (s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse('crits.samples.views.detail', args=[related_md5]) return render_to_response("file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render_to_response('file_upload_response.html', {'response': json.dumps({'success': False, 'form': form.as_table()})}, RequestContext(request))
def to_stix(self, username=None): """ Converts a CRITs event to a STIX document. The resulting document includes all related emails, samples, and indicators converted to CybOX Observable objects. Returns the STIX document and releasability constraints. (NOTE: the following statement is untrue until the releasability checking is finished, which includes setting releasability on all CRITs objects.) Raises UnreleasableEventError if the releasability on the relationships and the event do not share any common releasability sources. """ from crits.emails.email import Email from crits.samples.sample import Sample from crits.indicators.indicator import Indicator from cybox.common import Time, ToolInformationList, ToolInformation from cybox.core import Observables from stix.common import StructuredText from stix.core import STIXPackage, STIXHeader from stix.common import InformationSource from stix.common.identity import Identity stix_indicators = [] stix_observables = [] final_objects = [] # create a list of sources to send as part of the results. # list should be limited to the sources this user is allowed to use. # this list should be used along with the list of objects to set the # appropriate source's 'released' key to True for each object. final_sources = [] user_source_list = user_sources(username) for f in self.releasability: if f.name in user_source_list: final_sources.append(f.name) final_sources = set(final_sources) # TODO: eventually we can use class_from_id instead of the if block # but only once we support all CRITs types. for r in self.relationships: obj = None if r.rel_type == Email._meta['crits_type']: obj = Email.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_cybox() stix_observables.append(ind[0]) elif r.rel_type == Sample._meta['crits_type']: obj = Sample.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_cybox() for i in ind: stix_observables.append(i) elif r.rel_type == Indicator._meta['crits_type']: #NOTE: Currently this will raise an exception if there # are multiple indicators with the same value. # Should be fixed automatically once we transition # indicators to be related based on ObjectId rather # than value. obj = Indicator.objects(id=r.object_id, source__name__in=user_source_list).first() if obj: ind, releas = obj.to_stix_indicator() stix_indicators.append(ind) else: continue #Create a releasability list that is the intersection of # each related item's releasability with the event's # releasability. If the resulting set is empty, raise exception #TODO: Set releasability on all objects so that we actually # get results here instead of always raising an exception. if obj: releas_sources = set([rel.name for rel in releas]) final_sources = final_sources.intersection(releas_sources) #TODO: uncomment the following lines when objects have # releasability set. #if not final_sources: # raise UnreleasableEventError(r.value) # add to the final_objects list to send as part of the results final_objects.append(obj) tool_list = ToolInformationList() tool = ToolInformation("CRITs", "MITRE") tool.version = settings.CRITS_VERSION tool_list.append(tool) i_s = InformationSource( time=Time(produced_time= datetime.datetime.now()), identity = Identity(name=settings.COMPANY_NAME), tools = tool_list ) description = StructuredText(value=self.description) header = STIXHeader(information_source=i_s, description=description, package_intent=self.event_type, title=self.title) return (STIXPackage(indicators=stix_indicators, observables=Observables(stix_observables), stix_header=header, id_=self.event_id), final_sources, final_objects)
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source'] method = form.cleaned_data['method'] reference = form.cleaned_data['reference'] analyst = request.user.username if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response[ 'message'] = "Upload Failed. Unable to locate related sample." return render_to_response( "file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, method=method, reference=reference, file_format=form.cleaned_data['file_format'], password=form.cleaned_data['password'], user=analyst, campaign=campaign, confidence=confidence, related_md5=related_md5, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version) else: result = handle_uploaded_file( None, source, method=method, reference=reference, file_format=form.cleaned_data['file_format'], password=None, user=analyst, campaign=campaign, confidence=confidence, related_md5=related_md5, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), sha1=request.POST['sha1'].strip().lower(), sha256=request.POST['sha256'].strip().lower(), bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version) except ZipFileError, zfe: return render_to_response( 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'message': zfe.value }) }, RequestContext(request)) else: if len(result) > 1: filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits.samples.views.view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message} md5_response = result elif len(result) == 1: md5_response = None if not request.FILES: response['success'] = result[0].get('success', False) if (response['success'] == False): response['message'] = result[0].get( 'message', response.get('message')) else: md5_response = [result[0].get('object').md5] else: md5_response = [result[0]] response['success'] = True if md5_response != None: response['message'] = ( 'File uploaded successfully. <a href="%s">View Sample.</a>' % reverse('crits.samples.views.detail', args=md5_response)) if response['success']: if request.POST.get('email'): for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % ( s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse( 'crits.samples.views.detail', args=[related_md5]) return render_to_response("file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render_to_response( 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'form': form.as_table() }) }, RequestContext(request))
def upload_child(request, parent_md5): """ Upload a new child sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param parent_md5: The MD5 of the parent sample. :type parent_md5: str :returns: :class:`django.http.HttpResponse` """ new_samples = [] if request.method == "POST": form = EmailAttachForm(request.user.username, request.POST, request.FILES) if form.is_valid(): if request.FILES or 'filename' in request.POST and 'md5' in request.POST: # Child samples inherit all of the sources of the parent. parent = Sample.objects(md5=parent_md5).first() if not parent: return render_to_response( 'error.html', {'error': "Unable to find parent."}, RequestContext(request)) source = parent.source campaign_name = request.POST['campaign'] confidence = request.POST['confidence'] parent.campaign.append( EmbeddedCampaign(name=campaign_name, confidence=confidence, analyst=request.user.username)) campaigns = parent.campaign try: if request.FILES: new_samples = handle_uploaded_file( request.FILES["filedata"], source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME]) else: filename = request.POST['filename'].strip() md5 = request.POST['md5'].strip().lower() if not filename or not md5: error = "Need a file, or a filename and an md5." return render_to_response('error.html', {'error': error}, RequestContext(request)) else: new_samples = handle_uploaded_file( None, source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, filename=filename, bucket_list=form.cleaned_data[ form_consts.Common. BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], md5=md5) except ZipFileError, zfe: return render_to_response('error.html', {'error': zfe.value}, RequestContext(request)) else: return render_to_response( 'error.html', {'error': "Need a file, or a filename and an md5."}, RequestContext(request)) else: return render_to_response('error.html', {'error': 'form error'}, RequestContext(request))
def upload_child(request, parent_md5): """ Upload a new child sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param parent_md5: The MD5 of the parent sample. :type parent_md5: str :returns: :class:`django.http.HttpResponse` """ new_samples = [] if request.method == "POST": form = EmailAttachForm(request.user.username, request.POST, request.FILES) if form.is_valid(): if request.FILES or 'filename' in request.POST and 'md5' in request.POST: # Child samples inherit all of the sources of the parent. parent = Sample.objects(md5=parent_md5).first() if not parent: return render_to_response('error.html', {'error': "Unable to find parent."}, RequestContext(request)) source = parent.source campaign_name = request.POST['campaign'] confidence = request.POST['confidence'] parent.campaign.append(EmbeddedCampaign(name=campaign_name, confidence=confidence, analyst=request.user.username)) campaigns = parent.campaign try: if request.FILES: new_samples = handle_uploaded_file(request.FILES["filedata"], source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME]) else: filename = request.POST['filename'].strip() md5= request.POST['md5'].strip().lower() if not filename or not md5: error = "Need a file, or a filename and an md5." return render_to_response('error.html', {'error': error}, RequestContext(request)) else: new_samples = handle_uploaded_file(None, source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, filename=filename, bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME], md5=md5) except ZipFileError, zfe: return render_to_response('error.html', {'error': zfe.value}, RequestContext(request)) else: return render_to_response('error.html', {'error': "Need a file, or a filename and an md5."}, RequestContext(request)) else: return render_to_response('error.html', {'error': 'form error'}, RequestContext(request))
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.core.crits_mongoengine import RelationshipType from crits.core.source_access import SourceAccess from crits.core.user_role import UserRole from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event, EventType from crits.indicators.indicator import Indicator, IndicatorAction from crits.ips.ip import IP from crits.objects.object_type import ObjectType from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData, RawDataType from crits.samples.backdoor import Backdoor from crits.samples.exploit import Exploit from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target if not _id: return None # make sure it's a string _id = str(_id) if type_ == 'Backdoor': return Backdoor.objects(id=_id).first() if type_ == 'Campaign': return Campaign.objects(id=_id).first() elif type_ == 'Certificate': return Certificate.objects(id=_id).first() elif type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Domain': return Domain.objects(id=_id).first() elif type_ == 'Email': return Email.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'EventType': return EventType.objects(id=_id).first() elif type_ == 'Exploit': return Exploit.objects(id=_id).first() elif type_ == 'Indicator': return Indicator.objects(id=_id).first() elif type_ == 'IndicatorAction': return IndicatorAction.objects(id=_id).first() elif type_ == 'IP': return IP.objects(id=_id).first() elif type_ == 'ObjectType': return ObjectType.objects(id=_id).first() elif type_ == 'PCAP': return PCAP.objects(id=_id).first() elif type_ == 'RawData': return RawData.objects(id=_id).first() elif type_ == 'RawDataType': return RawDataType.objects(id=_id).first() elif type_ == 'RelationshipType': return RelationshipType.objects(id=_id).first() elif type_ == 'Sample': return Sample.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'Screenshot': return Screenshot.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() else: return None
def __parse_object(self, obs_obj): """ Parse an observable object. :param obs_obj: The observable object to parse. :type obs_obj: CybOX object type. """ properties = obs_obj.properties type_ = properties._XSI_TYPE #would isinstance be preferable? #elif isinstance(defined_obj, # cybox.objects.email_message_object.EmailMessage): #XXX: Need to check the database for an existing Sample or Indicator # and handle accordingly, or risk blowing it away!!!! if type_ == 'FileObjectType': sample = Sample.from_cybox(properties, [self.source]) md5_ = sample.md5 # do we already have this sample? db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" return elif saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Found matching Artifact in SA" sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving File to SA" self.saved_artifacts[md5_] = (properties, None) elif type_ == 'EmailMessageObjectType': # we assume all emails coming in from TAXII are new emails. # there is no way to guarantee we found a dupe in the db. email = Email.from_cybox(properties, [self.source]) email.save(username=self.source_instance.analyst) self.emails.append(('Email', str(email.id))) elif type_ in ['URIObjectType', 'AddressObjectType']: indicator = Indicator.from_cybox(properties, [self.source]) ind_type = indicator.ind_type value = indicator.value db_indicator = Indicator.objects( Q(ind_type=ind_type) & Q(value=value)).first() if db_indicator: # flat out replacing cybox indicator object with one from db. # we add the source to track we got a copy from TAXII. indicator = db_indicator indicator.add_source(self.source) indicator.save(username=self.source_instance.analyst) self.indicators.append(('Indicator', str(indicator.id))) elif type_ == 'ArtifactObjectType': # XXX: Check properties.type_ to see if it is TYPE_FILE, # TYPE_MEMORY, from CybOX definitions. This isn't implemented # yet in Greg's code. Just parse the file blindly for now. #if properties.type_ == 'File': # sample = Sample.from_cybox(properties, [self.source]) #else: # print "XXX: got unknown artifact type %s" % properties.type_ data = base64.b64decode(properties.data) md5_ = md5(data).hexdigest() #print "Found Artifact" if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Only Artifact found in SA" return elif saved_obj._XSI_TYPE == 'FileObjectType': #print "Found matching File in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving Artifact to SA" self.saved_artifacts[md5_] = (properties, data)
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = {'success': False, 'message': 'Unknown error; unable to upload file.'} inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source_name'] source_method = form.cleaned_data['source_method'] source_reference = form.cleaned_data['source_reference'] source_tlp = form.cleaned_data['source_tlp'] user = request.user description = form.cleaned_data['description'] related_id = form.cleaned_data.get('related_id', None) related_type = form.cleaned_data.get('related_type', None) relationship_type = form.cleaned_data.get('relationship_type', None) if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response['message'] = ("Upload Failed. Unable to locate related sample. %s" % related_md5) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source elif related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: response['success'] = False response['message'] = ("Upload Failed. Unable to locate related Item") return render(request, "file_upload_response.html",{'response': json.dumps(response)}, ) else: if form.cleaned_data['inherit_campaigns']: if campaign: related_obj.campaign.append(EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_obj.campaign if form.cleaned_data['inherit_sources']: inherited_source = related_obj.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=form.cleaned_data['password'], user=user, campaign=campaign, confidence=confidence, related_md5=related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) else: result = handle_uploaded_file( None, source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=None, user=user, campaign=campaign, confidence=confidence, related_md5 = related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), sha1=request.POST['sha1'].strip().lower(), sha256=request.POST['sha256'].strip().lower(), bucket_list=form.cleaned_data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) except ZipFileError, zfe: return render(request, 'file_upload_response.html', {'response': json.dumps({'success': False, 'message': zfe.value})}) else: # zip file upload, etc; result is a list of strings (1 hash per file) if len(result) > 0 and not isinstance(result[0], dict): filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits-samples-views-view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message } md5_response = result # regular file upload; result is a list with a single dict else: response['success'] = result[0].get('success', False) response['message'] = result[0].get('message', response.get('message')) try: md5_response = [result[0].get('object').md5] except: md5_response = None if response['success']: if request.POST.get('email') and md5_response: for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % (s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse('crits-samples-views-detail', args=[related_md5]) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render(request, 'file_upload_response.html', {'response': json.dumps({'success': False, 'form': form.as_table()})})
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRITs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`crits.core.crits_mongoengine.CritsBaseAttributes` """ # doing this to avoid circular imports from crits.actors.actor import ActorThreatIdentifier, Actor from crits.backdoors.backdoor import Backdoor from crits.campaigns.campaign import Campaign from crits.certificates.certificate import Certificate from crits.comments.comment import Comment from crits.domains.domain import Domain from crits.emails.email import Email from crits.events.event import Event from crits.exploits.exploit import Exploit from crits.indicators.indicator import Indicator from crits.ips.ip import IP from crits.pcaps.pcap import PCAP from crits.raw_data.raw_data import RawData from crits.samples.sample import Sample from crits.screenshots.screenshot import Screenshot from crits.targets.target import Target # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit', 'Indicator', 'Screenshot'] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Actor': return Actor.objects(name=value).first() if type_ == 'Backdoor': return Backdoor.objects(id=value).first() elif type_ == 'ActorThreatIdentifier': return ActorThreatIdentifier.objects(name=value).first() elif type_ == 'Campaign': return Campaign.objects(name=value).first() elif type_ == 'Certificate': return Certificate.objects(md5=value).first() elif type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Domain': return Domain.objects(domain=value).first() elif type_ == 'Email': return Email.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'Exploit': return Exploit.objects(id=value).first() elif type_ == 'Indicator': return Indicator.objects(id=value).first() elif type_ == 'IP': return IP.objects(ip=value).first() elif type_ == 'PCAP': return PCAP.objects(md5=value).first() elif type_ == 'RawData': return RawData.objects(md5=value).first() elif type_ == 'Sample': return Sample.objects(md5=value).first() elif type_ == 'Screenshot': return Screenshot.objects(id=value).first() elif type_ == 'Target': target = Target.objects(email_address=value).first() if target: return target else: return Target.objects(email_address__iexact=value).first() else: return None
def _get_meta_count(self, meta_type, meta_val): query_field = "analysis.results.{0}".format(meta_type) query = {query_field: meta_val} total_count = Sample.objects(__raw__=query).only('md5').count() return total_count