def analyze(self, line): if not line or line[0].startswith("#"): return tokens = line.split(',') c2_domain = [] ips_c2 = [] names_servers = [] ip_names_servers = [] context_feed = [] if len(tokens) == 6: c2_domain = tokens[0] ips_c2 = tokens[1].split('|') names_servers = tokens[2].split('|') ip_names_servers = tokens[3].split('|') context_feed = tokens[4] m = BambenekOsintIpmaster.reg.match(context_feed) malware_family = '' if m: malware_family = m.group(1) context = { "status": context_feed, "name servers": names_servers, "source": self.name } tags = [malware_family] c2 = None if c2_domain: c2 = Hostname.get_or_create(value=c2_domain) c2.add_context(context) c2.tag(tags) c2.add_source('feed') for ip in ips_c2: if ip: ip_obs = Ip.get_or_create(value=ip) ip_obs.tag(tags) ip_obs.add_source('feed') if c2: c2.active_link_to(ip_obs, "IP", self.source) for name_server in names_servers: if name_server: ns_obs = Hostname.get_or_create(value=name_server) c2.active_link_to(ns_obs, 'NS', self.source) ns_obs.tag(tags) ns_obs.add_context(context) ns_obs.add_source('feed') for ip_ns in ip_names_servers: if ip_ns: ip_ns_obs = Ip.get_or_create(value=ip_ns) c2.active_link_to(ip_ns_obs, 'IP NS', self.source) ip_ns_obs.tag(tags) ip_ns_obs.add_context(context) ip_ns_obs.add_source('feed') else: logging.error('Parsing error in line: %s' % line)
def analyze(self, dict): context = dict date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)", dict['title']).group('datetime') try: context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") except ValueError: pass g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)', dict['description']) g = g.groupdict() context['version'] = g['version'] context['description'] = FeodoTracker.descriptions[g['version']] context['subfamily'] = FeodoTracker.variants[g['version']] context['source'] = self.name del context['title'] variant_tag = FeodoTracker.variants[g['version']].lower() try: if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", g['host']): new = Ip.get_or_create(value=g['host']) else: new = Hostname.get_or_create(value=g['host']) new.add_context(context) new.add_source("feed") new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2']) except ObservableValidationError as e: logging.error(e)
def analyze(self, line): fields = line.split('|') if len(fields) < 8: return context = {} ip = fields[0] context['name'] = fields[1] context['router-port'] = fields[2] context['directory-port'] = fields[3] context['flags'] = fields[4] context['uptime'] = fields[5] context['version'] = fields[6] context['contactinfo'] = fields[7] context['description'] = "Tor exit node: %s (%s)" % (context['name'], ip) context['source'] = self.name try: ip = Ip.get_or_create(value=fields[0]) ip.add_context(context) ip.add_source("feed") ip.tag(['tor']) except ObservableValidationError as e: logging.error(e)
def analyze(self, line): if not line or line[0].startswith("#"): return try: ip, domain, family, md5, link, date = tuple(map(strip, line)) context = { "first_seen": date, "family": family, "report": link, "source": self.name } c2 = None sample = None try: sample = Hash.get_or_create(value=md5) sample.add_context(context) sample.tag(family.lower()) except ObservableValidationError as e: logging.error("Invalid line: {}\nLine: {}".format(e, line)) try: if domain: if '/' in domain: c2 = Url.get_or_create(value=domain) else: c2 = Hostname.get_or_create(value=domain) elif ip: c2 = Ip.get_or_create(value=ip) else: return c2.add_context(context) c2.tag(['c2', family.lower()]) except ObservableValidationError as e: logging.error("Invalid line: {}\nLine: {}".format(e, line)) if c2 and sample: sample.active_link_to(c2, 'c2', self.name, clean_old=False) except ValueError: logging.error("Error unpacking line: {}".format(line))
def analyze(self, line): if not line or line[0].startswith("#"): return date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple( line) tags = [] tags += TYPE_DICT[_type] tags.append(family.lower()) context = { "first_seen": date, "status": status, "registrar": registrar, "countries": countries.split("|"), "asns": asns.split("|"), "source": self.name } try: url = Url.get_or_create(value=url.rstrip()) url.add_context(context) url.tag(tags) hostname = Observable.add_text(hostname) hostname.tag(tags + ['blocklist']) for ip in ips.split("|"): if ip != hostname and ip is not None and ip != '': try: i = Ip.get_or_create(value=ip) i.active_link_to( hostname, "First seen IP", self.name, clean_old=False) except ObservableValidationError as e: logging.error("Invalid Observable: {}".format(e)) except ObservableValidationError as e: logging.error("Invalid line: {}\nLine: {}".format(e, line))
def _get_threat_forensics_nodes_inner(self, evidence, general_context, tags): # create context from notes context = general_context.copy() _ctx = self._make_context_from_notes([evidence]) context.update(_ctx) # add evidence['type'] and unicify tags tags = [{ 'name': _ } for _ in set([evidence['type']] + [d['name'] for d in tags])] # create Tags in DB for _ in tags: Tag.get_or_create(name=_['name']) # threat_forensics = [] # technical hack: set optional comments values for optional in ['action', 'rule', 'path', 'rule']: if optional not in evidence['what']: evidence['what'][optional] = None # add attributes for the known evidence type if evidence['type'] in ['file', 'dropper']: if 'path' in evidence['what']: threat_forensics.append( File.get_or_create(value=evidence['what']['path'], context=[context])) if 'md5' in evidence['what']: threat_forensics.append( Hash.get_or_create(value=evidence['what']['md5'], context=[context])) if 'sha256' in evidence['what']: threat_forensics.append( Hash.get_or_create(value=evidence['what']['sha256'], context=[context])) elif evidence['type'] == 'cookie': pass elif evidence['type'] == 'dns': threat_forensics.append( Hostname.get_or_create(value=evidence['what']['host'], context=[context])) elif evidence['type'] == 'ids': threat_forensics.append( Text.get_or_create(value=evidence['what']['ids'], context=[context])) pass elif evidence['type'] == 'mutex': threat_forensics.append( Text.get_or_create(value=evidence['what']['name'], context=[context])) elif evidence['type'] == 'network': if 'ip' in evidence['what']: # FIXME port, type threat_forensics.append( Ip.get_or_create(value=evidence['what']['ip'], context=[context])) elif 'domain' in evidence['what']: threat_forensics.append( Hostname.get_or_create(value=evidence['what']['domain'], context=[context])) elif evidence['type'] == 'process': pass elif evidence['type'] == 'registry': # threat_forensics.append(evidence['what']['key']) # threat_forensics.append(evidence['what']['value']) pass elif evidence['type'] == 'url': # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/ threat_forensics.append( Url.get_or_create(value=evidence['what']['url'], context=[context])) # add note as tag because its a signature if 'note' in evidence: threat_forensics[-1].tag(evidence['note'].replace( '.', '_').strip('_')) # tag all of that for o in threat_forensics: o.tag([t['name'] for t in tags]) return threat_forensics
def analyze(observable, results): links = set() json_result = ThreatCrowdAPI.fetch(observable) json_string = json.dumps( json_result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) result = {} if isinstance(observable, Hostname): if 'resolutions' in json_result: result['ip on this domains'] = 0 for ip in json_result['resolutions']: if ip['ip_address'].strip() != observable.value: if ip['last_resolved'] != '0000-00-00': last_resolved = datetime.datetime.strptime( ip['last_resolved'], "%Y-%m-%d") try: new_ip = Ip.get_or_create( value=ip['ip_address'].strip()) links.update( new_ip.active_link_to( observable, 'IP', 'ThreatCrowd', last_resolved)) result['ip on this domains'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add subdomain {} to the database". format(ip['ip_address'])) if 'emails' in json_result: result['nb emails'] = 0 for email in json_result['emails']: try: new_email = Email.get_or_create(value=email) links.update( new_email.active_link_to( observable, 'Used by', 'ThreatCrowd')) result['nb emails'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add email {} to the database". format(email)) if 'subdomains' in json_result: result['nb subdomains'] = 0 for subdomain in json_result['subdomains']: try: new_domain = Hostname.get_or_create(value=subdomain) links.update( observable.active_link_to( new_domain, 'subdomain', 'ThreatCrowd')) result['nb subdomains'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add subdomain {} to the database". format(subdomain)) if isinstance(observable, Ip): if 'resolutions' in json_result: result['domains resolved'] = 0 for domain in json_result['resolutions']: if domain['domain'].strip() != observable.value: try: last_resolved = datetime.datetime.strptime( domain['last_resolved'], "%Y-%m-%d") new_domain = Hostname.get_or_create( value=domain['domain'].strip()) links.update( new_domain.active_link_to( observable, 'A Record', 'ThreatCrowd', last_resolved)) result['domains resolved'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add domain {} to the database". format(domain['domain'])) if 'hashes' in json_result and len(json_result['hashes']) > 0: result['malwares'] = 0 for h in json_result['hashes']: new_hash = Hash.get_or_create(value=h) links.update( new_hash.active_link_to( observable, 'hash', 'ThreatCrowd')) result['malwares'] += 1 if isinstance(observable, Email): if 'domains' in json_result and len(json_result) > 0: result['domains recorded by email'] = 0 for domain in json_result['domains']: new_domain = Hostname.get_or_create(value=domain) links.update( new_domain.active_link_to( observable, 'recorded by', 'ThreatCrowd')) result['domains recorded by email'] += 1 if isinstance(observable, Hash): result['nb c2'] = 0 if 'md5' in json_result: new_hash = Hash.get_or_create(value=json_result['md5']) links.update( new_hash.active_link_to(observable, 'md5', 'ThreadCrowd')) if 'sha1' in json_result: new_hash = Hash.get_or_create(value=json_result['sha1']) links.update( new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd')) if 'sha256' in json_result: new_hash = Hash.get_or_create(value=json_result['sha256']) links.update( new_hash.active_link_to( observable, 'sha256', 'ThreadCrowd')) if 'domains' in json_result and len(json_result['domains']): for domain in json_result['domains']: new_domain = Hostname.get_or_create(value=domain) links.update( observable.active_link_to( new_domain, 'c2', 'ThreatCrowd')) result['nb c2'] += 1 if 'ips' in json_result and len(json_result['ips']): for ip in json_result['ips']: new_ip = Ip.get_or_create(value=ip.strip()) links.update( observable.active_link_to(new_ip, 'c2', 'ThreatCrowd')) result['nb c2'] += 1 if 'permalink' in json_result: result['permalink'] = json_result['permalink'] result['source'] = 'threatcrowd_query' result['raw'] = json_string observable.add_context(result) return list(links)
def _query_and_filter_previous_new_threat_for_campaign( campaign_info, context): # get all threat for this campaign from the API # filter out the threat we already have in DB # return the net new threats # TODO: alternative solution, query by type, get all campaign threat, intersect sets # Q/A: why do i have to play with perf issues ? # only create Observables and link them when they do not exists. cls_action = { 'COMPLETE_URL': Url, 'NORMALIZED_URL': Url, 'ATTACHMENT': Hash, 'DOMAIN': Hostname, 'HOSTNAME': Hostname } threats = [] log.info("There are {nb} threat associated to campaign".format( nb=len(campaign_info['campaignMembers']))) for threat in campaign_info['campaignMembers']: # ATTACHMENT, COMPLETE_URL, NORMALIZED_URL, or DOMAIN # BUG #5: undocumentated value HOSTNAME, could be hostname or ip v = threat['threat'] # t = threat['threatTime'][:10] # last_seen ? create_t = datetime.strptime(threat['threatTime'], "%Y-%m-%dT%H:%M:%S.%fZ") # TODO threat['threatStatus'] in active, ... if threat['threatStatus'] != 'active': log.warning('Campaign threat - threatStatus %s unsupported', threat['threatStatus']) # FIXME Campaign threat - threatStatus falsePositive unsupported # threatStatus ? if threat['subType'] not in cls_action: log.error('Campaign threat - subtype %s unsupported', threat['subType']) continue cls = cls_action[threat['subType']] try: # if it exists, don't do anything. tags and context are the same cls.objects.get(value=v) except DoesNotExist: # otherwise return it to link to it. # threats.append(cls.get_or_create(value=v, context=[context], created=t)) # tags named argument in constructor does not work the same as .tag() try: o = cls.get_or_create(value=v, context=[context], created=create_t) o.tag([threat['type'], threat['subType']]) except DoesNotExist: # wtf log.error("{cls} {v} has a weird problem - FIXME".format( cls=cls, v=v)) except ObservableValidationError: try: if threat['subType'] == 'HOSTNAME': # could be an Ip o = Ip.get_or_create(value=v, context=[context], created=create_t) except ObservableValidationError as e: log.error(e) log.error(pprint.pformat(threat)) log.error("Campaign {name}".format( name=campaign_info['name'])) o = Text.get_or_create(value=v, context=[context], created=create_t) o.tag([threat['type'], threat['subType']]) threats.append(o) log.info("Found %d new threat on campaign, new to us", len(threats)) # there is a bug here... log.debug(", ".join( ["%s:%s" % (t.__class__.__name__, t.value) for t in threats])) return threats
def analyze(self, block, first_seen): # pylint: disable=arguments-differ """ block example {u"first_seen": u"2019-03-13 14:38:12", u"hash": {u"md5": u"e1513c048e520e8d5fc5999d82994ea7", u"sha1": u"f09f66c3bd4cd54cc030b7d102be32376fd993b5", u"sha256": u"bbb450f1f68735054af6d1c64bd3a7e62f9977d40eeb286340d8bc1dac6f7e7e"}, u"hash_seen": 1, u"id": u"5c8915d47a324f51d460e8e5", u"sample": {u"name": u"vdvdv.exe", u"size": u"600576"}, u"server": {u"AS": u"AS197695", u"country": u"ru", u"ip": u"37.140.192.146", u"url": u"byhlavash.chimkent.su/vdvdv.exe"}} """ url = block["server"]["url"] if "http" not in url: url = "http://" + url context = {} context["date_added"] = first_seen context["as"] = block["server"]["AS"] context["country"] = block["server"]["country"] context["ip"] = block["server"]["ip"] context["source"] = self.name context["md5"] = block["hash"]["md5"] context["sha1"] = block["hash"]["sha1"] context["sha256"] = block["hash"]["sha256"] url_data = None try: url_data = Url.get_or_create(value=url) url_data.add_context(context) url_data.add_source(self.name) except ObservableValidationError as e: logging.error(e) if block.get("server", {}).get("ip", ""): try: ip = Ip.get_or_create(value=block["server"]["ip"]) ip.add_context(context) ip.add_source(self.name) if url_data: url_data.active_link_to(ip, 'ip', self.name, clean_old=False) except ObservableValidationError as e: logging.error(e) if block.get("hash", []): # md5, sha1, sha256 for hash_type in block["hash"]: try: hash_data = Hash.get_or_create( value=block["hash"][hash_type]) hash_data.add_context(context) hash_data.add_source(self.name) if url_data: url_data.active_link_to(hash_data, 'hash', self.name, clean_old=False) except ObservableValidationError as e: logging.error(e)
def process_domain(domain, attributes): context = {"source": "VirusTotal"} links = set() timestamp_creation = attributes["creation_date"] context["first_seen"] = datetime.fromtimestamp( timestamp_creation).isoformat() context["whois"] = attributes["whois"] if "whois_date" in attributes: timestamp_whois_date = attributes["whois_date"] context["whois_date"] = datetime.fromtimestamp( timestamp_creation).isoformat() if "last_dns_records" in attributes: last_dns_records = attributes["last_dns_records"] for rr in last_dns_records: related_obs = None if rr["type"] == "A": related_obs = Ip.get_or_create(value=rr["value"]) elif rr["type"] == "MX": related_obs = Hostname.get_or_create(value=rr["value"]) elif rr["type"] == "SOA": related_obs = Hostname.get_or_create(value=rr["value"]) elif rr["type"] == "NS": related_obs = Hostname.get_or_create(value=rr["value"]) if related_obs: links.update( related_obs.active_link_to(domain, rr["type"], context["source"])) if "last_dns_records_date" in attributes: timestamp_lst_dns_record = attributes["last_dns_records_date"] context["last_dns_records_date"] = datetime.fromtimestamp( timestamp_lst_dns_record).isoformat() if "registrar" in attributes: context["registrar"] = attributes["registrar"] tags = attributes["tags"] if tags: domain.tag(tags) if "popularity_ranks" in attributes: alexa_rank = attributes["popularity_ranks"] if alexa_rank: context["alexa_rank"] = alexa_rank["Alexa"]["rank"] timestamp_rank = alexa_rank["Alexa"]["timestamp"] context["alexa_rank_date"] = datetime.fromtimestamp( timestamp_creation).isoformat() if "last_analysis_stats" in attributes: stats_analysis = attributes["last_analysis_stats"] for k, v in stats_analysis.items(): context[k] = v if "last_https_certificate" and "last_https_certificate_date" in attributes: context["last_https_certificate"] = attributes[ "last_https_certificate"] try: timestamp_https_cert = attributes[ "last_https_certificate_date"] context[ "last_https_certificate_date"] = datetime.fromtimestamp( timestamp_https_cert).isoformat() except TypeError or ValueError: pass domain.add_context(context) return links
def analyze(observable, results): links = set() json_result = ThreatCrowdAPI.fetch(observable) json_string = json.dumps(json_result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) result = {} if isinstance(observable, Hostname): if 'resolutions' in json_result: result['ip on this domains'] = 0 for ip in json_result['resolutions']: if ip['ip_address'].strip() != observable.value: if ip['last_resolved'] != '0000-00-00': last_resolved = datetime.datetime.strptime( ip['last_resolved'], "%Y-%m-%d") try: new_ip = Ip.get_or_create( value=ip['ip_address'].strip()) links.update( new_ip.active_link_to( observable, 'IP', 'ThreatCrowd', last_resolved)) result['ip on this domains'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add subdomain {} to the database" .format(ip['ip_address'])) if 'emails' in json_result: result['nb emails'] = 0 for email in json_result['emails']: try: new_email = Email.get_or_create(value=email) links.update( new_email.active_link_to(observable, 'Used by', 'ThreatCrowd')) result['nb emails'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add email {} to the database" .format(email)) if 'subdomains' in json_result: result['nb subdomains'] = 0 for subdomain in json_result['subdomains']: try: new_domain = Hostname.get_or_create(value=subdomain) links.update( observable.active_link_to(new_domain, 'subdomain', 'ThreatCrowd')) result['nb subdomains'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add subdomain {} to the database" .format(subdomain)) if isinstance(observable, Ip): if 'resolutions' in json_result: result['domains resolved'] = 0 for domain in json_result['resolutions']: if domain['domain'].strip() != observable.value: try: last_resolved = datetime.datetime.strptime( domain['last_resolved'], "%Y-%m-%d") new_domain = Hostname.get_or_create( value=domain['domain'].strip()) links.update( new_domain.active_link_to( observable, 'A Record', 'ThreatCrowd', last_resolved)) result['domains resolved'] += 1 except ObservableValidationError: logging.error( "An error occurred when trying to add domain {} to the database" .format(domain['domain'])) if 'hashes' in json_result and len(json_result['hashes']) > 0: result['malwares'] = 0 for h in json_result['hashes']: new_hash = Hash.get_or_create(value=h) links.update( new_hash.active_link_to(observable, 'hash', 'ThreatCrowd')) result['malwares'] += 1 if isinstance(observable, Email): if 'domains' in json_result and len(json_result) > 0: result['domains recorded by email'] = 0 for domain in json_result['domains']: new_domain = Hostname.get_or_create(value=domain) links.update( new_domain.active_link_to(observable, 'recorded by', 'ThreatCrowd')) result['domains recorded by email'] += 1 if isinstance(observable, Hash): result['nb c2'] = 0 if 'md5' in json_result: new_hash = Hash.get_or_create(value=json_result['md5']) links.update( new_hash.active_link_to(observable, 'md5', 'ThreadCrowd')) if 'sha1' in json_result: new_hash = Hash.get_or_create(value=json_result['sha1']) links.update( new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd')) if 'sha256' in json_result: new_hash = Hash.get_or_create(value=json_result['sha256']) links.update( new_hash.active_link_to(observable, 'sha256', 'ThreadCrowd')) if 'domains' in json_result and len(json_result['domains']): for domain in json_result['domains']: new_domain = Hostname.get_or_create(value=domain) links.update( observable.active_link_to(new_domain, 'c2', 'ThreatCrowd')) result['nb c2'] += 1 if 'ips' in json_result and len(json_result['ips']): for ip in json_result['ips']: new_ip = Ip.get_or_create(value=ip.strip()) links.update( observable.active_link_to(new_ip, 'c2', 'ThreatCrowd')) result['nb c2'] += 1 if 'permalink' in json_result: result['permalink'] = json_result['permalink'] result['source'] = 'threatcrowd_query' result['raw'] = json_string observable.add_context(result) return list(links)
def analyze(self, item): context = item date_string = re.search( r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime') try: context['date_added'] = datetime.strptime( date_string, "%Y-%m-%d %H:%M:%S") except ValueError: pass g = re.match( r'^Host: (?P<host>.+), Version: (?P<version>\w)', context['description']) g = g.groupdict() context['version'] = g['version'] context['description'] = FeodoTracker.descriptions[g['version']] context['subfamily'] = FeodoTracker.variants[g['version']] context['source'] = self.name del context['title'] new = None variant_tag = FeodoTracker.variants[g['version']].lower() try: if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", g['host']): new = Ip.get_or_create(value=g['host']) else: new = Hostname.get_or_create(value=g['host']) new.add_context(context) new.add_source("feed") new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2']) except ObservableValidationError as e: logging.error(e) try: url_fedeo = context['guid'] r = requests.get(url_fedeo) if r.status_code == 200: html_source = r.text soup = BeautifulSoup(html_source, 'html.parser') tab = soup.find('table', attrs='sortable') results = [] if tab: all_tr = tab.find_all('tr') for tr in all_tr: all_td = tr.find_all('td') if all_td and len(all_td) == 7: results.append({ 'timestamp': all_td[0].text, 'md5_hash': all_td[1].text, 'filesize': all_td[2].text, 'VT': all_td[3].text, 'Host': all_td[4].text, 'Port': all_td[5].text, 'SSL Certif or method': all_td[6].text }) for r in results: new_hash = Hash.get_or_create(value=r['md5_hash']) new_hash.add_context(context) new_hash.add_source('feed') new_hash.tag([ variant_tag, 'malware', 'crimeware', 'banker', 'payload' ]) new_hash.active_link_to( new, 'c2', self.name, clean_old=False) host = Url.get_or_create( value='https://%s:%s' % (g['host'], r['Port'])) host.add_source('feed') host.add_context(context) host.tag( [variant_tag, 'malware', 'crimeware', 'banker', 'c2']) new_hash.active_link_to( host, 'c2', self.name, clean_old=False) except ObservableValidationError as e: logging.error(e)
def _add_events_nodes(self, events, context, tags): log.debug('_add_events_nodes on {nb} events'.format(nb=len(events))) attach_unsupported = dict( [(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]]) event_nodes = list() for msg in events: create_t = datetime.strptime( msg['messageTime'], "%Y-%m-%dT%H:%M:%S.%fZ") # PPS unique value guid = Text.get_or_create( value='proofpoint://%s' % msg['GUID'], created=create_t, context=[context]) log.debug('Event {msg}'.format(msg=msg['messageID'])) message_contents = list() src_ip = Ip.get_or_create( value=msg['senderIP'], created=create_t, context=[context]) src_ip.tag(['MTA']) guid.active_link_to([src_ip], "MTA src ip", self.name) # new event event_nodes.append(guid) # if self.config['import_email_metadata']: # email details # messageID message_id = Email.get_or_create( value=msg['messageID'], created=create_t, context=[context]) guid.active_link_to([message_id], "seen in", self.name) # sender _s1 = Email.get_or_create( value=msg['sender'], created=create_t, context=[context]) _s1.tag(['sender']) guid.active_link_to([_s1], "sender", self.name) if 'headerFrom' in msg: # header From _s2 = Email.get_or_create( value=msg['headerFrom'], created=create_t, context=[context]) _s2.tag(['sender']) guid.active_link_to([_s2], "headerFrom", self.name) # FIXME is that a duplicate of attachment-malware ? # attachment events for attach in msg['messageParts']: if attach['sandboxStatus'] in ['THREAT']: md5 = Hash.get_or_create( value=attach['md5'], created=create_t, context=[context]) md5.tag([t['name'] for t in tags]) fname = File.get_or_create( value=attach['filename'], created=create_t, context=[context]) fname.tag([t['name'] for t in tags]) # this should be a DUP from threat_nodes in analyse() sha_threat = Hash.get_or_create( value=attach['sha256'], created=create_t, context=[context]) sha_threat.active_link_to([md5, fname], "relates", self.name) sha_threat.tag([t['name'] for t in tags]) message_contents.append(sha_threat) # link the 3 together elif attach['sandboxStatus'] in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]: attach_unsupported[attach['sandboxStatus']] += 1 log.debug(pprint.pformat(attach)) # add context to the hashes guid.active_link_to(message_contents, "delivers", self.name) _stats = ', '.join( "%s: %d" % (k, v) for k, v in attach_unsupported.items()) log.warning('Ignored unsupported attachments: %s', _stats) for o in event_nodes: o.tag([t['name'] for t in tags]) return event_nodes
def _get_threat_forensics_nodes_inner( self, evidence, general_context, tags): # create context from notes context = general_context.copy() _ctx = self._make_context_from_notes([evidence]) context.update(_ctx) # add evidence['type'] and unicify tags tags = [{ 'name': _ } for _ in set([evidence['type']] + [d['name'] for d in tags])] # create Tags in DB for _ in tags: Tag.get_or_create(name=_['name']) # threat_forensics = [] # technical hack: set optional comments values for optional in ['action', 'rule', 'path', 'rule']: if optional not in evidence['what']: evidence['what'][optional] = None # add attributes for the known evidence type if evidence['type'] in ['file', 'dropper']: if 'path' in evidence['what']: threat_forensics.append( File.get_or_create( value=evidence['what']['path'], context=[context])) if 'md5' in evidence['what']: threat_forensics.append( Hash.get_or_create( value=evidence['what']['md5'], context=[context])) if 'sha256' in evidence['what']: threat_forensics.append( Hash.get_or_create( value=evidence['what']['sha256'], context=[context])) elif evidence['type'] == 'cookie': pass elif evidence['type'] == 'dns': threat_forensics.append( Hostname.get_or_create( value=evidence['what']['host'], context=[context])) elif evidence['type'] == 'ids': threat_forensics.append( Text.get_or_create( value=evidence['what']['ids'], context=[context])) pass elif evidence['type'] == 'mutex': threat_forensics.append( Text.get_or_create( value=evidence['what']['name'], context=[context])) elif evidence['type'] == 'network': if 'ip' in evidence['what']: # FIXME port, type threat_forensics.append( Ip.get_or_create( value=evidence['what']['ip'], context=[context])) elif 'domain' in evidence['what']: threat_forensics.append( Hostname.get_or_create( value=evidence['what']['domain'], context=[context])) elif evidence['type'] == 'process': pass elif evidence['type'] == 'registry': # threat_forensics.append(evidence['what']['key']) # threat_forensics.append(evidence['what']['value']) pass elif evidence['type'] == 'url': # BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/ threat_forensics.append( Url.get_or_create( value=evidence['what']['url'], context=[context])) # add note as tag because its a signature if 'note' in evidence: threat_forensics[-1].tag( evidence['note'].replace('.', '_').strip('_')) # tag all of that for o in threat_forensics: o.tag([t['name'] for t in tags]) return threat_forensics
def _query_and_filter_previous_new_threat_for_campaign( campaign_info, context): # get all threat for this campaign from the API # filter out the threat we already have in DB # return the net new threats # TODO: alternative solution, query by type, get all campaign threat, intersect sets # Q/A: why do i have to play with perf issues ? # only create Observables and link them when they do not exists. cls_action = { 'COMPLETE_URL': Url, 'NORMALIZED_URL': Url, 'ATTACHMENT': Hash, 'DOMAIN': Hostname, 'HOSTNAME': Hostname } threats = [] log.info( "There are {nb} threat associated to campaign".format( nb=len(campaign_info['campaignMembers']))) for threat in campaign_info['campaignMembers']: # ATTACHMENT, COMPLETE_URL, NORMALIZED_URL, or DOMAIN # BUG #5: undocumentated value HOSTNAME, could be hostname or ip v = threat['threat'] # t = threat['threatTime'][:10] # last_seen ? create_t = datetime.strptime( threat['threatTime'], "%Y-%m-%dT%H:%M:%S.%fZ") # TODO threat['threatStatus'] in active, ... if threat['threatStatus'] != 'active': log.warning( 'Campaign threat - threatStatus %s unsupported', threat['threatStatus']) # FIXME Campaign threat - threatStatus falsePositive unsupported # threatStatus ? if threat['subType'] not in cls_action: log.error( 'Campaign threat - subtype %s unsupported', threat['subType']) continue cls = cls_action[threat['subType']] try: # if it exists, don't do anything. tags and context are the same cls.objects.get(value=v) except DoesNotExist: # otherwise return it to link to it. # threats.append(cls.get_or_create(value=v, context=[context], created=t)) # tags named argument in constructor does not work the same as .tag() try: o = cls.get_or_create( value=v, context=[context], created=create_t) o.tag([threat['type'], threat['subType']]) except DoesNotExist: # wtf log.error( "{cls} {v} has a weird problem - FIXME".format( cls=cls, v=v)) except ObservableValidationError: try: if threat['subType'] == 'HOSTNAME': # could be an Ip o = Ip.get_or_create( value=v, context=[context], created=create_t) except ObservableValidationError as e: log.error(e) log.error(pprint.pformat(threat)) log.error( "Campaign {name}".format( name=campaign_info['name'])) o = Text.get_or_create( value=v, context=[context], created=create_t) o.tag([threat['type'], threat['subType']]) threats.append(o) log.info("Found %d new threat on campaign, new to us", len(threats)) # there is a bug here... log.debug( ", ".join( ["%s:%s" % (t.__class__.__name__, t.value) for t in threats])) return threats