def analyze(self, line):
if not line or line[0].startswith("#"):
return
tokens = line.split(',')
c2_domain = []
ips_c2 = []
names_servers = []
ip_names_servers = []
context_feed = []
if len(tokens) == 6:
c2_domain = tokens[0]
ips_c2 = tokens[1].split('|')
names_servers = tokens[2].split('|')
ip_names_servers = tokens[3].split('|')
context_feed = tokens[4]
m = BambenekOsintIpmaster.reg.match(context_feed)
malware_family = ''
if m:
malware_family = m.group(1)
context = {
"status": context_feed,
"name servers": names_servers,
"source": self.name
}
tags = [malware_family]
c2 = None
if c2_domain:
c2 = Hostname.get_or_create(value=c2_domain)
c2.add_context(context)
c2.tag(tags)
c2.add_source('feed')
for ip in ips_c2:
if ip:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.tag(tags)
ip_obs.add_source('feed')
if c2:
c2.active_link_to(ip_obs, "IP", self.source)
for name_server in names_servers:
if name_server:
ns_obs = Hostname.get_or_create(value=name_server)
c2.active_link_to(ns_obs, 'NS', self.source)
ns_obs.tag(tags)
ns_obs.add_context(context)
ns_obs.add_source('feed')
for ip_ns in ip_names_servers:
if ip_ns:
ip_ns_obs = Ip.get_or_create(value=ip_ns)
c2.active_link_to(ip_ns_obs, 'IP NS', self.source)
ip_ns_obs.tag(tags)
ip_ns_obs.add_context(context)
ip_ns_obs.add_source('feed')
else:
logging.error('Parsing error in line: %s' % line)
def analyze(self, dict):
context = dict
date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)", dict['title']).group('datetime')
try:
context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)', dict['description'])
g = g.groupdict()
context['version'] = g['version']
context['description'] = FeodoTracker.descriptions[g['version']]
context['subfamily'] = FeodoTracker.variants[g['version']]
context['source'] = self.name
del context['title']
variant_tag = FeodoTracker.variants[g['version']].lower()
try:
if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", g['host']):
new = Ip.get_or_create(value=g['host'])
else:
new = Hostname.get_or_create(value=g['host'])
new.add_context(context)
new.add_source("feed")
new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
except ObservableValidationError as e:
logging.error(e)
def analyze(self, line):
fields = line.split('|')
if len(fields) < 8:
return
context = {}
ip = fields[0]
context['name'] = fields[1]
context['router-port'] = fields[2]
context['directory-port'] = fields[3]
context['flags'] = fields[4]
context['uptime'] = fields[5]
context['version'] = fields[6]
context['contactinfo'] = fields[7]
context['description'] = "Tor exit node: %s (%s)" % (context['name'], ip)
context['source'] = self.name
try:
ip = Ip.get_or_create(value=fields[0])
ip.add_context(context)
ip.add_source("feed")
ip.tag(['tor'])
except ObservableValidationError as e:
logging.error(e)
def analyze(self, line):
if not line or line[0].startswith("#"):
return
try:
ip, domain, family, md5, link, date = tuple(map(strip, line))
context = {
"first_seen": date,
"family": family,
"report": link,
"source": self.name
}
c2 = None
sample = None
try:
sample = Hash.get_or_create(value=md5)
sample.add_context(context)
sample.tag(family.lower())
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
try:
if domain:
if '/' in domain:
c2 = Url.get_or_create(value=domain)
else:
c2 = Hostname.get_or_create(value=domain)
elif ip:
c2 = Ip.get_or_create(value=ip)
else:
return
c2.add_context(context)
c2.tag(['c2', family.lower()])
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
if c2 and sample:
sample.active_link_to(c2, 'c2', self.name, clean_old=False)
except ValueError:
logging.error("Error unpacking line: {}".format(line))
def analyze(self, line):
if not line or line[0].startswith("#"):
return
date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
line)
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": date,
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
try:
url = Url.get_or_create(value=url.rstrip())
url.add_context(context)
url.tag(tags)
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
for ip in ips.split("|"):
if ip != hostname and ip is not None and ip != '':
try:
i = Ip.get_or_create(value=ip)
i.active_link_to(
hostname,
"First seen IP",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error("Invalid Observable: {}".format(e))
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
def _get_threat_forensics_nodes_inner(self, evidence, general_context,
tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(value=evidence['what']['path'],
context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['md5'],
context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['sha256'],
context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['host'],
context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['ids'],
context=[context]))
pass
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['name'],
context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(value=evidence['what']['ip'],
context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['domain'],
context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(value=evidence['what']['url'],
context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(evidence['note'].replace(
'.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(
json_result, sort_keys=True, indent=4, separators=(',', ': '))
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if 'resolutions' in json_result:
result['ip on this domains'] = 0
for ip in json_result['resolutions']:
if ip['ip_address'].strip() != observable.value:
if ip['last_resolved'] != '0000-00-00':
last_resolved = datetime.datetime.strptime(
ip['last_resolved'], "%Y-%m-%d")
try:
new_ip = Ip.get_or_create(
value=ip['ip_address'].strip())
links.update(
new_ip.active_link_to(
observable, 'IP', 'ThreatCrowd',
last_resolved))
result['ip on this domains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(ip['ip_address']))
if 'emails' in json_result:
result['nb emails'] = 0
for email in json_result['emails']:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(
observable, 'Used by', 'ThreatCrowd'))
result['nb emails'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database".
format(email))
if 'subdomains' in json_result:
result['nb subdomains'] = 0
for subdomain in json_result['subdomains']:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(
new_domain, 'subdomain', 'ThreatCrowd'))
result['nb subdomains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(subdomain))
if isinstance(observable, Ip):
if 'resolutions' in json_result:
result['domains resolved'] = 0
for domain in json_result['resolutions']:
if domain['domain'].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain['last_resolved'], "%Y-%m-%d")
new_domain = Hostname.get_or_create(
value=domain['domain'].strip())
links.update(
new_domain.active_link_to(
observable, 'A Record', 'ThreatCrowd',
last_resolved))
result['domains resolved'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database".
format(domain['domain']))
if 'hashes' in json_result and len(json_result['hashes']) > 0:
result['malwares'] = 0
for h in json_result['hashes']:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(
observable, 'hash', 'ThreatCrowd'))
result['malwares'] += 1
if isinstance(observable, Email):
if 'domains' in json_result and len(json_result) > 0:
result['domains recorded by email'] = 0
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(
observable, 'recorded by', 'ThreatCrowd'))
result['domains recorded by email'] += 1
if isinstance(observable, Hash):
result['nb c2'] = 0
if 'md5' in json_result:
new_hash = Hash.get_or_create(value=json_result['md5'])
links.update(
new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))
if 'sha1' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha1'])
links.update(
new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))
if 'sha256' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha256'])
links.update(
new_hash.active_link_to(
observable, 'sha256', 'ThreadCrowd'))
if 'domains' in json_result and len(json_result['domains']):
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(
new_domain, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'ips' in json_result and len(json_result['ips']):
for ip in json_result['ips']:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(
observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
result['source'] = 'threatcrowd_query'
result['raw'] = json_string
observable.add_context(result)
return list(links)
def _query_and_filter_previous_new_threat_for_campaign(
campaign_info, context):
# get all threat for this campaign from the API
# filter out the threat we already have in DB
# return the net new threats
# TODO: alternative solution, query by type, get all campaign threat, intersect sets
# Q/A: why do i have to play with perf issues ?
# only create Observables and link them when they do not exists.
cls_action = {
'COMPLETE_URL': Url,
'NORMALIZED_URL': Url,
'ATTACHMENT': Hash,
'DOMAIN': Hostname,
'HOSTNAME': Hostname
}
threats = []
log.info("There are {nb} threat associated to campaign".format(
nb=len(campaign_info['campaignMembers'])))
for threat in campaign_info['campaignMembers']:
# ATTACHMENT, COMPLETE_URL, NORMALIZED_URL, or DOMAIN
# BUG #5: undocumentated value HOSTNAME, could be hostname or ip
v = threat['threat']
# t = threat['threatTime'][:10] # last_seen ?
create_t = datetime.strptime(threat['threatTime'],
"%Y-%m-%dT%H:%M:%S.%fZ")
# TODO threat['threatStatus'] in active, ...
if threat['threatStatus'] != 'active':
log.warning('Campaign threat - threatStatus %s unsupported',
threat['threatStatus'])
# FIXME Campaign threat - threatStatus falsePositive unsupported
# threatStatus ?
if threat['subType'] not in cls_action:
log.error('Campaign threat - subtype %s unsupported',
threat['subType'])
continue
cls = cls_action[threat['subType']]
try:
# if it exists, don't do anything. tags and context are the same
cls.objects.get(value=v)
except DoesNotExist:
# otherwise return it to link to it.
# threats.append(cls.get_or_create(value=v, context=[context], created=t))
# tags named argument in constructor does not work the same as .tag()
try:
o = cls.get_or_create(value=v,
context=[context],
created=create_t)
o.tag([threat['type'], threat['subType']])
except DoesNotExist:
# wtf
log.error("{cls} {v} has a weird problem - FIXME".format(
cls=cls, v=v))
except ObservableValidationError:
try:
if threat['subType'] == 'HOSTNAME': # could be an Ip
o = Ip.get_or_create(value=v,
context=[context],
created=create_t)
except ObservableValidationError as e:
log.error(e)
log.error(pprint.pformat(threat))
log.error("Campaign {name}".format(
name=campaign_info['name']))
o = Text.get_or_create(value=v,
context=[context],
created=create_t)
o.tag([threat['type'], threat['subType']])
threats.append(o)
log.info("Found %d new threat on campaign, new to us", len(threats))
# there is a bug here...
log.debug(", ".join(
["%s:%s" % (t.__class__.__name__, t.value) for t in threats]))
return threats
def analyze(self, block, first_seen): # pylint: disable=arguments-differ
"""
block example
{u"first_seen": u"2019-03-13 14:38:12",
u"hash": {u"md5": u"e1513c048e520e8d5fc5999d82994ea7",
u"sha1": u"f09f66c3bd4cd54cc030b7d102be32376fd993b5",
u"sha256": u"bbb450f1f68735054af6d1c64bd3a7e62f9977d40eeb286340d8bc1dac6f7e7e"},
u"hash_seen": 1,
u"id": u"5c8915d47a324f51d460e8e5",
u"sample": {u"name": u"vdvdv.exe", u"size": u"600576"},
u"server": {u"AS": u"AS197695",
u"country": u"ru",
u"ip": u"37.140.192.146",
u"url": u"byhlavash.chimkent.su/vdvdv.exe"}}
"""
url = block["server"]["url"]
if "http" not in url:
url = "http://" + url
context = {}
context["date_added"] = first_seen
context["as"] = block["server"]["AS"]
context["country"] = block["server"]["country"]
context["ip"] = block["server"]["ip"]
context["source"] = self.name
context["md5"] = block["hash"]["md5"]
context["sha1"] = block["hash"]["sha1"]
context["sha256"] = block["hash"]["sha256"]
url_data = None
try:
url_data = Url.get_or_create(value=url)
url_data.add_context(context)
url_data.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if block.get("server", {}).get("ip", ""):
try:
ip = Ip.get_or_create(value=block["server"]["ip"])
ip.add_context(context)
ip.add_source(self.name)
if url_data:
url_data.active_link_to(ip,
'ip',
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
if block.get("hash", []):
# md5, sha1, sha256
for hash_type in block["hash"]:
try:
hash_data = Hash.get_or_create(
value=block["hash"][hash_type])
hash_data.add_context(context)
hash_data.add_source(self.name)
if url_data:
url_data.active_link_to(hash_data,
'hash',
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
def process_domain(domain, attributes):
context = {"source": "VirusTotal"}
links = set()
timestamp_creation = attributes["creation_date"]
context["first_seen"] = datetime.fromtimestamp(
timestamp_creation).isoformat()
context["whois"] = attributes["whois"]
if "whois_date" in attributes:
timestamp_whois_date = attributes["whois_date"]
context["whois_date"] = datetime.fromtimestamp(
timestamp_creation).isoformat()
if "last_dns_records" in attributes:
last_dns_records = attributes["last_dns_records"]
for rr in last_dns_records:
related_obs = None
if rr["type"] == "A":
related_obs = Ip.get_or_create(value=rr["value"])
elif rr["type"] == "MX":
related_obs = Hostname.get_or_create(value=rr["value"])
elif rr["type"] == "SOA":
related_obs = Hostname.get_or_create(value=rr["value"])
elif rr["type"] == "NS":
related_obs = Hostname.get_or_create(value=rr["value"])
if related_obs:
links.update(
related_obs.active_link_to(domain, rr["type"],
context["source"]))
if "last_dns_records_date" in attributes:
timestamp_lst_dns_record = attributes["last_dns_records_date"]
context["last_dns_records_date"] = datetime.fromtimestamp(
timestamp_lst_dns_record).isoformat()
if "registrar" in attributes:
context["registrar"] = attributes["registrar"]
tags = attributes["tags"]
if tags:
domain.tag(tags)
if "popularity_ranks" in attributes:
alexa_rank = attributes["popularity_ranks"]
if alexa_rank:
context["alexa_rank"] = alexa_rank["Alexa"]["rank"]
timestamp_rank = alexa_rank["Alexa"]["timestamp"]
context["alexa_rank_date"] = datetime.fromtimestamp(
timestamp_creation).isoformat()
if "last_analysis_stats" in attributes:
stats_analysis = attributes["last_analysis_stats"]
for k, v in stats_analysis.items():
context[k] = v
if "last_https_certificate" and "last_https_certificate_date" in attributes:
context["last_https_certificate"] = attributes[
"last_https_certificate"]
try:
timestamp_https_cert = attributes[
"last_https_certificate_date"]
context[
"last_https_certificate_date"] = datetime.fromtimestamp(
timestamp_https_cert).isoformat()
except TypeError or ValueError:
pass
domain.add_context(context)
return links
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(json_result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if 'resolutions' in json_result:
result['ip on this domains'] = 0
for ip in json_result['resolutions']:
if ip['ip_address'].strip() != observable.value:
if ip['last_resolved'] != '0000-00-00':
last_resolved = datetime.datetime.strptime(
ip['last_resolved'], "%Y-%m-%d")
try:
new_ip = Ip.get_or_create(
value=ip['ip_address'].strip())
links.update(
new_ip.active_link_to(
observable, 'IP', 'ThreatCrowd',
last_resolved))
result['ip on this domains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database"
.format(ip['ip_address']))
if 'emails' in json_result:
result['nb emails'] = 0
for email in json_result['emails']:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(observable, 'Used by',
'ThreatCrowd'))
result['nb emails'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database"
.format(email))
if 'subdomains' in json_result:
result['nb subdomains'] = 0
for subdomain in json_result['subdomains']:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(new_domain, 'subdomain',
'ThreatCrowd'))
result['nb subdomains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database"
.format(subdomain))
if isinstance(observable, Ip):
if 'resolutions' in json_result:
result['domains resolved'] = 0
for domain in json_result['resolutions']:
if domain['domain'].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain['last_resolved'], "%Y-%m-%d")
new_domain = Hostname.get_or_create(
value=domain['domain'].strip())
links.update(
new_domain.active_link_to(
observable, 'A Record', 'ThreatCrowd',
last_resolved))
result['domains resolved'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database"
.format(domain['domain']))
if 'hashes' in json_result and len(json_result['hashes']) > 0:
result['malwares'] = 0
for h in json_result['hashes']:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(observable, 'hash',
'ThreatCrowd'))
result['malwares'] += 1
if isinstance(observable, Email):
if 'domains' in json_result and len(json_result) > 0:
result['domains recorded by email'] = 0
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(observable, 'recorded by',
'ThreatCrowd'))
result['domains recorded by email'] += 1
if isinstance(observable, Hash):
result['nb c2'] = 0
if 'md5' in json_result:
new_hash = Hash.get_or_create(value=json_result['md5'])
links.update(
new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))
if 'sha1' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha1'])
links.update(
new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))
if 'sha256' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha256'])
links.update(
new_hash.active_link_to(observable, 'sha256',
'ThreadCrowd'))
if 'domains' in json_result and len(json_result['domains']):
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(new_domain, 'c2',
'ThreatCrowd'))
result['nb c2'] += 1
if 'ips' in json_result and len(json_result['ips']):
for ip in json_result['ips']:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(
observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
result['source'] = 'threatcrowd_query'
result['raw'] = json_string
observable.add_context(result)
return list(links)
def analyze(self, item):
context = item
date_string = re.search(
r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
try:
context['date_added'] = datetime.strptime(
date_string, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
g = re.match(
r'^Host: (?P<host>.+), Version: (?P<version>\w)',
context['description'])
g = g.groupdict()
context['version'] = g['version']
context['description'] = FeodoTracker.descriptions[g['version']]
context['subfamily'] = FeodoTracker.variants[g['version']]
context['source'] = self.name
del context['title']
new = None
variant_tag = FeodoTracker.variants[g['version']].lower()
try:
if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
g['host']):
new = Ip.get_or_create(value=g['host'])
else:
new = Hostname.get_or_create(value=g['host'])
new.add_context(context)
new.add_source("feed")
new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
except ObservableValidationError as e:
logging.error(e)
try:
url_fedeo = context['guid']
r = requests.get(url_fedeo)
if r.status_code == 200:
html_source = r.text
soup = BeautifulSoup(html_source, 'html.parser')
tab = soup.find('table', attrs='sortable')
results = []
if tab:
all_tr = tab.find_all('tr')
for tr in all_tr:
all_td = tr.find_all('td')
if all_td and len(all_td) == 7:
results.append({
'timestamp': all_td[0].text,
'md5_hash': all_td[1].text,
'filesize': all_td[2].text,
'VT': all_td[3].text,
'Host': all_td[4].text,
'Port': all_td[5].text,
'SSL Certif or method': all_td[6].text
})
for r in results:
new_hash = Hash.get_or_create(value=r['md5_hash'])
new_hash.add_context(context)
new_hash.add_source('feed')
new_hash.tag([
variant_tag, 'malware', 'crimeware', 'banker', 'payload'
])
new_hash.active_link_to(
new, 'c2', self.name, clean_old=False)
host = Url.get_or_create(
value='https://%s:%s' % (g['host'], r['Port']))
host.add_source('feed')
host.add_context(context)
host.tag(
[variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
new_hash.active_link_to(
host, 'c2', self.name, clean_old=False)
except ObservableValidationError as e:
logging.error(e)
def _add_events_nodes(self, events, context, tags):
log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
attach_unsupported = dict(
[(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]])
event_nodes = list()
for msg in events:
create_t = datetime.strptime(
msg['messageTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
# PPS unique value
guid = Text.get_or_create(
value='proofpoint://%s' % msg['GUID'],
created=create_t,
context=[context])
log.debug('Event {msg}'.format(msg=msg['messageID']))
message_contents = list()
src_ip = Ip.get_or_create(
value=msg['senderIP'], created=create_t, context=[context])
src_ip.tag(['MTA'])
guid.active_link_to([src_ip], "MTA src ip", self.name)
# new event
event_nodes.append(guid)
#
if self.config['import_email_metadata']:
# email details
# messageID
message_id = Email.get_or_create(
value=msg['messageID'], created=create_t, context=[context])
guid.active_link_to([message_id], "seen in", self.name)
# sender
_s1 = Email.get_or_create(
value=msg['sender'], created=create_t, context=[context])
_s1.tag(['sender'])
guid.active_link_to([_s1], "sender", self.name)
if 'headerFrom' in msg:
# header From
_s2 = Email.get_or_create(
value=msg['headerFrom'],
created=create_t,
context=[context])
_s2.tag(['sender'])
guid.active_link_to([_s2], "headerFrom", self.name)
# FIXME is that a duplicate of attachment-malware ?
# attachment events
for attach in msg['messageParts']:
if attach['sandboxStatus'] in ['THREAT']:
md5 = Hash.get_or_create(
value=attach['md5'],
created=create_t,
context=[context])
md5.tag([t['name'] for t in tags])
fname = File.get_or_create(
value=attach['filename'],
created=create_t,
context=[context])
fname.tag([t['name'] for t in tags])
# this should be a DUP from threat_nodes in analyse()
sha_threat = Hash.get_or_create(
value=attach['sha256'],
created=create_t,
context=[context])
sha_threat.active_link_to([md5, fname], "relates",
self.name)
sha_threat.tag([t['name'] for t in tags])
message_contents.append(sha_threat)
# link the 3 together
elif attach['sandboxStatus'] in ['UNSUPPORTED_TYPE',
'TOO_SMALL', None]:
attach_unsupported[attach['sandboxStatus']] += 1
log.debug(pprint.pformat(attach))
# add context to the hashes
guid.active_link_to(message_contents, "delivers", self.name)
_stats = ', '.join(
"%s: %d" % (k, v) for k, v in attach_unsupported.items())
log.warning('Ignored unsupported attachments: %s', _stats)
for o in event_nodes:
o.tag([t['name'] for t in tags])
return event_nodes
def _get_threat_forensics_nodes_inner(
self, evidence, general_context, tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(
value=evidence['what']['path'], context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['md5'], context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['sha256'], context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['host'], context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['ids'], context=[context]))
pass
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['name'], context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(
value=evidence['what']['ip'], context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['domain'], context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(
value=evidence['what']['url'], context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(
evidence['note'].replace('.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
def _query_and_filter_previous_new_threat_for_campaign(
campaign_info, context):
# get all threat for this campaign from the API
# filter out the threat we already have in DB
# return the net new threats
# TODO: alternative solution, query by type, get all campaign threat, intersect sets
# Q/A: why do i have to play with perf issues ?
# only create Observables and link them when they do not exists.
cls_action = {
'COMPLETE_URL': Url,
'NORMALIZED_URL': Url,
'ATTACHMENT': Hash,
'DOMAIN': Hostname,
'HOSTNAME': Hostname
}
threats = []
log.info(
"There are {nb} threat associated to campaign".format(
nb=len(campaign_info['campaignMembers'])))
for threat in campaign_info['campaignMembers']:
# ATTACHMENT, COMPLETE_URL, NORMALIZED_URL, or DOMAIN
# BUG #5: undocumentated value HOSTNAME, could be hostname or ip
v = threat['threat']
# t = threat['threatTime'][:10] # last_seen ?
create_t = datetime.strptime(
threat['threatTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
# TODO threat['threatStatus'] in active, ...
if threat['threatStatus'] != 'active':
log.warning(
'Campaign threat - threatStatus %s unsupported',
threat['threatStatus'])
# FIXME Campaign threat - threatStatus falsePositive unsupported
# threatStatus ?
if threat['subType'] not in cls_action:
log.error(
'Campaign threat - subtype %s unsupported',
threat['subType'])
continue
cls = cls_action[threat['subType']]
try:
# if it exists, don't do anything. tags and context are the same
cls.objects.get(value=v)
except DoesNotExist:
# otherwise return it to link to it.
# threats.append(cls.get_or_create(value=v, context=[context], created=t))
# tags named argument in constructor does not work the same as .tag()
try:
o = cls.get_or_create(
value=v, context=[context], created=create_t)
o.tag([threat['type'], threat['subType']])
except DoesNotExist:
# wtf
log.error(
"{cls} {v} has a weird problem - FIXME".format(
cls=cls, v=v))
except ObservableValidationError:
try:
if threat['subType'] == 'HOSTNAME': # could be an Ip
o = Ip.get_or_create(
value=v, context=[context], created=create_t)
except ObservableValidationError as e:
log.error(e)
log.error(pprint.pformat(threat))
log.error(
"Campaign {name}".format(
name=campaign_info['name']))
o = Text.get_or_create(
value=v, context=[context], created=create_t)
o.tag([threat['type'], threat['subType']])
threats.append(o)
log.info("Found %d new threat on campaign, new to us", len(threats))
# there is a bug here...
log.debug(
", ".join(
["%s:%s" % (t.__class__.__name__, t.value) for t in threats]))
return threats
