def analyze(self, item):
tags = []
json_string = item.to_json()
context = {"source": self.name}
f_vt = File.get_or_create(value="FILE:{}".format(item["sha256"]))
sha256 = Hash.get_or_create(value=item["sha256"])
md5 = Hash.get_or_create(value=item["md5"])
sha1 = Hash.get_or_create(value=item["sha1"])
f_vt.active_link_to(md5, "md5", self.name)
f_vt.active_link_to(sha1, "sha1", self.name)
f_vt.active_link_to(sha256, "sha256", self.name)
tags.append(item["ruleset_name"])
tags.append(item["type"])
context["raw"] = json_string
context["size"] = item["size"]
context["score vt"] = "%s/%s" % (item["positives"], item["total"])
f_vt.tag(tags)
f_vt.add_context(context)
def analyze(self, item):
tags = []
context = {"source": self.name}
# Parse value of interest
subject = item["attributes"]["rule_name"]
date = item["attributes"]["date"]
tags2 = item["attributes"]["tags"]
sha2 = re.search(regex, str(tags2)).group()
date_string = datetime.utcfromtimestamp(date).strftime(
"%d/%m/%Y %H:%M:%S")
tags2.remove(sha2)
# Update to Yeti DB
f_vt3 = File.get_or_create(value="FILE:{}".format(sha2))
sha256 = Hash.get_or_create(value=sha2)
f_vt3.active_link_to(sha256, "sha256", self.name)
tags.append(tags2)
tags.append(subject)
context["date_added"] = date_string
context["snippet"] = item["attributes"]["snippet"]
# context['source_country'] = item["attributes"]['source_country']
context["raw"] = item
f_vt3.tag(str(tags))
f_vt3.add_context(context)
def analyze(self, block):
if not block.get("sha256_hash"):
return
context = {}
context["date_added"] = block["first_seen_utc"]
context["source"] = self.name
context["filename"] = block["file_name"]
context["md5"] = block["md5_hash"]
context["sha1"] = block["sha1_hash"]
context["sha256"] = block["sha256_hash"]
context["imphash"] = block["imphash"]
context["ssdeep"] = block["ssdeep"]
malware_file = File.get_or_create(value="FILE:{}".format(block["sha256_hash"]))
malware_file.add_context(context)
malware_file.tag(block["mime_type"])
# md5, sha1, sha256
for hash_type in ("md5_hash", "sha1_hash", "sha256_hash"):
try:
hash_data = Hash.get_or_create(value=block[hash_type])
hash_data.add_context(context)
hash_data.add_source(self.name)
if block.get("tags"):
hash_data.tag(block["tags"])
malware_file.active_link_to(
hash_data, hash_type.split("_")[0], self.name
)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, item):
tags = []
json_string = item.to_json()
context = {'source': self.name}
f_vt = File.get_or_create(value='FILE:{}'.format(item['sha256']))
sha256 = Hash.get_or_create(value=item['sha256'])
md5 = Hash.get_or_create(value=item['md5'])
sha1 = Hash.get_or_create(value=item['sha1'])
f_vt.active_link_to(md5, 'md5', self.name)
f_vt.active_link_to(sha1, 'sha1', self.name)
f_vt.active_link_to(sha256, 'sha256', self.name)
tags.append(item['ruleset_name'])
tags.append(item['type'])
context['raw'] = json_string
context['size'] = item['size']
context['score vt'] = '%s/%s' % (item['positives'], item['total'])
f_vt.tag(tags)
f_vt.add_context(context)
def analyze(self, block):
if not block.get('sha256_hash'):
return
context = {}
context['date_added'] = block['first_seen_utc']
context['source'] = self.name
context['filename'] = block['file_name']
context['md5'] = block['md5_hash']
context['sha1'] = block['sha1_hash']
context['sha256'] = block['sha256_hash']
context['imphash'] = block['imphash']
context['ssdeep'] = block['ssdeep']
malware_file = File.get_or_create(
value='FILE:{}'.format(block['sha256_hash']))
malware_file.add_context(context)
malware_file.tag(block['mime_type'])
# md5, sha1, sha256
for hash_type in ('md5_hash', 'sha1_hash', 'sha256_hash'):
try:
hash_data = Hash.get_or_create(value=block[hash_type])
hash_data.add_context(context)
hash_data.add_source(self.name)
if block.get('tags'):
hash_data.tag(block['tags'])
malware_file.active_link_to(hash_data,
hash_type.split('_')[0], self.name)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, line):
md5_obs = False
sha256_obs = False
url_obs = False
malware_file = False
context = {'source': self.name}
first_seen, url, filetype, md5, sha256, signature = line
if url:
try:
url_obs = Url.get_or_create(value=url)
if signature != 'None':
url_obs.tag(signature)
url_obs.add_context(context)
url_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if sha256:
try:
malware_file = File.get_or_create(
value='FILE:{}'.format(sha256))
malware_file.add_context(context)
malware_file.tag(filetype)
sha256_obs = Hash.get_or_create(value=sha256)
sha256_obs.tag(filetype)
sha256_obs.add_context(context)
if signature != 'None':
sha256_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if md5:
try:
md5_obs = Hash.get_or_create(value=md5)
md5_obs.add_context(context)
md5_obs.tag(filetype)
if signature != 'None':
md5_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if malware_file:
if signature != 'None':
malware_file.tag(signature)
if md5_obs:
malware_file.active_link_to(md5_obs, 'md5', self.name)
if sha256_obs:
malware_file.active_link_to(sha256_obs, 'sha256', self.name)
if url_obs:
url_obs.active_link_to(malware_file, 'drops', self.name)
def analyze(self, item):
if not item or item[0].startswith("#"):
return
first_seen, url, filetype, md5, sha256, signature = item
if url:
try:
url_obs = Url.get_or_create(value=url)
context = {'first_seen': first_seen, 'source': self.name}
if signature != 'None':
url_obs.tag(signature)
url_obs.add_context(context)
url_obs.add_source('feed')
context_malware = {'source': self.name}
malware_file = File.get_or_create(
value='FILE:{}'.format(sha256))
malware_file.add_context(context_malware)
sha256 = Hash.get_or_create(value=sha256)
sha256.tag(filetype)
sha256.add_context(context_malware)
if signature != 'None':
sha256.tag(signature)
md5 = Hash.get_or_create(value=md5)
md5.add_context(context_malware)
md5.tag(filetype)
if signature != 'None':
md5.tag(signature)
malware_file.active_link_to(md5, 'md5', self.name)
malware_file.active_link_to(sha256, 'sha256', self.name)
if signature != 'None':
malware_file.tag(signature)
malware_file.tag(filetype)
url_obs.active_link_to(malware_file, 'drops', self.name)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, line):
ssdeep, imphash, sha256, sha1, md5 = line
context = {}
context['source'] = self.name
file_obs = False
try:
file_obs = File.get_or_create(value='FILE:{}'.format(sha256))
file_obs.add_context(context)
file_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
try:
md5_data = Hash.get_or_create(value=md5)
md5_data.add_context(context)
md5_data.add_source(self.name)
file_obs.active_link_to(md5_data, 'md5', self.name)
except ObservableValidationError as e:
logging.error(e)
try:
sha1_data = Hash.get_or_create(value=sha1)
sha1_data.add_context(context)
sha1_data.add_source(self.name)
file_obs.active_link_to(sha1_data, 'sha1', self.name)
except ObservableValidationError as e:
logging.error(e)
try:
sha256_data = Hash.get_or_create(value=sha256)
sha256_data.add_context(context)
sha256_data.add_source(self.name)
file_obs.active_link_to(sha256_data, 'sha256', self.name)
except ObservableValidationError as e:
logging.error(e)
def _add_events_nodes(self, events, context, tags):
log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
attach_unsupported = dict([
(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]
])
event_nodes = list()
for msg in events:
create_t = datetime.strptime(msg['messageTime'],
"%Y-%m-%dT%H:%M:%S.%fZ")
# PPS unique value
guid = Text.get_or_create(value='proofpoint://%s' % msg['GUID'],
created=create_t,
context=[context])
log.debug('Event {msg}'.format(msg=msg['messageID']))
message_contents = list()
src_ip = Ip.get_or_create(value=msg['senderIP'],
created=create_t,
context=[context])
src_ip.tag(['MTA'])
guid.active_link_to([src_ip], "MTA src ip", self.name)
# new event
event_nodes.append(guid)
#
if self.config['import_email_metadata']:
# email details
# messageID
message_id = Email.get_or_create(value=msg['messageID'],
created=create_t,
context=[context])
guid.active_link_to([message_id], "seen in", self.name)
# sender
_s1 = Email.get_or_create(value=msg['sender'],
created=create_t,
context=[context])
_s1.tag(['sender'])
guid.active_link_to([_s1], "sender", self.name)
if 'headerFrom' in msg:
# header From
_s2 = Email.get_or_create(value=msg['headerFrom'],
created=create_t,
context=[context])
_s2.tag(['sender'])
guid.active_link_to([_s2], "headerFrom", self.name)
# FIXME is that a duplicate of attachment-malware ?
# attachment events
for attach in msg['messageParts']:
if attach['sandboxStatus'] in ['THREAT']:
md5 = Hash.get_or_create(value=attach['md5'],
created=create_t,
context=[context])
md5.tag([t['name'] for t in tags])
fname = File.get_or_create(value=attach['filename'],
created=create_t,
context=[context])
fname.tag([t['name'] for t in tags])
# this should be a DUP from threat_nodes in analyse()
sha_threat = Hash.get_or_create(value=attach['sha256'],
created=create_t,
context=[context])
sha_threat.active_link_to([md5, fname], "relates",
self.name)
sha_threat.tag([t['name'] for t in tags])
message_contents.append(sha_threat)
# link the 3 together
elif attach['sandboxStatus'] in [
'UNSUPPORTED_TYPE', 'TOO_SMALL', None
]:
attach_unsupported[attach['sandboxStatus']] += 1
log.debug(pprint.pformat(attach))
# add context to the hashes
guid.active_link_to(message_contents, "delivers", self.name)
_stats = ', '.join("%s: %d" % (k, v)
for k, v in attach_unsupported.items())
log.warning('Ignored unsupported attachments: %s', _stats)
for o in event_nodes:
o.tag([t['name'] for t in tags])
return event_nodes
def _get_threat_forensics_nodes_inner(self, evidence, general_context,
tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(value=evidence['what']['path'],
context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['md5'],
context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['sha256'],
context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['host'],
context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['ids'],
context=[context]))
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['name'],
context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(value=evidence['what']['ip'],
context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['domain'],
context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(value=evidence['what']['url'],
context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(evidence['note'].replace(
'.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
def analyze(self, line):
md5_obs = False
sha256_obs = False
url_obs = False
malware_file = False
first_seen = line['firstseen']
url = line['url']
filetype = line['filetype']
md5_hash = line['md5']
sha256_hash = line['sha256']
signature = line['signature']
context = {'source': self.name, 'first_seen': first_seen}
if url:
try:
url_obs = Url.get_or_create(value=url)
if signature != 'None':
url_obs.tag(signature)
url_obs.add_context(context)
url_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if sha256_hash:
try:
malware_file = File.get_or_create(
value='FILE:{}'.format(sha256_hash))
malware_file.add_context(context)
malware_file.tag(filetype)
sha256_obs = Hash.get_or_create(value=sha256_hash)
sha256_obs.tag(filetype)
sha256_obs.add_context(context)
if signature != 'None':
sha256_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if md5_hash:
try:
md5_obs = Hash.get_or_create(value=md5_hash)
md5_obs.add_context(context)
md5_obs.tag(filetype)
if signature != 'None':
md5_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if malware_file:
if signature != 'None':
malware_file.tag(signature)
if md5_obs:
malware_file.active_link_to(md5_obs, 'md5', self.name)
if sha256_obs:
malware_file.active_link_to(sha256_obs, 'sha256', self.name)
if url_obs:
url_obs.active_link_to(malware_file, 'drops', self.name)
def analyze(self, line):
md5_obs = False
sha256_obs = False
url_obs = False
malware_file = False
first_seen = line["firstseen"]
url = line["url"]
filetype = line["filetype"]
md5_hash = line["md5"]
sha256_hash = line["sha256"]
signature = line["signature"]
context = {"source": self.name, "first_seen": first_seen}
if url:
try:
url_obs = Url.get_or_create(value=url)
if signature != "None":
url_obs.tag(signature)
url_obs.add_context(context)
url_obs.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
if sha256_hash:
try:
malware_file = File.get_or_create(
value="FILE:{}".format(sha256_hash))
malware_file.add_context(context)
malware_file.tag(filetype)
sha256_obs = Hash.get_or_create(value=sha256_hash)
sha256_obs.tag(filetype)
sha256_obs.add_context(context)
if signature != "None":
sha256_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if md5_hash:
try:
md5_obs = Hash.get_or_create(value=md5_hash)
md5_obs.add_context(context)
md5_obs.tag(filetype)
if signature != "None":
md5_obs.tag(signature)
except ObservableValidationError as e:
logging.error(e)
if malware_file:
if signature != "None":
malware_file.tag(signature)
if md5_obs:
malware_file.active_link_to(md5_obs, "md5", self.name)
if sha256_obs:
malware_file.active_link_to(sha256_obs, "sha256", self.name)
if url_obs:
url_obs.active_link_to(malware_file, "drops", self.name)
def _add_events_nodes(self, events, context, tags):
log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
attach_unsupported = dict(
[(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]])
event_nodes = list()
for msg in events:
create_t = datetime.strptime(
msg['messageTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
# PPS unique value
guid = Text.get_or_create(
value='proofpoint://%s' % msg['GUID'],
created=create_t,
context=[context])
log.debug('Event {msg}'.format(msg=msg['messageID']))
message_contents = list()
src_ip = Ip.get_or_create(
value=msg['senderIP'], created=create_t, context=[context])
src_ip.tag(['MTA'])
guid.active_link_to([src_ip], "MTA src ip", self.name)
# new event
event_nodes.append(guid)
#
if self.config['import_email_metadata']:
# email details
# messageID
message_id = Email.get_or_create(
value=msg['messageID'], created=create_t, context=[context])
guid.active_link_to([message_id], "seen in", self.name)
# sender
_s1 = Email.get_or_create(
value=msg['sender'], created=create_t, context=[context])
_s1.tag(['sender'])
guid.active_link_to([_s1], "sender", self.name)
if 'headerFrom' in msg:
# header From
_s2 = Email.get_or_create(
value=msg['headerFrom'],
created=create_t,
context=[context])
_s2.tag(['sender'])
guid.active_link_to([_s2], "headerFrom", self.name)
# FIXME is that a duplicate of attachment-malware ?
# attachment events
for attach in msg['messageParts']:
if attach['sandboxStatus'] in ['THREAT']:
md5 = Hash.get_or_create(
value=attach['md5'],
created=create_t,
context=[context])
md5.tag([t['name'] for t in tags])
fname = File.get_or_create(
value=attach['filename'],
created=create_t,
context=[context])
fname.tag([t['name'] for t in tags])
# this should be a DUP from threat_nodes in analyse()
sha_threat = Hash.get_or_create(
value=attach['sha256'],
created=create_t,
context=[context])
sha_threat.active_link_to([md5, fname], "relates",
self.name)
sha_threat.tag([t['name'] for t in tags])
message_contents.append(sha_threat)
# link the 3 together
elif attach['sandboxStatus'] in ['UNSUPPORTED_TYPE',
'TOO_SMALL', None]:
attach_unsupported[attach['sandboxStatus']] += 1
log.debug(pprint.pformat(attach))
# add context to the hashes
guid.active_link_to(message_contents, "delivers", self.name)
_stats = ', '.join(
"%s: %d" % (k, v) for k, v in attach_unsupported.items())
log.warning('Ignored unsupported attachments: %s', _stats)
for o in event_nodes:
o.tag([t['name'] for t in tags])
return event_nodes
def analyze(self, item):
first_seen = item["analysis_start_time"]
f_hyb = File.get_or_create(value="FILE:{}".format(item["sha256"]))
sha256 = Hash.get_or_create(value=item["sha256"])
f_hyb.active_link_to(sha256, "sha256", self.name)
tags = []
context = {"source": self.name, "date": first_seen}
if "vxfamily" in item:
context["vxfamily"] = item["vxfamily"]
if "tags" in item:
tags.extend(item["tags"])
if "threatlevel_human" in item:
context["threatlevel_human"] = item["threatlevel_human"]
if "threatlevel" in item:
context["threatlevel"] = item["threatlevel"]
if "type" in item:
context["type"] = item["type"]
if "size" in item:
context["size"] = item["size"]
if "vt_detect" in item:
context["virustotal_score"] = item["vt_detect"]
if "et_alerts_total" in item:
context["et_alerts_total"] = item["et_alerts_total"]
if "process_list" in item:
context["count_process_spawn"] = len(item["process_list"])
context["url"] = "https://www.hybrid-analysis.com" + item["reporturl"]
f_hyb.add_context(context)
f_hyb.tag(tags)
f_hyb.add_source("feed")
sha256.add_context(context)
md5 = Hash.get_or_create(value=item["md5"])
md5.add_source("feed")
md5.add_context(context)
f_hyb.active_link_to(md5, "md5", self.name)
sha1 = Hash.get_or_create(value=item["sha1"])
sha1.add_source("feed")
sha1.add_context(context)
f_hyb.active_link_to(sha1, "sha1", self.name)
if "domains" in item:
for domain in item["domains"]:
try:
new_host = Hostname.get_or_create(value=domain)
f_hyb.active_link_to(new_host, "C2", self.name)
logging.debug(domain)
new_host.add_context({
"source": self.name,
"contacted_by": f_hyb
})
new_host.add_source("feed")
except ObservableValidationError as e:
logging.error(e)
if "extracted_files" in item:
for extracted_file in item["extracted_files"]:
context_file_dropped = {"source": self.name}
if not "sha256" in extracted_file:
logging.error(extracted_file)
continue
new_file = File.get_or_create(
value="FILE:{}".format(extracted_file["sha256"]))
sha256_new_file = Hash.get_or_create(
value=extracted_file["sha256"])
sha256_new_file.add_source("feed")
new_file.active_link_to(sha256_new_file, "sha256", self.name)
context_file_dropped["virustotal_score"] = 0
context_file_dropped["size"] = extracted_file["file_size"]
if "av_matched" in extracted_file:
context_file_dropped["virustotal_score"] = extracted_file[
"av_matched"]
if "threatlevel_readable" in extracted_file:
context_file_dropped["threatlevel"] = extracted_file[
"threatlevel_readable"]
if "av_label" in extracted_file:
context_file_dropped["av_label"] = extracted_file[
"av_label"]
if "type_tags" in extracted_file:
new_file.tag(extracted_file["type_tags"])
new_file.add_context(context_file_dropped)
sha256_new_file.add_context(context_file_dropped)
new_file.add_source(self.name)
f_hyb.active_link_to(new_file, "drop", self.name)
def _get_threat_forensics_nodes_inner(self, evidence, general_context,
tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
"name": _
} for _ in set([evidence["type"]] + [d["name"] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_["name"])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ["action", "rule", "path", "rule"]:
if optional not in evidence["what"]:
evidence["what"][optional] = None
# add attributes for the known evidence type
if evidence["type"] in ["file", "dropper"]:
if "path" in evidence["what"]:
threat_forensics.append(
File.get_or_create(value=evidence["what"]["path"],
context=[context]))
if "md5" in evidence["what"]:
threat_forensics.append(
Hash.get_or_create(value=evidence["what"]["md5"],
context=[context]))
if "sha256" in evidence["what"]:
threat_forensics.append(
Hash.get_or_create(value=evidence["what"]["sha256"],
context=[context]))
elif evidence["type"] == "cookie":
pass
elif evidence["type"] == "dns":
threat_forensics.append(
Hostname.get_or_create(value=evidence["what"]["host"],
context=[context]))
elif evidence["type"] == "ids":
threat_forensics.append(
Text.get_or_create(value=evidence["what"]["ids"],
context=[context]))
elif evidence["type"] == "mutex":
threat_forensics.append(
Text.get_or_create(value=evidence["what"]["name"],
context=[context]))
elif evidence["type"] == "network":
if "ip" in evidence["what"]:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(value=evidence["what"]["ip"],
context=[context]))
elif "domain" in evidence["what"]:
threat_forensics.append(
Hostname.get_or_create(value=evidence["what"]["domain"],
context=[context]))
elif evidence["type"] == "process":
pass
elif evidence["type"] == "registry":
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence["type"] == "url":
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(value=evidence["what"]["url"],
context=[context]))
# add note as tag because its a signature
if "note" in evidence:
threat_forensics[-1].tag(evidence["note"].replace(
".", "_").strip("_"))
# tag all of that
for o in threat_forensics:
o.tag([t["name"] for t in tags])
return threat_forensics
def analyze(self, block): # pylint: disable=arguments-differ
"""
{
"sha256_hash": "7a094ad0ef65079505b37da09b647597143ea7fdfac8c838796cf294da3ff388",
"sha1_hash": "6c823dad668aa14147b48487ccc0b79a477a4fce",
"md5_hash": "b319c7702230a785be61331defc25120",
"first_seen": "2020-03-17 12:58:49",
"last_seen": null,
"file_name": "a3aca2964e.sys",
"file_size": 4755520,
"file_type_mime": "application\/x-dosexec",
"file_type": "sys",
"reporter": "mxdh000",
"anonymous": 0,
"signature": null,
"imphash": "9c3307eb75e37993d0067f7cc6f873a6",
"ssdeep": "98304:BCPd6KWRbNKPVJFbZT4uX\/74m+7+LcDK3yaf:BCFPPF5D4bKYDji",
"tags": [
"zyba-rootkit-2"
],
"intelligence": {
"clamav": null,
"downloads": "4",
"uploads": "1",
"mail": null
}
}
"""
if not block.get('sha256_hash'):
return
context = {}
context["date_added"] = block['first_seen']
context["source"] = self.name
context["filename"] = block["file_name"]
context["md5"] = block["md5_hash"]
context["sha1"] = block["sha1_hash"]
context["sha256"] = block["sha256_hash"]
context["imphash"] = block["hash"]["imphash"]
context["ssdeep"] = block["hash"]["ssdeep"]
malware_file = File.get_or_create(
value='FILE:{}'.format(block["sha256_hash"]))
malware_file.add_context(context)
malware_file.tag(block["file_type_mime"])
# md5, sha1, sha256
for hash_type in ("md5_hash", "sha1_hash", "sha256_hash"):
try:
hash_data = Hash.get_or_create(value=block[hash_type])
hash_data.add_context(context)
hash_data.add_source(self.name)
if block.get("tags"):
hash_data.tag(block["tags"])
malware_file.active_link_to(hash_data,
hash_type.split("_")[0], self.name)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, item):
if not item or item[0].startswith("#"):
return
first_seen, url, filetype, md5, sha256, signature = item
item_date = dateutil.parser.parse(first_seen)
max_age = yeti_config.get('limits', 'max_age')
limit_date = datetime.now() - timedelta(days=max_age)
if item_date < limit_date:
return
if url:
try:
url_obs = Url.get_or_create(value=url)
if signature != None:
tag = signature\
.replace(' ', '_')\
.replace('/', '_')\
.replace(':', '_')\
.replace('.', '-')\
.replace('!', '-')
url_obs.tag(tag)
context = {
'first_seen': first_seen,
'source': self.name
}
url_obs.add_context(context)
url_obs.add_source('feed')
context_malware = {
'source': self.name
}
malware_file = File.get_or_create(
value='FILE:{}'.format(sha256))
malware_file.add_context(context_malware)
sha256 = Hash.get_or_create(value=sha256)
sha256.tag(filetype)
sha256.add_context(context_malware)
if signature != 'None':
sha256.tag(signature)
md5 = Hash.get_or_create(value=md5)
md5.add_context(context_malware)
md5.tag(filetype)
if signature != 'None':
md5.tag(signature)
malware_file.active_link_to(md5, 'md5', self.name)
malware_file.active_link_to(sha256, 'sha256', self.name)
if signature != 'None':
malware_file.tag(signature)
malware_file.tag(filetype)
url_obs.active_link_to(malware_file, 'drops', self.name)
except ObservableValidationError as e:
logging.error(e)
def analyze(self, item, first_seen):
f_hyb = File.get_or_create(value='FILE:{}'.format(item['sha256']))
sha256 = Hash.get_or_create(value=item['sha256'])
f_hyb.active_link_to(sha256, 'sha256', self.name)
tags = []
context = {'source': self.name, 'date': first_seen}
if 'vxfamily' in item:
context['vxfamily'] = item['vxfamily']
if 'tags' in item:
tags.extend(item['tags'])
if 'threatlevel_human' in item:
context['threatlevel_human'] = item['threatlevel_human']
if 'threatlevel' in item:
context['threatlevel'] = item['threatlevel']
if 'type' in item:
context['type'] = item['type']
if 'size' in item:
context['size'] = item['size']
if 'vt_detect' in item:
context['virustotal_score'] = item['vt_detect']
if 'et_alerts_total' in item:
context['et_alerts_total'] = item['et_alerts_total']
if 'process_list' in item:
context['count_process_spawn'] = len(item['process_list'])
context['url'] = 'https://www.hybrid-analysis.com' + item['reporturl']
f_hyb.add_context(context)
f_hyb.tag(tags)
f_hyb.add_source('feed')
sha256.add_context(context)
md5 = Hash.get_or_create(value=item['md5'])
md5.add_source('feed')
md5.add_context(context)
f_hyb.active_link_to(md5, 'md5', self.name)
sha1 = Hash.get_or_create(value=item['sha1'])
sha1.add_source('feed')
sha1.add_context(context)
f_hyb.active_link_to(sha1, 'sha1', self.name)
if 'domains' in item:
for domain in item['domains']:
try:
new_host = Hostname.get_or_create(value=domain)
f_hyb.active_link_to(new_host, 'C2', self.name)
logging.debug(domain)
new_host.add_context({
'source': self.name,
'contacted_by': f_hyb
})
new_host.add_source('feed')
except ObservableValidationError as e:
logging.error(e)
if 'extracted_files' in item:
for extracted_file in item['extracted_files']:
context_file_dropped = {'source': self.name}
if not 'sha256' in extracted_file:
logging.error(extracted_file)
continue
new_file = File.get_or_create(
value='FILE:{}'.format(extracted_file['sha256']))
sha256_new_file = Hash.get_or_create(
value=extracted_file['sha256'])
sha256_new_file.add_source('feed')
new_file.active_link_to(sha256_new_file, 'sha256', self.name)
context_file_dropped['virustotal_score'] = 0
context_file_dropped['size'] = extracted_file['file_size']
if 'av_matched' in extracted_file:
context_file_dropped['virustotal_score'] = \
extracted_file['av_matched']
if 'threatlevel_readable' in extracted_file:
context_file_dropped['threatlevel'] = \
extracted_file['threatlevel_readable']
if 'av_label' in extracted_file:
context_file_dropped['av_label'] = extracted_file[
'av_label']
if 'type_tags' in extracted_file:
new_file.tag(extracted_file['type_tags'])
new_file.add_context(context_file_dropped)
sha256_new_file.add_context(context_file_dropped)
new_file.add_source(self.name)
f_hyb.active_link_to(new_file, 'drop', self.name)
def _get_threat_forensics_nodes_inner(
self, evidence, general_context, tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(
value=evidence['what']['path'], context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['md5'], context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['sha256'], context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['host'], context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['ids'], context=[context]))
pass
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['name'], context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(
value=evidence['what']['ip'], context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['domain'], context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(
value=evidence['what']['url'], context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(
evidence['note'].replace('.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics