def tag(self, new_tags, strict=False):
if isinstance(new_tags, (str, unicode)):
new_tags = [new_tags]
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
for new_tag in new_tags:
if new_tag.strip() != "":
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
Link.connect(self, e).add_history("Tagged")
if not self.modify(
{"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()
):
self.modify(push__tags=ObservableTag(name=tag.name))
tag.modify(inc__count=1)
return self.reload()
def tag(self, new_tags, strict=False, expiration=None):
"""Tags an observable.
An observable can be tagged to add more information as to what it represents.
Args:
new_tags:
An array of strings to tag the observable with.
strict:
Set to ``True`` to replace all existing tags with the ``new_tags``.
expiration:
Timedelta field after which the Tag will not be considered fresh anymore.
Returns:
A fresh Observable instance as reloaded from the database.
"""
new_tags = iterify(new_tags)
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
tagged = False
for new_tag in new_tags:
if new_tag.strip() != '':
tagged = True
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
if not expiration:
expiration = tag.default_expiration
extra_tags = tag.produces + [tag]
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
self.active_link_to(e, 'Tagged', 'tags', clean_old=False)
for tag in extra_tags:
if not self.modify(
{"tags__name": tag.name},
set__tags__S__fresh=True,
set__tags__S__last_seen=datetime.utcnow()):
self.modify(push__tags=ObservableTag(
name=tag.name, expiration=expiration))
tag.modify(inc__count=1)
if tagged:
self.update(set__last_tagged=datetime.utcnow())
return self.reload()
def tag(self, new_tags, strict=False, expiration=None):
"""Tags an observable.
An observable can be tagged to add more information as to what it represents.
Args:
new_tags:
An array of strings to tag the observable with.
strict:
Set to ``True`` to replace all existing tags with the ``new_tags``.
expiration:
Timedelta field after which the Tag will not be considered fresh anymore.
Returns:
A fresh Observable instance as reloaded from the database.
"""
new_tags = iterify(new_tags)
if strict:
remove = set([t.name for t in self.tags]) - set(new_tags)
for tag in remove:
self.modify(pull__tags__name=tag)
tagged = False
for new_tag in new_tags:
if new_tag.strip() != '':
tagged = True
new_tag = Tag(name=new_tag)
new_tag.clean()
try: # check if tag is a replacement
tag = Tag.objects.get(replaces=new_tag.name)
except DoesNotExist:
tag = Tag.get_or_create(name=new_tag.name)
if not expiration:
expiration = tag.default_expiration
extra_tags = tag.produces + [tag]
# search for related entities and link them
for e in Entity.objects(tags__in=[tag.name]):
self.active_link_to(e, 'Tagged', 'tags', clean_old=False)
for tag in extra_tags:
if not self.modify(
{"tags__name": tag.name}, set__tags__S__fresh=True,
set__tags__S__last_seen=datetime.utcnow()):
self.modify(
push__tags=ObservableTag(
name=tag.name, expiration=expiration))
tag.modify(inc__count=1)
if tagged:
self.update(set__last_tagged=datetime.utcnow())
return self.reload()
def post(self, id=None):
if not id:
data = request.json
data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
data['replaces'] = request.json['replaces'].split(',')
return render(Tag(**data).save().info())
else:
try:
data = request.json
data['produces'] = [Tag.get_or_create(name=t.strip()) for t in request.json['produces'].split(',') if t.strip()]
data['replaces'] = request.json['replaces'].split(',')
t = Tag.objects.get(id=id)
t.update(**data)
Observable.change_all_tags(t.name, data['name'])
return render({"status": "ok"})
except TagValidationError as e:
restful_abort(400, error=str(e))
except Exception as e:
import traceback
traceback.print_exc()
restful_abort(400, error='Must specify name and produces parameters')
def clean(self):
tags = []
for t in self.tags:
if t:
tags.append(Tag.get_or_create(name=t.lower().strip()))
self.tags = [t.name for t in tags]
db.drop_database('yeti')
## Populate database with initial values
MalwareFamily("mailer").save()
MalwareFamily("banker").save()
MalwareFamily("worm").save()
MalwareFamily("ransomware").save()
MalwareFamily("backdoor").save()
MalwareFamily("stealer").save()
MalwareFamily("passwordstealer").save()
MalwareFamily("rootkit").save()
MalwareFamily("trojan").save()
MalwareFamily("dropper").save()
t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])
Tag.get_or_create(name="crimeware").add_produces("malware")
Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save()
url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags
# print url.find_tags()
def clean(self):
tags = []
for t in self.tags:
if t:
tags.append(Tag.get_or_create(name=t.lower().strip()))
self.tags = [t.name for t in tags]
e.save()
e = Malware(name="Dyre").save()
e.family = trojan
e.save()
e = Exploit(name="CVE-2015-3113").save()
e = Malware(name="Teslacrypt").save()
e.family = ransomware
e.save()
e = Malware(name="Alphacrypt").save()
e.family = ransomware
e.save()
e = Malware(name="Locky").save()
e.family = ransomware
e.save()
t1 = Tag.get_or_create(name="zeus").add_produces(
["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])
Tag.get_or_create(name="crimeware").add_produces("malware")
et = ExportTemplate(name="Default")
et.template = "{{ obs.value }}\n"
et.save()
et = ExportTemplate(name="Bluecoat")
et.template = """define category cert_blocklist
{% for obs in elements %}{{ obs.value }}
{% endfor %}end
"""
def analyze(self, threat_meta):
threat = threat_meta['threat']
events = threat_meta['events']
log.info("%d messages blocked for threat %s", len(events),
threat['threatID'])
log.debug(pprint.pformat(threat))
# make tags and context
context = {
'source': self.name,
'event_type': 'email_blocked',
'tlp': 'green'
}
tags = [
Tag.get_or_create(name=threat['threatType']),
Tag.get_or_create(name=threat['classification'])
]
tags = [{'name': t.name} for t in tags]
# extract Url and/or Hash info from threat
threat_nodes = self._make_threat_nodes(threat, context, tags)
if threat_nodes is None:
return
# TODO verify if we want that. Indicators are probably in campaign_info
# get all forensics report for this threat
threat_forensics = self._get_threat_forensics_nodes(
threat, threat_nodes, context, tags)
if threat_forensics is not None:
# attach all node to threat_nodes
for _t_node in threat_nodes:
_t_node.active_link_to(threat_forensics, "Drops", self.name)
for _f in threat_forensics:
log.debug("{threatid} Drops {forensic}".format(
threatid=_t_node.value, forensic=_f.value))
# now attach each event to the threat_nodes
events_node = self._add_events_nodes(events, context, tags)
for email in events_node:
email.active_link_to(threat_nodes, "Delivers", self.name)
for _n in threat_nodes:
log.debug("{email} Delivers {n}".format(email=email.value,
n=_n.value))
# so now our main threat is in threats[].
# lets link to the campaign
campaign, campaign_info = self._make_entities(threat)
if campaign is not None:
# attach this threat to the campaign
log.info("Linking campaign to %d threat nodes", len(threat_nodes))
campaign.action(threat_nodes, self.name, "Delivers")
# campaign_info contains campaignMembers a list of threats (urls/malwares)
# unroll and fetch them too if not duplicates
# fields: id, subType, threat, threatStatus, threatTime, type
threats_nodes2 = self._query_and_filter_previous_new_threat_for_campaign(
campaign_info, context)
log.info("Linking campaign to %d new threat nodes",
len(threats_nodes2))
# campaign.action(threats_nodes2, self.name, "Delivers")
# even faster, dont look at clean_old
campaign.active_link_to(threats_nodes2,
"Delivers",
self.name,
clean_old=False)
return
def _get_threat_forensics_nodes_inner(self, evidence, general_context,
tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(value=evidence['what']['path'],
context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['md5'],
context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(value=evidence['what']['sha256'],
context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['host'],
context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['ids'],
context=[context]))
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(value=evidence['what']['name'],
context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(value=evidence['what']['ip'],
context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(value=evidence['what']['domain'],
context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(value=evidence['what']['url'],
context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(evidence['note'].replace(
'.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
def _get_threat_forensics_nodes_inner(self, evidence, general_context,
tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
"name": _
} for _ in set([evidence["type"]] + [d["name"] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_["name"])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ["action", "rule", "path", "rule"]:
if optional not in evidence["what"]:
evidence["what"][optional] = None
# add attributes for the known evidence type
if evidence["type"] in ["file", "dropper"]:
if "path" in evidence["what"]:
threat_forensics.append(
File.get_or_create(value=evidence["what"]["path"],
context=[context]))
if "md5" in evidence["what"]:
threat_forensics.append(
Hash.get_or_create(value=evidence["what"]["md5"],
context=[context]))
if "sha256" in evidence["what"]:
threat_forensics.append(
Hash.get_or_create(value=evidence["what"]["sha256"],
context=[context]))
elif evidence["type"] == "cookie":
pass
elif evidence["type"] == "dns":
threat_forensics.append(
Hostname.get_or_create(value=evidence["what"]["host"],
context=[context]))
elif evidence["type"] == "ids":
threat_forensics.append(
Text.get_or_create(value=evidence["what"]["ids"],
context=[context]))
elif evidence["type"] == "mutex":
threat_forensics.append(
Text.get_or_create(value=evidence["what"]["name"],
context=[context]))
elif evidence["type"] == "network":
if "ip" in evidence["what"]:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(value=evidence["what"]["ip"],
context=[context]))
elif "domain" in evidence["what"]:
threat_forensics.append(
Hostname.get_or_create(value=evidence["what"]["domain"],
context=[context]))
elif evidence["type"] == "process":
pass
elif evidence["type"] == "registry":
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence["type"] == "url":
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(value=evidence["what"]["url"],
context=[context]))
# add note as tag because its a signature
if "note" in evidence:
threat_forensics[-1].tag(evidence["note"].replace(
".", "_").strip("_"))
# tag all of that
for o in threat_forensics:
o.tag([t["name"] for t in tags])
return threat_forensics
def analyze(self, threat_meta):
threat = threat_meta['threat']
events = threat_meta['events']
log.info(
"%d messages blocked for threat %s", len(events),
threat['threatID'])
log.debug(pprint.pformat(threat))
# make tags and context
context = {
'source': self.name,
'event_type': 'email_blocked',
'tlp': 'green'
}
tags = [
Tag.get_or_create(name=threat['threatType']),
Tag.get_or_create(name=threat['classification'])
]
tags = [{'name': t.name} for t in tags]
# extract Url and/or Hash info from threat
threat_nodes = self._make_threat_nodes(threat, context, tags)
if threat_nodes is None:
return
# TODO verify if we want that. Indicators are probably in campaign_info
# get all forensics report for this threat
threat_forensics = self._get_threat_forensics_nodes(
threat, threat_nodes, context, tags)
if threat_forensics is not None:
# attach all node to threat_nodes
for _t_node in threat_nodes:
_t_node.active_link_to(threat_forensics, "Drops", self.name)
for _f in threat_forensics:
log.debug(
"{threatid} Drops {forensic}".format(
threatid=_t_node.value, forensic=_f.value))
# now attach each event to the threat_nodes
events_node = self._add_events_nodes(events, context, tags)
for email in events_node:
email.active_link_to(threat_nodes, "Delivers", self.name)
for _n in threat_nodes:
log.debug(
"{email} Delivers {n}".format(
email=email.value, n=_n.value))
# so now our main threat is in threats[].
# lets link to the campaign
campaign, campaign_info = self._make_entities(threat)
if campaign is not None:
# attach this threat to the campaign
log.info("Linking campaign to %d threat nodes", len(threat_nodes))
campaign.action(threat_nodes, self.name, "Delivers")
# campaign_info contains campaignMembers a list of threats (urls/malwares)
# unroll and fetch them too if not duplicates
# fields: id, subType, threat, threatStatus, threatTime, type
threats_nodes2 = self._query_and_filter_previous_new_threat_for_campaign(
campaign_info, context)
log.info(
"Linking campaign to %d new threat nodes", len(threats_nodes2))
# campaign.action(threats_nodes2, self.name, "Delivers")
# even faster, dont look at clean_old
campaign.active_link_to(
threats_nodes2, "Delivers", self.name, clean_old=False)
return
def _get_threat_forensics_nodes_inner(
self, evidence, general_context, tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(
value=evidence['what']['path'], context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['md5'], context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['sha256'], context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['host'], context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['ids'], context=[context]))
pass
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['name'], context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(
value=evidence['what']['ip'], context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['domain'], context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(
value=evidence['what']['url'], context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(
evidence['note'].replace('.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
