def match(self):
"""Match observables against Yeti's intelligence repository.
Takes an array of observables, expands them and tries to match them against specific indicators or known observables.
To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
the "expanded" observable array will also include the hostname ``google.com``.
:<json [string] observables: An array of observables to be analyzed
:>json [Entity] entities: Related ``Entity`` objects
:>json [Observable] known: ``Observable`` objects that are already present in database
:>json [Indicator] matches: ``Indicators`` that matched observables
:>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
:>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
"""
params = request.json
observables = params.pop('observables', [])
fetch_neighbors = params.pop('fetch_neighbors', True)
add_unknown = bool(params.pop('add_unknown', False))
if add_unknown and current_user.has_permission('observable', 'write'):
for o in observables:
Observable.add_text(o)
data = match_observables(observables, save_matches=add_unknown and current_user.has_permission('observable', 'write'), fetch_neighbors=fetch_neighbors)
return render(data)
def match(self):
"""Match observables against Yeti's intelligence repository.
Takes an array of observables, expands them and tries to match them against specific indicators or known observables.
To "expand" an observable means to enrich the query. For instance, if the arrays of observables contains the URL ``http://google.com``,
the "expanded" observable array will also include the hostname ``google.com``.
:<json [string] observables: An array of observables to be analyzed
:>json [Entity] entities: Related ``Entity`` objects
:>json [Observable] known: ``Observable`` objects that are already present in database
:>json [Indicator] matches: ``Indicators`` that matched observables
:>json Observable matches[].observable: The ``Observable`` object that matched the ``Indicator``
:>json string unknown: Array of observable strings that didn't match any ``Indicators`` and are unknown to Yeti
"""
params = request.json
observables = params.pop('observables', [])
add_unknown = bool(params.pop('add_unknown', False))
if add_unknown:
for o in observables:
Observable.add_text(o)
data = match_observables(observables, save_matches=add_unknown)
return render(data)
def match(self):
params = request.json
observables = params.pop('observables', [])
add_unknown = bool(params.pop('add_unknown', False))
if add_unknown:
for o in observables:
Observable.add_text(o)
data = match_observables(observables, save_matches=add_unknown)
return render(data)
def analyze(self, item):
observable = item['title']
description = item['description'].lower()
context = {}
context['description'] = "{} C2 server".format(description)
context['date_added'] = parser.parse(item['pubDate'])
context['source'] = self.name
try:
e = Observable.add_text(observable)
except ObservableValidationError as e:
logging.error(e)
return
e.add_context(context)
e.add_source("feed")
tags = ['malware', 'c2', description, 'crimeware']
if description == 'pony':
tags.extend(['stealer', 'dropper'])
elif description == 'athena':
tags.extend(['stealer', 'ddos'])
elif description in ['zeus', 'citadel']:
tags.extend(['banker'])
e.tag(tags)
def analyze(self, dict):
observable = dict['title']
description = dict['description'].lower()
context = {}
context['description'] = "{} C2 server".format(description)
context['date_added'] = datetime.strptime(dict['pubDate'], "%d-%m-%Y")
context['source'] = self.name
try:
e = Observable.add_text(observable)
except ObservableValidationError as e:
logging.error(e)
return
e.add_context(context)
e.add_source("feed")
tags = ['malware', 'c2', description, 'crimeware']
if description == 'pony':
tags.extend(['stealer', 'dropper'])
elif description == 'athena':
tags.extend(['stealer', 'ddos'])
elif description in ['zeus', 'citadel']:
tags.extend(['banker'])
e.tag(tags)
def analyze(observable, results):
links = set()
params = {'query': observable.value}
data = PassiveTotalApi.get('/dns/passive', results.settings, params)
for record in data['results']:
first_seen = datetime.strptime(
record['firstSeen'], "%Y-%m-%d %H:%M:%S")
last_seen = datetime.strptime(
record['lastSeen'], "%Y-%m-%d %H:%M:%S")
new = Observable.add_text(record['resolve'])
if isinstance(observable, Hostname):
links.update(
observable.link_to(
new, "{} record".format(record['recordType']),
'PassiveTotal', first_seen, last_seen))
else:
links.update(
new.link_to(
observable, "{} record".format(record['recordType']),
'PassiveTotal', first_seen, last_seen))
return list(links)
def analyze(observable, results):
links = set()
params = {"query": observable.value}
data = PassiveTotalApi.get("/dns/passive", results.settings, params)
for record in data["results"]:
first_seen = datetime.strptime(record["firstSeen"],
"%Y-%m-%d %H:%M:%S")
last_seen = datetime.strptime(record["lastSeen"],
"%Y-%m-%d %H:%M:%S")
new = Observable.add_text(record["resolve"])
if isinstance(observable, Hostname):
links.update(
observable.link_to(
new,
"{} record".format(record["recordType"]),
"PassiveTotal",
first_seen,
last_seen,
))
else:
links.update(
new.link_to(
observable,
"{} record".format(record["recordType"]),
"PassiveTotal",
first_seen,
last_seen,
))
return list(links)
def analyze(observable, results):
links = set()
params = {'query': observable.value}
data = PassiveTotalApi.get('/dns/passive', results.settings, params)
for record in data['results']:
first_seen = datetime.strptime(record['firstSeen'],
"%Y-%m-%d %H:%M:%S")
last_seen = datetime.strptime(record['lastSeen'],
"%Y-%m-%d %H:%M:%S")
new = Observable.add_text(record['resolve'])
if isinstance(observable, Hostname):
links.update(
observable.link_to(
new, "{} record".format(record['recordType']),
'PassiveTotal', first_seen, last_seen))
else:
links.update(
new.link_to(observable,
"{} record".format(record['recordType']),
'PassiveTotal', first_seen, last_seen))
return list(links)
def enrich(self):
return "ENRICH"
if request.method == "POST":
lines = request.form['bulk-text'].split('\n')
for l in lines:
obs = refang(l.split(',')[0])
tags = refang(l.split(',')[1:])
o = Observable.add_text(obs)
o.tag(tags)
return render_template('observable/query.html')
def analyze(self, item):
s_re = '\[([^\]]*)] Type: (\w+) - IP: (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})'
r = re.compile(s_re)
m = r.match(item['description'])
malware_family = ''
c2_IP = ''
if m:
malware_family = m.group(2)
c2_IP = m.group(3)
observable = item['title']
description = item['description'].lower()
context = {}
context['description'] = "{} C2 server".format(c2_IP)
context['date_added'] = parser.parse(item['pubDate'])
context['source'] = self.name
c2 = None
e = None
try:
e = Observable.add_text(observable)
if c2_IP:
c2 = Ip.get_or_create(value=c2_IP)
e.active_link_to(
c2,
"IP",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error(e)
logging.error(description)
return
tags = ['malware', 'c2', malware_family.lower(), 'crimeware']
if malware_family == 'pony':
tags.extend(['stealer', 'dropper'])
elif malware_family == 'athena':
tags.extend(['stealer', 'ddos'])
elif malware_family in ['zeus', 'citadel','lokibot']:
tags.extend(['banker'])
if e:
e.add_context(context)
e.add_source("feed")
e.tag(tags)
if c2:
c2.add_context(context)
c2.add_source("feed")
c2.tag(tags)
def analyze(observable, results):
links = set()
json_result = CirclPDNSApi.fetch(observable, results.settings)
json_string = json.dumps(json_result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
result = {}
result['source'] = 'circl_pdns_query'
result['raw'] = json_string
if isinstance(observable, Ip):
for record in json_result:
new = Observable.add_text(record['rrname'])
new.add_source('analytics')
links.update(
observable.link_to(
new,
source='Circl.lu Passive DNS',
description='{} record'.format(record['rrtype']),
first_seen=datetime.fromtimestamp(
record['time_first']),
last_seen=datetime.fromtimestamp(record['time_last'])))
elif isinstance(observable, Hostname):
for record in json_result:
new = Observable.add_text(record["rdata"])
new.add_source('analytics')
links.update(
observable.link_to(
new,
source='Circl.lu Passive DNS',
description='{} record'.format(record['rrtype']),
first_seen=datetime.fromtimestamp(
record['time_first']),
last_seen=datetime.fromtimestamp(record['time_last'])))
observable.add_context(result)
return list(links)
def analyze(self, line):
first_seen, _type, family, hostname, url, status, registrar, ips, asns, countries = line
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": parser.parse(first_seen),
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
url_obs = False
hostname_obs = False
try:
url_obs = Url.get_or_create(value=url.rstrip())
url_obs.add_context(context)
url_obs.tag(tags)
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
try:
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
for ip in ips.split("|"):
if ip != hostname and ip is not None and ip != '':
try:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.active_link_to((url_obs, hostname),
"ip",
self.name,
clean_old=False)
except (ObservableValidationError, UnicodeEncodeError) as e:
logging.error("Invalid Observable: {}".format(e))
for asn in asns.split("|"):
try:
asn_obs = AutonomousSystem.get_or_create(value=asn)
asn_obs.active_link_to((hostname, ip_obs),
"asn",
self.name,
clean_old=False)
except (ObservableValidationError,
UnicodeEncodeError) as e:
logging.error("Invalid Observable: {}".format(e))
def analyze(self, item, pub_date): # pylint: disable=arguments-differ
s_re = "\[([^\]]*)] Type: (\w+) - IP: (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})"
r = re.compile(s_re)
m = r.match(item["description"])
malware_family = ""
c2_IP = ""
if m:
malware_family = m.group(2)
c2_IP = m.group(3)
observable = item["title"]
description = item["description"].lower()
context = {}
context["description"] = "{} C2 server".format(c2_IP)
context["date_added"] = pub_date
context["source"] = self.name
c2 = None
e = None
try:
e = Observable.add_text(observable)
if c2_IP:
c2 = Ip.get_or_create(value=c2_IP)
e.active_link_to(c2, "IP", self.name, clean_old=False)
except ObservableValidationError as e:
logging.error(e)
logging.error(description)
return
tags = ["malware", "c2", malware_family.lower(), "crimeware"]
if malware_family == "pony":
tags.extend(["stealer", "dropper"])
elif malware_family == "athena":
tags.extend(["stealer", "ddos"])
elif malware_family in ["zeus", "citadel", "lokibot"]:
tags.extend(["banker"])
if e:
e.add_context(context)
e.add_source(self.name)
e.tag(tags)
if c2:
c2.add_context(context)
c2.add_source(self.name)
c2.tag(tags)
def analyze(self, line):
if not line or line[0].startswith("#"):
return
date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
line)
item_date = dateutil.parser.parse(date)
max_age = yeti_config.get('limits', 'max_age')
limit_date = datetime.now() - timedelta(days=max_age)
if item_date < limit_date:
return
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": date,
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
try:
url = Url.get_or_create(value=url.rstrip())
url.add_context(context)
url.tag(tags)
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
for ip in ips.split("|"):
if ip != hostname and ip is not None and ip != '':
try:
i = Ip.get_or_create(value=ip)
i.active_link_to(hostname,
"First seen IP",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error("Invalid Observable: {}".format(e))
except ObservableValidationError as e:
logging.error(e)
def rdata_lookup(observable, api_key):
links = set()
for record in DNSDBApi.lookup('rdata', observable, api_key):
new = Observable.add_text(record['rrname'])
new.add_source('analytics')
links.update(
new.link_to(observable,
source='DNSDB Passive DNS',
description='{} record'.format(record['rrtype']),
first_seen=record['first_seen'],
last_seen=record['last_seen']))
return list(links)
def rdata_lookup(observable, api_key):
links = set()
for record in DNSDBApi.lookup('rdata', observable, api_key):
new = Observable.add_text(record['rrname'])
new.add_source('analytics')
links.update(
new.link_to(
observable,
source='DNSDB Passive DNS',
description='{} record'.format(record['rrtype']),
first_seen=record['first_seen'],
last_seen=record['last_seen']))
return list(links)
def each(cls, hostname, rtype=None, results=[]):
generated = []
h = Hostname.get_or_create(value=hostname.value)
for rdata in results:
logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
try:
e = Observable.add_text(rdata)
e.add_source("analytics")
generated.append(e)
l = Link.connect(h, e)
l.add_history(tag=rtype, description='{} record'.format(rtype))
except ObservableValidationError as e:
logging.error("{} is not a valid datatype".format(rdata))
h.analysis_done(cls.__name__)
return generated
def rrset_lookup(hostname, api_key):
links = set()
for record in DNSDBApi.lookup('rrset', hostname, api_key):
for observable in record['rdata']:
observable = Observable.add_text(observable)
observable.add_source('analytics')
links.update(
hostname.link_to(
observable,
source='DNSDB Passive DNS',
description='{} record'.format(record['rrtype']),
first_seen=record['first_seen'],
last_seen=record['last_seen']))
return list(links)
def each(cls, hostname, rtype=None, results=[]):
generated = []
h = Hostname.get_or_create(value=hostname.value)
for rdata in results:
logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
try:
e = Observable.add_text(rdata)
e.add_source("analytics")
generated.append(e)
except ObservableValidationError as e:
logging.error("{} is not a valid datatype".format(rdata))
h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")
h.analysis_done(cls.__name__)
return generated
def rrset_lookup(hostname, api_key):
links = set()
for record in DNSDBApi.lookup('rrset', hostname, api_key):
for observable in record['rdata']:
observable = Observable.add_text(observable)
observable.add_source('analytics')
links.update(
hostname.link_to(observable,
source='DNSDB Passive DNS',
description='{} record'.format(
record['rrtype']),
first_seen=record['first_seen'],
last_seen=record['last_seen']))
return list(links)
def rdata_lookup(observable, api_key):
links = set()
for record in DNSDBApi.lookup("rdata", observable, api_key):
new = Observable.add_text(record["rrname"])
new.add_source("analytics")
links.update(
new.link_to(
observable,
source="DNSDB Passive DNS",
description="{} record".format(record["rrtype"]),
first_seen=record["first_seen"],
last_seen=record["last_seen"],
)
)
return list(links)
def analyze(self, line):
if not line or line[0].startswith("#"):
return
date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
line)
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": date,
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
try:
url = Url.get_or_create(value=url.rstrip())
url.add_context(context)
url.tag(tags)
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
for ip in ips.split("|"):
if ip != hostname and ip is not None and ip != '':
try:
i = Ip.get_or_create(value=ip)
i.active_link_to(
hostname,
"First seen IP",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error("Invalid Observable: {}".format(e))
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
def rrset_lookup(hostname, api_key):
links = set()
for record in DNSDBApi.lookup("rrset", hostname, api_key):
for observable in record["rdata"]:
observable = Observable.add_text(observable)
observable.add_source("analytics")
links.update(
hostname.link_to(
observable,
source="DNSDB Passive DNS",
description="{} record".format(record["rrtype"]),
first_seen=record["first_seen"],
last_seen=record["last_seen"],
)
)
return list(links)
def each(cls, hostname, rtype=None, results=[]):
generated = []
h = Hostname.get_or_create(value=hostname.value)
for rdata in results:
logging.debug("{} resolved to {} ({} record)".format(
h.value, rdata, rtype))
try:
e = Observable.add_text(rdata)
e.add_source("analytics")
generated.append(e)
except ObservableValidationError as e:
logging.error("{} is not a valid datatype".format(rdata))
h.active_link_to(generated, "{} record".format(rtype),
"ResolveHostnames")
h.analysis_done(cls.__name__)
return generated
def analyze(self, line):
if not line or line[0].startswith("#"):
return
date, _type, family, hostname, url, status, registrar, ips, asns, countries = tuple(
line)
tags = []
tags += TYPE_DICT[_type]
tags.append(family.lower())
context = {
"first_seen": date,
"status": status,
"registrar": registrar,
"countries": countries.split("|"),
"asns": asns.split("|"),
"source": self.name
}
try:
url = Url.get_or_create(value=url)
url.add_context(context)
url.tag(tags)
hostname = Observable.add_text(hostname)
hostname.tag(tags + ['blocklist'])
for ip in ips.split("|"):
if ip != hostname:
try:
i = Ip.get_or_create(value=ip)
i.active_link_to(hostname,
"First seen IP",
self.name,
clean_old=False)
except ObservableValidationError as e:
logging.error("Invalid Observable: {}".format(e))
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
def match_observables(observables):
# Remove empty observables
observables = [observable for observable in observables if observable]
extended_query = set(observables) | set(derive(observables))
added_entities = set()
data = {"matches": [], "unknown": set(observables), "entities": [], "known": [], "neighbors": []}
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for o, i in Indicator.search(extended_query):
o = Observable.add_text(o)
match = i.info()
match.update({"observable": o.info(), "related": [], "suggested_tags": set()})
for nodes in i.neighbors().values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description or l.tag}
match["related"].append(node_data)
# uniquely add node information to related entitites
if node.name not in added_entities:
nodeinfo = node.info()
nodeinfo['type'] = node.type
data["entities"].append(nodeinfo)
added_entities.add(node.name)
o_tags = o.get_tags()
[match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]
data["matches"].append(match)
del_from_set(data["unknown"], o.value)
return data
def __add_attribute(self, instance, attribute, context, tags):
if attribute["category"] == "External analysis":
return
if attribute.get("type") in self.TYPES_TO_IMPORT:
context["id"] = attribute["event_id"]
context["link"] = urljoin(
self.instances[instance]["url"],
"/events/{}".format(attribute["event_id"]),
)
context["comment"] = attribute["comment"]
obs = Observable.add_text(attribute["value"])
if attribute["category"]:
obs.tag(attribute["category"].replace(" ", "_"))
if tags:
obs.tag(tags)
obs.add_context(context)
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables, extended_query = derive(observables)
observables = list(observables)
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value
not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url'],
"context": o_info['context']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({
"observable": o.info() if o.id else o.value,
"related": [],
"suggested_tags": set()
})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {
"entity": node.type,
"name": node.name,
"link_description": l.description
}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
for tag in node.generate_tags():
if tag not in o_tags:
match["suggested_tags"].add(tag)
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
def match_observables(observables, save_matches=False, fetch_neighbors=True):
# Remove empty observables
observables = [refang(observable) for observable in observables if observable]
extended_query = set(observables) | set(derive(observables))
data = {
"matches": [],
"unknown": set(observables),
"entities": {},
"known": [],
"neighbors": [],
}
# add to "known"
for o in Observable.objects(value__in=list(extended_query)):
data['known'].append(o.info())
del_from_set(data['unknown'], o.value)
if fetch_neighbors:
for link, node in (o.incoming()):
if isinstance(node, Observable):
if (link.src.value not in extended_query or link.dst.value not in extended_query) and node.tags:
data['neighbors'].append((link.info(), node.info()))
for nodes in o.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"observables": []}
if 'observables' not in ent['matches']:
ent['matches']['observables'] = []
info = node.info()
o_info = o.info()
info['matched_observable'] = {
"value": o_info['value'],
"tags": [t['name'] for t in o_info['tags']],
"human_url": o_info['human_url'],
"url": o_info['url']
}
if info not in ent['matches']['observables']:
ent['matches']['observables'].append(info)
data['entities'][node.name] = ent
# add to "matches"
for o, i in Indicator.search(extended_query):
if save_matches:
o = Observable.add_text(o)
else:
o = Observable.guess_type(o)(value=o)
try:
o.validate()
except ObservableValidationError:
pass
try:
o = Observable.objects.get(value=o.value)
except Exception:
pass
match = i.info()
match.update({"observable": o.info(), "related": [], "suggested_tags": set()})
for nodes in i.neighbors("Entity").values():
for l, node in nodes:
# add node name and link description to indicator
node_data = {"entity": node.type, "name": node.name, "link_description": l.description}
match["related"].append(node_data)
# uniquely add node information to related entitites
ent = data['entities'].get(node.name, node.info())
if 'matches' not in ent:
ent['matches'] = {"indicators": []}
if 'indicators' not in ent['matches']:
ent['matches']['indicators'] = []
info = i.info()
info['matched_observable'] = o.value
if info not in ent['matches']['indicators']:
ent['matches']['indicators'].append(info)
data['entities'][node.name] = ent
o_tags = o.get_tags()
[match["suggested_tags"].add(tag) for tag in node.generate_tags() if tag not in o_tags]
data["matches"].append(match)
data['entities'] = data['entities'].values()
return data
et.save()
et = ExportTemplate(name="Bluecoat")
et.template = """define category cert_blocklist
{% for obs in elements %}{{ obs.value }}
{% endfor %}end
"""
et.save()
Export(name="TestExport",
acts_on="Url",
description="Test description",
frequency=timedelta(hours=1),
include_tags=[t1, t2],
template=et).save()
url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags
# print url.find_tags()
# import pdb; pdb.set_trace()
## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "3"
bartalex.tags = ["bartalex"]
bartalex.save()
dridex = Malware.get_or_create(name="Dridex")
MalwareFamily("rootkit").save()
MalwareFamily("trojan").save()
MalwareFamily("dropper").save()
t1 = Tag.get_or_create(name="zeus").add_produces(["crimeware", "banker", "malware"])
t2 = Tag.get_or_create(name="banker").add_produces(["crimeware", "malware"])
t3 = Tag.get_or_create(name="c2")
t3.add_replaces(["c&c", "cc"])
Tag.get_or_create(name="crimeware").add_produces("malware")
Export(name="TestExport", description="Test description", frequency=timedelta(hours=1), include_tags=[t1, t2]).save()
url = Observable.add_text("hxxp://zeuscpanel.com/gate.php")
url.tag(["zeus", "banker", "cc", "c2"])
print url.tags
# print url.find_tags()
# import pdb; pdb.set_trace()
## Create some instances of malware & co
bartalex = Malware.get_or_create(name="Bartalex")
bartalex.family = MalwareFamily.objects.get(name="dropper")
bartalex.killchain = "delivery"
bartalex.tags = ["bartalex"]
bartalex.save()
def analyze(self, item):
first_seen = item["first_seen_utc"]
ioc_value = item["ioc_value"]
ioc_type = item["ioc_type"]
threat_type = item["threat_type"]
malware_alias = item["malware_alias"]
malware_printable = item["malware_printable"]
last_seen_utc = item["last_seen_utc"]
confidence_level = item["confidence_level"]
reference = item["reference"]
reporter = item["reporter"]
tags = []
context = {"source": self.name}
context["first_seen"] = first_seen
if reference:
context["reference"] = reference
else:
context["reference"] = "Unknown"
if reporter:
context["reporter"] = reporter
else:
context["reporter"] = "Unknown"
if threat_type:
context["threat_type"] = threat_type
if item["tags"]:
tags.extend(item["tags"].split(","))
if malware_printable:
tags.append(malware_printable)
if malware_alias:
context["malware_alias"] = malware_alias
if last_seen_utc:
context["last_seen_utc"] = last_seen_utc
if confidence_level:
context["confidence_level"] = confidence_level
value = None
obs = None
try:
if "ip" in ioc_type:
value, port = ioc_value.split(":")
context["port"] = port
obs = Ip.get_or_create(value=value)
else:
obs = Observable.add_text(ioc_value)
except ObservableValidationError as e:
logging.error(e)
return
if obs:
obs.add_context(context)
obs.add_source(self.name)
if tags:
obs.tag(tags)
if malware_printable:
obs.tags
