def analyze(self, line):
if not line or line[0].startswith("#"):
return
tokens = line.split(',')
c2_domain = []
ips_c2 = []
names_servers = []
ip_names_servers = []
context_feed = []
if len(tokens) == 6:
c2_domain = tokens[0]
ips_c2 = tokens[1].split('|')
names_servers = tokens[2].split('|')
ip_names_servers = tokens[3].split('|')
context_feed = tokens[4]
m = BambenekOsintIpmaster.reg.match(context_feed)
malware_family = ''
if m:
malware_family = m.group(1)
context = {
"status": context_feed,
"name servers": names_servers,
"source": self.name
}
tags = [malware_family]
c2 = None
if c2_domain:
c2 = Hostname.get_or_create(value=c2_domain)
c2.add_context(context)
c2.tag(tags)
c2.add_source('feed')
for ip in ips_c2:
if ip:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.tag(tags)
ip_obs.add_source('feed')
if c2:
c2.active_link_to(ip_obs, "IP", self.source)
for name_server in names_servers:
if name_server:
ns_obs = Hostname.get_or_create(value=name_server)
c2.active_link_to(ns_obs, 'NS', self.source)
ns_obs.tag(tags)
ns_obs.add_context(context)
ns_obs.add_source('feed')
for ip_ns in ip_names_servers:
if ip_ns:
ip_ns_obs = Ip.get_or_create(value=ip_ns)
c2.active_link_to(ip_ns_obs, 'IP NS', self.source)
ip_ns_obs.tag(tags)
ip_ns_obs.add_context(context)
ip_ns_obs.add_source('feed')
else:
logging.error('Parsing error in line: %s' % line)
def analyze(self, line):
if not line or line[0].startswith("#"):
return
tokens = line.split(',')
c2_domain = []
ips_c2 = []
names_servers = []
ip_names_servers = []
context_feed = []
if len(tokens) == 6:
c2_domain = tokens[0]
ips_c2 = tokens[1].split('|')
names_servers = tokens[2].split('|')
ip_names_servers = tokens[3].split('|')
context_feed = tokens[4]
m = BambenekOsintIpmaster.reg.match(context_feed)
malware_family = ''
if m:
malware_family = m.group(1)
context = {
"status": context_feed,
"name servers": names_servers,
"source": self.name
}
tags = [malware_family]
c2 = None
if c2_domain:
c2 = Hostname.get_or_create(value=c2_domain)
c2.add_context(context)
c2.tag(tags)
c2.add_source('feed')
for ip in ips_c2:
if ip:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.tag(tags)
ip_obs.add_source('feed')
if c2:
c2.active_link_to(ip_obs, "IP", self.source)
for name_server in names_servers:
if name_server:
ns_obs = Hostname.get_or_create(value=name_server)
c2.active_link_to(ns_obs, 'NS', self.source)
ns_obs.tag(tags)
ns_obs.add_context(context)
ns_obs.add_source('feed')
for ip_ns in ip_names_servers:
if ip_ns:
ip_ns_obs = Ip.get_or_create(value=ip_ns)
c2.active_link_to(ip_ns_obs, 'IP NS', self.source)
ip_ns_obs.tag(tags)
ip_ns_obs.add_context(context)
ip_ns_obs.add_source('feed')
else:
logging.error('Parsing error in line: %s' % line)
def analyze(self, line):
tokens = line.split(",")
c2_domain = []
ips_c2 = []
names_servers = []
ip_names_servers = []
context_feed = []
if len(tokens) == 6:
c2_domain = tokens[0]
ips_c2 = tokens[1].split("|")
names_servers = tokens[2].split("|")
ip_names_servers = tokens[3].split("|")
context_feed = tokens[4]
m = BambenekOsintIpmaster.reg.match(context_feed)
malware_family = ""
if m:
malware_family = m.group(1)
context = {
"status": context_feed,
"name servers": names_servers,
"source": self.name,
}
tags = [malware_family]
c2 = None
if c2_domain:
c2 = Hostname.get_or_create(value=c2_domain)
c2.add_context(context)
c2.tag(tags)
c2.add_source("feed")
for ip in ips_c2:
if ip:
ip_obs = Ip.get_or_create(value=ip)
ip_obs.tag(tags)
ip_obs.add_source(self.name)
if c2:
c2.active_link_to(ip_obs, "IP", self.source)
for name_server in names_servers:
if name_server:
ns_obs = Hostname.get_or_create(value=name_server)
c2.active_link_to(ns_obs, "NS", self.source)
ns_obs.tag(tags)
ns_obs.add_context(context)
ns_obs.add_source(self.name)
for ip_ns in ip_names_servers:
if ip_ns:
ip_ns_obs = Ip.get_or_create(value=ip_ns)
c2.active_link_to(ip_ns_obs, "IP NS", self.source)
ip_ns_obs.tag(tags)
ip_ns_obs.add_context(context)
ip_ns_obs.add_source(self.name)
else:
logging.error("Parsing error in line: %s" % line)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
results.update(raw=pformat(result))
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = Text.get_or_create(value=result['asn'])
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def _process_data(json_result, observable):
links = set()
for key in ("undetected_communicating_file",
"detected_downloaded_file", "undetected_downloaded_file",
"detected_communicating_file"):
for file_hash in json_result.get(key, []):
new_hash = Hash.get_or_create(value=file_hash)
new_hash.tag(observable.get_tags())
links.update(
new_hash.active_link_to(observable, key, "malwares.com"))
for host in json_result.get("hostname_history", {}).get("list", []):
new_host = Hostname.get_or_create(value=host)
new_host.tag(observable.get_tags())
links.update(
new_host.active_link_to(observable, "hostname",
"malwares.com"))
for key in ("detected_url", "undetected_url", "distribution_url"):
for url in json_result.get(key, []):
new_url = Url.get_or_create(value=url)
new_url.tag(observable.get_tags())
links.update(
new_url.active_link_to(observable, key, "malwares.com"))
observable.add_context(json_result)
return list(links)
def analyze(observable, results):
links = set()
if isinstance(observable, Ip):
params = {"q": observable.value, "rt": 2}
json_result = ThreatMinerApi.fetch(observable, params, "host.php")
_results, result = aux_checker(json_result)
for r in _results:
o_hostname = Hostname.get_or_create(value=r.get("domain"))
links.update(observable.link_to(o_hostname,
description="a record", source="ThreatMiner",
first_seen=r["first_seen"], last_seen=r["last_seen"])
)
observable.add_context(result)
elif isinstance(observable, Hostname):
params = {"q": observable.value, "rt": 2}
json_result = ThreatMinerApi.fetch(observable, params, "domain.php")
_results, result = aux_checker(json_result)
for r in _results:
o_ip = Ip.get_or_create(value=r.get("ip"))
links.update(observable.link_to(
o_ip,
description="a record",
source="ThreatMiner",
first_seen=r["first_seen"],
last_seen=r["last_seen"]
))
observable.add_context(result)
return list(links)
def _process_data(json_result, observable):
links = set()
for page in json_result:
if not page.get("page"):
continue
# IP iocs has more data than the rest
if not isinstance(observable, Ip) and page['page'].get('ip'):
new_ip = Ip.get_or_create(value=page['page']['ip'])
links.update(
new_ip.active_link_to(observable, 'ip', 'UrlScanIo Query'))
if not isinstance(observable,
Hostname) and page['page'].get('domain'):
new_host = Hostname.get_or_create(value=page['page']['domain'])
links.update(
new_host.active_link_to(observable, 'hostname',
'UrlScanIo Query'))
if not isinstance(observable, Url) and page['page'].get('url'):
new_url = Url.get_or_create(value=page['page']['url'])
links.update(
new_url.active_link_to(observable, 'url',
'UrlScanIo Query'))
links.update(UrlScanIoApi._process_asn_data(page, observable))
def analyze(self, line):
line = line.strip()
sline = line.split()
try:
if line[0] != "#" and len(sline) > 3: # ignore comments and entries with no clear reference
if sline[0].isdigit():
sline.pop(0)
hostname = sline[0]
description = sline[1]
reference = sline[2]
if sline[3] == "relisted":
date = sline[4]
else:
date = sline[3]
context = {}
context["source"] = self.name
context["description"] = description # malware, EK, etc
context["reference"] = reference # GG safe browsing, blog, other blacklist, etc...
context["date_added"] = datetime.strptime(date, "%Y%m%d")
try:
hn = Hostname.get_or_create(value=hostname)
hn.add_context(context)
hn.add_source("feed")
hn.tag([description])
except ObservableValidationError as e:
logging.error(e)
except Exception:
pass
def analyze(self, dict):
context = dict
date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)", dict['title']).group('datetime')
try:
context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)', dict['description'])
g = g.groupdict()
context['version'] = g['version']
context['description'] = FeodoTracker.descriptions[g['version']]
context['subfamily'] = FeodoTracker.variants[g['version']]
context['source'] = self.name
del context['title']
variant_tag = FeodoTracker.variants[g['version']].lower()
try:
if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", g['host']):
new = Ip.get_or_create(value=g['host'])
else:
new = Hostname.get_or_create(value=g['host'])
new.add_context(context)
new.add_source("feed")
new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
except ObservableValidationError as e:
logging.error(e)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
results.update(raw=pformat(result))
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = Text.get_or_create(value=result['asn'])
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def analyze(observable, result):
links = set()
endpoint = "/ip_addresses/%s/resolutions" % observable.value
api_key = result.settings["virutotal_api_key"]
result = VirustotalApi.fetch(api_key, endpoint)
if result:
for data in result["data"]:
context = {"source": "VirusTotal PDNS"}
attributes = data["attributes"]
hostname = Hostname.get_or_create(
value=attributes["host_name"])
if "date" in attributes:
timestamp_date = attributes["date"]
date_last_resolv = datetime.fromtimestamp(
timestamp_date).isoformat()
context[hostname.value] = date_last_resolv
hostname.add_context({
"source": context["source"],
observable.value: date_last_resolv,
})
links.update(
hostname.active_link_to(observable, "resolved",
context["source"]))
return list(links)
def analyze(self, dict):
hostname, date_string = dict['title'].split(' ')
context = dict
context['source'] = self.name
context['status'] = re.search(r"Status: (?P<status>\S+)", dict['description']).group('status')
context['date_added'] = datetime.strptime(date_string, "%Y-%m-%d")
try:
context['SBL'] = re.search(r"SBL: (?P<sbl>\S+),", dict['description']).group('sbl')
except AttributeError as e:
pass
context['description'] = "Palevotracker C2"
try:
context['status'] = re.search(r"Status: (?P<status>\S+),", dict['description']).group('status')
except AttributeError as e:
pass
try:
hn = Hostname.get_or_create(value=hostname)
hn.add_context(context)
hn.add_source("feed")
hn.tag(['palevo', 'c2', 'malware', 'crimeware', 'worm'])
except ObservableValidationError as e:
logging.error(e)
def analyze(self, dict):
context = dict
date_string = re.search(r"\((?P<datetime>[\d\- :]+)\)",
dict['title']).group('datetime')
try:
context['date_added'] = datetime.strptime(date_string,
"%Y-%m-%d %H:%M:%S")
except ValueError:
pass
g = re.match(r'^Host: (?P<host>.+), Version: (?P<version>\w)',
dict['description'])
g = g.groupdict()
context['version'] = g['version']
context['description'] = FeodoTracker.descriptions[g['version']]
context['subfamily'] = FeodoTracker.variants[g['version']]
context['source'] = self.name
del context['title']
variant_tag = FeodoTracker.variants[g['version']].lower()
try:
if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
g['host']):
new = Ip.get_or_create(value=g['host'])
else:
new = Hostname.get_or_create(value=g['host'])
new.add_context(context)
new.add_source("feed")
new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
except ObservableValidationError as e:
logging.error(e)
def whois_links(observable, whois):
links = set()
to_extract = [
{
"field": "organization",
"type": Text,
"label": "Registrant Organization",
},
{
"field": "registrar",
"type": Text,
"label": "Registrar",
},
{
"field": "contactEmail",
"type": Email,
"label": "Registrant Email",
},
{
"field": "name",
"type": Text,
"label": "Registrant Name",
},
{
"field": "telephone",
"type": Text,
"label": "Registrant Phone",
"record_type": "phone",
},
]
for field in to_extract:
if field["field"] in whois and whois[field["field"]] != "N/A":
node = field["type"].get_or_create(value=whois[field["field"]])
if field["type"] == Text:
if "record_type" in field:
node.update(record_type=field["record_type"])
else:
node.update(record_type=field["field"])
links.update(
observable.active_link_to(node, field["label"],
"PassiveTotal"))
if "nameServers" in whois:
nameservers = []
for ns in whois["nameServers"]:
if ns not in ["No nameserver", "not.defined"]:
try:
nameservers.append(Hostname.get_or_create(value=ns))
except Exception as e:
print(e.with_traceback())
if nameservers:
links.update(
observable.active_link_to(nameservers, "NS record",
"PassiveTotal"))
return list(links)
def analyze(self, cnc):
try:
cnc_data = Hostname.get_or_create(value=cnc)
cnc_data.add_context({"source": self.name})
cnc_data.add_source(self.name)
except ObservableValidationError as e:
logging.error(e)
def analyze(observable, results):
links = set()
context = {
"source": "malwares.com",
}
params = {
"api_key": results.settings["malwares_api_key"],
"hostname": observable.value,
}
json_result = MalwaresApi.fetch(observable, params, "hostname/info")
if json_result:
context["raw"] = json_result
for key in (
"undetected_communicating_file",
"detected_downloaded_file",
"undetected_downloaded_file",
"detected_communicating_file",
):
item = json_result[key]
for f in item["list"]:
new_hash = Hash.get_or_create(value=f["sha256"])
links.update(
new_hash.active_link_to(observable, key,
"malwares.com"))
context[key] = item["total"]
for host in json_result.get("hostname_history",
{}).get("list", []):
new_host = Hostname.get_or_create(value=host)
new_host.tag(observable.get_tags())
links.update(
new_host.active_link_to(observable, "hostname",
"malwares.com"))
for key in ("detected_url", "undetected_url"):
item = json_result[key]
for i in item["list"]:
try:
new_url = Url.get_or_create(value=i["url"])
new_url.tag(observable.get_tags())
links.update(
new_url.active_link_to(observable, key,
"malwares.com"))
except ObservableValidationError:
logging.error("Url is not valid %s" % i["url"])
context[key] = item["total"]
ip_history = json_result["ip_history"]
for i in ip_history["list"]:
ip = Ip.get_or_create(value=i["ip"])
links.update(
ip.active_link_to(observable, "ip_story", "malwares.com"))
context["ip_story"] = ip_history["total"]
return links
def analyze(observable, results):
links = set()
context = {
"source": "malwares.com",
}
params = {
"api_key": results.settings["malwares_api_key"],
"ip": observable.value,
}
json_result = MalwaresApi.fetch(observable, params, "ip/info")
for key in (
"undetected_communicating_file",
"detected_downloaded_file",
"undetected_downloaded_file",
"detected_communicating_file",
):
for item in json_result[key]["list"]:
h = Hash.get_or_create(value=item["sha256"])
links.update(
h.active_link_to(observable, key, context["source"]))
h.add_context({
"source": context["source"],
"firs_seen": item["date"]
})
context[key] = json_result[key]["total"]
for key in ("detected_url", "undetected_url"):
for item in json_result[key]["list"]:
url = Url.get_or_create(value=item["url"])
links.update(
url.active_link_to(observable, key, context["source"]))
url.add_context({
"source": context["source"],
"firs_seen": item["date"]
})
context[key] = json_result[key]["total"]
hostname_history = json_result["hostname_history"]
for item in hostname_history["list"]:
try:
hostname = Hostname.get_or_create(value=item["hostname"])
links.update(
hostname.active_link_to(observable, "hostname_history",
context["source"]))
hostname.add_context({
"source": context["source"],
"firs_seen": item["date"]
})
except ObservableValidationError:
logging.error("%s is not a hostname valid" % item["hostname"])
context["hostname_history"] = hostname_history["total"]
observable.add_context(context)
return links
def check_type(txt):
from core.observables import Hostname
try:
localpart, hostname = txt.split("@")
except Exception:
return False
return bool(
Hostname.check_type(hostname)
and re.match("^[a-zA-Z0-9!#%&'*+-/=?^_`{|}~]+$", localpart))
def analyze(observable, results):
links = set()
data = DomainToolsApi.get("/{}/host-domains/".format(observable.value), results.settings)
results.update(raw=json.dumps(data, indent=2))
for record in data['response']['ip_addresses']['domain_names']:
node = Hostname.get_or_create(value=record)
links.update(node.active_link_to(observable, 'A record', 'DomainTools'))
return list(links)
def whois_links(observable, whois):
links = set()
to_extract = [
{
"field": "organization",
"type": Text,
"label": "Registrant Organization",
},
{
"field": "registrar",
"type": Text,
"label": "Registrar",
},
{
"field": "contactEmail",
"type": Email,
"label": "Registrant Email",
},
{
"field": "name",
"type": Text,
"label": "Registrant Name",
},
{
"field": "telephone",
"type": Text,
"label": "Registrant Phone",
"record_type": "phone"
}
]
for field in to_extract:
if field['field'] in whois and whois[field['field']] != 'N/A':
node = field['type'].get_or_create(value=whois[field['field']])
if field['type'] == Text:
if "record_type" in field:
node.update(record_type=field['record_type'])
else:
node.update(record_type=field['field'])
links.update(observable.active_link_to(node, field['label'], 'PassiveTotal'))
if 'nameServers' in whois:
nameservers = []
for ns in whois['nameServers']:
if ns not in ["No nameserver", "not.defined"]:
try:
nameservers.append(Hostname.get_or_create(value=ns))
except Exception, e:
print e
if nameservers:
links.update(observable.active_link_to(nameservers, "NS record", 'PassiveTotal'))
def analyze(observable, results):
links = set()
data = DomainToolsApi.get("/{}/name-server-domains".format(observable.value), results.settings)
results.update(raw=json.dumps(data, indent=2))
for record in data['response']['primary_domains'] + data['response']['secondary_domains']:
node = Hostname.get_or_create(value=record)
links.update(node.active_link_to(observable, 'NS record', 'DomainTools'))
return list(links)
def analyze(observable, results):
links = set()
params = {"query": observable.value, "field": "nameserver"}
data = PassiveTotalApi.get("/whois/search", results.settings, params)
for record in data["results"]:
domain = Hostname.get_or_create(value=record["domain"])
links.update(whois_links(domain, record))
return list(links)
def analyze(observable, results):
links = set()
params = {'query': observable.value, 'field': 'nameserver'}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
domain = Hostname.get_or_create(value=record['domain'])
links.update(whois_links(domain, record))
return list(links)
def analyze(observable, results):
links = []
params = {'terms': observable.value, 'mode': 'purchase'}
data = DomainToolsApi.get("/reverse-whois/", results.settings, params)
for domain in data['response']['domains']:
node = Hostname.get_or_create(value=domain)
links += node.active_link_to(
observable, 'Registrant Information', 'DomainTools')
return links
def analyze(observable, results):
links = set()
params = {
'query': observable.value,
'field': 'email'
}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
print record
domain = Hostname.get_or_create(value=record['domain'])
links.update(domain.active_link_to(observable, "Registrant Email", 'PassiveTotal'))
for ns in record['nameServers']:
if ns != "No nameserver":
ns = Hostname.get_or_create(value=ns)
links.update(domain.active_link_to(ns, "NS record", 'PassiveTotal'))
return list(links)
def analyze(observable, results):
links = set()
params = {'query': observable.value, 'field': 'nameserver'}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
domain = Hostname.get_or_create(value=record['domain'])
links.update(whois_links(domain, record))
return list(links)
def analyze(observable, results):
links = []
params = {'terms': observable.value, 'mode': 'purchase'}
data = DomainToolsApi.get("/reverse-whois/", results.settings, params)
for domain in data['response']['domains']:
node = Hostname.get_or_create(value=domain)
links += node.active_link_to(observable, 'Registrant Information',
'DomainTools')
return links
def analyze(observable, results):
links = []
params = {"terms": observable.value, "mode": "purchase"}
data = DomainToolsApi.get("/reverse-whois/", results.settings, params)
for domain in data["response"]["domains"]:
node = Hostname.get_or_create(value=domain)
links += node.active_link_to(
observable, "Registrant Information", "DomainTools"
)
return links
def analyze(self, line):
line = line.strip()
parts = line.split()
hostname = str(parts[1]).strip()
context = {'source': self.name}
try:
host = Hostname.get_or_create(value=hostname)
host.add_context(context)
host.add_source(self.name)
host.tag(['malware', 'blocklist'])
except ObservableValidationError as e:
logging.error(e)
def analyze(observable, results):
links = set()
params = {
'query': '*.{}'.format(observable.value)
}
data = PassiveTotalApi.get('/enrichment/subdomains', results.settings, params)
for record in data['subdomains']:
subdomain = Hostname.get_or_create(value='{}.{}'.format(record, observable.value))
links.update(observable.active_link_to(subdomain, "Subdomain", 'PassiveTotal'))
return list(links)
def analyze(observable, results):
links = set()
params = {
'query': '*.{}'.format(observable.value)
}
data = PassiveTotalApi.get('/enrichment/subdomains', results.settings, params)
for record in data['subdomains']:
subdomain = Hostname.get_or_create(value='{}.{}'.format(record, observable.value))
links.update(observable.active_link_to(subdomain, "Subdomain", 'PassiveTotal'))
return list(links)
def analyze_hostname(cls, hostname, results):
"""Specific analyzer for Hostname observables."""
links = set()
result = []
for rec in itertools.chain(
db.passive.get(
db.passive.searchdns(hostname.value, subdomains=True)),
db.passive.get(
db.passive.searchdns(hostname.value,
reverse=True,
subdomains=True)),
):
LOG.debug("%s.analyze_hostname: record %r", cls.__name__, rec)
host = Hostname.get_or_create(value=rec["value"])
if "addr" in rec:
links.update(
Ip.get_or_create(value=rec["addr"]).link_to(
host,
"dns-%s" % rec["source"].split("-", 1)[0],
"IVRE - DNS-%s" % rec["source"],
first_seen=rec["firstseen"],
last_seen=rec["lastseen"],
))
else:
links.update(
host.link_to(
Hostname.get_or_create(value=rec["targetval"]),
"dns-%s" % rec["source"].split("-", 1)[0],
"IVRE - DNS-%s" % rec["source"],
first_seen=rec["firstseen"],
last_seen=rec["lastseen"],
))
result.append(rec)
results.update(raw=pformat(result))
return list(links)
def analyze(observable, results):
links = set()
params = {"query": "*.{}".format(observable.value)}
data = PassiveTotalApi.get("/enrichment/subdomains", results.settings,
params)
for record in data["subdomains"]:
subdomain = Hostname.get_or_create(
value="{}.{}".format(record, observable.value))
links.update(
observable.active_link_to(subdomain, "Subdomain",
"PassiveTotal"))
return list(links)
def analyze(observable, results):
links = set()
if isinstance(observable, Ip):
params = {"q": observable.value, "rt": 2}
json_result = ThreatMinerApi.fetch(observable, params, "host.php")
_results, result = aux_checker(json_result)
for r in _results:
try:
o_hostname = Hostname.get_or_create(value=r.get("domain"))
links.update(
observable.link_to(
o_hostname,
description="a record",
source="ThreatMiner",
first_seen=parser.parse(r["first_seen"]),
last_seen=parser.parse(r["last_seen"]),
))
except ObservableValidationError as e:
logging.error("Caught an exception: {}".format(e))
observable.add_context(result)
elif isinstance(observable, Hostname):
params = {"q": observable.value, "rt": 2}
json_result = ThreatMinerApi.fetch(observable, params,
"domain.php")
_results, result = aux_checker(json_result)
for r in _results:
try:
o_ip = Ip.get_or_create(value=r.get("ip"))
links.update(
observable.link_to(
o_ip,
description="a record",
source="ThreatMiner",
first_seen=r["first_seen"],
last_seen=r["last_seen"],
))
except ObservableValidationError as e:
logging.error("Caught an exception: {}".format(e))
observable.add_context(result)
return list(links)
def analyze(self, line):
line = line.strip()
sline = line.split()
hostname = sline[0]
context = {}
context['source'] = self.name
context['provider'] = sline[0]
try:
hostname = Hostname.get_or_create(value=hostname)
hostname.add_context(context)
hostname.add_source(self.name)
hostname.tag('dyndns')
except ObservableValidationError:
pass
def analyze(self, line):
try:
line = line.strip()
parts = line.split()
hostname = str(parts[1]).strip()
context = {"source": self.name}
try:
host = Hostname.get_or_create(value=hostname)
host.add_context(context)
host.add_source(self.name)
host.tag(["fraud", "blocklist"])
except ObservableValidationError as e:
logging.error(e)
except Exception as e:
logging.debug(e)
def each(cls, hostname, rtype=None, results=[]):
generated = []
h = Hostname.get_or_create(value=hostname.value)
for rdata in results:
logging.info("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
try:
e = Observable.add_text(rdata)
e.add_source("analytics")
generated.append(e)
l = Link.connect(h, e)
l.add_history(tag=rtype, description='{} record'.format(rtype))
except ObservableValidationError as e:
logging.error("{} is not a valid datatype".format(rdata))
h.analysis_done(cls.__name__)
return generated
def analyze(observable, result):
links = set()
endpoint = "/domains/%s/subdomains" % observable.value
api_key = result.settings["virutotal_api_key"]
result = VirustotalApi.fetch(api_key, endpoint)
if result:
for data in result["data"]:
context = {"source": "VirusTotal"}
attributes = data["attributes"]
sub_domain = Hostname.get_or_create(value=data["id"])
links.update(
VirustotalApi.process_domain(sub_domain, attributes))
links.update(
sub_domain.active_link_to(observable, "subdomain",
context["source"]))
return list(links)
def each(cls, hostname, rtype=None, results=[]):
generated = []
h = Hostname.get_or_create(value=hostname.value)
for rdata in results:
logging.debug("{} resolved to {} ({} record)".format(h.value, rdata, rtype))
try:
e = Observable.add_text(rdata)
e.add_source("analytics")
generated.append(e)
except ObservableValidationError as e:
logging.error("{} is not a valid datatype".format(rdata))
h.active_link_to(generated, "{} record".format(rtype), "ResolveHostnames")
h.analysis_done(cls.__name__)
return generated
def analyze(self, line):
if not line or line[0].startswith("#"):
return
try:
ip, domain, family, md5, link, date = tuple(map(strip, line))
context = {
"first_seen": date,
"family": family,
"report": link,
"source": self.name
}
c2 = None
sample = None
try:
sample = Hash.get_or_create(value=md5)
sample.add_context(context)
sample.tag(family.lower())
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
try:
if domain:
if '/' in domain:
c2 = Url.get_or_create(value=domain)
else:
c2 = Hostname.get_or_create(value=domain)
elif ip:
c2 = Ip.get_or_create(value=ip)
else:
return
c2.add_context(context)
c2.tag(['c2', family.lower()])
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
if c2 and sample:
sample.active_link_to(c2, 'c2', self.name, clean_old=False)
except ValueError:
logging.error("Error unpacking line: {}".format(line))
def analyze(self, line):
if not line or line[0].startswith("#"):
return
try:
ip, domain, family, md5, link, date = tuple(map(strip, line))
context = {
"first_seen": date,
"family": family,
"report": link,
"source": self.name
}
c2 = None
sample = None
try:
sample = Hash.get_or_create(value=md5)
sample.add_context(context)
sample.tag(family.lower())
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
try:
if domain:
if '/' in domain:
c2 = Url.get_or_create(value=domain)
else:
c2 = Hostname.get_or_create(value=domain)
elif ip:
c2 = Ip.get_or_create(value=ip)
else:
return
c2.add_context(context)
c2.tag(['c2', family.lower()])
except ObservableValidationError as e:
logging.error("Invalid line: {}\nLine: {}".format(e, line))
if c2 and sample:
sample.active_link_to(c2, 'c2', self.name, clean_old=False)
except ValueError:
logging.error("Error unpacking line: {}".format(line))
def analyze(self, line):
line = line.strip()
sline = line.split()
try:
if line[0] != '#':
hostname = sline[0]
context = {}
context['source'] = self.name
context['provider'] = sline[0]
hostname = Hostname.get_or_create(value=hostname)
hostname.add_context(context)
hostname.add_source("feed")
hostname.tag('dyndns')
except Exception:
pass
def analyze(observable, results):
links = set()
if isinstance(observable, Email):
field = 'email'
elif observable.record_type:
field = observable.record_type
else:
raise ValueError("Could not determine field for this observable")
params = {'query': observable.value, 'field': field}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
domain = Hostname.get_or_create(value=record['domain'])
links.update(whois_links(domain, record))
return list(links)
def each(cls, hostname, rtype=None, results=[]):
parts = extract(hostname.value)
if parts.suffix in SUSPICIOUS_TLDS:
hostname.tag('suspicious_tld')
if parts.subdomain != '':
hostname.update(domain=False)
domain = Hostname.get_or_create(value=parts.registered_domain, domain=True)
domain.add_source("analytics")
hostname.active_link_to(domain, "domain", "ProcessHostnames", clean_old=False)
if domain.has_tag('dyndns'):
hostname.tag('dyndns')
return domain
else:
hostname.update(domain=True)
return None
def analyze(observable, results):
links = set()
params = {
'query': observable.value,
'field': 'nameserver'
}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
domain = Hostname.get_or_create(value=record['domain'])
links.update(domain.active_link_to(observable, "NS record", 'PassiveTotal'))
registrant_email = get_value_at(record, 'registrant.email')
if registrant_email:
registrant = Email.get_or_create(value=registrant_email)
links.update(domain.active_link_to(registrant, "Registrant Email", 'PassiveTotal'))
return list(links)
def analyze(self, line):
if line.startswith('#'):
return
try:
line = line.strip()
parts = line.split()
hostname = str(parts[1]).strip()
context = {'source': self.name}
try:
host = Hostname.get_or_create(value=hostname)
host.add_context(context)
host.add_source('feed')
host.tag(['phishing', 'blocklist'])
except ObservableValidationError as e:
logging.error(e)
except Exception as e:
logging.debug(e)
def each(cls, hostname, rtype=None, results=[]):
parts = extract(hostname.value)
if parts.suffix in SUSPICIOUS_TLDS:
hostname.tag('suspicious_tld')
if parts.subdomain != '':
hostname.update(domain=False)
domain = Hostname.get_or_create(value=parts.registered_domain, domain=True)
domain.add_source("analytics")
l = Link.connect(hostname, domain)
l.add_history(tag='domain')
if domain.has_tag('dyndns'):
hostname.tag('dyndns')
return domain
else:
hostname.update(domain=True)
return None
def analyze(self, item):
context = item
date_string = re.search(
r"\((?P<datetime>[\d\- :]+)\)", context['title']).group('datetime')
try:
context['date_added'] = datetime.strptime(
date_string, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
g = re.match(
r'^Host: (?P<host>.+), Version: (?P<version>\w)',
context['description'])
g = g.groupdict()
context['version'] = g['version']
context['description'] = FeodoTracker.descriptions[g['version']]
context['subfamily'] = FeodoTracker.variants[g['version']]
context['source'] = self.name
del context['title']
new = None
variant_tag = FeodoTracker.variants[g['version']].lower()
try:
if re.search(r"[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}",
g['host']):
new = Ip.get_or_create(value=g['host'])
else:
new = Hostname.get_or_create(value=g['host'])
new.add_context(context)
new.add_source("feed")
new.tag([variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
except ObservableValidationError as e:
logging.error(e)
try:
url_fedeo = context['guid']
r = requests.get(url_fedeo)
if r.status_code == 200:
html_source = r.text
soup = BeautifulSoup(html_source, 'html.parser')
tab = soup.find('table', attrs='sortable')
results = []
if tab:
all_tr = tab.find_all('tr')
for tr in all_tr:
all_td = tr.find_all('td')
if all_td and len(all_td) == 7:
results.append({
'timestamp': all_td[0].text,
'md5_hash': all_td[1].text,
'filesize': all_td[2].text,
'VT': all_td[3].text,
'Host': all_td[4].text,
'Port': all_td[5].text,
'SSL Certif or method': all_td[6].text
})
for r in results:
new_hash = Hash.get_or_create(value=r['md5_hash'])
new_hash.add_context(context)
new_hash.add_source('feed')
new_hash.tag([
variant_tag, 'malware', 'crimeware', 'banker', 'payload'
])
new_hash.active_link_to(
new, 'c2', self.name, clean_old=False)
host = Url.get_or_create(
value='https://%s:%s' % (g['host'], r['Port']))
host.add_source('feed')
host.add_context(context)
host.tag(
[variant_tag, 'malware', 'crimeware', 'banker', 'c2'])
new_hash.active_link_to(
host, 'c2', self.name, clean_old=False)
except ObservableValidationError as e:
logging.error(e)
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(
json_result, sort_keys=True, indent=4, separators=(',', ': '))
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if 'resolutions' in json_result:
result['ip on this domains'] = 0
for ip in json_result['resolutions']:
if ip['ip_address'].strip() != observable.value:
if ip['last_resolved'] != '0000-00-00':
last_resolved = datetime.datetime.strptime(
ip['last_resolved'], "%Y-%m-%d")
try:
new_ip = Ip.get_or_create(
value=ip['ip_address'].strip())
links.update(
new_ip.active_link_to(
observable, 'IP', 'ThreatCrowd',
last_resolved))
result['ip on this domains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(ip['ip_address']))
if 'emails' in json_result:
result['nb emails'] = 0
for email in json_result['emails']:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(
observable, 'Used by', 'ThreatCrowd'))
result['nb emails'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database".
format(email))
if 'subdomains' in json_result:
result['nb subdomains'] = 0
for subdomain in json_result['subdomains']:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(
new_domain, 'subdomain', 'ThreatCrowd'))
result['nb subdomains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(subdomain))
if isinstance(observable, Ip):
if 'resolutions' in json_result:
result['domains resolved'] = 0
for domain in json_result['resolutions']:
if domain['domain'].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain['last_resolved'], "%Y-%m-%d")
new_domain = Hostname.get_or_create(
value=domain['domain'].strip())
links.update(
new_domain.active_link_to(
observable, 'A Record', 'ThreatCrowd',
last_resolved))
result['domains resolved'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database".
format(domain['domain']))
if 'hashes' in json_result and len(json_result['hashes']) > 0:
result['malwares'] = 0
for h in json_result['hashes']:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(
observable, 'hash', 'ThreatCrowd'))
result['malwares'] += 1
if isinstance(observable, Email):
if 'domains' in json_result and len(json_result) > 0:
result['domains recorded by email'] = 0
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(
observable, 'recorded by', 'ThreatCrowd'))
result['domains recorded by email'] += 1
if isinstance(observable, Hash):
result['nb c2'] = 0
if 'md5' in json_result:
new_hash = Hash.get_or_create(value=json_result['md5'])
links.update(
new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))
if 'sha1' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha1'])
links.update(
new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))
if 'sha256' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha256'])
links.update(
new_hash.active_link_to(
observable, 'sha256', 'ThreadCrowd'))
if 'domains' in json_result and len(json_result['domains']):
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(
new_domain, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'ips' in json_result and len(json_result['ips']):
for ip in json_result['ips']:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(
observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
result['source'] = 'threatcrowd_query'
result['raw'] = json_string
observable.add_context(result)
return list(links)
def _get_threat_forensics_nodes_inner(
self, evidence, general_context, tags):
# create context from notes
context = general_context.copy()
_ctx = self._make_context_from_notes([evidence])
context.update(_ctx)
# add evidence['type'] and unicify tags
tags = [{
'name': _
} for _ in set([evidence['type']] + [d['name'] for d in tags])]
# create Tags in DB
for _ in tags:
Tag.get_or_create(name=_['name'])
#
threat_forensics = []
# technical hack: set optional comments values
for optional in ['action', 'rule', 'path', 'rule']:
if optional not in evidence['what']:
evidence['what'][optional] = None
# add attributes for the known evidence type
if evidence['type'] in ['file', 'dropper']:
if 'path' in evidence['what']:
threat_forensics.append(
File.get_or_create(
value=evidence['what']['path'], context=[context]))
if 'md5' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['md5'], context=[context]))
if 'sha256' in evidence['what']:
threat_forensics.append(
Hash.get_or_create(
value=evidence['what']['sha256'], context=[context]))
elif evidence['type'] == 'cookie':
pass
elif evidence['type'] == 'dns':
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['host'], context=[context]))
elif evidence['type'] == 'ids':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['ids'], context=[context]))
pass
elif evidence['type'] == 'mutex':
threat_forensics.append(
Text.get_or_create(
value=evidence['what']['name'], context=[context]))
elif evidence['type'] == 'network':
if 'ip' in evidence['what']:
# FIXME port, type
threat_forensics.append(
Ip.get_or_create(
value=evidence['what']['ip'], context=[context]))
elif 'domain' in evidence['what']:
threat_forensics.append(
Hostname.get_or_create(
value=evidence['what']['domain'], context=[context]))
elif evidence['type'] == 'process':
pass
elif evidence['type'] == 'registry':
# threat_forensics.append(evidence['what']['key'])
# threat_forensics.append(evidence['what']['value'])
pass
elif evidence['type'] == 'url':
# BUG yeti-#115 ObservableValidationError: Invalid URL: http://xxxxx-no-tld/
threat_forensics.append(
Url.get_or_create(
value=evidence['what']['url'], context=[context]))
# add note as tag because its a signature
if 'note' in evidence:
threat_forensics[-1].tag(
evidence['note'].replace('.', '_').strip('_'))
# tag all of that
for o in threat_forensics:
o.tag([t['name'] for t in tags])
return threat_forensics
