def pt(self, domain, conf, verbose):
client = EnrichmentRequest(conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
print('## Searching subdomains in Passive Total')
res = client.get_subdomains(query=domain)
for d in res['subdomains']:
print('%s.%s' % (d, domain))
class EnrichmentTestCase(unittest.TestCase):
"""Test case for DNS methods."""
formats = ['json', 'xml', 'csv', 'text', 'table']
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = EnrichmentRequest('--No-User--', '--No-Key--')
def teardown_class(self):
self.patcher.stop()
def test_enrichment(self):
"""Test various actions for enrichment."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_enrichment(**payload)
assert (response['queryValue'])
def test_process_enrichment(self):
"""Test processing enrichment data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_enrichment(**payload)
wrapped = Response(response)
assert (wrapped.queryValue) == 'passivetotal.org'
def test_osint(self):
"""Test getting unique passive DNS records."""
payload = {'query': 'xxxvideotube.org'}
response = self.client.get_osint(**payload)
wrapped = Response(response)
assert (response['results'])
record = wrapped.results.pop(0)
record = Response(record)
assert (record.source) == 'RiskIQ'
assert (
record.sourceUrl
) == "https://www.riskiq.com/blog/riskiq-labs/post/a-brief-encounter-with-slempo"
def test_malware(self):
"""Test processing malware."""
payload = {'query': 'noorno.com'}
response = self.client.get_malware(**payload)
wrapped = Response(response)
assert (response['results'])
record = wrapped.results.pop(0)
record = Response(record)
assert (record.source) == 'Threatexpert'
assert (record.sample) == "7ebf1e2d0c89b1c8124275688c9e8e98"
def test_subdomains(self):
"""Test processing subdomains."""
payload = {'query': '*.passivetotal.org'}
response = self.client.get_subdomains(**payload)
wrapped = Response(response)
assert (wrapped.queryValue) == '*.passivetotal.org'
assert ('www' in wrapped.subdomains)
class EnrichmentTestCase(unittest.TestCase):
"""Test case for DNS methods."""
formats = ['json', 'xml', 'csv', 'text', 'table']
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = EnrichmentRequest('--No-User--', '--No-Key--')
def teardown_class(self):
self.patcher.stop()
def test_enrichment(self):
"""Test various actions for enrichment."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_enrichment(**payload)
assert(response['queryValue'])
def test_process_enrichment(self):
"""Test processing enrichment data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_enrichment(**payload)
wrapped = Response(response)
assert (wrapped.queryValue) == 'passivetotal.org'
def test_osint(self):
"""Test getting unique passive DNS records."""
payload = {'query': 'xxxvideotube.org'}
response = self.client.get_osint(**payload)
wrapped = Response(response)
assert (response['results'])
record = wrapped.results.pop(0)
record = Response(record)
assert (record.source) == 'RiskIQ'
assert (record.sourceUrl) == "https://www.riskiq.com/blog/riskiq-labs/post/a-brief-encounter-with-slempo"
def test_malware(self):
"""Test processing malware."""
payload = {'query': 'noorno.com'}
response = self.client.get_malware(**payload)
wrapped = Response(response)
assert (response['results'])
record = wrapped.results.pop(0)
record = Response(record)
assert (record.source) == 'Threatexpert'
assert (record.sample) == "7ebf1e2d0c89b1c8124275688c9e8e98"
def test_subdomains(self):
"""Test processing subdomains."""
payload = {'query': '*.passivetotal.org'}
response = self.client.get_subdomains(**payload)
wrapped = Response(response)
assert (wrapped.queryValue) == '*.passivetotal.org'
assert ('www' in wrapped.subdomains)
def load_client(context):
"""Get an instance of a loaded client."""
username = context.getTransformSetting('username')
api_key = context.getTransformSetting('aKey')
test_status = context.getTransformSetting('test_local')
if test_status and test_status == 'True':
server = context.getTransformSetting('server')
version = context.getTransformSetting('version')
return EnrichmentRequest(username, api_key, server, version)
else:
return EnrichmentRequest(username, api_key, headers=gen_debug(request))
def run(self):
Analyzer.run(self)
data = self.getData()
try:
# enrichment service
if self.service == 'enrichment':
enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key)
result = enrichment_request.get_enrichment(query=data)
self.report(result)
# malware service
elif self.service == 'malware':
enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key)
result = enrichment_request.get_malware(query=data)
self.report(result)
# osint service
elif self.service == 'osint':
enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key)
result = enrichment_request.get_osint(query=data)
self.report(result)
# passive dns service
elif self.service == 'passive_dns':
dns_request = DnsRequest(username=self.username, api_key=self.api_key)
result = dns_request.get_passive_dns(query=data)
self.report(result)
# ssl certificate details service
elif self.service == 'ssl_certificate_details':
ssl_request = SslRequest(username=self.username, api_key=self.api_key)
result = ssl_request.get_ssl_certificate_details(query=data)
self.report(result)
# ssl certificate history service
elif self.service == 'ssl_certificate_history':
ssl_request = SslRequest(username=self.username, api_key=self.api_key)
result = ssl_request.get_ssl_certificate_history(query=data)
self.report(result)
# unique resolutions service
elif self.service == 'unique_resolutions':
dns_request = DnsRequest(username=self.username, api_key=self.api_key)
result = dns_request.get_unique_resolutions(query=data)
self.report(result)
# whois details service
elif self.service == 'whois_details':
whois_request = WhoisRequest(username=self.username, api_key=self.api_key)
result = whois_request.get_whois_details(query=data)
self.report(result)
else:
self.error('Unknown PassiveTotal service')
except Exception as e:
self.unexpectedError(e)
def __init__(self):
try:
self.clients = {
'ssl': SslRequest.from_config(),
'dns': DnsRequest.from_config(),
'enrichment': EnrichmentRequest.from_config(),
'whois': WhoisRequest.from_config(),
'attribute': AttributeRequest.from_config(),
}
except Exception:
self.clients = None
def secondLevelCheck( type , record ):
if record is not None:
client = EnrichmentRequest('Passive Total Username', 'Passive Total Password')
if type == "ip":
payload = {'query': record['ip']}
response = client.get_malware(**payload)
#Malware hashes have been associated with this query
if len(response['results']) > 0:
#include and alarm
my_logger.debug('IP,' + record["ip"] + ',' + record["timestamp"] + ',' + record["OTXCount"] + ',' + len(response["results"]) + ';' + response)
else:
my_logger.debug('IP,' + record["ip"] + ',' + record["timestamp"] + ',' + record["OTXCount"] + ',' + ';')
if type == "DNS":
payload = {'query': record['url']}
response = client.get_malware(**payload)
#Malware hashes have been associated with this query
if len(response['results']) > 0:
#include and alarm
my_logger.debug('DNS,' + record["url"] + ',' + record["timestamp"] + ',' + record["OTXCount"] + ',' + len(response["results"]) + ';' + response)
else:
my_logger.debug('DNS,' + record["url"] + ',' + record["timestamp"] + ',' + record["OTXCount"] + ',' + ';')
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == 'whois':
client = WhoisRequest(conf['PassiveTotal']['username'],
conf['PassiveTotal']['key'])
if args.domain:
raw_results = client.search_whois_by_field(query=unbracket(
args.domain.strip()),
field="domain")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
elif args.file:
with open(args.file, 'r') as infile:
data = infile.read().split()
print(
"Domain|Date|Registrar|name|email|Phone|organization|Street|City|Postal Code|State|Country"
)
for d in data:
do = unbracket(d.strip())
# FIXME: bulk request here
raw_results = client.search_whois_by_field(
query=do, field="domain")
if "results" not in raw_results:
print("%s|||||||||||" % bracket(do))
else:
if len(raw_results["results"]) == 0:
print("%s|||||||||||" % bracket(do))
else:
r = raw_results["results"][0]
if "registered" in r:
dd = datetime.datetime.strptime(
r["registered"],
"%Y-%m-%dT%H:%M:%S.%f%z")
ddo = dd.strftime("%m/%d/%Y %H:%M:%S")
else:
ddo = ""
print(
"%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" %
(bracket(do), ddo, r["registrar"]
if "registrar" in r else "",
r["registrant"]["name"]
if "name" in r["registrant"] else "",
r["registrant"]["email"]
if "email" in r["registrant"] else "",
r["registrant"]["telephone"]
if "telephone" in r["registrant"] else "",
r["registrant"]["organization"] if
"organization" in r["registrant"] else "",
r["registrant"]["street"]
if "street" in r["registrant"] else "",
r["registrant"]["city"]
if "city" in r["registrant"] else "",
r["registrant"]["postalCode"] if
"postalCode" in r["registrant"] else "",
r["registrant"]["state"]
if "state" in r["registrant"] else "",
r["registrant"]["country"]
if "country" in r["registrant"] else ""))
elif args.email:
raw_results = client.search_whois_by_field(
query=args.email.strip(), field="email")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
self.parser.print_help()
elif args.subcommand == "dns":
client = DnsRequest(conf['PassiveTotal']['username'],
conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(query=unbracket(
args.DOMAIN), )
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
elif args.subcommand == "malware":
client = EnrichmentRequest(conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
if args.domain:
raw_results = client.get_malware(query=args.domain)
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
elif args.file:
with open(args.file, 'r') as infile:
data = infile.read().split()
domain_list = list(set([a.strip() for a in data]))
if len(domain_list) < 51:
raw_results = client.get_bulk_malware(
query=domain_list)
if "results" not in raw_results or not raw_results[
"success"]:
print("Request failed")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
sys.exit(1)
else:
results = raw_results["results"]
else:
results = {}
bulk_size = 50
i = 0
while i * bulk_size < len(domain_list):
raw_results = client.get_bulk_malware(
query=domain_list[i * bulk_size:(i + 1) *
bulk_size])
if "results" not in raw_results or not raw_results[
"success"]:
print("Request failed")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
sys.exit(1)
else:
results.update(raw_results["results"])
i += 1
if args.raw:
print(
json.dumps(results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
print("Domain|Date|Sample|Source|Source URL")
for domain in results:
if "results" in results[domain]:
for sample in results[domain]["results"]:
print("%s|%s|%s|%s|%s" %
(domain, sample["collectionDate"],
sample["sample"], sample["source"],
sample["sourceUrl"]))
else:
self.parser.print_help()
elif args.subcommand == "osint":
# FIXME: add research of projects
client = EnrichmentRequest(conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
if args.domain:
raw_results = client.get_osint(query=args.domain)
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
elif args.file:
with open(args.file, 'r') as infile:
data = infile.read().split()
domain_list = list(set([a.strip() for a in data]))
if len(domain_list) < 51:
raw_results = client.get_bulk_osint(query=domain_list)
if "results" not in raw_results or not raw_results[
"success"]:
print("Request failed")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
sys.exit(1)
else:
results = raw_results["results"]
else:
results = {}
bulk_size = 50
i = 0
while i * bulk_size < len(domain_list):
raw_results = client.get_bulk_osint(
query=domain_list[i * bulk_size:(i + 1) *
bulk_size])
if "results" not in raw_results or not raw_results[
"success"]:
print("Request failed")
print(
json.dumps(raw_results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
sys.exit(1)
else:
results.update(raw_results["results"])
i += 1
if args.raw:
print(
json.dumps(results,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
print("Domain|Source|URL|Tags")
for domain in results:
if "results" in results[domain]:
for report in results[domain]["results"]:
print("%s|%s|%s|%s" %
(domain, report["source"],
report["source_url"], " / ".join(
report["tags"])))
else:
self.parser.print_help()
else:
self.parser.print_help()
else:
self.parser.print_help()
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == "intel":
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' % args.DOMAIN)
passive_dns = []
urls = []
malware = []
files = []
# MISP
misp_e = plugins['misp'].test_config(conf)
if misp_e:
print('[+] Downloading MISP information...')
server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key'])
misp_results = server.search('attributes', value=unbracket(args.DOMAIN))
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
try:
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(IndicatorTypes.DOMAIN, unbracket(args.DOMAIN))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"ip": r['hostname'],
"first": parse(r["first"]).astimezone(pytz.utc),
"last": parse(r["last"]).astimezone(pytz.utc),
"source" : "OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
if "result" in r:
urls.append({
"date": parse(r["date"]).astimezone(pytz.utc),
"url": r["url"],
"ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "" ,
"source": "OTX"
})
else:
urls.append({
"date": parse(r["date"]).astimezone(pytz.utc),
"url": r["url"],
"ip": "",
"source": "OTX"
})
except AttributeError:
print('OTX crashed ¯\_(ツ)_/¯')
# UrlScan
us = UrlScan()
print('[+] Downloading UrlScan information....')
res = us.search(args.DOMAIN)
for r in res['results']:
urls.append({
"date": parse(r["task"]["time"]).astimezone(pytz.utc),
"url": r["page"]["url"],
"ip": r["page"]["ip"] if "ip" in r["page"] else "",
"source": "UrlScan"
})
# UrlHaus
uh_e = plugins['urlhaus'].test_config(conf)
if uh_e:
print("[+] Checking urlhaus...")
try:
urlhaus = UrlHaus(conf["UrlHaus"]["key"])
res = urlhaus.get_host(unbracket(args.DOMAIN))
except UrlHausError:
print("Error with the query")
else:
if "urls" in res:
for r in res['urls']:
urls.append({
"date": parse(r["date_added"]).astimezone(pytz.utc),
"url": r["url"],
"ip":"",
"source": "UrlHaus"
})
# CIRCL
circl_e = plugins['circl'].test_config(conf)
if circl_e:
print('[+] Downloading CIRCL passive DNS information....')
x = pypdns.PyPDNS(
basic_auth=(
conf['Circl']['user'],
conf['Circl']['pass']
)
)
res = x.query(unbracket(args.DOMAIN))
for answer in res:
passive_dns.append({
"ip": answer['rdata'],
"first": answer['time_first'].astimezone(pytz.utc),
"last": answer['time_last'].astimezone(pytz.utc),
"source" : "CIRCL"
})
# BinaryEdge
be_e = plugins['binaryedge'].test_config(conf)
if be_e:
print('[+] Downloading BinaryEdge information....')
try:
be = BinaryEdge(conf['BinaryEdge']['key'])
res = be.domain_dns(unbracket(args.DOMAIN))
for d in res['events']:
if "A" in d:
for a in d['A']:
passive_dns.append({
"ip": a,
"first": parse(d['updated_at']).astimezone(pytz.utc),
"last": parse(d['updated_at']).astimezone(pytz.utc),
"source" : "BinaryEdge"
})
except BinaryEdgeException:
print('You need a paid BinaryEdge subscription for this request')
# RobTex
print('[+] Downloading Robtex information....')
try:
rob = Robtex()
res = rob.get_pdns_domain(args.DOMAIN)
for d in res:
if d['rrtype'] in ['A', 'AAAA']:
passive_dns.append({
'first': d['time_first_o'].astimezone(pytz.utc),
'last': d['time_last_o'].astimezone(pytz.utc),
'ip': d['rrdata'],
'source': 'Robtex'
})
except RobtexError:
print("Robtex query failed")
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
try:
pt_osint = {}
ptout = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first": parse(res["firstSeen"]).astimezone(pytz.utc),
"last": parse(res["lastSeen"]).astimezone(pytz.utc),
"ip": res["resolve"],
"source": "PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(query=unbracket(args.DOMAIN))
# Get malware
raw_results = client2.get_malware(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash': r["sample"],
'date': parse(r['collectionDate']).astimezone(pytz.utc),
'source' : 'PT (%s)' % r["source"]
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_domain_report(unbracket(args.DOMAIN))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first": parse(r["last_resolved"]).astimezone(pytz.utc),
"last": parse(r["last_resolved"]).astimezone(pytz.utc),
"ip": r["ip_address"],
"source": "VT"
})
if "undetected_downloaded_samples" in res['results']:
for r in res['results']['undetected_downloaded_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '',
'source' : 'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results']['undetected_referrer_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '',
'source' : 'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results']['detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc),
'source' : 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results']['detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc),
'source' : 'VT'
})
if "detected_urls" in res['results']:
for r in res['results']['detected_urls']:
urls.append({
'date': parse(r['scan_date']).astimezone(pytz.utc),
'url': r['url'],
'ip': '',
'source': 'VT'
})
else:
vt_e = False
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
try:
print('[+] Downloading Threat Grid....')
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.DOMAIN), type='domain')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts']).astimezone(pytz.utc)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source' : 'ThreatGrid'
})
already.append(r['sample_sha256'])
except ThreatGridError as e:
print("Failed to connect to Threat Grid: %s" % e.message)
print('[+] Downloading ThreatMiner....')
tm = ThreatMiner()
response = tm.get_report(unbracket(args.DOMAIN))
if response['status_code'] == '200':
tmm = response['results']
else:
tmm = []
if response['status_code'] == '404':
print("Request to ThreatMiner failed: {}".format(response['status_message']))
response = tm.get_related_samples(unbracket(args.DOMAIN))
if response['status_code'] == '200':
for r in response['results']:
malware.append({
'hash': r,
'date': None,
'source': 'ThreatMiner'
})
print('----------------- Intelligence Report')
if misp_e:
if len(misp_results['Attribute']) > 0:
print('MISP:')
for event in misp_results['Attribute']:
print("- {} - {}".format(
event['Event']['id'],
event['Event']['info']
))
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print('- %s (%s - %s)' % (
p['name'],
p['created'][:10],
"https://otx.alienvault.com/pulse/" + p['id']
)
)
else:
print('OTX: Not found in any pulse')
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print("PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"]))
else:
print("PT: %s" % (pt_osint["results"][0]["sourceUrl"]))
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("- %s %s" % (r["name"], r["sourceUrl"]))
else:
print("- %s" % (r["sourceUrl"]))
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
# ThreatMiner
if len(tmm) > 0:
print("ThreatMiner:")
for r in tmm:
print("- {} {} - {}".format(
r['year'],
r['filename'],
r['URL']
))
if len(malware) > 0:
print('----------------- Malware')
for r in malware:
print("[%s] %s %s" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d") if r["date"] else ""
)
)
if len(files) > 0:
print('----------------- Files')
for r in files:
if r['date'] != '':
print("[%s] %s (%s)" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d")
)
)
else:
print("[%s] %s" % (
r["source"],
r["hash"],
)
)
if len(urls) > 0:
print('----------------- Urls')
for r in sorted(urls, key=lambda x: x["date"], reverse=True):
print("[%s] %s - %s %s" % (
r["source"],
r["url"],
r["ip"],
r["date"].strftime("%Y-%m-%d")
)
)
# TODO: add ASN + location info here
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True):
print("[+] %-40s (%s -> %s)(%s)" % (
r["ip"],
r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"),
r["source"]
)
)
else:
self.parser.print_help()
else:
self.parser.print_help()
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == 'info':
print("Not implemented yet")
elif args.subcommand == "intel":
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' % args.DOMAIN)
passive_dns = []
urls = []
malware = []
files = []
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(IndicatorTypes.DOMAIN, unbracket(args.DOMAIN))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"ip": r['hostname'],
"first": parse(r["first"]),
"last": parse(r["last"]),
"source" : "OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
if "result" in r:
urls.append({
"date": parse(r["date"]),
"url": r["url"],
"ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "" ,
"source": "OTX"
})
else:
urls.append({
"date": parse(r["date"]),
"url": r["url"],
"ip": "",
"source": "OTX"
})
# CIRCL
circl_e = plugins['circl'].test_config(conf)
if circl_e:
print('[+] Downloading CIRCL passive DNS information....')
x = pypdns.PyPDNS(
basic_auth=(
conf['Circl']['user'],
conf['Circl']['pass']
)
)
res = x.query(unbracket(args.DOMAIN))
for answer in res:
passive_dns.append({
"ip": answer['rdata'],
"first": answer['time_first'],
"last": answer['time_last'],
"source" : "CIRCL"
})
# BinaryEdge
be_e = plugins['binaryedge'].test_config(conf)
if be_e:
print('[+] Downloading BinaryEdge information....')
be = BinaryEdge(conf['BinaryEdge']['key'])
res = be.domain_dns(unbracket(args.DOMAIN))
for d in res['events']:
if "A" in d:
for a in d['A']:
passive_dns.append({
"ip": a,
"first": parse(d['updated_at']),
"last": parse(d['updated_at']),
"source" : "BinaryEdge"
})
# RobTex
print('[+] Downloading Robtex information....')
rob = Robtex()
res = rob.get_pdns_domain(args.DOMAIN)
for d in res:
if d['rrtype'] in ['A', 'AAAA']:
passive_dns.append({
'first': d['time_first_o'],
'last': d['time_last_o'],
'ip': d['rrdata'],
'source': 'Robtex'
})
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
try:
pt_osint = {}
ptout = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first": parse(res["firstSeen"]),
"last": parse(res["lastSeen"]),
"ip": res["resolve"],
"source": "PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(query=unbracket(args.DOMAIN))
# Get malware
raw_results = client2.get_malware(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash': r["sample"],
'date': parse(r['collectionDate']),
'source' : 'PT (%s)' % r["source"]
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_domain_report(unbracket(args.DOMAIN))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first": parse(r["last_resolved"]),
"last": parse(r["last_resolved"]),
"ip": r["ip_address"],
"source": "VT"
})
if "undetected_downloaded_samples" in res['results']:
for r in res['results']['undetected_downloaded_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']) if 'date' in r else '',
'source' : 'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results']['undetected_referrer_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']) if 'date' in r else '',
'source' : 'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results']['detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source' : 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results']['detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source' : 'VT'
})
if "detected_urls" in res['results']:
for r in res['results']['detected_urls']:
urls.append({
'date': parse(r['scan_date']),
'url': r['url'],
'ip': '',
'source': 'VT'
})
else:
vt_e = False
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
try:
print('[+] Downloading Threat Grid....')
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.DOMAIN), type='domain')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts'])
d = d.replace(tzinfo=None)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source' : 'ThreatGrid'
})
already.append(r['sample_sha256'])
except ThreatGridError as e:
print("Failed to connect to Threat Grid: %s" % e.message)
# TODO: Add MISP
print('----------------- Intelligence Report')
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print(' -%s (%s - %s)' % (
p['name'],
p['created'][:10],
"https://otx.alienvault.com/pulse/" + p['id']
)
)
else:
print('OTX: Not found in any pulse')
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print("PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"]))
else:
print("PT: %s" % (pt_osint["results"][0]["sourceUrl"]))
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("-%s %s" % (r["name"], r["sourceUrl"]))
else:
print("-%s" % (r["sourceUrl"]))
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
if len(malware) > 0:
print('----------------- Malware')
for r in sorted(malware, key=lambda x: x["date"]):
print("[%s] %s %s" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d")
)
)
if len(files) > 0:
print('----------------- Files')
for r in files:
if r['date'] != '':
print("[%s] %s (%s)" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d")
)
)
else:
print("[%s] %s" % (
r["source"],
r["hash"],
)
)
if len(urls) > 0:
print('----------------- Urls')
for r in sorted(urls, key=lambda x: x["date"], reverse=True):
print("[%s] %s - %s %s" % (
r["source"],
r["url"],
r["ip"],
r["date"].strftime("%Y-%m-%d")
)
)
# TODO: add ASN + location info here
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True):
print("[+] %-40s (%s -> %s)(%s)" % (
r["ip"],
r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"),
r["source"]
)
)
else:
self.parser.print_help()
else:
self.parser.print_help()
def intel(self, type, query, data, conf):
if type == "domain":
print("[+] Checking Passive Total...")
try:
pt_osint = {}
ptout = False
client = DnsRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]["key"],
)
raw_results = client.get_passive_dns(query=query)
if "results" in raw_results:
for res in raw_results["results"]:
data["passive_dns"].append({
"first":
parse(res["firstSeen"]).astimezone(pytz.utc),
"last":
parse(res["lastSeen"]).astimezone(pytz.utc),
"ip":
res["resolve"],
"source":
"PT",
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]["key"],
)
# Get OSINT
pt_osint = client2.get_osint(query=query)
if "results" in pt_osint:
for r in pt_osint["results"]:
data["reports"].append({
"date":
"",
"title":
r["name"] if "name" in r else "",
"url":
r["sourceUrl"],
"source":
"PT"
})
# Get malware
raw_results = client2.get_malware(query=query)
if "results" in raw_results:
for r in raw_results["results"]:
data["malware"].append({
"hash":
r["sample"],
"date":
parse(r["collectionDate"]).astimezone(
pytz.utc),
"source":
"PT (%s)" % r["source"],
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
elif type == "ip":
print("[+] Checking Passive Total...")
try:
pt_osint = {}
ptout = False
client = DnsRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]["key"],
)
raw_results = client.get_passive_dns(query=query)
if "results" in raw_results:
for res in raw_results["results"]:
data["passive_dns"].append({
"first":
parse(res["firstSeen"]).astimezone(pytz.utc),
"last":
parse(res["lastSeen"]).astimezone(pytz.utc),
"domain":
res["resolve"],
"source":
"PT",
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]["key"],
)
# Get OSINT
pt_osint = client2.get_osint(query=query)
if "results" in pt_osint:
for r in pt_osint["results"]:
data["reports"].append({
"date":
"",
"title":
r["name"] if "name" in r else "",
"url":
r["sourceUrl"],
"source":
"PT"
})
# Get malware
raw_results = client2.get_malware(query=query)
if "results" in raw_results:
for r in raw_results["results"]:
data["malware"].append({
"hash":
r["sample"],
"date":
parse(r["collectionDate"]).astimezone(
pytz.utc),
"source":
"PT (%s)" % r["source"],
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = EnrichmentRequest('--No-User--', '--No-Key--')
def run(self):
data = self.get_data()
try:
# enrichment service
if self.service == 'enrichment':
enrichment_request = EnrichmentRequest(username=self.username,
api_key=self.api_key)
result = enrichment_request.get_enrichment(query=data)
self.report(result)
# malware service
elif self.service == 'malware':
enrichment_request = EnrichmentRequest(username=self.username,
api_key=self.api_key)
result = enrichment_request.get_malware(query=data)
self.report(result)
# osint service
elif self.service == 'osint':
enrichment_request = EnrichmentRequest(username=self.username,
api_key=self.api_key)
result = enrichment_request.get_osint(query=data)
self.report(result)
# passive dns service
elif self.service == 'passive_dns':
dns_request = DnsRequest(username=self.username,
api_key=self.api_key)
result = dns_request.get_passive_dns(query=data)
self.report(result)
# ssl certificate details service
elif self.service == 'ssl_certificate_details':
ssl_request = SslRequest(username=self.username,
api_key=self.api_key)
result = ssl_request.get_ssl_certificate_details(query=data)
self.report(result)
# ssl certificate history service
elif self.service == 'ssl_certificate_history':
ssl_request = SslRequest(username=self.username,
api_key=self.api_key)
result = ssl_request.get_ssl_certificate_history(query=data)
print(len(result['results']))
if len(result['results']
) == 1 and result['results'][0]['ipAddresses'] == 'N/A':
print("ok")
self.report({'results': []})
else:
self.report(result)
# unique resolutions service
elif self.service == 'unique_resolutions':
dns_request = DnsRequest(username=self.username,
api_key=self.api_key)
result = dns_request.get_unique_resolutions(query=data)
self.report(result)
# whois details service
elif self.service == 'whois_details':
whois_request = WhoisRequest(username=self.username,
api_key=self.api_key)
result = whois_request.get_whois_details(query=data)
self.report(result)
# components service
elif self.service == 'components':
host_attr_request = HostAttributeRequest(
username=self.username, api_key=self.api_key)
result = host_attr_request.get_components(query=data)
self.report(result)
# trackers service
elif self.service == 'trackers':
host_attr_request = HostAttributeRequest(
username=self.username, api_key=self.api_key)
result = host_attr_request.get_trackers(query=data)
self.report(result)
# host pairs service
elif self.service == 'host_pairs':
host_attr_request = HostAttributeRequest(
username=self.username, api_key=self.api_key)
result = host_attr_request.get_host_pairs(query=data,
direction='parents')
children = host_attr_request.get_host_pairs(
query=data, direction='children')
result['totalRecords'] += children['totalRecords']
result['results'] = result['results'] + children['results']
self.report(result)
else:
self.error('Unknown PassiveTotal service')
except Exception as e:
self.unexpectedError(e)
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = EnrichmentRequest('--No-User--', '--No-Key--')
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == 'info':
if not is_ip(unbracket(args.IP)):
print("Invalid IP address")
sys.exit(1)
# FIXME: move code here in a library
ip = unbracket(args.IP)
try:
ipy = IP(ip)
except ValueError:
print('Invalid IP format, quitting...')
return
ipinfo = self.ipinfo(ip)
print('MaxMind: Located in %s, %s' %
(ipinfo['city'], ipinfo['country']))
if ipinfo['asn'] == 0:
print("MaxMind: IP not found in the ASN database")
else:
print('MaxMind: ASN%i, %s' %
(ipinfo['asn'], ipinfo['asn_name']))
print('CAIDA Type: %s' % ipinfo['asn_type'])
try:
asndb2 = pyasn.pyasn(self.asncidr)
res = asndb2.lookup(ip)
except OSError:
print("Configuration files are not available")
print("Please run harpoon update before using harpoon")
sys.exit(1)
if res[1] is None:
print("IP not found in ASN database")
else:
# Search for name
f = open(self.asnname, 'r')
found = False
line = f.readline()
name = ''
while not found and line != '':
s = line.split('|')
if s[0] == str(res[0]):
name = s[1].strip()
found = True
line = f.readline()
print('ASN %i - %s (range %s)' % (res[0], name, res[1]))
if ipinfo['hostname'] != '':
print('Hostname: %s' % ipinfo['hostname'])
if ipinfo['specific'] != '':
print("Specific: %s" % ipinfo['specific'])
if ipy.iptype() == "PRIVATE":
"Private IP"
print("")
if ipy.version() == 4:
print("Censys:\t\thttps://censys.io/ipv4/%s" % ip)
print("Shodan:\t\thttps://www.shodan.io/host/%s" % ip)
print("IP Info:\thttp://ipinfo.io/%s" % ip)
print("BGP HE:\t\thttps://bgp.he.net/ip/%s" % ip)
print(
"IP Location:\thttps://www.iplocation.net/?query=%s" %
ip)
elif args.subcommand == "intel":
if not is_ip(unbracket(args.IP)):
print("Invalid IP address")
sys.exit(1)
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' %
unbracket(args.IP))
passive_dns = []
urls = []
malware = []
files = []
# MISP
misp_e = plugins['misp'].test_config(conf)
if misp_e:
print('[+] Downloading MISP information...')
server = ExpandedPyMISP(conf['Misp']['url'],
conf['Misp']['key'])
misp_results = server.search('attributes',
value=unbracket(args.IP))
# Binary Edge
be_e = plugins['binaryedge'].test_config(conf)
if be_e:
try:
print('[+] Downloading BinaryEdge information...')
be = BinaryEdge(conf['BinaryEdge']['key'])
# FIXME: this only get the first page
res = be.domain_ip(unbracket(args.IP))
for d in res["events"]:
passive_dns.append({
"domain":
d['domain'],
"first":
parse(d['updated_at']).astimezone(pytz.utc),
"last":
parse(d['updated_at']).astimezone(pytz.utc),
"source":
"BinaryEdge"
})
except BinaryEdgeException:
print(
'BinaryEdge request failed, you need a paid subscription'
)
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(
IndicatorTypes.IPv4, unbracket(args.IP))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"domain":
r['hostname'],
"first":
parse(r["first"]).astimezone(pytz.utc),
"last":
parse(r["last"]).astimezone(pytz.utc),
"source":
"OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
if "result" in r:
urls.append({
"date":
parse(r["date"]).astimezone(pytz.utc),
"url":
r["url"],
"ip":
r["result"]["urlworker"]["ip"] if "ip"
in r["result"]["urlworker"] else "",
"source":
"OTX"
})
else:
urls.append({
"date":
parse(r["date"]).astimezone(pytz.utc),
"url":
r["url"],
"ip":
"",
"source":
"OTX"
})
# RobTex
print('[+] Downloading Robtex information....')
rob = Robtex()
try:
res = rob.get_ip_info(unbracket(args.IP))
except RobtexError:
print("Error with Robtex")
else:
for d in ["pas", "pash", "act", "acth"]:
if d in res:
for a in res[d]:
passive_dns.append({
'first':
a['date'].astimezone(pytz.utc),
'last':
a['date'].astimezone(pytz.utc),
'domain':
a['o'],
'source':
'Robtex'
})
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
out_pt = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'],
conf['PassiveTotal']['key'])
try:
raw_results = client.get_passive_dns(
query=unbracket(args.IP))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first":
parse(res["firstSeen"]).astimezone(
pytz.utc),
"last":
parse(res["lastSeen"]).astimezone(
pytz.utc),
"domain":
res["resolve"],
"source":
"PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("Quota exceeded for Passive Total")
out_pt = True
pt_osint = {}
except requests.exceptions.ReadTimeout:
print("Timeout on Passive Total requests")
if not out_pt:
try:
client2 = EnrichmentRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(
query=unbracket(args.IP))
# Get malware
raw_results = client2.get_malware(
query=unbracket(args.IP))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash':
r["sample"],
'date':
parse(r['collectionDate']),
'source':
'PT (%s)' % r["source"]
})
except requests.exceptions.ReadTimeout:
print("Timeout on Passive Total requests")
# Urlhaus
uh_e = plugins['urlhaus'].test_config(conf)
if uh_e:
print("[+] Checking urlhaus data...")
try:
urlhaus = UrlHaus(conf["UrlHaus"]["key"])
res = urlhaus.get_host(unbracket(args.IP))
except UrlHausError:
print("Error with the query")
else:
if "urls" in res:
for r in res['urls']:
urls.append({
"date":
parse(r["date_added"]).astimezone(
pytz.utc),
"url":
r["url"],
"source":
"UrlHaus"
})
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_ip_report(unbracket(args.IP))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first":
parse(r["last_resolved"]).astimezone(
pytz.utc),
"last":
parse(r["last_resolved"]).astimezone(
pytz.utc),
"domain":
r["hostname"],
"source":
"VT"
})
if "undetected_downloaded_samples" in res[
'results']:
for r in res['results'][
'undetected_downloaded_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results'][
'undetected_referrer_samples']:
if 'date' in r:
files.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
else:
#FIXME : should consider data without dates
files.append({
'hash':
r['sha256'],
'date':
datetime.datetime(1970, 1, 1),
'source':
'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results'][
'detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results'][
'detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
else:
vt_e = False
print('[+] Downloading GreyNoise information....')
gn = GreyNoise()
try:
greynoise = gn.query_ip(unbracket(args.IP))
except GreyNoiseError:
greynoise = []
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
print('[+] Downloading Threat Grid....')
try:
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.IP), type='ip')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts'])
d = d.replace(tzinfo=None)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source': 'TG'
})
already.append(r['sample_sha256'])
except ThreatGridError as e:
print("Error with threat grid: {}".format(e.message))
# ThreatMiner
print('[+] Downloading ThreatMiner....')
tm = ThreatMiner()
response = tm.get_report(unbracket(args.IP))
if response['status_code'] == '200':
tmm = response['results']
else:
tmm = []
if response['status_code'] != '404':
print("Request to ThreatMiner failed: {}".format(
response['status_message']))
response = tm.get_related_samples(unbracket(args.IP))
if response['status_code'] == '200':
for r in response['results']:
malware.append({
'hash': r,
'date': None,
'source': 'ThreatMiner'
})
print('----------------- Intelligence Report')
ctor = CommandTor()
tor_list = ctor.get_list()
if tor_list:
if unbracket(args.IP) in tor_list:
print("{} is a Tor Exit node".format(unbracket(
args.IP)))
else:
print("Impossible to reach the Tor Exit Node list")
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print('- %s (%s - %s)' %
(p['name'], p['created'][:10],
"https://otx.alienvault.com/pulse/" +
p['id']))
else:
print('OTX: Not found in any pulse')
if misp_e:
if len(misp_results['Attribute']) > 0:
print('MISP:')
for event in misp_results['Attribute']:
print("- {} - {}".format(event['Event']['id'],
event['Event']['info']))
if len(greynoise) > 0:
print("GreyNoise: IP identified as")
for r in greynoise:
print("\t%s (%s -> %s)" %
(r["name"], r["first_seen"], r["last_updated"]))
else:
print("GreyNoise: Not found")
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print(
"PT: %s %s" %
(pt_osint["results"][0]["name"],
pt_osint["results"][0]["sourceUrl"]))
else:
print("PT: %s" %
pt_osint["results"][0]["sourceUrl"])
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("-%s %s" %
(r["name"], r["sourceUrl"]))
else:
print("-%s" % r["sourceUrl"])
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
# ThreatMiner
if len(tmm) > 0:
print("ThreatMiner:")
for r in tmm:
print("- {} {} - {}".format(r['year'], r['filename'],
r['URL']))
if len(malware) > 0:
print('----------------- Malware')
for r in malware:
print("[%s] %s %s" % (r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")
if r["date"] else ""))
if len(files) > 0:
print('----------------- Files')
for r in sorted(files, key=lambda x: x["date"]):
print("[%s] %s %s" % (r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")))
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns,
key=lambda x: x["first"],
reverse=True):
print("[+] %-40s (%s -> %s)(%s)" %
(r["domain"], r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"), r["source"]))
if len(urls) > 0:
print('----------------- Urls')
for r in sorted(urls,
key=lambda x: x["date"],
reverse=True):
print("[%s] %s - %s" %
(r["source"], r["url"],
r["date"].strftime("%Y-%m-%d")))
else:
self.parser.print_help()
else:
self.parser.print_help()
There are times when it's difficult to tell which items have been tagged as something malicious or suspicious. This script will take an initial starting point and print out any tagged items along with their tags. """ __author__ = 'Brandon Dixon ([email protected])' __version__ = '1.0.0' __description__ = "Surface tagged items from a passive DNS query" __keywords__ = ['pdns', 'tags', 'triage', 'analysis'] import sys from passivetotal.libs.dns import DnsRequest from passivetotal.libs.enrichment import EnrichmentRequest query = sys.argv[1] client = DnsRequest.from_config() enricher = EnrichmentRequest.from_config() def main(): """Take an initial seed and identify OSINT tags.""" initial_seed = client.get_unique_resolutions(query=query) all_records = initial_seed.get('results', list()) all_records += query for item in all_records: tmp = enricher.get_enrichment(query=item) tags = tmp.get('tags', list()) if len(tags) > 0: print("%s - %s" % (item, ', '.join(tags))) if __name__ == "__main__":
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == 'info':
if not is_ip(unbracket(args.IP)):
print("Invalid IP address")
sys.exit(1)
# FIXME: move code here in a library
ip = unbracket(args.IP)
try:
ipy = IP(ip)
except ValueError:
print('Invalid IP format, quitting...')
return
ipinfo = self.ipinfo(ip)
print('MaxMind: Located in %s, %s' %
(ipinfo['city'], ipinfo['country']))
if ipinfo['asn'] == 0:
print("MaxMind: IP not found in the ASN database")
else:
print('MaxMind: ASN%i, %s' %
(ipinfo['asn'], ipinfo['asn_name']))
asndb2 = pyasn.pyasn(self.asncidr)
res = asndb2.lookup(ip)
if res[1] is None:
print("IP not found in ASN database")
else:
# Search for name
f = open(self.asnname, 'r')
found = False
line = f.readline()
name = ''
while not found and line != '':
s = line.split('|')
if s[0] == str(res[0]):
name = s[1].strip()
found = True
line = f.readline()
print('ASN %i - %s (range %s)' % (res[0], name, res[1]))
if ipinfo['hostname'] != '':
print('Hostname: %s' % ipinfo['hostname'])
if ipinfo['specific'] != '':
print("Specific: %s" % ipinfo['specific'])
if ipy.iptype() == "PRIVATE":
"Private IP"
print("")
if ipy.version() == 4:
print("Censys:\t\thttps://censys.io/ipv4/%s" % ip)
print("Shodan:\t\thttps://www.shodan.io/host/%s" % ip)
print("IP Info:\thttp://ipinfo.io/%s" % ip)
print("BGP HE:\t\thttps://bgp.he.net/ip/%s" % ip)
print(
"IP Location:\thttps://www.iplocation.net/?query=%s" %
ip)
elif args.subcommand == "intel":
if not is_ip(unbracket(args.IP)):
print("Invalid IP address")
sys.exit(1)
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' %
unbracket(args.IP))
passive_dns = []
urls = []
malware = []
files = []
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(
IndicatorTypes.IPv4, unbracket(args.IP))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"domain": r['hostname'],
"first": parse(r["first"]),
"last": parse(r["last"]),
"source": "OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
urls.append(r)
# RobTex
print('[+] Downloading Robtex information....')
rob = Robtex()
res = rob.get_ip_info(unbracket(args.IP))
for d in ["pas", "pash", "act", "acth"]:
if d in res:
for a in res[d]:
passive_dns.append({
'first': a['date'],
'last': a['date'],
'domain': a['o'],
'source': 'Robtex'
})
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
out_pt = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'],
conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(
query=unbracket(args.IP))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first": parse(res["firstSeen"]),
"last": parse(res["lastSeen"]),
"domain": res["resolve"],
"source": "PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("Quota exceeded for Passive Total")
out_pt = True
pt_osint = {}
if not out_pt:
client2 = EnrichmentRequest(
conf["PassiveTotal"]["username"],
conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(query=unbracket(args.IP))
# Get malware
raw_results = client2.get_malware(
query=unbracket(args.IP))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash':
r["sample"],
'date':
parse(r['collectionDate']),
'source':
'PT (%s)' % r["source"]
})
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_ip_report(unbracket(args.IP))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first":
parse(r["last_resolved"]),
"last":
parse(r["last_resolved"]),
"domain":
r["hostname"],
"source":
"VT"
})
if "undetected_downloaded_samples" in res[
'results']:
for r in res['results'][
'undetected_downloaded_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results'][
'undetected_referrer_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results'][
'detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results'][
'detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']),
'source': 'VT'
})
else:
vt_e = False
print('[+] Downloading GreyNoise information....')
gn = GreyNoise()
try:
greynoise = gn.query_ip(unbracket(args.IP))
except GreyNoiseError:
greynoise = []
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
print('[+] Downloading Threat Grid....')
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.IP), type='ip')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts'])
d = d.replace(tzinfo=None)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source': 'TG'
})
already.append(r['sample_sha256'])
# TODO: Add MISP
print('----------------- Intelligence Report')
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print(' -%s (%s - %s)' %
(p['name'], p['created'][:10],
"https://otx.alienvault.com/pulse/" +
p['id']))
else:
print('OTX: Not found in any pulse')
if len(greynoise) > 0:
print("GreyNoise: IP identified as")
for r in greynoise:
print("\t%s (%s -> %s)" %
(r["name"], r["first_seen"], r["last_updated"]))
else:
print("GreyNoise: Not found")
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print(
"PT: %s %s" %
(pt_osint["results"][0]["name"],
pt_osint["results"][0]["sourceUrl"]))
else:
print("PT: %s" %
pt_osint["results"][0]["sourceUrl"])
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("-%s %s" %
(r["name"], r["sourceUrl"]))
else:
print("-%s" % r["sourceUrl"])
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
if len(malware) > 0:
print('----------------- Malware')
for r in sorted(malware, key=lambda x: x["date"]):
print("[%s] %s %s" % (r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")))
if len(files) > 0:
print('----------------- Files')
for r in sorted(files, key=lambda x: x["date"]):
print("[%s] %s %s" % (r["source"], r["hash"],
r["date"].strftime("%Y-%m-%d")))
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns,
key=lambda x: x["first"],
reverse=True):
print("[+] %-40s (%s -> %s)(%s)" %
(r["domain"], r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"), r["source"]))
else:
self.parser.print_help()
else:
self.parser.print_help()
import json
import argparse
from passivetotal.libs.dns import DnsRequest
from passivetotal.libs.enrichment import EnrichmentRequest
def get_config():
conf_file = os.path.join(os.path.expanduser("~"),
".config/passivetotal/api_config.json")
if os.path.isfile(conf_file):
with open(conf_file, 'r') as f:
conf = json.loads(f.read())
else:
print('No config file')
sys.exit(1)
return conf
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='List subdomains for a domain')
parser.add_argument('DOMAIN', help='Domain')
args = parser.parse_args()
conf = get_config()
client = EnrichmentRequest(conf['username'], conf['api_key'])
raw_results = client.get_subdomains(query=args.DOMAIN)
for s in raw_results['subdomains']:
print(s + '.' + raw_results['primaryDomain'])
def call_osint(args):
client = EnrichmentRequest.from_config()
return client.get_osint(query=args.query)
logger.info("Starting command processing")
input_events, dummyresults, settings = splunk.Intersplunk.getOrganizedResults(
)
keywords, options = splunk.Intersplunk.getKeywordsAndOptions()
query_value = options.get("query", "")
logger.info("Query target: %s" % query_value)
logger.debug("Raw options: %s" % str(options))
configuration = get_config("passivetotal", "api-setup")
username = configuration.get('username', None)
api_key = configuration.get('apikey', None)
output_events = list()
enrichment = EnrichmentRequest(
username, api_key,
headers=build_headers()).get_enrichment(query=query_value)
if 'error' in enrichment:
raise Exception(
"Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans."
)
classification = ActionsClient(
username, api_key).get_classification_status(query=query_value,
headers=build_headers())
tmp = classification.get('classification', 'unknown').replace('_', '-')
if tmp == '':
tmp = 'unknown'
enrichment['tags'].append(tmp)
classification_lookup = {
'non-malicious': 1,
'suspicious': 2,
import sys
from passivetotal.libs.attributes import AttributeRequest
from passivetotal.libs.enrichment import EnrichmentRequest
def show_tagged(direction, enriched):
for host, data in enriched.get("results", {}).items():
if len(data['tags']) == 0:
continue
print(data['queryValue'], ','.join(data['tags']))
query = sys.argv[1]
direction = sys.argv[2]
result_key = {'parents': 'parent', 'children': 'child'}
if len(sys.argv) != 3:
print("Usage: python host_pair_sentinel.py <query> <parents|children>")
sys.exit(1)
if direction not in ['children', 'parents']:
print("[!] Direction must be 'children' or 'parents' to work")
sys.exit(1)
client = AttributeRequest.from_config()
matches = client.get_host_attribute_pairs(query=query, direction=direction)
hostnames = [x[result_key[direction]] for x in matches.get("results", list())]
client = EnrichmentRequest.from_config()
enriched = client.get_bulk_enrichment(query=hostnames)
show_tagged(direction, enriched)
def passivs_get_subdomain(query):
client = EnrichmentRequest(username=username,api_key=api_key)
result = client.get_subdomains(query=query)
_ = result['subdomains']
subdomains = [[passive_get_ip(i)] for i in _ ]
def passivs_get_subdomain(query):
client = EnrichmentRequest(username=username, api_key=api_key)
result = client.get_subdomains(query=query)
_ = result['subdomains']
subdomains = [[passive_get_ip(i)] for i in _]
There are times when it's difficult to tell which items have been tagged as something malicious or suspicious. This script will take an initial starting point and print out any tagged items along with their tags. """ __author__ = 'Brandon Dixon ([email protected])' __version__ = '1.0.0' __description__ = "Surface tagged items from a passive DNS query" __keywords__ = ['pdns', 'tags', 'triage', 'analysis'] import sys from passivetotal.libs.dns import DnsRequest from passivetotal.libs.enrichment import EnrichmentRequest query = sys.argv[1] client = DnsRequest.from_config() enricher = EnrichmentRequest.from_config() def main(): """Take an initial seed and identify OSINT tags.""" initial_seed = client.get_unique_resolutions(query=query) all_records = initial_seed.get('results', list()) all_records += query for item in all_records: tmp = enricher.get_enrichment(query=item) tags = tmp.get('tags', list()) if len(tags) > 0: print("%s - %s" % (item, ', '.join(tags))) if __name__ == "__main__": main()
from passivetotal.libs.attributes import AttributeRequest
from passivetotal.libs.enrichment import EnrichmentRequest
def show_tagged(direction, enriched):
for host, data in enriched.get("results", {}).iteritems():
if len(data['tags']) == 0:
continue
print data['queryValue'], ','.join(data['tags'])
query = sys.argv[1]
direction = sys.argv[2]
result_key = {'parents': 'parent', 'children': 'child'}
if len(sys.argv) != 3:
print "Usage: python host_pair_sentinel.py <query> <parents|children>"
sys.exit(1)
if direction not in ['children', 'parents']:
print "[!] Direction must be 'children' or 'parents' to work"
sys.exit(1)
client = AttributeRequest.from_config()
matches = client.get_host_attribute_pairs(query=query, direction=direction)
hostnames = [x[result_key[direction]] for x in matches.get("results", list())]
client = EnrichmentRequest.from_config()
enriched = client.get_bulk_enrichment(query=hostnames)
show_tagged(direction, enriched)
sys.exit(1)
return conf
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Get hashes from PT for domains')
parser.add_argument('FILE', help='File with list of domains')
args = parser.parse_args()
conf = get_config()
with open(args.FILE, 'r') as f:
domains = list(set([d.strip() for d in f.read().split()]))
client = EnrichmentRequest(conf['username'], conf['api_key'])
for domain in domains:
if domain == '':
continue
print(f"################ {domain}")
try:
raw_results = client.get_malware(query=domain)
if raw_results['success']:
for s in raw_results['results']:
print(s)
else:
print("Request failed")
except:
print("Something failed")