def run(self, conf, args, plugins):
if 'subcommand' in args:
us = UrlScan()
if args.subcommand == 'search':
# Search
res = us.search(args.QUERY)
if args.raw:
print(json.dumps(res, sort_keys=True, indent=4))
else:
if len(res['results']) > 0:
for r in res['results']:
print(
"{} - {} - {} - https://urlscan.io/result/{}".
format(
r["task"]["time"], r["page"]["url"],
r["page"]["ip"]
if "ip" in r["page"] else "", r["_id"]))
else:
print("No results for this query")
elif args.subcommand == 'view':
print(json.dumps(us.view(args.UID), sort_keys=True, indent=4))
else:
self.parser.print_help()
else:
self.parser.print_help()
def intel(self, type, query, data, conf):
if type in ["domain", "ip"]:
print('[+] Checking UrlScan...')
us = UrlScan()
res = us.search(query)
if 'results' in res:
for r in res['results']:
data["urls"].append({
"date":
parse(r["task"]["time"]).astimezone(pytz.utc),
"url":
r["page"]["url"],
"ip":
r["page"]["ip"] if "ip" in r["page"] else "",
"source":
"UrlScan"
})
def run(self, conf, args, plugins):
if 'subcommand' in args:
# Optional key
try:
key = conf['UrlScan']['key']
if key.strip() != "":
us = UrlScan(key)
else:
us = UrlScan()
except KeyError:
us = UrlScan()
if args.subcommand == 'search':
# Search
try:
res = us.search(args.QUERY)
except UrlScanError as e:
print("Error: {}".format(e.message))
else:
if args.raw:
print(json.dumps(res, sort_keys=True, indent=4))
else:
if len(res['results']) > 0:
for r in res['results']:
print(
"{} - {} - {} - https://urlscan.io/result/{}"
.format(
r["task"]["time"], r["page"]["url"],
r["page"]["ip"] if "ip" in r["page"]
else "", r["_id"]))
else:
print("No results for this query")
elif args.subcommand == 'view':
try:
print(
json.dumps(us.view(args.UID), sort_keys=True,
indent=4))
except UrlScanError as e:
print("Error: {}".format(e.message))
elif args.subcommand == 'list':
with open(args.FILE) as f:
data = f.read().split('\n')
for d in data:
d = d.strip()
if d == "":
continue
print("##### {}".format(d))
try:
res = us.search(d)
except UrlScanQuotaExceeded as e:
duration = int(e.message[-13:-9]) + 10
print("Out of quota, waiting for {} seconds".format(
duration))
time.sleep(duration)
res = us.search(d)
if 'results' in res:
if len(res['results']) > 0:
for r in res['results']:
print(
"{} - {} - {} - https://urlscan.io/result/{}"
.format(
r["task"]["time"], r["page"]["url"],
r["page"]["ip"] if "ip" in r["page"]
else "", r["_id"]))
else:
print("Nothing found")
else:
print("Nothing found")
# brief sleeping time to avoid overloading URL Scan
time.sleep(0.5)
elif args.subcommand == 'quota':
if us.api_key:
print(json.dumps(us.quota(), sort_keys=True, indent=4))
else:
print(
"You need to configure a UrlScan key to check your quota"
)
else:
self.parser.print_help()
else:
self.parser.print_help()
def run(self, conf, args, plugins):
if 'subcommand' in args:
if args.subcommand == "intel":
# Start with MISP and OTX to get Intelligence Reports
print('###################### %s ###################' % args.DOMAIN)
passive_dns = []
urls = []
malware = []
files = []
# MISP
misp_e = plugins['misp'].test_config(conf)
if misp_e:
print('[+] Downloading MISP information...')
server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key'])
misp_results = server.search('attributes', value=unbracket(args.DOMAIN))
# OTX
otx_e = plugins['otx'].test_config(conf)
if otx_e:
print('[+] Downloading OTX information....')
try:
otx = OTXv2(conf["AlienVaultOtx"]["key"])
res = otx.get_indicator_details_full(IndicatorTypes.DOMAIN, unbracket(args.DOMAIN))
otx_pulses = res["general"]["pulse_info"]["pulses"]
# Get Passive DNS
if "passive_dns" in res:
for r in res["passive_dns"]["passive_dns"]:
passive_dns.append({
"ip": r['hostname'],
"first": parse(r["first"]).astimezone(pytz.utc),
"last": parse(r["last"]).astimezone(pytz.utc),
"source" : "OTX"
})
if "url_list" in res:
for r in res["url_list"]["url_list"]:
if "result" in r:
urls.append({
"date": parse(r["date"]).astimezone(pytz.utc),
"url": r["url"],
"ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "" ,
"source": "OTX"
})
else:
urls.append({
"date": parse(r["date"]).astimezone(pytz.utc),
"url": r["url"],
"ip": "",
"source": "OTX"
})
except AttributeError:
print('OTX crashed ¯\_(ツ)_/¯')
# UrlScan
us = UrlScan()
print('[+] Downloading UrlScan information....')
res = us.search(args.DOMAIN)
for r in res['results']:
urls.append({
"date": parse(r["task"]["time"]).astimezone(pytz.utc),
"url": r["page"]["url"],
"ip": r["page"]["ip"] if "ip" in r["page"] else "",
"source": "UrlScan"
})
# UrlHaus
uh_e = plugins['urlhaus'].test_config(conf)
if uh_e:
print("[+] Checking urlhaus...")
try:
urlhaus = UrlHaus(conf["UrlHaus"]["key"])
res = urlhaus.get_host(unbracket(args.DOMAIN))
except UrlHausError:
print("Error with the query")
else:
if "urls" in res:
for r in res['urls']:
urls.append({
"date": parse(r["date_added"]).astimezone(pytz.utc),
"url": r["url"],
"ip":"",
"source": "UrlHaus"
})
# CIRCL
circl_e = plugins['circl'].test_config(conf)
if circl_e:
print('[+] Downloading CIRCL passive DNS information....')
x = pypdns.PyPDNS(
basic_auth=(
conf['Circl']['user'],
conf['Circl']['pass']
)
)
res = x.query(unbracket(args.DOMAIN))
for answer in res:
passive_dns.append({
"ip": answer['rdata'],
"first": answer['time_first'].astimezone(pytz.utc),
"last": answer['time_last'].astimezone(pytz.utc),
"source" : "CIRCL"
})
# BinaryEdge
be_e = plugins['binaryedge'].test_config(conf)
if be_e:
print('[+] Downloading BinaryEdge information....')
try:
be = BinaryEdge(conf['BinaryEdge']['key'])
res = be.domain_dns(unbracket(args.DOMAIN))
for d in res['events']:
if "A" in d:
for a in d['A']:
passive_dns.append({
"ip": a,
"first": parse(d['updated_at']).astimezone(pytz.utc),
"last": parse(d['updated_at']).astimezone(pytz.utc),
"source" : "BinaryEdge"
})
except BinaryEdgeException:
print('You need a paid BinaryEdge subscription for this request')
# RobTex
print('[+] Downloading Robtex information....')
try:
rob = Robtex()
res = rob.get_pdns_domain(args.DOMAIN)
for d in res:
if d['rrtype'] in ['A', 'AAAA']:
passive_dns.append({
'first': d['time_first_o'].astimezone(pytz.utc),
'last': d['time_last_o'].astimezone(pytz.utc),
'ip': d['rrdata'],
'source': 'Robtex'
})
except RobtexError:
print("Robtex query failed")
# PT
pt_e = plugins['pt'].test_config(conf)
if pt_e:
try:
pt_osint = {}
ptout = False
print('[+] Downloading Passive Total information....')
client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key'])
raw_results = client.get_passive_dns(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for res in raw_results["results"]:
passive_dns.append({
"first": parse(res["firstSeen"]).astimezone(pytz.utc),
"last": parse(res["lastSeen"]).astimezone(pytz.utc),
"ip": res["resolve"],
"source": "PT"
})
if "message" in raw_results:
if "quota_exceeded" in raw_results["message"]:
print("PT quota exceeded")
ptout = True
if not ptout:
client2 = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key'])
# Get OSINT
# TODO: add PT projects here
pt_osint = client2.get_osint(query=unbracket(args.DOMAIN))
# Get malware
raw_results = client2.get_malware(query=unbracket(args.DOMAIN))
if "results" in raw_results:
for r in raw_results["results"]:
malware.append({
'hash': r["sample"],
'date': parse(r['collectionDate']).astimezone(pytz.utc),
'source' : 'PT (%s)' % r["source"]
})
except requests.exceptions.ReadTimeout:
print("PT: Time Out")
# VT
vt_e = plugins['vt'].test_config(conf)
if vt_e:
if conf["VirusTotal"]["type"] != "public":
print('[+] Downloading VT information....')
vt = PrivateApi(conf["VirusTotal"]["key"])
res = vt.get_domain_report(unbracket(args.DOMAIN))
if "results" in res:
if "resolutions" in res['results']:
for r in res["results"]["resolutions"]:
passive_dns.append({
"first": parse(r["last_resolved"]).astimezone(pytz.utc),
"last": parse(r["last_resolved"]).astimezone(pytz.utc),
"ip": r["ip_address"],
"source": "VT"
})
if "undetected_downloaded_samples" in res['results']:
for r in res['results']['undetected_downloaded_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '',
'source' : 'VT'
})
if "undetected_referrer_samples" in res['results']:
for r in res['results']['undetected_referrer_samples']:
files.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '',
'source' : 'VT'
})
if "detected_downloaded_samples" in res['results']:
for r in res['results']['detected_downloaded_samples']:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc),
'source' : 'VT'
})
if "detected_referrer_samples" in res['results']:
for r in res['results']['detected_referrer_samples']:
if "date" in r:
malware.append({
'hash': r['sha256'],
'date': parse(r['date']).astimezone(pytz.utc),
'source' : 'VT'
})
if "detected_urls" in res['results']:
for r in res['results']['detected_urls']:
urls.append({
'date': parse(r['scan_date']).astimezone(pytz.utc),
'url': r['url'],
'ip': '',
'source': 'VT'
})
else:
vt_e = False
tg_e = plugins['threatgrid'].test_config(conf)
if tg_e:
try:
print('[+] Downloading Threat Grid....')
tg = ThreatGrid(conf['ThreatGrid']['key'])
res = tg.search_samples(unbracket(args.DOMAIN), type='domain')
already = []
if 'items' in res:
for r in res['items']:
if r['sample_sha256'] not in already:
d = parse(r['ts']).astimezone(pytz.utc)
malware.append({
'hash': r["sample_sha256"],
'date': d,
'source' : 'ThreatGrid'
})
already.append(r['sample_sha256'])
except ThreatGridError as e:
print("Failed to connect to Threat Grid: %s" % e.message)
print('[+] Downloading ThreatMiner....')
tm = ThreatMiner()
response = tm.get_report(unbracket(args.DOMAIN))
if response['status_code'] == '200':
tmm = response['results']
else:
tmm = []
if response['status_code'] == '404':
print("Request to ThreatMiner failed: {}".format(response['status_message']))
response = tm.get_related_samples(unbracket(args.DOMAIN))
if response['status_code'] == '200':
for r in response['results']:
malware.append({
'hash': r,
'date': None,
'source': 'ThreatMiner'
})
print('----------------- Intelligence Report')
if misp_e:
if len(misp_results['Attribute']) > 0:
print('MISP:')
for event in misp_results['Attribute']:
print("- {} - {}".format(
event['Event']['id'],
event['Event']['info']
))
if otx_e:
if len(otx_pulses):
print('OTX:')
for p in otx_pulses:
print('- %s (%s - %s)' % (
p['name'],
p['created'][:10],
"https://otx.alienvault.com/pulse/" + p['id']
)
)
else:
print('OTX: Not found in any pulse')
if pt_e:
if "results" in pt_osint:
if len(pt_osint["results"]):
if len(pt_osint["results"]) == 1:
if "name" in pt_osint["results"][0]:
print("PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"]))
else:
print("PT: %s" % (pt_osint["results"][0]["sourceUrl"]))
else:
print("PT:")
for r in pt_osint["results"]:
if "name" in r:
print("- %s %s" % (r["name"], r["sourceUrl"]))
else:
print("- %s" % (r["sourceUrl"]))
else:
print("PT: Nothing found!")
else:
print("PT: Nothing found!")
# ThreatMiner
if len(tmm) > 0:
print("ThreatMiner:")
for r in tmm:
print("- {} {} - {}".format(
r['year'],
r['filename'],
r['URL']
))
if len(malware) > 0:
print('----------------- Malware')
for r in malware:
print("[%s] %s %s" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d") if r["date"] else ""
)
)
if len(files) > 0:
print('----------------- Files')
for r in files:
if r['date'] != '':
print("[%s] %s (%s)" % (
r["source"],
r["hash"],
r["date"].strftime("%Y-%m-%d")
)
)
else:
print("[%s] %s" % (
r["source"],
r["hash"],
)
)
if len(urls) > 0:
print('----------------- Urls')
for r in sorted(urls, key=lambda x: x["date"], reverse=True):
print("[%s] %s - %s %s" % (
r["source"],
r["url"],
r["ip"],
r["date"].strftime("%Y-%m-%d")
)
)
# TODO: add ASN + location info here
if len(passive_dns) > 0:
print('----------------- Passive DNS')
for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True):
print("[+] %-40s (%s -> %s)(%s)" % (
r["ip"],
r["first"].strftime("%Y-%m-%d"),
r["last"].strftime("%Y-%m-%d"),
r["source"]
)
)
else:
self.parser.print_help()
else:
self.parser.print_help()