def new_scan(host,
publish="off",
start_new="on",
all="done",
ignoreMismatch="on"):
"""This function requests SSL Labs to run new scan for the target domain."""
if helpers.is_ip(host):
print(
red("[!] Your target host must be a domain, not an IP address! \
SSL Labs will onyl scan domains."))
exit()
else:
path = "analyze"
payload = {
'host': host,
'publish': publish,
'start_new': start_new,
'all': all,
'ignoreMismatch': ignoreMismatch
}
results = request_api(path, payload)
payload.pop('start_new')
while results['status'] != 'READY' and results['status'] != 'ERROR':
print("Scan in progress, please wait for the results.")
time.sleep(30)
results = request_api(path, payload)
return results
def run_urlvoid_lookup(self, domain):
"""Collect reputation data from URLVoid for the target domain. This returns an ElementTree
object.
A free API key is required.
"""
if not helpers.is_ip(domain):
try:
if self.urlvoid_api_key != "":
print(
green("[+] Checking reputation for {} with URLVoid".
format(domain)))
url = "http://api.urlvoid.com/api1000/{}/host/{}"\
.format(self.urlvoid_api_key, domain)
response = requests.get(url)
tree = ET.fromstring(response.content)
return tree
else:
print(
green(
"[-] No URLVoid API key, so skipping this test."))
return None
except Exception as error:
print(red("[!] Could not load URLVoid for reputation check!"))
print(red("L.. Details: {}".format(error)))
return None
else:
print(
red("[!] Target is not a domain, so skipping URLVoid queries.")
)
def prepare_scope(self,
ip_list,
domain_list,
scope_file=None,
domain=None):
"""Function to split the user's scope file into IP addresses and domain names."""
# Generate the scope lists from the supplied scope file, if there is one
scope = []
if scope_file:
scope = self.DC.generate_scope(scope_file)
if domain:
# Just in case the domain is not in the scope file, it's added here
if not any(domain in d for d in scope):
print(
yellow(
"[*] The provided domain, {}, was not found in your scope file, so \
it has been added to the scope for OSINT.".format(domain)))
scope.append(domain)
# Create lists of IP addresses and domain names from scope
for item in scope:
if helpers.is_ip(item):
ip_list.append(item)
elif item == "":
pass
else:
domain_list.append(item)
for target in scope:
self.c.execute("INSERT INTO hosts VALUES (NULL,?,?,?)",
(target, True, "Scope File"))
self.conn.commit()
return scope, ip_list, domain_list
def prepare_scope(self, scope_file, domain=None):
"""Function to split the user's scope file into IP addresses and domain names."""
# Generate the scope lists from the supplied scope file, if there is one
scope = []
scope = self.DC.generate_scope(scope_file)
if domain:
# Just in case the domain is not in the scope file, it's added here
if not any(domain in d for d in scope):
print(yellow("[*] The provided domain, {}, was not found in your scope file, so \
it has been added to the scope for OSINT.".format(domain)))
scope.append(domain)
# Create lists of IP addresses and domain names from scope
for item in scope:
if helpers.is_ip(item):
self.ip_addresses.append(item)
elif item == "":
pass
else:
self.domains_list.append(item)
# Record the scope being used in the database
self.c.execute('''CREATE TABLE 'ReportScope' ('Target' text)''')
for target in scope:
self.c.execute("INSERT INTO ReportScope VALUES (?)", (target,))
self.conn.commit()
return scope, self.ip_addresses, self.domains_list
def run_urlvoid_lookup(self,domain):
"""Collect reputation data from URLVoid for the target domain. This returns an ElementTree
object.
A URLVoid API key is required.
Parameters:
domain The domain name to check with URLVoid
"""
if not helpers.is_ip(domain):
try:
if self.urlvoid_api_key != "":
url = "http://api.urlvoid.com/api1000/{}/host/{}".format(self.urlvoid_api_key,domain)
response = requests.get(url,timeout=self.requests_timeout)
tree = ET.fromstring(response.content)
return tree
else:
click.secho("[*] No URLVoid API key, so skipping this test.",fg="green")
return None
except Exception as error:
click.secho("[!] Could not load URLVoid for reputation check!",fg="red")
click.secho("L.. Details: {}".format(error),fg="red")
return None
else:
click.secho("[!] Target is not a domain, so skipping URLVoid queries.",fg="red")
def run_urlvoid_lookup(self, domain):
"""Collect reputation data from URLVoid for the target domain. This returns an ElementTree
object.
A URLVoid API key is required.
Parameters:
domain The domain name to check with URLVoid
"""
if not helpers.is_ip(domain):
try:
if self.urlvoid_api_key != "":
url = "http://api.urlvoid.com/api1000/{}/host/{}".format(
self.urlvoid_api_key, domain)
response = requests.get(url, timeout=self.requests_timeout)
tree = ET.fromstring(response.content)
return tree
else:
click.secho(
"[*] No URLVoid API key, so skipping this test.",
fg="green")
return None
except Exception as error:
click.secho("[!] Could not load URLVoid for reputation check!",
fg="red")
click.secho("L.. Details: {}".format(error), fg="red")
return None
else:
click.secho(
"[!] Target is not a domain, so skipping URLVoid queries.",
fg="red")
def lookup_robtex_ipinfo(self, ip_address):
"""Function to lookup information about a target IP address with Robtex."""
if helpers.is_ip(ip_address):
request = requests.get(self.robtex_api + ip_address)
ip_json = request.json()
return ip_json
else:
print(red("[!] The provided IP for Robtex address is invalid!"))
def prepare_scope(scope_file, expanded_scope):
"""Parse IP ranges inside the provided scope file to expand IP ranges. This supports ranges
with hyphens, underscores, and CIDRs.
Parameters:
scope_file A file containing domain name and IP addresses/ranges
expanded_scope A list object for storing to expanded scope list
"""
try:
with open(scope_file, "r") as scope_file:
for target in scope_file:
target = target.rstrip()
# Record individual IPs and expand CIDRs
if helpers.is_ip(target):
ip_list = list(IPNetwork(target))
for address in sorted(ip_list):
str_address = str(address)
expanded_scope.append(str_address)
# Sort IP ranges from domain names and expand the ranges
if not helpers.is_domain(target):
# Check for hyphenated ranges like those accepted by Nmap, e.g. 192.168.1.1-50
if "-" in target:
target = target.rstrip()
parts = target.split("-")
startrange = parts[0]
b = parts[0]
dot_split = b.split(".")
temp = "."
# Join the values using a "." so it makes a valid IP
combine = dot_split[0], dot_split[1], dot_split[
2], parts[1]
endrange = temp.join(combine)
# Calculate the IP range
ip_list = list(iter_iprange(startrange, endrange))
# Iterate through the range and remove ip_list
for x in ip_list:
temp = str(x)
expanded_scope.append(temp)
# Check if range has an underscore, e.g. 192.168.1.2_192.168.1.155
elif "_" in target:
target = target.rstrip()
parts = target.split("_")
startrange = parts[0]
endrange = parts[1]
ip_list = list(iter_iprange(startrange, endrange))
for address in ip_list:
str_address = str(address)
expanded_scope.append(str_address)
else:
expanded_scope.append(target.rstrip())
click.secho(
"[+] Scope list expanded to {} items. Proceeding with verification \
checks.".format(len(expanded_scope)),
fg="green")
except IOError as error:
click.secho("[!] Parsing of scope file failed!", fg="red")
click.secho("L.. Details: {}".format(error), fg="red")
def prepare_scope(scope_file,expanded_scope):
"""Parse IP ranges inside the provided scope file to expand IP ranges. This supports ranges
with hyphens, underscores, and CIDRs.
Parameters:
scope_file A file containing domain name and IP addresses/ranges
expanded_scope A list object for storing to expanded scope list
"""
try:
with open(scope_file,"r") as scope_file:
for target in scope_file:
target = target.rstrip()
# Record individual IPs and expand CIDRs
if helpers.is_ip(target):
ip_list = list(IPNetwork(target))
for address in sorted(ip_list):
str_address = str(address)
expanded_scope.append(str_address)
# Sort IP ranges from domain names and expand the ranges
if not helpers.is_domain(target):
# Check for hyphenated ranges like those accepted by Nmap, e.g. 192.168.1.1-50
if "-" in target:
target = target.rstrip()
parts = target.split("-")
startrange = parts[0]
b = parts[0]
dot_split = b.split(".")
temp = "."
# Join the values using a "." so it makes a valid IP
combine = dot_split[0],dot_split[1],dot_split[2],parts[1]
endrange = temp.join(combine)
# Calculate the IP range
ip_list = list(iter_iprange(startrange,endrange))
# Iterate through the range and remove ip_list
for x in ip_list:
temp = str(x)
expanded_scope.append(temp)
# Check if range has an underscore, e.g. 192.168.1.2_192.168.1.155
elif "_" in target:
target = target.rstrip()
parts = target.split("_")
startrange = parts[0]
endrange = parts[1]
ip_list = list(iter_iprange(startrange,endrange))
for address in ip_list:
str_address = str(address)
expanded_scope.append(str_address)
else:
expanded_scope.append(target.rstrip())
click.secho("[+] Scope list expanded to {} items. Proceeding with verification \
checks.".format(len(expanded_scope)),fg="green")
except IOError as error:
click.secho("[!] Parsing of scope file failed!",fg="red")
click.secho("L.. Details: {}".format(error),fg="red")
def perform_whois(expanded_scope, output):
"""Look-up the provided IP address in the ARIN database.
Parameters:
expanded_scope A list of domain name and IP addresses (no ranges)
output A list object for storing the output
"""
total_addresses = len(expanded_scope)
with click.progressbar(expanded_scope,
label='Collecting information on addresses',
length=total_addresses) as bar:
for address in bar:
if not helpers.is_ip(address):
pass
else:
# Try to send GET request to the ARIN REST API for IP values
try:
r = requests.get("http://whois.arin.net/rest/ip/" +
address + ".json",
timeout=10)
tmp = r.json()
try:
name = tmp['net']['customerRef']['@name']
# start = tmp['net']['netBlocks']['netBlock']['startAddress']['$']
# end = tmp['net']['netBlocks']['netBlock']['endAddress']['$']
hostname = reverse_lookup(address)
cn = get_certificate(address)
output[address] = address, name, hostname, cn
except:
# The formatting of ARIN data may change if an org is used for the contact
name = tmp['net']['orgRef']['@name']
# start = tmp['net']['netBlocks']['netBlock']['startAddress']['$']
# end = tmp['net']['netBlocks']['netBlock']['endAddress']['$']
hostname = reverse_lookup(address)
cn = get_certificate(address)
output[address] = address, name, hostname, cn
except:
pass
# Pause for just a sec to not destroy ARIN with requests
sleep(1)
def results_from_cache(host,
publish="off",
start_new="off",
from_cache="on",
all="done"):
"""This function returns results from SSL Labs' cache (previously run scans)."""
if helpers.is_ip(host):
print(
red("[!] Your target host must be a domain, not an IP address! \
SSL Labs will onyl scan domains."))
exit()
else:
path = "analyze"
payload = {
'host': host,
'publish': publish,
'start_new': start_new,
'from_cache': from_cache,
'all': all
}
data = request_api(path, payload)
return data
def _graph_hosts(self):
"""Convert the hosts table into Neo4j graph nodes."""
self.c.execute("SELECT host_address,in_scope_file,source FROM hosts")
all_hosts = self.c.fetchall()
for row in all_hosts:
if row[1] == 0:
scoped = False
else:
scoped = True
if helpers.is_ip(row[0]):
query = """
MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'})
RETURN x
""" % (row[0], scoped, row[2])
helpers.execute_query(self.neo4j_driver, query)
else:
query = """
MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'})
RETURN x
""" % (row[0], scoped, row[2])
helpers.execute_query(self.neo4j_driver, query)
def perform_whois(expanded_scope,output):
"""Look-up the provided IP address in the ARIN database.
Parameters:
expanded_scope A list of domain name and IP addresses (no ranges)
output A list object for storing the output
"""
total_addresses = len(expanded_scope)
with click.progressbar(expanded_scope,
label='Collecting information on addresses',
length=total_addresses) as bar:
for address in bar:
if not helpers.is_ip(address):
pass
else:
# Try to send GET request to the ARIN REST API for IP values
try:
r = requests.get("http://whois.arin.net/rest/ip/" + address + ".json",timeout=10)
tmp = r.json()
try:
name = tmp['net']['customerRef']['@name']
# start = tmp['net']['netBlocks']['netBlock']['startAddress']['$']
# end = tmp['net']['netBlocks']['netBlock']['endAddress']['$']
hostname = reverse_lookup(address)
cn = get_certificate(address)
output[address] = address,name,hostname,cn
except:
# The formatting of ARIN data may change if an org is used for the contact
name = tmp['net']['orgRef']['@name']
# start = tmp['net']['netBlocks']['netBlock']['startAddress']['$']
# end = tmp['net']['netBlocks']['netBlock']['endAddress']['$']
hostname = reverse_lookup(address)
cn = get_certificate(address)
output[address] = address,name,hostname,cn
except:
pass
# Pause for just a sec to not destroy ARIN with requests
sleep(1)
def create_cymon_worksheet(self, target):
"""Function to check the provided the target against Cymon.io's database of threat feeds
and then print the results.
"""
if helpers.is_ip(target):
domains_results, ip_results = self.DC.search_cymon_ip(target)
if domains_results:
print(yellow("\n[+] Associated Domains:"))
# Print out associated domains for the IP
for result in domains_results:
print("URL:\t %s" % result['name'])
print("Created: %s" % result['created'])
print("Updated: %s\n" % result['updated'])
if ip_results:
print(yellow("[+] Recorded Malicious Events:"))
# Print out security events for the IP
for result in ip_results:
print("Title:\t\t %s" % result['title'])
print("Description:\t %s" % result['description'])
print("Created:\t %s" % result['created'])
print("Updated:\t %s" % result['updated'])
print("Details:\t %s\n" % result['details_url'])
else:
results = self.DC.search_cymon_domain(target)
# Print out information for the domain
if results:
print(yellow("\n[+] Cymon.io events for %s" % target))
print("URL:\t %s" % results['name'])
print("Created: %s" % results['created'])
print("Updated: %s" % results['updated'])
for source in results['sources']:
print("Source:\t {}".format(source))
for ip in results['ips']:
print("IP:\t {}".format(ip))
print("")
print(green("[+] Cymon search completed!"))
def _graph_hosts(self):
"""Convert the hosts table into Neo4j graph nodes."""
self.c.execute("SELECT host_address,in_scope_file,source FROM hosts")
all_hosts = self.c.fetchall()
with click.progressbar(all_hosts,
label="Creating Domain and IP nodes",
length=len(all_hosts)) as bar:
for row in bar:
if row[1] == 0:
scoped = False
else:
scoped = True
if helpers.is_ip(row[0]):
query = """
MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'})
RETURN x
""" % (row[0], scoped, row[2])
helpers.execute_query(self.neo4j_driver, query)
else:
query = """
MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'})
RETURN x
""" % (row[0], scoped, row[2])
helpers.execute_query(self.neo4j_driver, query)
def _graph_hosts(self):
"""Convert the hosts table into Neo4j graph nodes."""
self.c.execute("SELECT host_address,in_scope_file,source FROM hosts")
all_hosts = self.c.fetchall()
with click.progressbar(all_hosts,
label="Creating Domain and IP nodes",
length=len(all_hosts)) as bar:
for row in bar:
if row[1] == 0:
scoped = False
else:
scoped = True
if helpers.is_ip(row[0]):
query = """
MERGE (x:IP {Address:'%s', Scoped:'%s', Source:'%s'})
RETURN x
"""% (row[0],scoped,row[2])
helpers.execute_query(self.neo4j_driver,query)
else:
query = """
MERGE (x:Domain {Name:'%s', Scoped:'%s', Source:'%s'})
RETURN x
""" % (row[0],scoped,row[2])
helpers.execute_query(self.neo4j_driver,query)
def create_domain_report_table(self, scope, ip_addresses, domains_list, verbose):
"""Function to generate a domain report consisting of information like DNS records and
subdomains.
"""
# Create the DNS table for holding the domains' DNS records
self.c.execute('''CREATE TABLE 'DNS'
('Domain' text, 'NSRecords' text, 'ARecords' text, 'MXRecords' text,
'TXTRecords' text, 'SOARecords' text, 'VulnerableCacheSnooping' text)''')
# Get the DNS records for each domain
for domain in domains_list:
# Get the NS records
try:
temp = []
ns_records_list = self.DC.get_dns_record(domain, "NS")
for rdata in ns_records_list.response.answer:
for item in rdata.items:
temp.append(item.to_text())
ns_records = ", ".join(temp)
# Record name server that resolve cached queries
vulnerable_dns_servers = []
for nameserver in temp:
result = self.DC.check_dns_cache(nameserver.strip("."))
if result:
vulnerable_dns_servers.append(result)
except:
ns_records = "None"
# Get the A records
try:
temp = []
a_records = self.DC.get_dns_record(domain, "A")
for rdata in a_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
a_records = ", ".join(temp)
except:
a_records = "None"
# Get the MX records
try:
temp = []
mx_records = self.DC.get_dns_record(domain, "MX")
for rdata in mx_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
mx_records = ", ".join(temp)
except:
mx_records = "None"
# Get the TXT records
try:
temp = []
txt_records = self.DC.get_dns_record(domain, "TXT")
for rdata in txt_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
txt_records = ", ".join(temp)
except:
txt_records = "None"
# Get the SOA records
try:
temp = []
soa_records = self.DC.get_dns_record(domain, "SOA")
for rdata in soa_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
soa_records = ", ".join(temp)
except:
soa_records = "None"
# INSERT the DNS records into the table
self.c.execute("INSERT INTO 'DNS' VALUES (?,?,?,?,?,?,?)",
(domain, ns_records, a_records, mx_records, txt_records, soa_records,
", ".join(vulnerable_dns_servers)))
self.conn.commit()
sys.exit()
# Create the Subdomains table for recording subdomain info for each domain
self.c.execute('''CREATE TABLE 'Subdomains'
('Domain' text, 'Subdomain' text, 'IP' text, 'ASN' text, 'Provider' text,
'DomainFrontable' text)''')
# Collect the subdomain information from DNS Dumpster and NetCraft
for domain in domains_list:
print(green("[+] Checking DNS Dumpster and NetCraft for {}".format(domain)))
dumpster_results = []
netcraft_results = []
try:
dumpster_results = self.DC.check_dns_dumpster(domain)
except:
print(red("[!] There was a problem collecting results from DNS Dumpster for {}.".format(domain)))
try:
netcraft_results = self.DC.check_netcraft(domain)
except:
print(red("[!] There was a problem collecting results from NetCraft for {}.".format(domain)))
if dumpster_results:
# See if we can save the domain map from DNS Dumpster
if dumpster_results['image_data']:
with open("reports/" + domain + "_Domain_Map.png", "wb") as fh:
fh.write(base64.decodebytes(dumpster_results['image_data']))
# Record the info from DNS Dumpster
for result in dumpster_results['dns_records']['host']:
if result['reverse_dns']:
# TODO: Reverse DNS
subdomain = result['domain']
ip = result['ip']
asn = result['as']
provider = result['provider']
else:
subdomain = result['domain']
ip = result['ip']
asn = result['as']
provider = result['provider']
# Check the subdomain for domain fronting possibilties
frontable = self.DC.check_domain_fronting(result['domain'])
# INSERT the subdomain info into the table
self.c.execute("INSERT INTO Subdomains VALUES (?,?,?,?,?,?)",
(domain, subdomain, ip, asn, provider, frontable))
self.conn.commit()
# INSERT the subdomain info collected from NetCraft
if netcraft_results:
for result in netcraft_results:
frontable = self.DC.check_domain_fronting(result)
self.c.execute("INSERT INTO Subdomains VALUES (?,?,NULL,NULL,NULL,?)",
(domain, result, frontable))
self.conn.commit()
# Create IPHistory table for historical data collected from NetCraft
self.c.execute('''CREATE TABLE 'IPHistory'
('Domain' text, 'Netblock Owner' text, 'IP' text)''')
for domain in domains_list:
ip_history = []
try:
ip_history = self.DC.fetch_netcraft_domain_history(domain)
except:
print(red("[!] There was a problem collecting domain history from NetCraft for {}.".format(domain)))
if ip_history:
for result in ip_history:
net_owner = result[0]
ip = result[1]
self.c.execute("INSERT INTO IPHistory VALUES (?,?,?)",
(domain, net_owner, ip))
self.conn.commit()
# Create the WhoisData table
self.c.execute('''CREATE TABLE 'WhoisData'
('Domain' text, 'Registrar' text, 'Expiration' text, 'Organization' text,
'Registrant' text, 'AdminContact' text, 'TectContact' text, 'Address' text,
'DNSSec' text)''')
# The whois lookups are only for domain names
for domain in domains_list:
try:
# Run whois lookup
print(green("[+] Running whois for {}".format(domain)))
results = self.DC.run_whois(domain)
# Log whois results to domain report
if results:
# Check if more than one expiration date is returned
if isinstance(results['expiration_date'], datetime.date):
expiration_date = results['expiration_date']
# We have a list, so break-up list into human readable dates and times
else:
expiration_date = []
for date in results['expiration_date']:
expiration_date.append(date.strftime("%Y-%m-%d %H:%M:%S"))
expiration_date = ", ".join(expiration_date)
registrar = results['registrar']
org = results['org']
registrant = results['registrant']
admin_email = results['admin_email']
tech_email = results['tech_email']
address = results['address'].rstrip()
dnssec = ', '.join(results['dnssec'])
self.c.execute("INSERT INTO WhoisData VALUES (?,?,?,?,?,?,?,?,?)",
(domain, registrar, expiration_date, org, registrant,
admin_email, tech_email, address, dnssec))
self.conn.commit()
except Exception as error:
print(red("[!] There was an error running whois for {}!".format(domain)))
print(red("L.. Details: {}".format(error)))
# Create RDAP table
self.c.execute('''CREATE TABLE 'RDAPData'
('IP' text, 'RDAPSource' text, 'Organization' text, 'NetworkCIDRs' text,
'ASN' text, 'CountryCode' text, 'RobtexRelatedDomains' text)''')
# The RDAP lookups are only for IPs, but we get the IPs for each domain name, too
for target in scope:
try:
# Slightly change output and recorded target if it's a domain
if helpers.is_ip(target):
target_ip = target
for_output = target
print(green("[+] Running RDAP lookup for {}".format(for_output)))
elif target == "":
pass
else:
target_ip = socket.gethostbyname(target)
for_output = "{} ({})".format(target_ip, target)
print(green("[+] Running RDAP lookup for {}".format(for_output)))
# Log RDAP lookups
results = self.DC.run_rdap(target_ip)
if results:
rdap_source = results['asn_registry']
org = results['network']['name']
net_cidr = results['network']['cidr']
asn = results['asn']
country_code = results['asn_country_code']
# TODO: Convert Verbose mode output into something easily recorded in the DB
# # Verbose mode is optional to allow users to NOT be overwhelmed by contact data
# if verbose:
# row += 1
# for object_key, object_dict in results['objects'].items():
# if results['objects'] is not None:
# for item in results['objects']:
# name = results['objects'][item]['contact']['name']
# if name is not None:
# dom_worksheet.write(row, 1, "Contact Name:")
# dom_worksheet.write(row, 2, name)
# row += 1
# title = results['objects'][item]['contact']['title']
# if title is not None:
# dom_worksheet.write(row, 1, "Contact's Title:")
# dom_worksheet.write(row, 2, title)
# row += 1
# role = results['objects'][item]['contact']['role']
# if role is not None:
# dom_worksheet.write(row, 1, "Contact's Role:")
# dom_worksheet.write(row, 2, role)
# row += 1
# email = results['objects'][item]['contact']['email']
# if email is not None:
# dom_worksheet.write(row, 1, "Contact's Email:")
# dom_worksheet.write(row, 2, email[0]['value'])
# row += 1
# phone = results['objects'][item]['contact']['phone']
# if phone is not None:
# dom_worksheet.write(row, 1, "Contact's Phone:")
# dom_worksheet.write(row, 2, phone[0]['value'])
# row += 1
# address = results['objects'][item]['contact']['address']
# if address is not None:
# dom_worksheet.write(row, 1, "Contact's Address:")
# dom_worksheet.write(row, 2, address[0]['value'])
# row += 1
# Check Robtex for results for the current target
robtex = self.DC.lookup_robtex_ipinfo(target_ip)
if robtex:
results = []
for result in robtex['pas']:
results.append(result['o'])
robtex_results = ", ".join(results)
else:
robtex_results = "None"
self.c.execute("INSERT INTO RDAPData VALUES (?,?,?,?,?,?,?)",
(for_output, rdap_source, org, net_cidr, asn, country_code,
robtex_results))
self.conn.commit()
except Exception as error:
print(red("[!] The RDAP lookup failed for {}!".format(target)))
print(red("L.. Details: {}".format(error)))
# Create the URLVoid table
self.c.execute('''CREATE TABLE 'URLVoidResults'
('Domain' text, 'IP' text, 'Hostname(s)' text, 'DomainAge' text,
'GoogleRank' text, 'AlexaRank' text, 'ASN' text, 'AsnName' text,
'MaliciousCount' text, 'MaliciousEngines' text)''')
# Check each domain with URLVoid for reputation and some Alexa data
for domain in domains_list:
tree = self.DC.run_urlvoid_lookup(domain)
count = ""
engines = ""
if tree is not None:
# Check to see if urlvoid shows the domain flagged by any engines
try:
for child in tree:
malicious_check = child.tag
if malicious_check == "detections":
detections = tree[1]
engines = detections[0]
count = ET.tostring(detections[1], method='text').rstrip().decode('ascii')
temp = []
for engine in engines:
temp.append(ET.tostring(engine, method='text').rstrip().decode('ascii'))
engines = ", ".join(temp)
print(yellow("[*] URLVoid found malicious activity reported for \
{}!".format(domain)))
rep_data = tree[0]
ip_data = rep_data[11]
target = ET.tostring(rep_data[0], method='text').rstrip().decode('ascii')
ip_add = ET.tostring(ip_data[0], method='text').rstrip().decode('ascii')
hostnames = ET.tostring(ip_data[1], method='text').rstrip().decode('ascii')
domain_age = ET.tostring(rep_data[3], method='text').rstrip().decode('ascii')
google_rank = ET.tostring(rep_data[4], method='text').rstrip().decode('ascii')
alexa_rank = ET.tostring(rep_data[5], method='text').rstrip().decode('ascii')
asn = ET.tostring(ip_data[2], method='text').rstrip().decode('ascii')
asn_name = ET.tostring(ip_data[3], method='text').rstrip().decode('ascii')
self.c.execute("INSERT INTO URLVoidResults VALUES (?,?,?,?,?,?,?,?,?,?)",
(target, ip_add, hostnames, domain_age, google_rank,
alexa_rank, asn, asn_name, count, engines))
self.conn.commit()
except:
print(red("[!] There was an error getting the data for {}.".format(domain)))
def create_domain_report_table(self, scope, ip_list, domain_list, verbose):
"""Function to generate a domain report consisting of information like DNS records and
subdomains.
"""
# Get the DNS records for each domain
for domain in domain_list:
vulnerable_dns_servers = []
# Get the NS records
try:
temp = []
ns_records_list = self.DC.get_dns_record(domain, "NS")
for rdata in ns_records_list.response.answer:
for item in rdata.items:
temp.append(item.to_text())
ns_records = ", ".join(x.strip(".") for x in temp)
# Record name server that resolve cached queries
for nameserver in temp:
result = self.DC.check_dns_cache(nameserver.strip("."))
if result:
vulnerable_dns_servers.append(result)
except:
ns_records = "None"
# Get the A records
try:
temp = []
a_records = self.DC.get_dns_record(domain, "A")
for rdata in a_records.response.answer:
for item in rdata.items:
# Add A record IP to a temp list
temp.append(item.to_text())
# Check if this a known IP and add it to hosts if not
self.c.execute(
"SELECT count(*) FROM hosts WHERE host_address=?",
(item.to_text(), ))
res = self.c.fetchone()
if res[0] == 0:
self.c.execute(
"INSERT INTO 'hosts' VALUES (Null,?,?,?)",
(item.to_text(), False,
"Scope File Domain DNS"))
self.conn.commit()
# Also add it to our list of IP addresses
ip_list.append(item.to_text())
a_records = ", ".join(temp)
except:
a_records = "None"
# Get the MX records
try:
temp = []
mx_records = self.DC.get_dns_record(domain, "MX")
for rdata in mx_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
mx_records = ", ".join(x.strip(".") for x in temp)
except:
mx_records = "None"
# Get the TXT records
try:
temp = []
txt_records = self.DC.get_dns_record(domain, "TXT")
for rdata in txt_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
txt_records = ", ".join(temp)
except:
txt_records = "None"
# Get the SOA records
try:
temp = []
soa_records = self.DC.get_dns_record(domain, "SOA")
for rdata in soa_records.response.answer:
for item in rdata.items:
temp.append(item.to_text())
soa_records = ", ".join(temp)
except:
soa_records = "None"
# Get the _DMARC TXT record
try:
temp = []
dmarc_record = self.DC.get_dns_record("_dmarc." + domain,
"TXT")
for rdata in dmarc_record.response.answer:
for item in rdata.items:
temp.append(item.to_text())
dmarc_record = ", ".join(temp)
except:
dmarc_record = "None"
# INSERT the DNS records into the table
self.c.execute(
"INSERT INTO 'dns' VALUES (NULL,?,?,?,?,?,?,?,?)",
(domain, ns_records, a_records, mx_records, txt_records,
soa_records, dmarc_record, ", ".join(vulnerable_dns_servers)))
self.conn.commit()
# Collect the subdomain information from DNS Dumpster and NetCraft
for domain in domain_list:
dumpster_results = []
netcraft_results = []
try:
dumpster_results = self.DC.check_dns_dumpster(domain)
except:
print(
red("[!] There was a problem collecting results from DNS Dumpster for {}."
.format(domain)))
try:
netcraft_results = self.DC.check_netcraft(domain)
except:
print(
red("[!] There was a problem collecting results from NetCraft for {}."
.format(domain)))
if dumpster_results:
# See if we can save the domain map from DNS Dumpster
if dumpster_results['image_data']:
with open("reports/" + domain + "_Domain_Map.png",
"wb") as fh:
fh.write(
base64.decodebytes(dumpster_results['image_data']))
# Record the info from DNS Dumpster
for result in dumpster_results['dns_records']['host']:
if result['reverse_dns']:
subdomain = result['domain']
ip = result['ip']
# asn = result['as']
# provider = result['provider']
else:
subdomain = result['domain']
ip = result['ip']
# asn = result['as']
# provider = result['provider']
# Check if this a known IP and add it to hosts if not
self.c.execute(
"SELECT count(*) FROM hosts WHERE host_address=?",
(result['ip'], ))
res = self.c.fetchone()
if res[0] == 0:
self.c.execute(
"INSERT INTO 'hosts' VALUES (Null,?,?,?)",
(result['ip'], False, "DNS Dumpster"))
self.conn.commit()
# Also add it to our list of IP addresses
ip_list.append(result['ip'])
# Check the subdomain for domain fronting possibilties
frontable = self.DC.check_domain_fronting(result['domain'])
# INSERT the subdomain info into the table
# self.c.execute("INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?,?,?)",
# (domain, subdomain, ip, asn, provider, frontable, "DNS Dumpster"))
self.c.execute(
"INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)",
(domain, subdomain, ip, frontable, "DNS Dumpster"))
self.conn.commit()
# INSERT the subdomain info collected from NetCraft
if netcraft_results:
for result in netcraft_results:
if not result in dumpster_results['dns_records']['host']:
try:
ip_address = socket.gethostbyname(result)
# Check if this a known IP and add it to hosts if not
self.c.execute(
"SELECT count(*) FROM hosts WHERE host_address=?",
(ip_address, ))
res = self.c.fetchone()
if res[0] == 0:
self.c.execute(
"INSERT INTO 'hosts' VALUES (Null,?,?,?)",
(ip_address, False, "Netcraft DNS"))
self.conn.commit()
# Also add it to our list of IP addresses
ip_list.append(ip_address)
except:
ip_address = "Lookup Failed"
frontable = self.DC.check_domain_fronting(result)
self.c.execute(
"INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)",
(domain, result, ip_address, frontable,
"Netcraft"))
self.conn.commit()
# Try to collect certificate data for the domain
try:
cert_data = self.DC.run_censys_search_cert(domain)
for cert in cert_data['results']:
self.c.execute(
"INSERT INTO 'certificates' VALUES (NULL,?,?,?)",
(domain, cert["parsed.subject_dn"],
cert["parsed.issuer_dn"]))
self.conn.commit()
cert_subdomains = self.DC.parse_cert_subdomains(cert_data)
for sub in cert_subdomains:
if not sub in dumpster_results['dns_records'][
'host'] or sub in netcraft_results:
# Check for wildcard subdomains from certificates and ignore them for this
if not "*" in sub:
try:
ip_address = socket.gethostbyname(sub)
# Check if this a known IP and add it to hosts if not
self.c.execute(
"SELECT count(*) FROM hosts WHERE host_address=?",
(ip_address, ))
res = self.c.fetchone()
if res[0] == 0:
self.c.execute(
"INSERT INTO 'hosts' VALUES (Null,?,?,?)",
(ip_address, False,
"Certificate Lookup"))
self.conn.commit()
# Also add it to our list of IP addresses
ip_list.append(ip_address)
except:
ip_address = "Lookup Failed"
frontable = self.DC.check_domain_fronting(sub)
self.c.execute(
"INSERT INTO 'subdomains' VALUES (NULL,?,?,?,?,?)",
(domain, sub, ip_address, frontable,
"Certificate"))
self.conn.commit()
except:
pass
# Take a break for Censys's rate limits
sleep(self.sleep)
for domain in domain_list:
ip_history = []
try:
ip_history = self.DC.fetch_netcraft_domain_history(domain)
except:
print(
red("[!] There was a problem collecting domain history from NetCraft for {}."
.format(domain)))
if ip_history:
for result in ip_history:
net_owner = result[0]
ip_address = result[1]
self.c.execute(
"INSERT INTO ip_history VALUES (NULL,?,?,?)",
(domain, net_owner, ip_address))
self.conn.commit()
# Check if this a known IP and add it to hosts if not
self.c.execute(
"SELECT count(*) FROM hosts WHERE host_address=?",
(ip_address, ))
res = self.c.fetchone()
if res[0] == 0:
self.c.execute(
"INSERT INTO 'hosts' VALUES (Null,?,?,?)",
(ip_address, False, "Netcraft Domain IP History"))
self.conn.commit()
# Also add it to our list of IP addresses
ip_list.append(ip_address)
# The whois lookups are only for domain names
for domain in domain_list:
try:
# Run whois lookup
results = self.DC.run_whois(domain)
if results:
# Check if more than one expiration date is returned
if isinstance(results['expiration_date'], datetime.date):
expiration_date = results['expiration_date']
# We have a list, so break-up list into human readable dates and times
else:
expiration_date = []
for date in results['expiration_date']:
expiration_date.append(
date.strftime("%Y-%m-%d %H:%M:%S"))
expiration_date = ", ".join(expiration_date)
registrar = results['registrar']
org = results['org']
registrant = results['registrant']
admin_email = results['admin_email']
tech_email = results['tech_email']
address = results['address'].rstrip()
if results['dnssec'] == "unsigned":
dnssec = results['dnssec']
else:
dnssec = ', '.join(results['dnssec'])
self.c.execute(
"INSERT INTO whois_data VALUES (NULL,?,?,?,?,?,?,?,?,?)",
(domain, registrar, expiration_date, org, registrant,
admin_email, tech_email, address, dnssec))
self.conn.commit()
except Exception as error:
print(
red("[!] There was an error running whois for {}!".format(
domain)))
print(red("L.. Details: {}".format(error)))
# The RDAP lookups are only for IPs, but we get the IPs for each domain name, too
self.c.execute("SELECT host_address FROM hosts")
collected_hosts = self.c.fetchall()
# for target in scope:
for target in collected_hosts:
try:
# Slightly change output and record target if it's a domain
target = target[0]
if helpers.is_ip(target):
target_ip = target
# for_output = target
elif target == "":
pass
else:
target_ip = socket.gethostbyname(target)
# for_output = "{} ({})".format(target_ip, target)
# Log RDAP lookups
results = self.DC.run_rdap(target_ip)
if results:
rdap_source = results['asn_registry']
org = results['network']['name']
net_cidr = results['network']['cidr']
asn = results['asn']
country_code = results['asn_country_code']
# TODO: Convert Verbose mode output into something easily recorded in the DB
# # Verbose mode is optional to allow users to NOT be overwhelmed by contact data
# if verbose:
# row += 1
# for object_key, object_dict in results['objects'].items():
# if results['objects'] is not None:
# for item in results['objects']:
# name = results['objects'][item]['contact']['name']
# if name is not None:
# dom_worksheet.write(row, 1, "Contact Name:")
# dom_worksheet.write(row, 2, name)
# row += 1
# title = results['objects'][item]['contact']['title']
# if title is not None:
# dom_worksheet.write(row, 1, "Contact's Title:")
# dom_worksheet.write(row, 2, title)
# row += 1
# role = results['objects'][item]['contact']['role']
# if role is not None:
# dom_worksheet.write(row, 1, "Contact's Role:")
# dom_worksheet.write(row, 2, role)
# row += 1
# email = results['objects'][item]['contact']['email']
# if email is not None:
# dom_worksheet.write(row, 1, "Contact's Email:")
# dom_worksheet.write(row, 2, email[0]['value'])
# row += 1
# phone = results['objects'][item]['contact']['phone']
# if phone is not None:
# dom_worksheet.write(row, 1, "Contact's Phone:")
# dom_worksheet.write(row, 2, phone[0]['value'])
# row += 1
# address = results['objects'][item]['contact']['address']
# if address is not None:
# dom_worksheet.write(row, 1, "Contact's Address:")
# dom_worksheet.write(row, 2, address[0]['value'])
# row += 1
# Check Robtex for results for the current target
robtex = self.DC.lookup_robtex_ipinfo(target_ip)
if robtex:
results = []
for result in robtex['pas']:
results.append(result['o'])
robtex_results = ", ".join(results)
else:
robtex_results = "None"
self.c.execute(
"INSERT INTO rdap_data VALUES (NULL,?,?,?,?,?,?,?)",
(target_ip, rdap_source, org, net_cidr, asn,
country_code, robtex_results))
self.conn.commit()
except Exception as error:
print(red("[!] The RDAP lookup failed for {}!".format(target)))
print(red("L.. Details: {}".format(error)))
