def process_urls(data):
i, urls, timeout = data
blacklist = [
'Date', 'Connection', 'Content-Type', 'Content-Length', 'Keep-Alive',
'Content-Encoding', 'Vary'
]
new_headers = {}
for u in urls:
display("Processing %s" % u)
try:
res = requests.get(u, timeout=int(timeout), verify=False)
for k in res.headers.keys():
if k not in blacklist:
if not new_headers.get(u, False):
new_headers[u] = []
new_headers[u].append("%s: %s" % (k, res.headers[k]))
except KeyboardInterrupt:
display_warning("Got Ctrl+C, exiting")
sys.exit(1)
except Exception as e:
display_error("%s no good, skipping: %s" % (u, e))
return (i, new_headers)
def process_output(self, cmds):
'''
Process the output generated by the earlier commands.
'''
# Cycle through all of the targets we ran earlier
for c in cmds:
output_file = c['output']
target = c['target']
# Read file
data = open(output_file).read().split('\n')
# Quick and dirty way to filter out headers and blank lines, as well
# as duplicates
res = list(
set([
d for d in data if 'Domain,Cname,Provider' not in d and d
]))
if res:
# Load up the DB entry.
created, subdomain = self.Domain.find_or_create(domain=target)
# Process results
for d in res:
results = d.split(',')
if results[3] == "false":
display_warning(
"Hosting found at {} for {}, not vulnerable.".
format(target, results[2]))
elif results[3] == "true":
display_new("{} vulnerable to {}!".format(
target, results[2]))
if not subdomain.meta[self.name].get(
'vulnerable', False):
subdomain.meta[self.name]['vulnerable'] = []
subdomain.meta[self.name]['vulnerable'].append(d)
else:
display_warning("Not sure of result: {}".format(data))
# This is a little hackish, but needed to get data to save
t = dict(subdomain.meta)
self.Domain.commit()
subdomain.meta = t
self.Domain.commit()
def process_output(self, cmds):
display_warning(
"There is currently no post-processing for this module. For the juicy results, refer to the output file paths."
)
def run(self, args):
if not args.api_key:
display_error("You must supply an API key to use shodan!")
return
if args.search:
ranges = [args.search]
if args.import_db:
ranges = []
if args.rescan:
if args.fast:
ranges += [
"net:{}".format(c.cidr) for c in self.ScopeCidr.all()
]
else:
cidrs = [c.cidr for c in self.ScopeCidr.all()]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active")
]
else:
if args.fast:
ranges += [
"net:{}".format(c.cidr)
for c in self.ScopeCidr.all(tool=self.name)
]
else:
cidrs = [
c.cidr for c in self.ScopeCidr.all(tool=self.name)
]
for c in cidrs:
ranges += [str(i) for i in IPNetwork(c)]
if not args.cidr_only:
ranges += [
"{}".format(i.ip_address)
for i in self.IPAddress.all(scope_type="active",
tool=self.name)
]
api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
api_search_url = (
"https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
for r in ranges:
time.sleep(1)
if ":" in r:
display("Doing Shodan search: {}".format(r))
try:
results = json.loads(
requests.get(api_search_url.format(args.api_key, r,
1)).text)
if results.get("error") and "request timed out" in results[
"error"]:
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
total = len(results["matches"])
matches = []
i = 1
while total > 0:
display("Adding {} results from page {}".format(total, i))
matches += results["matches"]
i += 1
try:
time.sleep(1)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
i)).text)
if (results.get("error")
and "request timed out" in results["error"]):
display_warning(
"Timeout occurred on Shodan's side.. trying again in 5 seconds."
)
results = json.loads(
requests.get(
api_search_url.format(args.api_key, r,
1)).text)
total = len(results["matches"])
except Exception as e:
display_error("Something went wrong: {}".format(e))
total = 0
pdb.set_trace()
for res in matches:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP, port_number=port_str, proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.meta["shodan_data"] = res
port.save()
else:
try:
results = json.loads(
requests.get(api_host_url.format(r,
args.api_key)).text)
except Exception as e:
display_error("Something went wrong: {}".format(e))
next
# pdb.set_trace()
if results.get("data", False):
display("{} results found for: {}".format(
len(results["data"]), r))
for res in results["data"]:
ip_str = res["ip_str"]
port_str = res["port"]
transport = res["transport"]
display("Processing IP: {} Port: {}/{}".format(
ip_str, port_str, transport))
created, IP = self.IPAddress.find_or_create(
ip_address=ip_str)
IP.meta["shodan_data"] = results
created, port = self.Port.find_or_create(
ip_address=IP,
port_number=port_str,
proto=transport)
if created:
svc = ""
if res.get("ssl", False):
svc = "https"
elif res.get("http", False):
svc = "http"
else:
svc = ""
port.service_name = svc
port.meta["shodan_data"] = res
port.save()
self.IPAddress.commit()
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=False,
**kwargs):
created, d = super(DomainRepository,
self).find_or_create(only_tool, **kwargs)
display("Processing %s" % d.domain)
if created:
# If this is a new subdomain, set scoping info based on what is passed to the function initially.
d.in_scope = in_scope
d.passive_scope = passive_scope
base_domain = '.'.join(
[t for t in tldextract.extract(d.domain)[1:] if t])
BaseDomains = BaseDomainRepository(self.db, "")
# If the base domain is new, it'll inherit the same scoping permissions.
created, bd = BaseDomains.find_or_create(
only_tool,
passive_scope=d.passive_scope,
in_scope=in_scope,
domain=base_domain)
if created:
display_new(
"The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s"
% (base_domain, bd.in_scope, bd.passive_scope))
else:
# If the base domain already exists, then the subdomain inherits the scope info from the base domain.
d.passive_scope = bd.passive_scope
d.in_scope = bd.in_scope
d.base_domain = bd
# Get all IPs that this domain resolves to.
ips = []
try:
answers = dns.resolver.query(d.domain, 'A')
for a in answers:
ips.append(a.address)
except:
# If something goes wrong with DNS, we end up here
pass
if not ips:
display_warning("No IPs discovered for %s" % d.domain)
for i in ips:
IPAddresses = IPRepository(self.db, "")
display("Processing IP address %s" % i)
created, ip = IPAddresses.find_or_create(
only_tool,
in_scope=d.in_scope,
passive_scope=d.passive_scope,
ip_address=i)
# If the IP is in scope, then the domain should be
if ip.in_scope:
d.in_scope = ip.in_scope
ip.passive_scope = True
d.passive_scope = True
# display("%s marked active scope due to IP being marked active." % d.domain)
elif ip.passive_scope:
d.passive_scope = ip.passive_scope
d.ip_addresses.append(ip)
display_new(
"%s is being added to the database. Active Scope: %s Passive Scope: %s"
% (d.domain, d.in_scope, d.passive_scope))
# Final sanity check - if a domain is active scoped, it should also be passively scoped.
if d.in_scope:
d.passive_scope = True
return created, d
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=True,
**kwargs):
created, ip = super(IPRepository,
self).find_or_create(only_tool, **kwargs)
if created:
# If newly created then will determine scoping based on parent options and if in a scoped cidr.
ip_str = ip.ip_address
ip.passive_scope = passive_scope
# If the parent domain is active scope, then this also is.
if in_scope:
ip.in_scope = in_scope
else:
# Go through ScopeCIDR table and see if this IP is in a CIDR in scope
ScopeCidrs = ScopeCIDRRepository(self.db, "")
addr = IPAddress(ip.ip_address)
cidrs = ScopeCidrs.all()
# pdb.set_trace()
for c in cidrs:
if addr in IPNetwork(c.cidr):
ip.in_scope = True
# Final sanity check - if an IP is active scoped, it should also be passive scoped.
if ip.in_scope:
ip.passive_scope = True
ip.update()
# Build CIDR info - mainly for reporting
res = False
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), "Non-Public Subnet"], )
if res:
cidr_data = res
else:
while True:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except:
res = IPWhois(ip_str).lookup_whois()
if res["nets"]:
break
else:
display_warning(
"The networks didn't populate from whois. Usually retrying after a couple of seconds resolves this. Sleeping for 5 seconds and trying again."
)
time.sleep(5)
cidr_data = []
for n in res["nets"]:
if "," in n["cidr"]:
for cidr_str in n["cidr"].split(", "):
cidr_data.append([cidr_str, n["description"]])
else:
cidr_data.append([n["cidr"], n["description"]])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
try:
cidr_len = len(IPNetwork(cidr_data[0][0]))
except:
pdb.set_trace()
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(matching_cidr[1], matching_cidr[0]))
CIDR = CIDRRepository(self.db, "")
created, cidr = CIDR.find_or_create(only_tool=True,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = matching_cidr[1]
cidr.update()
ip.cidr = cidr
ip.update()
display_new(
"IP address %s added to database. Active Scope: %s Passive Scope: %s"
% (ip.ip_address, ip.in_scope, ip.passive_scope))
return created, ip