def run(self, args):
if args.import_db:
if args.rescan:
svc = self.Port.all(service_name='http')
svc += self.Port.all(service_name='https')
else:
svc = self.Port.all(service_name='http', tool=self.name)
svc += self.Port.all(service_name='https', tool=self.name)
data = []
for s in svc:
if s.ip_address.in_scope:
urls = [
"%s://%s:%s" % (s.service_name,
s.ip_address.ip_address, s.port_number)
]
for d in s.ip_address.domains:
urls.append("%s://%s:%s" %
(s.service_name, d.domain, s.port_number))
data.append([s.id, urls, args.timeout])
pool = ThreadPool(int(args.threads))
results = pool.map(process_urls, data)
display_new("Adding headers to the database")
for i, headers in results:
created, svc = self.Port.find_or_create(id=i)
svc.meta['headers'] = headers
svc.update()
self.Port.commit()
def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
def find_or_create(self,
only_tool=False,
passive_scope=True,
in_scope=False,
**kwargs):
created, port = super(PortRepository,
self).find_or_create(only_tool, **kwargs)
if created:
display_new("Port {} added to database for IP {}".format(
port.port_number, port.ip_address.ip_address))
return created, port
def process_output(self, cmds):
'''
Process the output generated by the earlier commands.
'''
# Cycle through all of the targets we ran earlier
for c in cmds:
output_file = c['output']
target = c['target']
# Read file
data = open(output_file).read().split('\n')
# Quick and dirty way to filter out headers and blank lines, as well
# as duplicates
res = list(
set([
d for d in data if 'Domain,Cname,Provider' not in d and d
]))
if res:
# Load up the DB entry.
created, subdomain = self.Domain.find_or_create(domain=target)
# Process results
for d in res:
results = d.split(',')
if results[3] == "false":
display_warning(
"Hosting found at {} for {}, not vulnerable.".
format(target, results[2]))
elif results[3] == "true":
display_new("{} vulnerable to {}!".format(
target, results[2]))
if not subdomain.meta[self.name].get(
'vulnerable', False):
subdomain.meta[self.name]['vulnerable'] = []
subdomain.meta[self.name]['vulnerable'].append(d)
else:
display_warning("Not sure of result: {}".format(data))
# This is a little hackish, but needed to get data to save
t = dict(subdomain.meta)
self.Domain.commit()
subdomain.meta = t
self.Domain.commit()
def reclassify_domain(self, bd):
if bd.meta.get('whois', False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta['whois'])
res = raw_input("Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] ")
if res.lower() == 'a':
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == 'p':
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error("Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(bd.domain))
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=True,
**kwargs):
created, ip = super(IPRepository,
self).find_or_create(only_tool, **kwargs)
if created:
# If newly created then will determine scoping based on parent options and if in a scoped cidr.
ip_str = ip.ip_address
ip.passive_scope = passive_scope
# If the parent domain is active scope, then this also is.
if in_scope:
ip.in_scope = in_scope
else:
# Go through ScopeCIDR table and see if this IP is in a CIDR in scope
ScopeCidrs = ScopeCIDRRepository(self.db, "")
addr = IPAddress(ip.ip_address)
cidrs = ScopeCidrs.all()
# pdb.set_trace()
for c in cidrs:
if addr in IPNetwork(c.cidr):
ip.in_scope = True
# Final sanity check - if an IP is active scoped, it should also be passive scoped.
if ip.in_scope:
ip.passive_scope = True
ip.update()
# Build CIDR info - mainly for reporting
res = False
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
res = ([str(cidr), 'Non-Public Subnet'], )
if res:
cidr_data = res
else:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except:
res = IPWhois(ip_str).lookup_whois()
cidr_data = []
for n in res['nets']:
if ',' in n['cidr']:
for cidr_str in n['cidr'].split(', '):
cidr_data.append([cidr_str, n['description']])
else:
cidr_data.append([n['cidr'], n['description']])
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
cidr_len = len(IPNetwork(cidr_data[0][0]))
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
display("Processing CIDR from whois: %s - %s" %
(matching_cidr[1], matching_cidr[0]))
CIDR = CIDRRepository(self.db, "")
created, cidr = CIDR.find_or_create(only_tool=True,
cidr=matching_cidr[0])
if created:
display_new("CIDR %s added to database" % cidr.cidr)
cidr.org_name = matching_cidr[1]
cidr.update()
ip.cidr = cidr
ip.update()
display_new(
"IP address %s added to database. Active Scope: %s Passive Scope: %s"
% (ip.ip_address, ip.in_scope, ip.passive_scope))
return created, ip
def find_or_create(self,
only_tool=False,
in_scope=False,
passive_scope=False,
**kwargs):
created, d = super(DomainRepository,
self).find_or_create(only_tool, **kwargs)
display("Processing %s" % d.domain)
if created:
# If this is a new subdomain, set scoping info based on what is passed to the function initially.
d.in_scope = in_scope
d.passive_scope = passive_scope
base_domain = '.'.join(
[t for t in tldextract.extract(d.domain)[1:] if t])
BaseDomains = BaseDomainRepository(self.db, "")
# If the base domain is new, it'll inherit the same scoping permissions.
created, bd = BaseDomains.find_or_create(
only_tool,
passive_scope=d.passive_scope,
in_scope=in_scope,
domain=base_domain)
if created:
display_new(
"The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s"
% (base_domain, bd.in_scope, bd.passive_scope))
else:
# If the base domain already exists, then the subdomain inherits the scope info from the base domain.
d.passive_scope = bd.passive_scope
d.in_scope = bd.in_scope
d.base_domain = bd
# Get all IPs that this domain resolves to.
ips = []
try:
answers = dns.resolver.query(d.domain, 'A')
for a in answers:
ips.append(a.address)
except:
# If something goes wrong with DNS, we end up here
pass
if not ips:
display_warning("No IPs discovered for %s" % d.domain)
for i in ips:
IPAddresses = IPRepository(self.db, "")
display("Processing IP address %s" % i)
created, ip = IPAddresses.find_or_create(
only_tool,
in_scope=d.in_scope,
passive_scope=d.passive_scope,
ip_address=i)
# If the IP is in scope, then the domain should be
if ip.in_scope:
d.in_scope = ip.in_scope
ip.passive_scope = True
d.passive_scope = True
# display("%s marked active scope due to IP being marked active." % d.domain)
elif ip.passive_scope:
d.passive_scope = ip.passive_scope
d.ip_addresses.append(ip)
display_new(
"%s is being added to the database. Active Scope: %s Passive Scope: %s"
% (d.domain, d.in_scope, d.passive_scope))
# Final sanity check - if a domain is active scoped, it should also be passively scoped.
if d.in_scope:
d.passive_scope = True
return created, d
