class Module(ToolTemplate):
name = "DNSRecon"
binary_name = "dnsrecon"
"""
This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries.
It can also run over IP ranges, looking for additional domains in the PTR records.
"""
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d','--domain', help="Target domain for dnsRecon")
self.options.add_argument('-f', '--file', help="Import domains from file")
self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true")
self.options.add_argument('-r', '--range', help="Range to scan for PTR records")
self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true")
self.options.add_argument('--rescan', help="Rescan domains already scanned", action="store_true")
# self.options.add_argument('--import_output_xml', help="Import XML file")
# self.options.add_argument('--import_output_json', help="Import json file")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain, passive_scope=True)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d, passive_scope=True)
targets.append(domain.domain)
elif args.import_database:
domains = self.BaseDomain.all(scope_type="passive", tool=self.name)
for domain in domains:
targets.append(domain.domain)
elif args.range:
targets.append(args.range)
elif args.import_range:
cidrs = self.ScopeCIDR.all(tool=self.name)
for cidr in cidrs:
targets.append(cidr.cidr)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] )
else:
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({'target':t, 'output':os.path.join(self.path, t.replace("/", "_") + ".json")})
return res
def build_cmd(self, args):
command = self.binary
if args.domain or args.file or args.import_database:
command += " -d {target} -j {output} "
else:
command += " -s -r {target} -j {output} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
for c in cmds:
target = c['target']
output_path = c['output']
try:
res = json.loads(open(output_path).read())
except IOError:
display_error("DnsRecon failed for {}".format(target))
if " -d " in res[0]['arguments']:
created, dbrec = self.Domain.find_or_create(domain=target)
dbrec.dns = res
dbrec.save()
for record in res:
domain = None
if record.get("type") == "A" or record.get("type") == "PTR":
domain = record.get("name").lower().replace("www.","")
elif record.get("type") == "MX":
domain = record.get("exchange").lower().replace("www.","")
elif record.get("type") == "SRV" or record.get("type" == "NS"):
domain = record.get("target").lower().replace("www.","")
elif record.get("type") == "SOA":
domain = record.get("mname").lower().replace("www.","")
if domain:
created, domain_obj = self.Domain.find_or_create(domain=domain)
self.Domain.commit()
class Module(ModuleTemplate):
'''
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
'''
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
'-f',
'--import_file',
help="File containing domains to import. One per line")
self.options.add_argument('-d',
'--domain',
help="Single domain to import")
self.options.add_argument(
'-i',
'--import_ips',
help="File containing IPs and ranges, one per line.")
self.options.add_argument('-Id',
'--import_database_domains',
help='Import domains from database',
action="store_true")
self.options.add_argument('-Ii',
'--import_database_ips',
help='Import IPs from database',
action="store_true")
self.options.add_argument(
'--force',
help="Force processing again, even if already processed",
action="store_true")
def run(self, args):
if args.import_file:
domains = open(args.import_file)
for line in domains:
if line.strip():
self.process_domain(line.strip(), force_scope=True)
self.Domain.commit()
if args.domain:
self.process_domain(args.domain, force_scope=True)
self.Domain.commit()
if args.import_database_domains:
if args.force:
domains = self.Domain.all()
else:
domains = self.Domain.all(tool=self.name)
for d in domains:
# pdb.set_trace()
self.process_domain(d.domain)
self.Domain.commit()
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if '/' in line or '-' in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if '/' in args.import_ips or '-' in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_database_ips:
for ip in self.IPAddress.all():
self.process_ip(ip.ip_address)
self.Domain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, 'A')
for a in answers:
ips.append(a.address)
return ips
except:
return []
def process_domain(self, domain_str, force_scope=False):
# First check if the root domain exists, and if it doesn't, add it
created, domain = self.Domain.find_or_create(
only_tool=True, domain=domain_str, force_in_scope=force_scope)
# if not created:
# # print("%s already processed, skipping." % domain_str)
# return
print("Processing %s" % domain_str)
# Next get ip addresses of domain
ips = self.get_domain_ips(domain_str)
for i in ips:
ip = self.process_ip(i, force_scope=force_scope)
domain.ip_addresses.append(ip)
domain.save()
def process_ip(self, ip_str, force_scope=False):
created, ip = self.IPAddress.find_or_create(only_tool=True,
ip_address=ip_str,
force_in_scope=force_scope)
if created:
print(" - Found New IP: %s" % ip_str)
res = self.check_private_subnets(ip_str)
if res:
cidr_data = res
else:
try:
res = IPWhois(ip_str).lookup_whois(get_referral=True)
except:
res = IPWhois(ip_str).lookup_whois()
cidr_data = []
for n in res['nets']:
if ',' in n['cidr']:
for cidr_str in n['cidr'].split(', '):
cidr_data.append([cidr_str, n['description']])
else:
cidr_data.append([n['cidr'], n['description']])
try:
cidr_data = [
cidr_d for cidr_d in cidr_data
if IPAddress(ip_str) in IPNetwork(cidr_d[0])
]
except:
pdb.set_trace()
cidr_len = len(IPNetwork(cidr_data[0][0]))
matching_cidr = cidr_data[0]
for c in cidr_data:
if len(IPNetwork(c[0])) < cidr_len:
matching_cidr = c
print("New CIDR found: %s - %s" %
(matching_cidr[1], matching_cidr[0]))
cidr = self.CIDR.find_or_create(only_tool=True,
cidr=matching_cidr[0],
org_name=matching_cidr[1])[1]
ip.cidr = cidr
ip.save()
# else:
# print(" - IP Already processed: %s" % ip_str)
return ip
def check_private_subnets(self, ip_str):
for cidr in private_subnets:
if IPAddress(ip_str) in cidr:
return ([str(cidr), 'Non-Public Subnet'], )
return False
def process_cidr(self, line):
if '/' in line:
print("Adding %s to scoped CIDRs" % line.strip())
self.ScopeCIDR.find_or_create(cidr=line.strip())
elif '-' in line:
start_ip, end_ip = line.strip().replace(' ', '').split('-')
if '.' not in end_ip:
end_ip = '.'.join(start_ip.split('.')[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
print("Adding %s to scoped CIDRs" % str(c))
self.ScopeCIDR.find_or_create(cidr=str(c))
class Module(ModuleTemplate):
"""
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
"""
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-d",
"--import_domains",
help=
"Either domain to import or file containing domains to import. One per line",
)
self.options.add_argument(
"-i",
"--import_ips",
help=
"Either IP/range to import or file containing IPs and ranges, one per line.",
)
self.options.add_argument(
"-a",
"--active",
help="Set scoping on imported data as active",
action="store_true",
)
self.options.add_argument(
"-p",
"--passive",
help="Set scoping on imported data as passive",
action="store_true",
)
self.options.add_argument(
"-sc",
"--scope_cidrs",
help=
"Cycle through out of scope networks and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument(
"-sb",
"--scope_base_domains",
help=
"Cycle through out of scope base domains and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument("--descope",
help="Descope an IP, domain, or CIDR")
self.options.add_argument(
"-Ii",
"--import_database_ips",
help="Import IPs from database",
action="store_true",
)
self.options.add_argument(
"--force",
help="Force processing again, even if already processed",
action="store_true",
)
def run(self, args):
self.in_scope = args.active
self.passive_scope = args.passive
if args.descope:
if "/" in args.descope:
self.descope_cidr(args.descope)
elif check_string(args.descope):
pass
else:
self.descope_ip(args.descope)
# Check if in ScopeCIDR and remove if found
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if "/" in line or "-" in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if "/" in args.import_ips or "-" in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
self.Domain.commit()
except IOError:
self.process_domain(args.import_domains.strip())
self.Domain.commit()
if args.scope_base_domains:
base_domains = self.BaseDomain.all(in_scope=False,
passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
self.BaseDomain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, "A")
for a in answers:
ips.append(a.address)
return ips
except:
return []
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(
only_tool=True,
domain=domain_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if (domain.in_scope != self.in_scope
or domain.passive_scope != self.passive_scope):
display(
"Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (domain_str, self.in_scope, self.passive_scope))
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(
only_tool=True,
ip_address=ip_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display(
"IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (ip_str, self.in_scope, self.passive_scope))
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
def scope_ips(self):
IPAddresses = self.IPAddress.all()
def reclassify_domain(self, bd):
if bd.meta.get("whois", False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta["whois"])
res = raw_input(
"Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] "
)
if res.lower() == "a":
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == "p":
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error(
"Unfortunately, there is no whois information for {}. Please populate it using the Whois module"
.format(bd.domain))
def descope_ip(self, ip):
ip = self.IPAddress.all(ip_address=ip)
if ip:
for i in ip:
display("Removing IP {} from scope".format(i.ip_address))
i.in_scope = False
i.passive_scope = False
i.update()
for d in i.domains:
in_scope_ips = [
ipa for ipa in d.ip_addresses
if ipa.in_scope or ipa.passive_scope
]
if not in_scope_ips:
display(
"Domain {} has no more scoped IPs. Removing from scope."
.format(d.domain))
d.in_scope = False
d.passive_scope = False
self.IPAddress.commit()
def descope_cidr(self, cidr):
CIDR = self.ScopeCIDR.all(cidr=cidr)
if CIDR:
for c in CIDR:
display("Removing {} from ScopeCIDRs".format(c.cidr))
c.delete()
cnet = IPNetwork(cidr)
for ip in self.IPAddress.all():
if IPAddress(ip.ip_address) in cnet:
self.descope_ip(ip.ip_address)
class Module(ToolTemplate):
name = "Sublist3r"
binary_name = "sublist3r"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Domain to brute force")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({"target": t, "output": output})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd["output"]
if os.path.isfile(output_path):
data = open(output_path).read().split("\n")
else:
display_error("{} not found.".format(output_path))
next
for d in data:
new_domain = d.split(":")[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
class Module(ToolTemplate):
name = "aquatone-discover"
binary_name = "aquatone-discover"
def __init__(self, db):
self.db = db
self.Domain = DomainRepository(db, self.name)
self.BaseDomain = BaseDomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Target domain for aquatone")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-r",
"--rescan",
help="Run aquatone on hosts that have already been processed.",
action="store_true",
)
self.options.set_defaults(timeout=None)
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [(target, output), (target, output), etc, etc]
"""
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domainsFile = open(args.file).read().split("\n")
for d in domainsFile:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
all_domains = self.BaseDomain.all(scope_type="passive")
else:
all_domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in all_domains:
targets.append(d.domain)
else:
print("You need to supply domain(s).")
output_path = os.path.join(self.base_config["PROJECT"]["base_path"],
"aquatone")
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target": t,
"output": "{}/{}/hosts.json".format(output_path, t)
})
return res
def build_cmd(self, args):
"""
Create the actual command that will be executed. Use {target} and {output} as placeholders.
"""
cmd = self.binary + " -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def pre_run(self, args):
output_path = self.base_config["PROJECT"]["base_path"]
self.orig_home = os.environ["HOME"]
os.environ["HOME"] = output_path
def process_output(self, cmds):
"""
Process the output generated by the earlier commands.
"""
for cmd in cmds:
data2 = json.loads(open(cmd["output"]).read())
for sub, ip in data2.items():
created = False
new_domain = sub.lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
def post_run(self, args):
os.environ["HOME"] = self.orig_home
class Module(ToolTemplate):
name = "GobusterDNS"
binary_name = "gobuster"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been brute forced",
action="store_true")
self.options.set_defaults(
timeout=600) # Kick the default timeout to 10 minutes
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
targets.append(d)
if args.import_database:
if args.rescan:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive")
]
else:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive",
tool=self.name)
]
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
'target':
t,
'output':
os.path.join(output_path,
t.replace('/', '_') + "-dns.txt")
})
return res
def build_cmd(self, args):
cmd = self.binary + " -m dns "
cmd += " -o {output} -u {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for c in cmds:
target = c['target']
output_path = c['output']
if os.path.isfile(output_path):
data = open(output_path).read().split('\n')
else:
display_error("{} not found.".format(output_path))
return
for d in data:
if 'Found: ' in d:
new_domain = d.split(' ')[1].lower()
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
class Module(ToolTemplate):
name = "Sublist3r"
binary_name = 'sublist3r'
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output_path = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({'target': t, 'output': output_path})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd['output']
data = open(output_path).read().split('\n')
for d in data:
new_domain = d.split(':')[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
class Module(ToolTemplate):
'''
This tool will check various subdomains for domain takeover vulnerabilities.
'''
name = "Tko-subs"
binary_name = "tko-subs"
def __init__(self, db):
self.db = db
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('--data',
help="Path to the providers_data.csv file")
self.options.add_argument('-d',
'--domain',
help="Domain to run the tool against.")
self.options.add_argument('-i',
'--importdb',
help="Import subdomains from the database.",
action="store_true")
self.options.add_argument('--rescan',
help="Rescan already processed entries",
action="store_true")
def get_targets(self, args):
'''
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [{'target':target, 'output':output}, {'target':target, 'output':output}, etc, etc]
'''
# Create an empty list to add targets to
domains = []
# Check if a domain has been explicitly passed, and if so add it to targets
if args.domain:
domains.append(args.domain)
# Check if the --import option was passed and if so get data from the domain.
# The scoping is set to "active" since the tool could potentially make a
# request from the server.
if args.importdb:
if args.rescan:
all_domains = self.Domain.all(scope_type="active")
else:
all_domains = self.Domain.all(scope_type="active",
tool=self.name)
for d in all_domains:
domains.append(d.domain)
# Set the output_path base, as a junction of the base_path and the path name supplied
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
# Create the path if it doesn't already exist
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
# Create the final targets list, with output paths added.
for d in domains:
res.append({'target': d, 'output': os.path.join(output_path, d)})
return res
def build_cmd(self, args):
'''
Create the actual command that will be executed. Use {target} and {output} as placeholders.
'''
cmd = self.binary + " -domain {target} -output {output} "
# Add in any extra arguments passed in the extra_args parameter
if args.tool_args:
cmd += args.tool_args
# Add that data parameter in there
cmd += " -data " + args.data
return cmd
def process_output(self, cmds):
'''
Process the output generated by the earlier commands.
'''
# Cycle through all of the targets we ran earlier
for c in cmds:
output_file = c['output']
target = c['target']
# Read file
data = open(output_file).read().split('\n')
# Quick and dirty way to filter out headers and blank lines, as well
# as duplicates
res = list(
set([
d for d in data if 'Domain,Cname,Provider' not in d and d
]))
if res:
# Load up the DB entry.
created, subdomain = self.Domain.find_or_create(domain=target)
# Process results
for d in res:
results = d.split(',')
if results[3] == "false":
display_warning(
"Hosting found at {} for {}, not vulnerable.".
format(target, results[2]))
elif results[3] == "true":
display_new("{} vulnerable to {}!".format(
target, results[2]))
if not subdomain.meta[self.name].get(
'vulnerable', False):
subdomain.meta[self.name]['vulnerable'] = []
subdomain.meta[self.name]['vulnerable'].append(d)
else:
display_warning("Not sure of result: {}".format(data))
# This is a little hackish, but needed to get data to save
t = dict(subdomain.meta)
self.Domain.commit()
subdomain.meta = t
self.Domain.commit()
class Module(ModuleTemplate):
name = "GobusterDir"
def __init__(self, db):
self.db = db
self.IPAddress = IPRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.Port = PortRepository(db, self.name)
self.Url = UrlRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-P',
'--auth_password',
help="Password for basic auth")
self.options.add_argument('-U',
'--auth_username',
help="Username for basic auth")
self.options.add_argument('-a',
'--user_agent',
help="User agent string")
self.options.add_argument('-c',
'--cookies',
help="Cookies to use for the requests")
self.options.add_argument('-f',
'--forward_slash',
help="Add a forward slash to URL",
action="store_true")
self.options.add_argument('-t', '--threads', help="Number of threads")
self.options.add_argument('-p',
'--proxy',
help="Proxy to use for requests")
self.options.add_argument('-r',
'--redirects',
help="Follow redirects",
action="store_true")
self.options.add_argument(
'-s',
'--scan_codes',
help="Positive scan codes (default '200,204,301,302,307')")
self.options.add_argument('-x',
'--extension',
help="File extension to search for")
self.options.add_argument('-u', '--url', help="URL to brute force")
self.options.add_argument('-w', '--wordlist', help="Wordlist to use")
self.options.add_argument('--file', help="Import URLs from file")
self.options.add_argument('-i',
'--import_database',
help="Import URLs from database",
action="store_true")
self.options.add_argument(
'-o',
'--output_path',
help=
"Path which will contain program output (relative to base_path in config",
default="gobuster")
self.options.add_argument(
'--force',
help="Rescan domains that have already been brute forced",
action="store_true")
self.options.add_argument('--output_file',
help="Name of file to output results to",
default="output.txt")
self.options.add_argument(
'--process_threads',
help="Number of threads master tool will use",
default="10")
def run(self, args):
if not args.wordlist:
print("Wordlist is required!")
return
if not args.binary:
self.binary = which.run('gobuster')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"Gobuster binary not found. Please explicitly provide path with --binary"
)
# pdb.set_trace()
if args.url:
self.brute_force_domain(args.url, args)
self.Domain.commit()
elif args.file:
urls = open(args.file).read().split('\n')
for u in urls:
if u:
self.brute_force_domain(u, args)
self.Domain.commit()
elif args.import_database:
urls = get_urls.run(self.db)
self.brute_force_domains(urls, args)
def brute_force_domain(self, url, args):
res = urlparse(url)
host = res.netloc.split(':')[0]
if ':' in res.netloc:
port = int(res.netloc.split(':')[1])
elif res.scheme == 'https':
port = 443
else:
port = 80
# if re.match( r'(\d*)\.(\d*)\.(\d*)\.(\d)', host, re.M|re.I):
# created, ip = self.IPAddress.find_or_create(ip_address=host)
# else:
# created, domain = self.Domain.find_or_create(domain=host)
# ip = domain.ip_addresses[0]
# created, port = self.Port.find_or_create(port_number=port, ip_address_id = ip.id)
command_args = " -m dir"
if args.auth_username and args.auth_password:
command_args += " -U %s -P %s " % (args.auth_username,
args.auth_password)
if args.threads:
command_args += " -t " + args.threads
command_args += " -k -w " + args.wordlist
if args.user_agent:
command_args += " -a %s " % args.user_agent
if args.cookies:
command_args += " -c %s " % args.cookies
if args.proxy:
command_args += " -p %s " % args.proxy
if args.redirects:
command_args += " -r "
if args.scan_codes:
command_args += " -s %s " % args.scan_codes
if args.extension:
command_args += " -x %s " % args.extension
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path,
"%s-dir.txt" % url.split('/')[-1])
command_args += " -o %s " % output_path
cmd = shlex.split(self.binary + command_args + " -l -u " + url)
print("Executing: %s" % ' '.join(cmd))
Popen(cmd, shell=False).wait()
def brute_force_domains(self, urls, args):
commands = []
for url in urls:
res = urlparse(url)
host = res.netloc.split(':')[0]
if ':' in res.netloc:
port = int(res.netloc.split(':')[1])
elif res.scheme == 'https':
port = 443
else:
port = 80
# if re.match( r'(\d*)\.(\d*)\.(\d*)\.(\d)', host, re.M|re.I):
# created, ip = self.IPAddress.find_or_create(ip_address=host)
# else:
# created, domain = self.Domain.find_or_create(domain=host)
# ip = domain.ip_addresses[0]
# created, port = self.Port.find_or_create(port_number=port, ip_address_id = ip.id)
command_args = " -m dir"
if args.auth_username and args.auth_password:
command_args += " -U %s -P %s " % (args.auth_username,
args.auth_password)
if args.threads:
command_args += " -t " + args.threads
command_args += " -k -w " + args.wordlist
if args.user_agent:
command_args += " -a %s " % args.user_agent
if args.cookies:
command_args += " -c %s " % args.cookies
if args.proxy:
command_args += " -p %s " % args.proxy
if args.redirects:
command_args += " -r "
if args.scan_codes:
command_args += " -s %s " % args.scan_codes
if args.extension:
command_args += " -x %s " % args.extension
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'],
args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path,
"%s-dir.txt" % url.split('/')[-1])
command_args += " -o %s " % output_path
commands.append(
shlex.split(self.binary + command_args + " -l -u " + url))
pool = ThreadPool(int(args.process_threads))
pool.map(run_cmd, commands)
class Module(ModuleTemplate):
'''
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
'''
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db,self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d', '--import_domains', help="Either domain to import or file containing domains to import. One per line")
self.options.add_argument('-i', '--import_ips', help="Either IP/range to import or file containing IPs and ranges, one per line.")
self.options.add_argument('-a', '--active', help='Set scoping on imported data as active', action="store_true")
self.options.add_argument('-p', '--passive', help='Set scoping on imported data as passive', action="store_true")
self.options.add_argument('-sc', '--scope_cidrs', help='Cycle through out of scope networks and decide if you want to add them in scope', action="store_true")
self.options.add_argument('-sb', '--scope_base_domains', help='Cycle through out of scope base domains and decide if you want to add them in scope', action="store_true")
self.options.add_argument('-Ii', '--import_database_ips', help='Import IPs from database', action="store_true")
self.options.add_argument('--force', help="Force processing again, even if already processed", action="store_true")
def run(self, args):
self.in_scope = args.active
self.passive_scope = args.passive
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if '/' in line or '-' in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if '/' in args.import_ips or '-' in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
self.Domain.commit()
except IOError:
self.process_domain(args.import_domains.strip())
self.Domain.commit()
if args.scope_base_domains:
base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
self.BaseDomain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, 'A')
for a in answers:
ips.append(a.address)
return ips
except:
return []
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(only_tool=True, domain=domain_str, in_scope=self.in_scope, passive_scope=self.passive_scope)
if not created:
if domain.in_scope != self.in_scope or domain.passive_scope != self.passive_scope:
display("Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (domain_str, self.in_scope, self.passive_scope))
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(only_tool=True, ip_address=ip_str, in_scope=in_scope, passive_scope=self.passive_scope)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display("IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (ip_str, self.in_scope, self.passive_scope))
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def process_cidr(self, line):
display("Processing %s" % line)
if '/' in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
elif '-' in line:
start_ip, end_ip = line.strip().replace(' ', '').split('-')
if '.' not in end_ip:
end_ip = '.'.join(start_ip.split('.')[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" % line.strip())
cidr.in_scope = True
cidr.update()
def scope_ips(self):
IPAddresses = self.IPAddress.all()
def reclassify_domain(self, bd):
if bd.meta.get('whois', False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta['whois'])
res = raw_input("Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] ")
if res.lower() == 'a':
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == 'p':
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error("Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(bd.domain))