def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
class Report(ReportTemplate):
'''
This is a domain summary report. It shows base domain, then
all of the subdomains, along with any resolved IPs.
'''
name = "DomainSummaryReport"
markdown = ['###', '-']
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db, self.name)
def run(self, args):
# Cidrs = self.CIDR.
results = []
basedomains = self.BaseDomain.all()
for b in basedomains:
domain_data = []
for d in b.subdomains:
domain_data.append("%s (%s)" % (d.domain, ', '.join(
[i.ip_address for i in d.ip_addresses])))
results.append(b.domain)
for d in sorted(domain_data):
results.append('\t' + d)
self.process_output(results, args)
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name)
class Report(ReportTemplate):
'''
Displays WHOIS records for domains and IPs.
'''
name = "WhoisReport"
markdown = ['##', '###', '`']
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db)
self.ScopeCIDR = ScopeCIDRRepository(db)
def run(self, args):
# Cidrs = self.CIDR.
results = []
domains = self.BaseDomain.all()
domain_data = {}
for d in domains:
if d.meta.get('whois', False):
domain_data[d.domain] = d.meta['whois']
cidr_data = {}
CIDRs = self.ScopeCIDR.all()
for c in CIDRs:
if c.meta.get('whois', False):
cidr_data[c.cidr] = c.meta['whois']
# pdb.set_trace()
domain_blacklist = [
'Please note: ', 'URL of the ICANN WHOIS', '>>>', 'Notes:',
'whitelisting here:', 'NOTICE: ', 'TERMS OF USE: ',
'by the following', 'to: (1) allow'
]
results.append("ARIN Registration")
for c in sorted(cidr_data.keys()):
results.append("\t" + c)
for l in cidr_data[c].split('\n'):
if ": " in l and l[0] != '#':
results.append('\t\t' + l)
results.append("Domain Registration")
for d in sorted(domain_data.keys()):
results.append("\t" + d.upper())
for l in domain_data[d].split('\n'):
if ": " in l:
clean = True
for b in domain_blacklist:
if b in l:
clean = False
if clean:
results.append('\t\t' + l)
self.process_output(results, args)
class Report(ReportTemplate):
'''
This report displays data related to discovered user accounts.
'''
name = "UserReport"
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db)
self.User = UserRepository(db)
self.Cred = CredRepository(db)
def set_options(self):
super(Report, self).set_options()
self.options.add_argument('-u1', '--usernames_passwords', help="Prints out username/password pairs", action="store_true")
self.options.add_argument('-u2', '--emails_passwords', help="Prints out email/password pairs", action="store_true")
self.options.add_argument('-u3', '--emails', help="Prints out e-mail addresses", action="store_true")
self.options.add_argument('-u4', '--accounts', help="Prints out user accounts", action="store_true")
def run(self, args):
results = []
domains = self.BaseDomain.all()
for d in domains:
for user in d.users:
if args.emails:
results.append(user.email)
elif args.accounts:
results.append(user.email.split('@')[0])
else:
for cred in user.creds:
if cred.password and cred.password != "None":
if args.emails_passwords:
txt = "%s:%s" % (user.email, cred.password)
elif args.usernames_passwords:
txt = "%s:%s" % (user.email.split('@')[0], cred.password)
if txt not in results:
results.append(txt)
self.process_output(results, args)
class Report(ReportTemplate):
"""
This report displays all of the various Base Domains, Domains, and IP Addresses with scoping information
"""
markdown = ["###", "`"]
name = "CertReport"
def __init__(self, db):
self.IPAddress = IPRepository(db)
self.Domains = DomainRepository(db)
self.BaseDomains = BaseDomainRepository(db)
self.CIDRs = CIDRRepository(db)
def set_options(self):
super(Report, self).set_options()
def run(self, args):
results = []
base_domains = self.BaseDomains.all()
for b in base_domains:
results.append("%s\tActive Scope: %s\tPassive Scope: %s" %
(b.domain, b.in_scope, b.passive_scope))
for d in b.subdomains:
results.append("\t%s\tActive Scope: %s\tPassive Scope: %s" %
(d.domain, d.in_scope, d.passive_scope))
cidrs = self.CIDRs.all()
results.append("\n\n")
for c in cidrs:
results.append("%s - %s" % (c.cidr, c.org_name))
for ip in c.ip_addresses:
results.append("\t%s\tActive Scope: %s\tPassive Scope: %s" %
(ip.ip_address, ip.in_scope, ip.passive_scope))
self.process_output(results, args)
class Module(ToolTemplate):
name = "Whois"
binary_name = "whois"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d", "--domain", help="Domain to query")
self.options.add_argument("-c", "--cidr", help="CIDR to query")
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
self.options.add_argument(
"-a",
"--all_data",
help="Scan all data in database, regardless of scope",
action="store_true",
)
self.options.add_argument(
"-i",
"--import_database",
help="Run WHOIS on all domains and CIDRs in database",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append({"domain": domain.domain, "cidr": ""})
elif args.cidr:
created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr)
targets.append({"domain": "", "cidr": cidr.cidr.split("/")[0]})
elif args.import_database:
if args.all_data:
scope_type = ""
else:
scope_type = "passive"
if args.rescan:
domains = self.BaseDomain.all(scope_type=scope_type)
cidrs = self.ScopeCidr.all()
else:
domains = self.BaseDomain.all(scope_type=scope_type,
tool=self.name)
cidrs = self.ScopeCidr.all(tool=self.name)
for domain in domains:
targets.append({"domain": domain.domain, "cidr": ""})
for cidr in cidrs:
targets.append({"domain": "", "cidr": cidr.cidr.split("/")[0]})
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t["output"] = os.path.join(output_path, t["domain"] + t["cidr"])
return targets
def build_cmd(self, args):
if not args.tool_args:
args.tool_args = ""
cmd = ('bash -c "' + self.binary + " {domain}{cidr} " +
args.tool_args + '> {output}" ')
return cmd
def process_output(self, cmds):
display("Importing data to database")
for cmd in cmds:
if cmd["cidr"]:
_, cidr = self.ScopeCidr.find_or_create(cidr=cmd["cidr"])
cidr.meta["whois"] = open(cmd["output"]).read()
display(cidr.meta["whois"])
cidr.update()
elif cmd["domain"]:
_, domain = self.BaseDomain.find_or_create(
domain=cmd["domain"])
domain.meta["whois"] = open(cmd["output"]).read()
display(domain.meta["whois"])
domain.update()
self.BaseDomain.commit()
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db)
self.ScopeCIDR = ScopeCIDRRepository(db)
class Module(ModuleTemplate):
"""
Unfortunately, this is a Windows only deal. So we output a list of commands
to be run, and have a way of ingesting the output. Since metadata is more
junk then good data, items have to be manually verified
"""
name = "PowerMeta"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-c",
"--command",
help="Print out a list of commands to run",
action="store_true",
)
self.options.add_argument("-ps",
"--powershell",
help="Filename for a .ps1 command output")
self.options.add_argument(
"-i",
"--import_file",
help="import CSV file generated by PowerMeta")
self.options.add_argument(
"-f",
"--import_folder",
help="import CSV files generated by PowerMeta")
def run(self, args):
# pdb.set_trace()
if args.command:
res = []
res.append("Import-Module PowerMeta.ps1")
res += [
"Invoke-PowerMeta -TargetDomain %s -Download -Extract -ExtractAllToCsv %s.csv"
% (b.domain, b.domain) for b in self.BaseDomain.all()
]
if args.powershell:
open(args.powershell, "w").write("\r\n".join(res))
print("Command saved as %s" % args.powershell)
else:
print("\n".join(res))
if args.import_file:
self.process_domain(args.import_file, args)
if args.import_folder:
if args.import_folder[-1] == "/":
args.import_folder = args.import_folder[:-1]
files = glob.glob(args.import_folder + "/*.csv")
for f in files:
print("Processing %s" % f)
self.process_domain(f, args)
def process_domain(self, csvfile, args):
domain = csvfile.split("/")[-1].split(".csv")[0]
domain_obj = self.BaseDomain.find(domain=domain)
if not domain_obj:
print("Error: Domain not found: %s" % domain)
return
csvdata = open(csvfile).read().decode("utf-16").encode("utf-8")
csvreader = csv.reader(BytesIO(csvdata), delimiter=",", quotechar='"')
headers = csvreader.next()
searchable_headers = ["Author", "Creator"]
indexes = [
headers.index(s) for s in searchable_headers if s in headers
]
data = []
for row in csvreader:
for i in indexes:
data.append(row[i])
data = list(set(data)) # Dedup
for d in data:
if d.strip():
res = raw_input("Is %s a valid name? [y/N] " % d)
if res and res[0].lower() == "y":
if " " in d:
if ", " in d:
first_name = d.split(", ")[1]
last_name = d.split(", ")[0]
else:
first_name = d.split(" ")[0]
last_name = " ".join(d.split(" ")[1:])
created, user = self.User.find_or_create(
first_name=first_name, last_name=last_name)
if created:
print("New user created")
user.domain = domain_obj
# else:
# created, user = self.User.find_or_create(username=d)
# if created:
# print("New username created")
# user.domain = domain_obj
user.update()
self.User.commit()
class Report(ReportTemplate):
"""
Displays WHOIS records for domains and IPs.
"""
name = "WhoisReport"
markdown = ["##", "###", "`"]
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db)
self.ScopeCIDR = ScopeCIDRRepository(db)
def run(self, args):
# Cidrs = self.CIDR.
results = []
if args.scope != "all":
domains = self.BaseDomain.all(scope_type=args.scope)
else:
domains = self.BaseDomain.all()
domain_data = {}
for d in domains:
if d.meta.get("whois", False):
domain_data[d.domain] = d.meta["whois"]
cidr_data = {}
CIDRs = self.ScopeCIDR.all()
for c in CIDRs:
if c.meta.get("whois", False):
cidr_data[c.cidr] = c.meta["whois"]
# pdb.set_trace()
domain_blacklist = [
"Please note: ",
"URL of the ICANN WHOIS",
">>>",
"Notes:",
"whitelisting here:",
"NOTICE: ",
"TERMS OF USE: ",
"by the following",
"to: (1) allow",
]
results.append("ARIN Registration")
for c in sorted(cidr_data.keys()):
results.append("\t" + c)
for l in cidr_data[c].split("\n"):
if ": " in l and l[0] != "#":
results.append("\t\t" + l)
results.append("Domain Registration")
for d in sorted(domain_data.keys()):
results.append("\t" + d.upper())
for l in domain_data[d].split("\n"):
if ": " in l:
clean = True
for b in domain_blacklist:
if b in l:
clean = False
if clean:
results.append("\t\t" + l)
self.process_output(results, args)
class Module(ModuleTemplate):
name = "GobusterDNS"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-fw',
'--force_wildcard',
help="Continues after hitting wildcard cert",
action="store_true")
self.options.add_argument('-t', '--threads', help="Number of threads")
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-w', '--wordlist', help="Wordlist to use")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-o',
'--output_path',
help=
"Path which will contain program output (relative to base_path in config",
default="gobuster")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been brute forced",
action="store_true")
def run(self, args):
if not args.wordlist:
print("Wordlist is required!")
return
if not args.binary:
self.binary = which.run('gobuster')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"Gobuster binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
self.brute_force_domain(domain, args)
self.BaseDomain.commit()
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
self.brute_force_domain(domain, args)
self.BaseDomain.commit()
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all()
else:
domains = self.BaseDomain.all(tool=self.name)
for domain in domains:
self.brute_force_domain(domain, args)
domain.set_tool(self.name)
self.BaseDomain.commit()
def brute_force_domain(self, domain_obj, args):
domain = domain_obj.domain
command_args = " -m dns"
if args.force_wildcard:
command_args += " -fw"
if args.threads:
command_args += " -t " + args.threads
command_args += " -w " + args.wordlist
cmd = shlex.split(self.binary + command_args + " -u " + domain)
print("Executing: %s" % ' '.join(cmd))
res = check_output(cmd)
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, "%s-dns.txt" % domain)
open(output_path, 'w').write(res)
data = res.split('\n')
for d in data:
if 'Found: ' in d:
new_domain = d.split(' ')[1].lower()
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
if created:
print("New subdomain found: %s" % new_domain)
class Module(ToolTemplate):
name = "Whois"
binary_name = "whois"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d', '--domain', help="Domain to query")
self.options.add_argument('-c', '--cidr', help="CIDR to query")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
self.options.add_argument(
'--import_database',
help="Run WHOIS on all domains and CIDRs in database",
action="store_true")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append({'domain': domain.domain, 'cidr': ''})
elif args.cidr:
created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr)
targets.append({'domain': '', 'cidr': cidr.cidr.split('/')[0]})
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
cidrs = self.ScopeCidr.all(scope_type="passive")
else:
domains = self.BaseDomain.all(scope_type="passive",
tool=self.name)
cidrs = self.ScopeCidr.all(scope_type="passive",
tool=self.name)
for domain in domains:
targets.append({'domain': domain.domain, 'cidr': ''})
for cidr in cidrs:
targets.append({'domain': '', 'cidr': cidr.cidr.split('/')[0]})
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t['output'] = os.path.join(output_path, t['domain'] + t['cidr'])
return targets
def build_cmd(self, args):
if not args.tool_args:
args.tool_args = ""
cmd = "bash -c \"" + self.binary + " {domain}{cidr} " + args.tool_args + "> {output}\" "
return cmd
def process_output(self, cmds):
display("Importing data to database")
for cmd in cmds:
if cmd['cidr']:
_, cidr = self.ScopeCidr.find_or_create(cidr=cmd['cidr'])
cidr.meta['whois'] = open(cmd['output']).read()
display(cidr.meta['whois'])
cidr.update()
elif cmd['domain']:
_, domain = self.BaseDomain.find_or_create(
domain=cmd['domain'])
domain.meta['whois'] = open(cmd['output']).read()
display(domain.meta['whois'])
domain.update()
self.BaseDomain.commit()
class Module(ToolTemplate):
name = "DNSRecon"
binary_name = "dnsrecon"
"""
This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries.
It can also run over IP ranges, looking for additional domains in the PTR records.
"""
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d','--domain', help="Target domain for dnsRecon")
self.options.add_argument('-f', '--file', help="Import domains from file")
self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true")
self.options.add_argument('-r', '--range', help="Range to scan for PTR records")
self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true")
self.options.add_argument('--rescan', help="Rescan domains already scanned", action="store_true")
# self.options.add_argument('--import_output_xml', help="Import XML file")
# self.options.add_argument('--import_output_json', help="Import json file")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain, passive_scope=True)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d, passive_scope=True)
targets.append(domain.domain)
elif args.import_database:
domains = self.BaseDomain.all(scope_type="passive", tool=self.name)
for domain in domains:
targets.append(domain.domain)
elif args.range:
targets.append(args.range)
elif args.import_range:
cidrs = self.ScopeCIDR.all(tool=self.name)
for cidr in cidrs:
targets.append(cidr.cidr)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] )
else:
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({'target':t, 'output':os.path.join(self.path, t.replace("/", "_") + ".json")})
return res
def build_cmd(self, args):
command = self.binary
if args.domain or args.file or args.import_database:
command += " -d {target} -j {output} "
else:
command += " -s -r {target} -j {output} "
if args.tool_args:
command += args.tool_args
return command
def process_output(self, cmds):
for c in cmds:
target = c['target']
output_path = c['output']
try:
res = json.loads(open(output_path).read())
except IOError:
display_error("DnsRecon failed for {}".format(target))
if " -d " in res[0]['arguments']:
created, dbrec = self.Domain.find_or_create(domain=target)
dbrec.dns = res
dbrec.save()
for record in res:
domain = None
if record.get("type") == "A" or record.get("type") == "PTR":
domain = record.get("name").lower().replace("www.","")
elif record.get("type") == "MX":
domain = record.get("exchange").lower().replace("www.","")
elif record.get("type") == "SRV" or record.get("type" == "NS"):
domain = record.get("target").lower().replace("www.","")
elif record.get("type") == "SOA":
domain = record.get("mname").lower().replace("www.","")
if domain:
created, domain_obj = self.Domain.find_or_create(domain=domain)
self.Domain.commit()
class Module(ModuleTemplate):
name = "LinkedInt"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Domain to add onto email")
self.options.add_argument('-c',
'--company_id',
help="Company ID to search")
self.options.add_argument('-C',
'--restrict',
help="Restrict to company filter",
action="store_true")
self.options.add_argument('-k',
'--keywords',
help="Keywords to search for")
self.options.add_argument(
'-o',
'--output_path',
help=
"Path which will contain program output (relative to base_path in config",
default=self.name)
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run('LinkedInt.py')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"LinkedInt binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
self.process_domain(domain, args)
self.BaseDomain.commit()
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path,
"%s-linkedint" % domain.replace('.', '_'))
command_args = " -o %s" % output_path
command_args += " -e %s" % domain
command_args += " -u %s" % args.keywords
if args.company_id:
command_args += " -i %s " % args.company_id
if args.restrict:
command_args += " -c "
# if args.threads:
# command_args += " -t " + args.threads
current_dir = os.getcwd()
new_dir = '/'.join(self.binary.split('/')[:-1])
os.chdir(new_dir)
cmd = shlex.split("python " + self.binary + command_args)
print("Executing: %s" % ' '.join(cmd))
res = subprocess.Popen(cmd).wait()
os.chdir(current_dir)
count = 0
with open(output_path + '.csv') as csvfile:
csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
for row in csvreader:
count += 1
if self.User.find(email=row[3]):
created, user = self.User.find_or_create(
email=remove_binary(row[3]))
user.first_name = remove_binary(row[0])
user.last_name = remove_binary(row[1])
user.job_title = remove_binary(row[4])
user.location = remove_binary(row[5])
else:
created, user = self.User.find_or_create(
first_name=remove_binary(row[0]),
last_name=remove_binary(row[1]),
domain=domain_obj)
if created:
print("New user: %s %s" %
(remove_binary(row[0]), remove_binary(row[1])))
user.email = remove_binary(row[3])
user.job_title = remove_binary(row[4])
user.location = remove_binary(row[5])
user.update()
print("%s found and imported" % count)
self.User.commit()
class Module(ToolTemplate):
name = "TheHarvester"
binary_name = "theharvester"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d', '--domain', help="Domain to harvest")
self.options.add_argument('-f', '--file', help="Import domains from file")
self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true")
self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain)
targets.append({'target':domain.domain})
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
targets.append({'target':domain.domain})
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name, scope_type="passive")
for d in domains:
targets.append({'target':d.domain})
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t['output'] = os.path.join(output_path, "%s-theharvester" % t['target'].replace('.', '_') )
return targets
def build_cmd(self, args):
cmd = self.binary + " -f {output} -b all -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
try:
data = xmltodict.parse(open(cmd['output'] + '.xml').read())
except:
data = None
if data:
if data['theHarvester'].get('email', False):
if type(data['theHarvester']['email']) == list:
emails = data['theHarvester']['email']
else:
emails = [data['theHarvester']['email']]
for e in emails:
created, user = self.User.find_or_create(email=e)
_, domain = self.BaseDomain.find_or_create(e.split('@')[1])
user.domain = domain
user.update()
if created:
display_new("New email: %s" % e)
if data['theHarvester'].get('host', False):
if type(data['theHarvester']['host']) == list:
hosts = data['theHarvester']['host']
else:
hosts = [data['theHarvester']['host']]
for d in hosts:
created, domain = self.Domain.find_or_create(domain=d['hostname'])
if data['theHarvester'].get('vhost', False):
if type(data['theHarvester']['vhost']) == list:
hosts = data['theHarvester']['vhost']
else:
hosts = [data['theHarvester']['vhost']]
for d in hosts:
created, domain = self.Domain.find_or_create(domain=d['hostname'])
self.BaseDomain.commit()
class Module(ToolTemplate):
name = "GobusterDNS"
binary_name = "gobuster"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been brute forced",
action="store_true")
self.options.set_defaults(
timeout=600) # Kick the default timeout to 10 minutes
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
targets.append(d)
if args.import_database:
if args.rescan:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive")
]
else:
targets += [
b.domain for b in self.BaseDomain.all(scope_type="passive",
tool=self.name)
]
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
'target':
t,
'output':
os.path.join(output_path,
t.replace('/', '_') + "-dns.txt")
})
return res
def build_cmd(self, args):
cmd = self.binary + " -m dns "
cmd += " -o {output} -u {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for c in cmds:
target = c['target']
output_path = c['output']
if os.path.isfile(output_path):
data = open(output_path).read().split('\n')
else:
display_error("{} not found.".format(output_path))
return
for d in data:
if 'Found: ' in d:
new_domain = d.split(' ')[1].lower()
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
class Module(ToolTemplate):
name = "Sublist3r"
binary_name = 'sublist3r'
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output_path = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({'target': t, 'output': output_path})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd['output']
data = open(output_path).read().split('\n')
for d in data:
new_domain = d.split(':')[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
class Module(ModuleTemplate):
name = "Fierce"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Target domain for Fierce")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-o',
'--output_path',
help="Relative path (to the base directory) to store Fierce output",
default="fierceFiles")
self.options.add_argument('-x',
'--fiercefile',
help="Fierce output name")
self.options.add_argument('--threads',
help="Number of threads to use",
default="10")
self.options.add_argument('--force',
help="Force overwrite",
action="store_true")
self.options.add_argument('--import_file', help="Import Fierce file")
def run(self, args):
if not args.binary:
self.binary = which.run('fierce')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"Fierce binary not found. Please explicitly provide path with --binary"
)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config['PROJECT']['base_path'],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config['PROJECT']['base_path'],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.import_database:
domains = self.BaseDomain.all(tool=self.name)
for domain in domains:
self.process_domain(domain, args)
domain.set_tool(self.name)
self.BaseDomain.commit()
elif args.import_file:
fiercefile = args.import_file
self.process_data(fiercefile, args)
self.BaseDomain.commit()
else:
print("You need to supply some options to do something.")
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
print("Processing %s" % domain)
if args.fiercefile:
fiercefile = os.path.join(self.path, args.fiercefile)
else:
fiercefile = os.path.join(self.path, domain + ".txt")
if os.path.isfile(fiercefile):
if not args.force:
answered = False
while answered == False:
rerun = raw_input(
"Would you like to [r]un Fierce again and overwrite the file, [p]arse the file, or change the file [n]ame? "
)
if rerun.lower() == 'r':
answered = True
elif rerun.lower() == 'p':
answered = True
return fiercefile
elif rerun.lower() == 'n':
new = False
while new == False:
newFile = raw_input("enter a new file name: ")
if not os.path.exists(
os.path.join(self.path, newFile)):
fiercefile = os.path.join(path, newFile)
answered = True
new = True
else:
print "That file exists as well"
command = self.binary
if args.threads:
command += " -threads %s " % args.threads
command += " -dns {} -file {} ".format(domain, fiercefile)
subprocess.Popen(command, shell=True).wait()
try:
self.process_data(fiercefile, args)
except IOError:
print("Fierce for %s failed." % domain)
return None
def process_data(self, fiercefile, args):
self.fiercefile = fiercefile
print "Reading", self.fiercefile
fierceOutput = ''
with open(self.fiercefile, 'r') as fiercefile:
for line in fiercefile:
fierceOutput += line
domains = []
if "Now performing" in fierceOutput:
hosts = re.findall("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$",
fierceOutput, re.MULTILINE)
if hosts:
for host in hosts:
#print host
domain = host.split("\t")[1].lower().replace(
"www.", "").rstrip(".")
if domain not in domains:
domains.append(domain)
elif "Whoah, it worked" in fierceOutput:
print "Zone transfer found!"
hosts = re.findall(".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",
fierceOutput, re.MULTILINE)
if hosts:
for host in hosts:
domain = host.split("\t")[0].lower().replace(
"www.", "").rstrip(".")
if domain not in domains:
domains.append(domain)
else:
print "Unable to process {}".format(fiercefile)
if domains:
for _domain in domains:
created, domain = self.Domain.find_or_create(domain=_domain)
return
class Module(ModuleTemplate):
"""
Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
CIDR info.
"""
name = "Ingestor"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.IPAddress = IPRepository(db, self.name)
self.CIDR = CIDRRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-d",
"--import_domains",
help=
"Either domain to import or file containing domains to import. One per line",
)
self.options.add_argument(
"-i",
"--import_ips",
help=
"Either IP/range to import or file containing IPs and ranges, one per line.",
)
self.options.add_argument(
"-a",
"--active",
help="Set scoping on imported data as active",
action="store_true",
)
self.options.add_argument(
"-p",
"--passive",
help="Set scoping on imported data as passive",
action="store_true",
)
self.options.add_argument(
"-sc",
"--scope_cidrs",
help=
"Cycle through out of scope networks and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument(
"-sb",
"--scope_base_domains",
help=
"Cycle through out of scope base domains and decide if you want to add them in scope",
action="store_true",
)
self.options.add_argument("--descope",
help="Descope an IP, domain, or CIDR")
self.options.add_argument(
"-Ii",
"--import_database_ips",
help="Import IPs from database",
action="store_true",
)
self.options.add_argument(
"--force",
help="Force processing again, even if already processed",
action="store_true",
)
def run(self, args):
self.in_scope = args.active
self.passive_scope = args.passive
if args.descope:
if "/" in args.descope:
self.descope_cidr(args.descope)
elif check_string(args.descope):
pass
else:
self.descope_ip(args.descope)
# Check if in ScopeCIDR and remove if found
if args.import_ips:
try:
ips = open(args.import_ips)
for line in ips:
if line.strip():
if "/" in line or "-" in line:
self.process_cidr(line)
else:
self.process_ip(line.strip(), force_scope=True)
self.Domain.commit()
except IOError:
if "/" in args.import_ips or "-" in args.import_ips:
self.process_cidr(args.import_ips)
else:
self.process_ip(args.import_ips.strip(), force_scope=True)
self.Domain.commit()
if args.import_domains:
try:
domains = open(args.import_domains)
for line in domains:
if line.strip():
self.process_domain(line.strip())
self.Domain.commit()
except IOError:
self.process_domain(args.import_domains.strip())
self.Domain.commit()
if args.scope_base_domains:
base_domains = self.BaseDomain.all(in_scope=False,
passive_scope=False)
for bd in base_domains:
self.reclassify_domain(bd)
self.BaseDomain.commit()
def get_domain_ips(self, domain):
ips = []
try:
answers = dns.resolver.query(domain, "A")
for a in answers:
ips.append(a.address)
return ips
except:
return []
def process_domain(self, domain_str):
created, domain = self.Domain.find_or_create(
only_tool=True,
domain=domain_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if (domain.in_scope != self.in_scope
or domain.passive_scope != self.passive_scope):
display(
"Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (domain_str, self.in_scope, self.passive_scope))
domain.in_scope = self.in_scope
domain.passive_scope = self.passive_scope
domain.update()
def process_ip(self, ip_str, force_scope=True):
created, ip = self.IPAddress.find_or_create(
only_tool=True,
ip_address=ip_str,
in_scope=self.in_scope,
passive_scope=self.passive_scope,
)
if not created:
if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
display(
"IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
% (ip_str, self.in_scope, self.passive_scope))
ip.in_scope = self.in_scope
ip.passive_scope = self.passive_scope
ip.update()
return ip
def process_cidr(self, line):
display("Processing %s" % line)
if "/" in line:
created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
elif "-" in line:
start_ip, end_ip = line.strip().replace(" ", "").split("-")
if "." not in end_ip:
end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])
cidrs = iprange_to_cidrs(start_ip, end_ip)
for c in cidrs:
created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
if created:
display_new("Adding %s to scoped CIDRs in database" %
line.strip())
cidr.in_scope = True
cidr.update()
def scope_ips(self):
IPAddresses = self.IPAddress.all()
def reclassify_domain(self, bd):
if bd.meta.get("whois", False):
display_new("Whois data found for {}".format(bd.domain))
print(bd.meta["whois"])
res = raw_input(
"Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] "
)
if res.lower() == "a":
bd.in_scope = True
bd.passive_scope = True
elif res.lower() == "p":
bd.in_scope = False
bd.passive_scope = True
else:
bd.in_scope = False
bd.passive_scope = False
bd.save()
else:
display_error(
"Unfortunately, there is no whois information for {}. Please populate it using the Whois module"
.format(bd.domain))
def descope_ip(self, ip):
ip = self.IPAddress.all(ip_address=ip)
if ip:
for i in ip:
display("Removing IP {} from scope".format(i.ip_address))
i.in_scope = False
i.passive_scope = False
i.update()
for d in i.domains:
in_scope_ips = [
ipa for ipa in d.ip_addresses
if ipa.in_scope or ipa.passive_scope
]
if not in_scope_ips:
display(
"Domain {} has no more scoped IPs. Removing from scope."
.format(d.domain))
d.in_scope = False
d.passive_scope = False
self.IPAddress.commit()
def descope_cidr(self, cidr):
CIDR = self.ScopeCIDR.all(cidr=cidr)
if CIDR:
for c in CIDR:
display("Removing {} from ScopeCIDRs".format(c.cidr))
c.delete()
cnet = IPNetwork(cidr)
for ip in self.IPAddress.all():
if IPAddress(ip.ip_address) in cnet:
self.descope_ip(ip.ip_address)
def __init__(self, db):
self.IPAddress = IPRepository(db)
self.Domains = DomainRepository(db)
self.BaseDomains = BaseDomainRepository(db)
self.CIDRs = CIDRRepository(db)
class Module(ModuleTemplate):
name = "Whois"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d', '--domain', help="Domain to query")
self.options.add_argument('-c', '--cidr', help="CIDR to query")
self.options.add_argument('-t',
'--threads',
help='Number of threads to run',
default="1")
self.options.add_argument(
'-o',
'--output_path',
help=
"Path which will contain program output (relative to base_path in config",
default="whois")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
self.options.add_argument(
'--import_database',
help="Run WHOIS on all domains and CIDRs in database",
action="store_true")
def run(self, args):
self.args = args
if not args.binary:
self.binary = which.run('whois')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"whois binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
self.process_domains(domains=[domain], ips=[])
elif args.cidr:
created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr)
self.process_domains(domains=[], ips=[domain])
elif args.import_database:
domains = self.BaseDomain.all()
cidrs = self.ScopeCidr.all()
self.process_domains(domains=domains, cidrs=cidrs)
self.ScopeCidr.commit()
def process_domains(self, domains, cidrs):
args = self.args
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
commands = []
for domain in domains:
name = domain.domain
file_path = os.path.join(output_path, "%s.txt" % name)
command_args = " "
cmd = shlex.split(self.binary + command_args + name)
commands.append([cmd, file_path, name])
for cidr in cidrs:
name = cidr.cidr.split('/')[0]
file_path = os.path.join(output_path, "%s.txt" % name)
command_args = " "
cmd = shlex.split(self.binary + command_args + name)
commands.append([cmd, file_path, cidr.cidr])
pool = ThreadPool(int(args.threads))
res = pool.map(run_cmd, commands)
print("Importing data to database")
res_d = {}
for r in res:
if r:
res_d[r[0]] = r[1]
domains = self.BaseDomain.all()
for d in domains:
if res_d.get(d.domain, False):
d.meta['whois'] = res_d[d.domain]
d.update()
pdb.set_trace()
for c in self.ScopeCidr.all():
if res_d.get(c.cidr, False):
c.meta['whois'] = res_d[c.cidr]
c.update()
class Module(ToolTemplate):
name = "Sublist3r"
binary_name = "sublist3r"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Domain to brute force")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output = os.path.join(output_path, "%s-sublist3r.txt" % t)
res.append({"target": t, "output": output})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd["output"]
if os.path.isfile(output_path):
data = open(output_path).read().split("\n")
else:
display_error("{} not found.".format(output_path))
next
for d in data:
new_domain = d.split(":")[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.ScopeCidr = ScopeCIDRRepository(db, self.name)
class Module(ToolTemplate):
name = "aquatone-discover"
binary_name = "aquatone-discover"
def __init__(self, db):
self.db = db
self.Domain = DomainRepository(db, self.name)
self.BaseDomain = BaseDomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d",
"--domain",
help="Target domain for aquatone")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-r",
"--rescan",
help="Run aquatone on hosts that have already been processed.",
action="store_true",
)
self.options.set_defaults(timeout=None)
def get_targets(self, args):
"""
This module is used to build out a target list and output file list, depending on the arguments. Should return a
list in the format [(target, output), (target, output), etc, etc]
"""
targets = []
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.file:
domainsFile = open(args.file).read().split("\n")
for d in domainsFile:
if d:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
targets.append(domain.domain)
elif args.import_database:
if args.rescan:
all_domains = self.BaseDomain.all(scope_type="passive")
else:
all_domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in all_domains:
targets.append(d.domain)
else:
print("You need to supply domain(s).")
output_path = os.path.join(self.base_config["PROJECT"]["base_path"],
"aquatone")
if not os.path.exists(output_path):
os.makedirs(output_path)
res = []
for t in targets:
res.append({
"target": t,
"output": "{}/{}/hosts.json".format(output_path, t)
})
return res
def build_cmd(self, args):
"""
Create the actual command that will be executed. Use {target} and {output} as placeholders.
"""
cmd = self.binary + " -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def pre_run(self, args):
output_path = self.base_config["PROJECT"]["base_path"]
self.orig_home = os.environ["HOME"]
os.environ["HOME"] = output_path
def process_output(self, cmds):
"""
Process the output generated by the earlier commands.
"""
for cmd in cmds:
data2 = json.loads(open(cmd["output"]).read())
for sub, ip in data2.items():
created = False
new_domain = sub.lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
self.Domain.commit()
def post_run(self, args):
os.environ["HOME"] = self.orig_home
class Module(ModuleTemplate):
name = "LinkedInt"
binary_name = "linkedint.py"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-b",
"--binary",
help="Path to binary for LinkedInt",
default=self.binary_name,
)
self.options.add_argument("-d", "--domain", help="Domain to add onto email")
self.options.add_argument("-c", "--company_id", help="Company ID to search")
self.options.add_argument(
"-C", "--restrict", help="Restrict to company filter", action="store_true"
)
self.options.add_argument(
"-e",
"--email_format",
help="Format for emails: auto,full,firstlast,firstmlast,flast,first.last,fmlast,lastfirst, default is auto",
default="auto",
)
self.options.add_argument("-k", "--keywords", help="Keywords to search for")
self.options.add_argument(
"-o",
"--output_path",
help="Path which will contain program output (relative to base_path in config",
default=self.name,
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
self.options.add_argument(
"--smart_shuffle",
help="Provide a list of keywords. The tool will run once with all of the keywords, then run again excluding all of the keywords. This is useful for bypassing the 1k limit. Keywords must be comma separated.",
)
self.options.add_argument(
"--top", help="Use the top X keywords from the job titles for smart shuffle"
)
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run("LinkedInt.py")
else:
self.binary = which.run(args.binary)
if not self.binary:
display_error(
"LinkedInt binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain)
if args.top:
titles = [
user.job_title.split(" at ")[0]
for user in domain.users
if user.job_title
]
words = []
for t in titles:
words += [w.lower() for w in t.split(" ")]
word_count = Counter(words).most_common()
display("Using the top %s words:" % args.top)
res = []
for w in word_count[: int(args.top)]:
display("\t{}\t{}".format(w[0], w[1]))
res.append(w[0])
# pdb.set_trace()
args.smart_shuffle = ",".join(res)
if args.smart_shuffle:
args.keywords = " OR ".join(
['"{}"'.format(i) for i in args.smart_shuffle.split(",")]
)
self.process_domain(domain, args)
args.keywords = " AND ".join(
['-"{}"'.format(i) for i in args.smart_shuffle.split(",")]
)
self.process_domain(domain, args)
else:
self.process_domain(domain, args)
self.BaseDomain.commit()
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:]
)
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path
)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(
output_path, "%s-linkedint" % domain.replace(".", "_")
)
command_args = " -o %s" % output_path
command_args += " -e %s" % domain
if args.keywords:
command_args += " -u '%s'" % args.keywords
if args.company_id:
command_args += " -i %s " % args.company_id
if args.restrict:
command_args += " -c "
# if args.threads:
# command_args += " -t " + args.threads
if args.email_format:
command_args += " -f " + args.email_format
current_dir = os.getcwd()
new_dir = "/".join(self.binary.split("/")[:-1])
os.chdir(new_dir)
cmd = shlex.split("python " + self.binary + command_args)
print("Executing: %s" % " ".join(cmd))
res = subprocess.Popen(cmd).wait()
os.chdir(current_dir)
count = 0
with open(output_path + ".csv") as csvfile:
csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
for row in csvreader:
count += 1
if self.User.find(email=row[3]):
created, user = self.User.find_or_create(
email=remove_binary(row[3])
)
user.first_name = remove_binary(row[0])
user.last_name = remove_binary(row[1])
user.job_title = remove_binary(row[4])
user.location = remove_binary(row[5])
else:
created, user = self.User.find_or_create(
first_name=remove_binary(row[0]),
last_name=remove_binary(row[1]),
domain=domain_obj,
)
if created:
print(
"New user: %s %s"
% (remove_binary(row[0]), remove_binary(row[1]))
)
user.email = remove_binary(row[3])
user.job_title = remove_binary(row[4])
user.location = remove_binary(row[5])
user.update()
print("%s found and imported" % count)
self.User.commit()
class Module(ModuleTemplate):
name = "TheHarvester"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-t', '--threads', help="Number of threads")
self.options.add_argument('-d', '--domain', help="Domain to brute force")
self.options.add_argument('-f', '--file', help="Import domains from file")
self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true")
self.options.add_argument('-o', '--output_path', help="Path which will contain program output (relative to base_path in config", default=self.name)
self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true")
def run(self, args):
if not args.binary:
self.binary = which.run('theharvester')
else:
self.binary = which.run(args.binary)
if not self.binary:
print("TheHarvester binary not found. Please explicitly provide path with --binary")
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all()
else:
domains = self.BaseDomain.all(tool=self.name)
for d in domains:
self.process_domain(d, args)
d.set_tool(self.name)
self.BaseDomain.commit()
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
if args.output_path[0] == "/":
output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, "%s-theharvester" % domain.replace('.', '_') )
command_args = " -f %s" % output_path
command_args += " -b all "
# if args.threads:
# command_args += " -t " + args.threads
cmd = shlex.split(self.binary + command_args + " -d " + domain)
print("Executing: %s" % ' '.join(cmd))
res = subprocess.Popen(cmd).wait()
try:
data = xmltodict.parse(open(output_path + '.xml').read())
except:
data = None
if data:
if data['theHarvester'].get('email', False):
if type(data['theHarvester']['email']) == list:
emails = data['theHarvester']['email']
else:
emails = [data['theHarvester']['email']]
for e in emails:
created, user = self.User.find_or_create(email=e)
user.domain = domain_obj
user.update()
if created:
print("New email: %s" % e)
if data['theHarvester'].get('host', False):
if type(data['theHarvester']['host']) == list:
hosts = data['theHarvester']['host']
else:
hosts = [data['theHarvester']['host']]
for d in hosts:
created, domain = self.Domain.find_or_create(domain=d['hostname'])
if created:
print("New domain: %s" % d['hostname'])
if data['theHarvester'].get('vhost', False):
if type(data['theHarvester']['vhost']) == list:
hosts = data['theHarvester']['vhost']
else:
hosts = [data['theHarvester']['vhost']]
for d in hosts:
created, domain = self.Domain.find_or_create(domain=d['hostname'])
if created:
print("New domain: %s" % d['hostname'])
class Module(ToolTemplate):
name = "Fierce"
binary_name = "fierce"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d',
'--domain',
help="Target domain for Fierce")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'--rescan',
help="Force rescan of already scanned domains",
action="store_true")
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
if args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
targets.append(args.domain)
if args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for domain in domains:
targets.append(domain.domain)
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config['PROJECT']['base_path'],
args.output_path[1:])
else:
self.path = os.path.join(self.base_config['PROJECT']['base_path'],
args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
res = []
for t in targets:
res.append({
'target': t,
'output': os.path.join(self.path, t + ".txt")
})
return res
def build_cmd(self, args):
command = self.binary
command += " -dns {target} -file {output} "
if args.tool_args:
command += args.tool_args
pdb.set_trace()
return command
def process_output(self, cmds):
for c in cmds:
target = c['target']
output_path = c['output']
try:
fierceOutput = open(output_path).read()
except IOError:
display_error(
"The output file for {} was not found. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file."
.format(target))
continue
domains = []
if "Now performing" in fierceOutput:
hosts = re.findall("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$",
fierceOutput, re.MULTILINE)
if hosts:
for host in hosts:
#print host
domain = host.split("\t")[1].lower().replace(
"www.", "").rstrip(".")
if domain not in domains:
domains.append(domain)
elif "Whoah, it worked" in fierceOutput:
print "Zone transfer found!"
hosts = re.findall(
".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", fierceOutput,
re.MULTILINE)
if hosts:
for host in hosts:
domain = host.split("\t")[0].lower().replace(
"www.", "").rstrip(".")
if domain not in domains:
domains.append(domain)
else:
display_error(
"Unable to process {}. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file."
.format(output_path))
if domains:
for _domain in domains:
created, domain = self.Domain.find_or_create(
domain=_domain)
class Module(ModuleTemplate):
name = "DNSRecon"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-d','--domain', help="Target domain for dnsRecon")
self.options.add_argument('-f', '--file', help="Import domains from file")
self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true")
self.options.add_argument('-o', '--output_path', help="Relative path (to the base directory) to store dnsRecon XML output", default="dnsReconFiles")
self.options.add_argument('-x', '--dns_recon_file', help="dnsRecon XML output name")
self.options.add_argument('-r', '--range', help="Range to scan for PTR records")
self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true")
self.options.add_argument('--threads', help="Number of threads to use", default="10")
self.options.add_argument('--force', help="Force overwrite", action="store_true")
self.options.add_argument('--import_output_xml', help="Import XML file")
self.options.add_argument('--import_output_json', help="Import json file")
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run('dnsrecon')
else:
self.binary = which.run(args.binary)
if not self.binary:
print("Dnsrecon binary not found. Please explicitly provide path with --binary")
if args.output_path[0] == "/":
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] )
else:
self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(self.path):
os.makedirs(self.path)
if args.domain:
created, domain = self.BaseDomain.find_or_create(domain=args.domain)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.import_database:
domains = self.BaseDomain.all()
for domain in domains:
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.range:
self.process_range(args.range, args)
self.BaseDomain.commit()
elif args.import_range:
cidrs = self.ScopeCIDR.all()
for cidr in cidrs:
self.process_range(cidr.cidr, args)
self.BaseDomain.commit()
elif args.import_output_json:
data = open(args.import_output_json).read()
self.process_data(data, args, data_type="json")
self.BaseDomain.commit()
elif args.import_output_xml:
data = open(args.import_output_xml).read()
self.process_data(data, args, data_type="json")
self.BaseDomain.commit()
else:
print("You need to supply some options to do something.")
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
print("Processing %s" % domain)
if args.dns_recon_file:
dns_recon_file = os.path.join(self.path,args.dnsReconFile)
else:
dns_recon_file = os.path.join(self.path,domain+".json")
if os.path.isfile(dns_recon_file):
if not args.force:
answered = False
while answered == False:
rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ")
if rerun.lower() == 'r':
answered = True
elif rerun.lower() == 'p':
answered = True
return dnsReconFile
elif rerun.lower() == 'n':
new = False
while new == False:
newFile = raw_input("enter a new file name: ")
if not os.path.exists(os.path.join(self.path,newFile)):
dns_recon_file = os.path.join(path,newFile)
answered = True
new = True
else:
print "That file exists as well"
command = self.binary
if args.threads:
command += " --threads %s " % args.threads
command += " -d {} -j {} ".format(domain, dns_recon_file)
subprocess.Popen(command, shell=True).wait()
try:
res = json.loads(open(dns_recon_file).read())
domain_obj.dns = {'data':res}
domain_obj.save()
self.process_data(res, args, data_type='json')
except IOError:
print("DnsRecon for %s failed." % domain)
return None
def process_data(self, data, args, data_type="json"):
if data_type == "xml":
tree = ET.fromstring(data)
root = tree.getroot()
records = root.findall("record")
else:
records = data
for record in data:
created = False
domain = ""
if record.get("type") == "A":
created, domain = self.Domain.find_or_create(domain=record.get("name").lower().replace("www.",""))
if record.get("type") == "A" :
domain = record.get("name").lower().replace("www.","")
#ip = record.get("address") #take what nslookup(get_domain_ip) says instead
elif record.get("type") == "MX":
domain = record.get("exchange").lower().replace("www.","")
elif record.get("type") == "SRV":
domain = record.get("target").lower().replace("www.","")
if domain:
created, domain_obj = self.Domain.find_or_create(domain=domain)
if created:
print("New domain found: " + domain)
def process_range(self, cidr, args):
print("Processing %s" % cidr)
if args.dns_recon_file:
dns_recon_file = os.path.join(self.path,args.dnsReconFile)
else:
dns_recon_file = os.path.join(self.path,cidr.replace('/', '_')+".json")
if os.path.isfile(dns_recon_file):
if not args.force:
answered = False
while answered == False:
rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ")
if rerun.lower() == 'r':
answered = True
elif rerun.lower() == 'p':
answered = True
return dnsReconFile
elif rerun.lower() == 'n':
new = False
while new == False:
newFile = raw_input("enter a new file name: ")
if not os.path.exists(os.path.join(self.path,newFile)):
dns_recon_file = os.path.join(path,newFile)
answered = True
new = True
else:
print "That file exists as well"
command = self.binary
if args.threads:
command += " --threads %s " % args.threads
command += " -s -r {} -j {} ".format(cidr, dns_recon_file)
subprocess.Popen(command, shell=True).wait()
try:
res = json.loads(open(dns_recon_file).read())
for r in res[1:]:
domain = r['name']
created, domain_obj = self.Domain.find_or_create(domain=domain)
if created:
print("New domain found: %s" % domain)
self.process_data(res, args, data_type='json')
except IOError:
print("DnsRecon for %s failed." % domain)
return None
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
class Module(ModuleTemplate):
name = "Sublist3r"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument('-t', '--threads', help="Number of threads")
self.options.add_argument('-d',
'--domain',
help="Domain to brute force")
self.options.add_argument('-f',
'--file',
help="Import domains from file")
self.options.add_argument('-i',
'--import_database',
help="Import domains from database",
action="store_true")
self.options.add_argument(
'-o',
'--output_path',
help=
"Path which will contain program output (relative to base_path in config",
default="sublist3r")
self.options.add_argument(
'-s',
'--rescan',
help="Rescan domains that have already been scanned",
action="store_true")
def run(self, args):
if not args.binary:
self.binary = which.run('sublist3r')
else:
self.binary = which.run(args.binary)
if not self.binary:
print(
"Sublist3r binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.file:
domains = open(args.file).read().split('\n')
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
self.process_domain(domain, args)
self.BaseDomain.commit()
elif args.import_database:
domains = self.BaseDomain.all(tool=self.name)
for d in domains:
self.process_domain(d, args)
d.set_tool(self.name)
self.BaseDomain.commit()
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config['PROJECT']['base_path'], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path, "%s-sublist3r.txt" % domain)
command_args = " -o %s" % output_path
if args.threads:
command_args += " -t " + args.threads
cmd = shlex.split(self.binary + command_args + " -d " + domain)
print("Executing: %s" % ' '.join(cmd))
res = subprocess.Popen(cmd).wait()
try:
data = open(output_path).read().split('\n')
for d in data:
new_domain = d.split(':')[0].lower()
if new_domain:
created, subdomain = self.Domain.find_or_create(
domain=new_domain)
if created:
print("New subdomain found: %s" % new_domain)
except IOError:
print("No results found.")
