class Module(ModuleTemplate): name = "GobusterDNS" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-fw', '--force_wildcard', help="Continues after hitting wildcard cert", action="store_true") self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-w', '--wordlist', help="Wordlist to use") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default="gobuster") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been brute forced", action="store_true") def run(self, args): if not args.wordlist: print("Wordlist is required!") return if not args.binary: self.binary = which.run('gobuster') else: self.binary = which.run(args.binary) if not self.binary: print( "Gobuster binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.brute_force_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.brute_force_domain(domain, args) self.BaseDomain.commit() elif args.import_database: if args.rescan: domains = self.BaseDomain.all() else: domains = self.BaseDomain.all(tool=self.name) for domain in domains: self.brute_force_domain(domain, args) domain.set_tool(self.name) self.BaseDomain.commit() def brute_force_domain(self, domain_obj, args): domain = domain_obj.domain command_args = " -m dns" if args.force_wildcard: command_args += " -fw" if args.threads: command_args += " -t " + args.threads command_args += " -w " + args.wordlist cmd = shlex.split(self.binary + command_args + " -u " + domain) print("Executing: %s" % ' '.join(cmd)) res = check_output(cmd) if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-dns.txt" % domain) open(output_path, 'w').write(res) data = res.split('\n') for d in data: if 'Found: ' in d: new_domain = d.split(' ')[1].lower() created, subdomain = self.Domain.find_or_create( domain=new_domain) if created: print("New subdomain found: %s" % new_domain)
class Module(ToolTemplate): name = "DNSRecon" binary_name = "dnsrecon" """ This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries. It can also run over IP ranges, looking for additional domains in the PTR records. """ def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d','--domain', help="Target domain for dnsRecon") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-r', '--range', help="Range to scan for PTR records") self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true") self.options.add_argument('--rescan', help="Rescan domains already scanned", action="store_true") # self.options.add_argument('--import_output_xml', help="Import XML file") # self.options.add_argument('--import_output_json', help="Import json file") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain, passive_scope=True) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d, passive_scope=True) targets.append(domain.domain) elif args.import_database: domains = self.BaseDomain.all(scope_type="passive", tool=self.name) for domain in domains: targets.append(domain.domain) elif args.range: targets.append(args.range) elif args.import_range: cidrs = self.ScopeCIDR.all(tool=self.name) for cidr in cidrs: targets.append(cidr.cidr) if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] ) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) res = [] for t in targets: res.append({'target':t, 'output':os.path.join(self.path, t.replace("/", "_") + ".json")}) return res def build_cmd(self, args): command = self.binary if args.domain or args.file or args.import_database: command += " -d {target} -j {output} " else: command += " -s -r {target} -j {output} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): for c in cmds: target = c['target'] output_path = c['output'] try: res = json.loads(open(output_path).read()) except IOError: display_error("DnsRecon failed for {}".format(target)) if " -d " in res[0]['arguments']: created, dbrec = self.Domain.find_or_create(domain=target) dbrec.dns = res dbrec.save() for record in res: domain = None if record.get("type") == "A" or record.get("type") == "PTR": domain = record.get("name").lower().replace("www.","") elif record.get("type") == "MX": domain = record.get("exchange").lower().replace("www.","") elif record.get("type") == "SRV" or record.get("type" == "NS"): domain = record.get("target").lower().replace("www.","") elif record.get("type") == "SOA": domain = record.get("mname").lower().replace("www.","") if domain: created, domain_obj = self.Domain.find_or_create(domain=domain) self.Domain.commit()
class Module(ModuleTemplate): name = "LinkedInt" binary_name = "linkedint.py" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-b", "--binary", help="Path to binary for LinkedInt", default=self.binary_name, ) self.options.add_argument("-d", "--domain", help="Domain to add onto email") self.options.add_argument("-c", "--company_id", help="Company ID to search") self.options.add_argument( "-C", "--restrict", help="Restrict to company filter", action="store_true" ) self.options.add_argument( "-e", "--email_format", help="Format for emails: auto,full,firstlast,firstmlast,flast,first.last,fmlast,lastfirst, default is auto", default="auto", ) self.options.add_argument("-k", "--keywords", help="Keywords to search for") self.options.add_argument( "-o", "--output_path", help="Path which will contain program output (relative to base_path in config", default=self.name, ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) self.options.add_argument( "--smart_shuffle", help="Provide a list of keywords. The tool will run once with all of the keywords, then run again excluding all of the keywords. This is useful for bypassing the 1k limit. Keywords must be comma separated.", ) self.options.add_argument( "--top", help="Use the top X keywords from the job titles for smart shuffle" ) def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run("LinkedInt.py") else: self.binary = which.run(args.binary) if not self.binary: display_error( "LinkedInt binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) if args.top: titles = [ user.job_title.split(" at ")[0] for user in domain.users if user.job_title ] words = [] for t in titles: words += [w.lower() for w in t.split(" ")] word_count = Counter(words).most_common() display("Using the top %s words:" % args.top) res = [] for w in word_count[: int(args.top)]: display("\t{}\t{}".format(w[0], w[1])) res.append(w[0]) # pdb.set_trace() args.smart_shuffle = ",".join(res) if args.smart_shuffle: args.keywords = " OR ".join( ['"{}"'.format(i) for i in args.smart_shuffle.split(",")] ) self.process_domain(domain, args) args.keywords = " AND ".join( ['-"{}"'.format(i) for i in args.smart_shuffle.split(",")] ) self.process_domain(domain, args) else: self.process_domain(domain, args) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:] ) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path ) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join( output_path, "%s-linkedint" % domain.replace(".", "_") ) command_args = " -o %s" % output_path command_args += " -e %s" % domain if args.keywords: command_args += " -u '%s'" % args.keywords if args.company_id: command_args += " -i %s " % args.company_id if args.restrict: command_args += " -c " # if args.threads: # command_args += " -t " + args.threads if args.email_format: command_args += " -f " + args.email_format current_dir = os.getcwd() new_dir = "/".join(self.binary.split("/")[:-1]) os.chdir(new_dir) cmd = shlex.split("python " + self.binary + command_args) print("Executing: %s" % " ".join(cmd)) res = subprocess.Popen(cmd).wait() os.chdir(current_dir) count = 0 with open(output_path + ".csv") as csvfile: csvreader = csv.reader(csvfile, delimiter=",", quotechar='"') for row in csvreader: count += 1 if self.User.find(email=row[3]): created, user = self.User.find_or_create( email=remove_binary(row[3]) ) user.first_name = remove_binary(row[0]) user.last_name = remove_binary(row[1]) user.job_title = remove_binary(row[4]) user.location = remove_binary(row[5]) else: created, user = self.User.find_or_create( first_name=remove_binary(row[0]), last_name=remove_binary(row[1]), domain=domain_obj, ) if created: print( "New user: %s %s" % (remove_binary(row[0]), remove_binary(row[1])) ) user.email = remove_binary(row[3]) user.job_title = remove_binary(row[4]) user.location = remove_binary(row[5]) user.update() print("%s found and imported" % count) self.User.commit()
class Module(ModuleTemplate): name = "Whois" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to query") self.options.add_argument('-c', '--cidr', help="CIDR to query") self.options.add_argument('-t', '--threads', help='Number of threads to run', default="1") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default="whois") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") self.options.add_argument( '--import_database', help="Run WHOIS on all domains and CIDRs in database", action="store_true") def run(self, args): self.args = args if not args.binary: self.binary = which.run('whois') else: self.binary = which.run(args.binary) if not self.binary: print( "whois binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domains(domains=[domain], ips=[]) elif args.cidr: created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr) self.process_domains(domains=[], ips=[domain]) elif args.import_database: domains = self.BaseDomain.all() cidrs = self.ScopeCidr.all() self.process_domains(domains=domains, cidrs=cidrs) self.ScopeCidr.commit() def process_domains(self, domains, cidrs): args = self.args if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) commands = [] for domain in domains: name = domain.domain file_path = os.path.join(output_path, "%s.txt" % name) command_args = " " cmd = shlex.split(self.binary + command_args + name) commands.append([cmd, file_path, name]) for cidr in cidrs: name = cidr.cidr.split('/')[0] file_path = os.path.join(output_path, "%s.txt" % name) command_args = " " cmd = shlex.split(self.binary + command_args + name) commands.append([cmd, file_path, cidr.cidr]) pool = ThreadPool(int(args.threads)) res = pool.map(run_cmd, commands) print("Importing data to database") res_d = {} for r in res: if r: res_d[r[0]] = r[1] domains = self.BaseDomain.all() for d in domains: if res_d.get(d.domain, False): d.meta['whois'] = res_d[d.domain] d.update() pdb.set_trace() for c in self.ScopeCidr.all(): if res_d.get(c.cidr, False): c.meta['whois'] = res_d[c.cidr] c.update()
class Module(ToolTemplate): name = "Whois" binary_name = "whois" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to query") self.options.add_argument('-c', '--cidr', help="CIDR to query") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") self.options.add_argument( '--import_database', help="Run WHOIS on all domains and CIDRs in database", action="store_true") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append({'domain': domain.domain, 'cidr': ''}) elif args.cidr: created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr) targets.append({'domain': '', 'cidr': cidr.cidr.split('/')[0]}) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") cidrs = self.ScopeCidr.all(scope_type="passive") else: domains = self.BaseDomain.all(scope_type="passive", tool=self.name) cidrs = self.ScopeCidr.all(scope_type="passive", tool=self.name) for domain in domains: targets.append({'domain': domain.domain, 'cidr': ''}) for cidr in cidrs: targets.append({'domain': '', 'cidr': cidr.cidr.split('/')[0]}) if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t['output'] = os.path.join(output_path, t['domain'] + t['cidr']) return targets def build_cmd(self, args): if not args.tool_args: args.tool_args = "" cmd = "bash -c \"" + self.binary + " {domain}{cidr} " + args.tool_args + "> {output}\" " return cmd def process_output(self, cmds): display("Importing data to database") for cmd in cmds: if cmd['cidr']: _, cidr = self.ScopeCidr.find_or_create(cidr=cmd['cidr']) cidr.meta['whois'] = open(cmd['output']).read() display(cidr.meta['whois']) cidr.update() elif cmd['domain']: _, domain = self.BaseDomain.find_or_create( domain=cmd['domain']) domain.meta['whois'] = open(cmd['output']).read() display(domain.meta['whois']) domain.update() self.BaseDomain.commit()
class Module(ToolTemplate): name = "Whois" binary_name = "whois" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to query") self.options.add_argument("-c", "--cidr", help="CIDR to query") self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) self.options.add_argument( "-a", "--all_data", help="Scan all data in database, regardless of scope", action="store_true", ) self.options.add_argument( "-i", "--import_database", help="Run WHOIS on all domains and CIDRs in database", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append({"domain": domain.domain, "cidr": ""}) elif args.cidr: created, cidr = self.ScopeCIDR.find_or_create(cidr=args.cidr) targets.append({"domain": "", "cidr": cidr.cidr.split("/")[0]}) elif args.import_database: if args.all_data: scope_type = "" else: scope_type = "passive" if args.rescan: domains = self.BaseDomain.all(scope_type=scope_type) cidrs = self.ScopeCidr.all() else: domains = self.BaseDomain.all(scope_type=scope_type, tool=self.name) cidrs = self.ScopeCidr.all(tool=self.name) for domain in domains: targets.append({"domain": domain.domain, "cidr": ""}) for cidr in cidrs: targets.append({"domain": "", "cidr": cidr.cidr.split("/")[0]}) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t["output"] = os.path.join(output_path, t["domain"] + t["cidr"]) return targets def build_cmd(self, args): if not args.tool_args: args.tool_args = "" cmd = ('bash -c "' + self.binary + " {domain}{cidr} " + args.tool_args + '> {output}" ') return cmd def process_output(self, cmds): display("Importing data to database") for cmd in cmds: if cmd["cidr"]: _, cidr = self.ScopeCidr.find_or_create(cidr=cmd["cidr"]) cidr.meta["whois"] = open(cmd["output"]).read() display(cidr.meta["whois"]) cidr.update() elif cmd["domain"]: _, domain = self.BaseDomain.find_or_create( domain=cmd["domain"]) domain.meta["whois"] = open(cmd["output"]).read() display(domain.meta["whois"]) domain.update() self.BaseDomain.commit()
class Module(ToolTemplate): name = "TheHarvester" binary_name = "theharvester" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to harvest") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) targets.append({'target':domain.domain}) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) targets.append({'target':domain.domain}) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append({'target':d.domain}) if args.output_path[0] == "/": output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t['output'] = os.path.join(output_path, "%s-theharvester" % t['target'].replace('.', '_') ) return targets def build_cmd(self, args): cmd = self.binary + " -f {output} -b all -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: try: data = xmltodict.parse(open(cmd['output'] + '.xml').read()) except: data = None if data: if data['theHarvester'].get('email', False): if type(data['theHarvester']['email']) == list: emails = data['theHarvester']['email'] else: emails = [data['theHarvester']['email']] for e in emails: created, user = self.User.find_or_create(email=e) _, domain = self.BaseDomain.find_or_create(e.split('@')[1]) user.domain = domain user.update() if created: display_new("New email: %s" % e) if data['theHarvester'].get('host', False): if type(data['theHarvester']['host']) == list: hosts = data['theHarvester']['host'] else: hosts = [data['theHarvester']['host']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if data['theHarvester'].get('vhost', False): if type(data['theHarvester']['vhost']) == list: hosts = data['theHarvester']['vhost'] else: hosts = [data['theHarvester']['vhost']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) self.BaseDomain.commit()
class Module(ModuleTemplate): name = "DNSRecon" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d','--domain', help="Target domain for dnsRecon") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-o', '--output_path', help="Relative path (to the base directory) to store dnsRecon XML output", default="dnsReconFiles") self.options.add_argument('-x', '--dns_recon_file', help="dnsRecon XML output name") self.options.add_argument('-r', '--range', help="Range to scan for PTR records") self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true") self.options.add_argument('--threads', help="Number of threads to use", default="10") self.options.add_argument('--force', help="Force overwrite", action="store_true") self.options.add_argument('--import_output_xml', help="Import XML file") self.options.add_argument('--import_output_json', help="Import json file") def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run('dnsrecon') else: self.binary = which.run(args.binary) if not self.binary: print("Dnsrecon binary not found. Please explicitly provide path with --binary") if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] ) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all() for domain in domains: self.process_domain(domain, args) self.BaseDomain.commit() elif args.range: self.process_range(args.range, args) self.BaseDomain.commit() elif args.import_range: cidrs = self.ScopeCIDR.all() for cidr in cidrs: self.process_range(cidr.cidr, args) self.BaseDomain.commit() elif args.import_output_json: data = open(args.import_output_json).read() self.process_data(data, args, data_type="json") self.BaseDomain.commit() elif args.import_output_xml: data = open(args.import_output_xml).read() self.process_data(data, args, data_type="json") self.BaseDomain.commit() else: print("You need to supply some options to do something.") def process_domain(self, domain_obj, args): domain = domain_obj.domain print("Processing %s" % domain) if args.dns_recon_file: dns_recon_file = os.path.join(self.path,args.dnsReconFile) else: dns_recon_file = os.path.join(self.path,domain+".json") if os.path.isfile(dns_recon_file): if not args.force: answered = False while answered == False: rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ") if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return dnsReconFile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists(os.path.join(self.path,newFile)): dns_recon_file = os.path.join(path,newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " --threads %s " % args.threads command += " -d {} -j {} ".format(domain, dns_recon_file) subprocess.Popen(command, shell=True).wait() try: res = json.loads(open(dns_recon_file).read()) domain_obj.dns = {'data':res} domain_obj.save() self.process_data(res, args, data_type='json') except IOError: print("DnsRecon for %s failed." % domain) return None def process_data(self, data, args, data_type="json"): if data_type == "xml": tree = ET.fromstring(data) root = tree.getroot() records = root.findall("record") else: records = data for record in data: created = False domain = "" if record.get("type") == "A": created, domain = self.Domain.find_or_create(domain=record.get("name").lower().replace("www.","")) if record.get("type") == "A" : domain = record.get("name").lower().replace("www.","") #ip = record.get("address") #take what nslookup(get_domain_ip) says instead elif record.get("type") == "MX": domain = record.get("exchange").lower().replace("www.","") elif record.get("type") == "SRV": domain = record.get("target").lower().replace("www.","") if domain: created, domain_obj = self.Domain.find_or_create(domain=domain) if created: print("New domain found: " + domain) def process_range(self, cidr, args): print("Processing %s" % cidr) if args.dns_recon_file: dns_recon_file = os.path.join(self.path,args.dnsReconFile) else: dns_recon_file = os.path.join(self.path,cidr.replace('/', '_')+".json") if os.path.isfile(dns_recon_file): if not args.force: answered = False while answered == False: rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ") if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return dnsReconFile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists(os.path.join(self.path,newFile)): dns_recon_file = os.path.join(path,newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " --threads %s " % args.threads command += " -s -r {} -j {} ".format(cidr, dns_recon_file) subprocess.Popen(command, shell=True).wait() try: res = json.loads(open(dns_recon_file).read()) for r in res[1:]: domain = r['name'] created, domain_obj = self.Domain.find_or_create(domain=domain) if created: print("New domain found: %s" % domain) self.process_data(res, args, data_type='json') except IOError: print("DnsRecon for %s failed." % domain) return None
class Module(ModuleTemplate): name = "Sublist3r" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default="sublist3r") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def run(self, args): if not args.binary: self.binary = which.run('sublist3r') else: self.binary = which.run(args.binary) if not self.binary: print( "Sublist3r binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all(tool=self.name) for d in domains: self.process_domain(d, args) d.set_tool(self.name) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-sublist3r.txt" % domain) command_args = " -o %s" % output_path if args.threads: command_args += " -t " + args.threads cmd = shlex.split(self.binary + command_args + " -d " + domain) print("Executing: %s" % ' '.join(cmd)) res = subprocess.Popen(cmd).wait() try: data = open(output_path).read().split('\n') for d in data: new_domain = d.split(':')[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) if created: print("New subdomain found: %s" % new_domain) except IOError: print("No results found.")
class Module(ToolTemplate): name = "aquatone-discover" binary_name = "aquatone-discover" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) self.BaseDomain = BaseDomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Target domain for aquatone") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-r", "--rescan", help="Run aquatone on hosts that have already been processed.", action="store_true", ) self.options.set_defaults(timeout=None) def get_targets(self, args): """ This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [(target, output), (target, output), etc, etc] """ targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domainsFile = open(args.file).read().split("\n") for d in domainsFile: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: all_domains = self.BaseDomain.all(scope_type="passive") else: all_domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in all_domains: targets.append(d.domain) else: print("You need to supply domain(s).") output_path = os.path.join(self.base_config["PROJECT"]["base_path"], "aquatone") if not os.path.exists(output_path): os.makedirs(output_path) res = [] for t in targets: res.append({ "target": t, "output": "{}/{}/hosts.json".format(output_path, t) }) return res def build_cmd(self, args): """ Create the actual command that will be executed. Use {target} and {output} as placeholders. """ cmd = self.binary + " -d {target} " if args.tool_args: cmd += args.tool_args return cmd def pre_run(self, args): output_path = self.base_config["PROJECT"]["base_path"] self.orig_home = os.environ["HOME"] os.environ["HOME"] = output_path def process_output(self, cmds): """ Process the output generated by the earlier commands. """ for cmd in cmds: data2 = json.loads(open(cmd["output"]).read()) for sub, ip in data2.items(): created = False new_domain = sub.lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit() def post_run(self, args): os.environ["HOME"] = self.orig_home
class Module(ModuleTemplate): name = "TheHarvester" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-o', '--output_path', help="Path which will contain program output (relative to base_path in config", default=self.name) self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def run(self, args): if not args.binary: self.binary = which.run('theharvester') else: self.binary = which.run(args.binary) if not self.binary: print("TheHarvester binary not found. Please explicitly provide path with --binary") if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: if args.rescan: domains = self.BaseDomain.all() else: domains = self.BaseDomain.all(tool=self.name) for d in domains: self.process_domain(d, args) d.set_tool(self.name) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-theharvester" % domain.replace('.', '_') ) command_args = " -f %s" % output_path command_args += " -b all " # if args.threads: # command_args += " -t " + args.threads cmd = shlex.split(self.binary + command_args + " -d " + domain) print("Executing: %s" % ' '.join(cmd)) res = subprocess.Popen(cmd).wait() try: data = xmltodict.parse(open(output_path + '.xml').read()) except: data = None if data: if data['theHarvester'].get('email', False): if type(data['theHarvester']['email']) == list: emails = data['theHarvester']['email'] else: emails = [data['theHarvester']['email']] for e in emails: created, user = self.User.find_or_create(email=e) user.domain = domain_obj user.update() if created: print("New email: %s" % e) if data['theHarvester'].get('host', False): if type(data['theHarvester']['host']) == list: hosts = data['theHarvester']['host'] else: hosts = [data['theHarvester']['host']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if created: print("New domain: %s" % d['hostname']) if data['theHarvester'].get('vhost', False): if type(data['theHarvester']['vhost']) == list: hosts = data['theHarvester']['vhost'] else: hosts = [data['theHarvester']['vhost']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if created: print("New domain: %s" % d['hostname'])
class Module(ToolTemplate): name = "Sublist3r" binary_name = "sublist3r" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to brute force") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output = os.path.join(output_path, "%s-sublist3r.txt" % t) res.append({"target": t, "output": output}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd["output"] if os.path.isfile(output_path): data = open(output_path).read().split("\n") else: display_error("{} not found.".format(output_path)) next for d in data: new_domain = d.split(":")[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit()
class Module(ToolTemplate): name = "Sublist3r" binary_name = 'sublist3r' def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output_path = os.path.join(output_path, "%s-sublist3r.txt" % t) res.append({'target': t, 'output': output_path}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd['output'] data = open(output_path).read().split('\n') for d in data: new_domain = d.split(':')[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit()
class Module(ModuleTemplate): name = "Fierce" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Target domain for Fierce") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help="Relative path (to the base directory) to store Fierce output", default="fierceFiles") self.options.add_argument('-x', '--fiercefile', help="Fierce output name") self.options.add_argument('--threads', help="Number of threads to use", default="10") self.options.add_argument('--force', help="Force overwrite", action="store_true") self.options.add_argument('--import_file', help="Import Fierce file") def run(self, args): if not args.binary: self.binary = which.run('fierce') else: self.binary = which.run(args.binary) if not self.binary: print( "Fierce binary not found. Please explicitly provide path with --binary" ) if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all(tool=self.name) for domain in domains: self.process_domain(domain, args) domain.set_tool(self.name) self.BaseDomain.commit() elif args.import_file: fiercefile = args.import_file self.process_data(fiercefile, args) self.BaseDomain.commit() else: print("You need to supply some options to do something.") def process_domain(self, domain_obj, args): domain = domain_obj.domain print("Processing %s" % domain) if args.fiercefile: fiercefile = os.path.join(self.path, args.fiercefile) else: fiercefile = os.path.join(self.path, domain + ".txt") if os.path.isfile(fiercefile): if not args.force: answered = False while answered == False: rerun = raw_input( "Would you like to [r]un Fierce again and overwrite the file, [p]arse the file, or change the file [n]ame? " ) if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return fiercefile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists( os.path.join(self.path, newFile)): fiercefile = os.path.join(path, newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " -threads %s " % args.threads command += " -dns {} -file {} ".format(domain, fiercefile) subprocess.Popen(command, shell=True).wait() try: self.process_data(fiercefile, args) except IOError: print("Fierce for %s failed." % domain) return None def process_data(self, fiercefile, args): self.fiercefile = fiercefile print "Reading", self.fiercefile fierceOutput = '' with open(self.fiercefile, 'r') as fiercefile: for line in fiercefile: fierceOutput += line domains = [] if "Now performing" in fierceOutput: hosts = re.findall("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: #print host domain = host.split("\t")[1].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) elif "Whoah, it worked" in fierceOutput: print "Zone transfer found!" hosts = re.findall(".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: domain = host.split("\t")[0].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) else: print "Unable to process {}".format(fiercefile) if domains: for _domain in domains: created, domain = self.Domain.find_or_create(domain=_domain) return
class Module(ModuleTemplate): name = "LinkedInt" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to add onto email") self.options.add_argument('-c', '--company_id', help="Company ID to search") self.options.add_argument('-C', '--restrict', help="Restrict to company filter", action="store_true") self.options.add_argument('-k', '--keywords', help="Keywords to search for") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default=self.name) self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run('LinkedInt.py') else: self.binary = which.run(args.binary) if not self.binary: print( "LinkedInt binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-linkedint" % domain.replace('.', '_')) command_args = " -o %s" % output_path command_args += " -e %s" % domain command_args += " -u %s" % args.keywords if args.company_id: command_args += " -i %s " % args.company_id if args.restrict: command_args += " -c " # if args.threads: # command_args += " -t " + args.threads current_dir = os.getcwd() new_dir = '/'.join(self.binary.split('/')[:-1]) os.chdir(new_dir) cmd = shlex.split("python " + self.binary + command_args) print("Executing: %s" % ' '.join(cmd)) res = subprocess.Popen(cmd).wait() os.chdir(current_dir) count = 0 with open(output_path + '.csv') as csvfile: csvreader = csv.reader(csvfile, delimiter=",", quotechar='"') for row in csvreader: count += 1 if self.User.find(email=row[3]): created, user = self.User.find_or_create( email=remove_binary(row[3])) user.first_name = remove_binary(row[0]) user.last_name = remove_binary(row[1]) user.job_title = remove_binary(row[4]) user.location = remove_binary(row[5]) else: created, user = self.User.find_or_create( first_name=remove_binary(row[0]), last_name=remove_binary(row[1]), domain=domain_obj) if created: print("New user: %s %s" % (remove_binary(row[0]), remove_binary(row[1]))) user.email = remove_binary(row[3]) user.job_title = remove_binary(row[4]) user.location = remove_binary(row[5]) user.update() print("%s found and imported" % count) self.User.commit()
class Module(ToolTemplate): name = "Subfinder" binary_name = "subfinder" def __init__(self, db): self.db = db self.BaseDomains = BaseDomainRepository(db, self.name) self.Domains = DomainRepository(db, self.name) self.IPs = IPRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-a", "--bruteforce-all", help="Brute-force subdomains." ) self.options.add_argument( "-d", "--domain", help="Domain to run subfinder against." ) self.options.add_argument( "-dL", "--domain-list", help="Read in a list of domains within the given file.", ) self.options.add_argument( "-i", "--db_domains", help="Import the domains from the database.", action="store_true", ) self.options.add_argument( "-r", "--resolvers", help="A list of resolvers(comma-separated) or a file containing a list of resolvers.", ) self.options.add_argument( "--rescan", help="Overwrite files without asking", action="store_true" ) self.options.add_argument( "-w", "--wordlist", help="The wordlist for when bruteforcing is selected." ) def get_targets(self, args): targets = [] outpath = "" if args.output_path: if not os.path.exists(args.output_path): os.makedirs(args.output_path) outpath = args.output_path if args.domain or args.db_domains: self.db_domain_file = self.__get_tempfile(args.domain, args) out_file = "database_domains.subfinder" if args.domain: created, domain = self.BaseDomains.find_or_create(domain=args.domain) out_file = os.path.join(outpath, "{}.subfinder".format(args.domain)) if not self.db_domain_file: return targets targets.append({"target": self.db_domain_file, "output": out_file}) elif args.domain_list: domains = io.open(args.domain_list, encoding="utf-8").read().split("\n") for d in domains: if d: created, domain = self.BaseDomains.find_or_create(domain=d) targets.append( { "target": args.domain_list, "output": os.path.join( outpath, "{}.subfinder".format(args.domain_list) ), } ) return targets def build_cmd(self, args): if args.binary: cmd = "{} ".format(args.binary) else: cmd = "{} ".format(self.binary_name) cmd = "{} -o {} -dL {}".format(cmd, "{output}", "{target}") return cmd def process_output(self, targets): for target in targets: with io.open(target["output"], encoding="utf-8") as fd: for line in fd: domain = line.strip() ips = get_domain_ip.run(domain) ip_obj = None _, dom = self.Domains.find_or_create(domain=domain) if ips: for ip in ips: _, ip_obj = self.IPs.find_or_create(ip_address=ip) if ip_obj: dom.ip_addresses.append(ip_obj) dom.save() self.BaseDomains.commit() self.IPs.commit() def post_run(self, args): # Remove the temporary db file if it was created. if getattr(self, "db_domain_file", None): try: os.unlink(self.db_domain_file) except IOError as e: print("Failed to remove the Subfinder db temp file: '{}'.".format(e)) def __get_tempfile(self, domain=None, args=None): # Create a temporary file and place all of the current database domains within the file. from tempfile import NamedTemporaryFile with NamedTemporaryFile(delete=False) as fd: if domain: fd.write("{}\n".format(domain).encode("utf-8")) else: # Go through the database and grab the domains adding them to the file. if args.rescan: domains = self.BaseDomains.all(passive_scope=True) else: domains = self.BaseDomains.all(tool=self.name, passive_scope=True) if domains: for domain in domains: fd.write("{}\n".format(domain.domain).encode("utf-8")) else: return None return fd.name