def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
class Report(ReportTemplate): """ This report displays all of the various Base Domains, Domains, and IP Addresses with scoping information """ markdown = ["###", "`"] name = "CertReport" def __init__(self, db): self.IPAddress = IPRepository(db) self.Domains = DomainRepository(db) self.BaseDomains = BaseDomainRepository(db) self.CIDRs = CIDRRepository(db) def set_options(self): super(Report, self).set_options() def run(self, args): results = [] base_domains = self.BaseDomains.all() for b in base_domains: results.append( "%s\tActive Scope: %s\tPassive Scope: %s" % (b.domain, b.in_scope, b.passive_scope) ) for d in b.subdomains: results.append( "\t%s\tActive Scope: %s\tPassive Scope: %s" % (d.domain, d.in_scope, d.passive_scope) ) cidrs = self.CIDRs.all() results.append("\n\n") for c in cidrs: results.append("%s - %s" % (c.cidr, c.org_name)) for ip in c.ip_addresses: results.append( "\t%s\tActive Scope: %s\tPassive Scope: %s" % (ip.ip_address, ip.in_scope, ip.passive_scope) ) self.process_output(results, args)
class Module(ModuleTemplate): name = "LinkedInt" binary_name = "linkedint.py" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-b", "--binary", help="Path to binary for LinkedInt", default=self.binary_name, ) self.options.add_argument("-d", "--domain", help="Domain to add onto email") self.options.add_argument("-c", "--company_id", help="Company ID to search") self.options.add_argument("-C", "--restrict", help="Restrict to company filter", action="store_true") self.options.add_argument( "-e", "--email_format", help= "Format for emails: auto,full,firstlast,firstmlast,flast,first.last,fmlast,lastfirst, default is auto", default="auto", ) self.options.add_argument("-k", "--keywords", help="Keywords to search for") self.options.add_argument( "-o", "--output_path", help= "Path which will contain program output (relative to base_path in config", default=self.name, ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) self.options.add_argument( "--smart_shuffle", help= "Provide a list of keywords. The tool will run once with all of the keywords, then run again excluding all of the keywords. This is useful for bypassing the 1k limit. Keywords must be comma separated.", ) self.options.add_argument( "--top", help="Use the top X keywords from the job titles for smart shuffle" ) self.options.add_argument( "--auto_keyword", help= "Generate a list of keywords from titles already discovered, and search repeatedly using the top x number of results (specified with --top).", action="store_true") def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run("LinkedInt.py") else: self.binary = which.run(args.binary) if not self.binary: display_error( "LinkedInt binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) if args.top: titles = [ user.job_title.split(" at ")[0] for user in domain.users if user.job_title ] words = [] for t in titles: words += [w.lower() for w in get_words(t)] word_count = Counter(words).most_common() display("Using the top %s words:" % args.top) res = [] for w in word_count[:int(args.top)]: display("\t{}\t{}".format(w[0], w[1])) res.append(w[0]) # pdb.set_trace() args.smart_shuffle = ",".join(res) if args.auto_keyword: if not args.top: display_error( "You must specify the top number of keywords using --top" ) else: if os.path.isfile('/tmp/armory_linkedinsearchqueries'): blacklist = open('/tmp/armory_linkedinsearchqueries' ).read().split('\n') else: blacklist = [] bfile = open('/tmp/armory_linkedinsearchqueries', 'a') for w in args.smart_shuffle.split(','): if w not in blacklist: args.keywords = w self.process_domain(domain, args) self.BaseDomain.commit() bfile.write('{}\n'.format(w)) else: display( "Skipped {} due to it already being searched.". format(w)) bfile.close() elif args.smart_shuffle: args.keywords = " OR ".join( ['"{}"'.format(i) for i in args.smart_shuffle.split(",")]) self.process_domain(domain, args) self.BaseDomain.commit() args.keywords = " AND ".join( ['-"{}"'.format(i) for i in args.smart_shuffle.split(",")]) self.process_domain(domain, args) self.BaseDomain.commit() else: self.process_domain(domain, args) self.BaseDomain.commit() self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-linkedint" % domain.replace(".", "_")) command_args = " -o %s" % output_path command_args += " -e %s" % domain if args.keywords: command_args += " -u '%s'" % args.keywords if args.company_id: command_args += " -i %s " % args.company_id if args.restrict: command_args += " -c " # if args.threads: # command_args += " -t " + args.threads if args.email_format: command_args += " -f " + args.email_format current_dir = os.getcwd() new_dir = "/".join(self.binary.split("/")[:-1]) os.chdir(new_dir) cmd = shlex.split("python2 " + self.binary + command_args) print("Executing: %s" % " ".join(cmd)) subprocess.Popen(cmd).wait() os.chdir(current_dir) count = 0 with open(output_path + ".csv") as csvfile: csvreader = csv.reader(csvfile, delimiter=",", quotechar='"') for row in csvreader: count += 1 created, user = self.User.find_or_create( email=remove_binary(row[3])) user.first_name = remove_binary(row[0]) user.last_name = remove_binary(row[1]).split(',')[0] user.job_title = remove_binary(row[4]) user.location = remove_binary(row[5]) if created: user.domain = domain_obj print("New user: %s %s" % (remove_binary(row[0]), remove_binary(row[1]))) user.update() print("%s found and imported" % count) self.User.commit()
class Module(ToolTemplate): ''' This module uses the Ruby version of Aquatone. You can usually install it with "gem install aquatone" ''' name = "Aquatone Takeover" binary_name = "aquatone-takeover" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) self.BaseDomain = BaseDomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-r", "--rescan", help="Run aquatone on hosts that have already been processed.", action="store_true", ) self.options.set_defaults(timeout=None) def get_targets(self, args): """ This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [(target, output), (target, output), etc, etc] """ targets = [] if args.import_database: if args.rescan: all_domains = self.BaseDomain.all(scope_type="passive") else: all_domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in all_domains: # We need to find all of the http/https ports and create the json file. output_path = os.path.join( self.base_config["PROJECT"]["base_path"], "output", "aquatone", d.domain, ) if not os.path.exists(output_path): os.makedirs(output_path) hosts_j = {} hosts = [] open_ports = [] urls = [] targets.append(d.domain) for s in d.subdomains: name = s.domain for ip in s.ip_addresses: hosts_j[name] = ip.ip_address port_list = [] for p in ip.ports: if "http" in p.service_name: hosts.append("{}.{}".format( name, ip.ip_address)) port_list.append(p.port_number) urls.append("{}://{}:{}/".format( p.service_name, name, p.port_number)) urls.append("{}://{}:{}/".format( p.service_name, ip.ip_address, p.port_number)) if port_list: open_ports.append("{},{}".format( ip.ip_address, ",".join([str(o) for o in port_list]))) open(os.path.join(output_path, "hosts.txt"), "w").write("\n".join(list(set(hosts)))) open(os.path.join(output_path, "urls.txt"), "w").write("\n".join(list(set(urls)))) open(os.path.join(output_path, "open_ports.txt"), "w").write("\n".join(list(set(open_ports)))) open(os.path.join(output_path, "hosts.json"), "w").write(json.dumps(hosts_j)) else: display_error("You need to supply domain(s).") res = [] for t in targets: res.append({"target": t}) return res def build_cmd(self, args): """ Create the actual command that will be executed. Use {target} and {output} as placeholders. """ cmd = self.binary + " -d {target} " if args.tool_args: cmd += args.tool_args return cmd def pre_run(self, args): output_path = os.path.join(self.base_config["PROJECT"]["base_path"], "output") self.orig_home = os.environ["HOME"] os.environ["HOME"] = output_path def process_output(self, cmds): """ Process the output generated by the earlier commands. """ for cmd in cmds: created, domain = self.BaseDomain.find_or_create( domain=cmd['target']) domain.set_tool(self.name) self.BaseDomain.commit() def post_run(self, args): display("Potential takeovers are stored in {}".format( os.environ["HOME"])) os.environ["HOME"] = self.orig_home
class Module(ToolTemplate): ''' PyMeta is a tool used for searching domains on various search engines, finding all of the relevant documents, and raiding the exif data to find users. ''' name = "PyMeta" binary_name = "pymeta" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to brute force") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: targets.append(d) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:] ) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path ) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output = os.path.join(output_path) res.append({"target": t, "output": output}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} -csv -f full " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd["output"] created, domain_obj = self.BaseDomain.find_or_create(domain=cmd["target"]) try: csvreader = csv.reader(open(os.path.join(cmd["output"], "pymeta_{}.csv".format(cmd["target"])))) if sys.version[0] == '2': headers = csvreader.next() else: headers = csvreader.__next__() searchable_headers = ["Author", "Creator", "Producer"] indexes = [headers.index(s) for s in searchable_headers if s in headers] data = [] for row in csvreader: for i in indexes: data.append(row[i]) data = list(set(data)) # Dedup for d in data: # pdb.set_trace() if d.strip() and len(d.split(' '))==2: res = raw_input("Is %s a valid name? [y/N] " % d) if res and res[0].lower() == "y": if " " in d: if ", " in d: first_name = d.split(", ")[1] last_name = d.split(", ")[0] else: first_name = d.split(" ")[0] last_name = " ".join(d.split(" ")[1:]) created, user = self.User.find_or_create( first_name=first_name, last_name=last_name ) if created: print("New user created") user.domain = domain_obj elif '@' in d: res = raw_input("Is %s a valid email address? [y/N] " % d) if res and res[0].lower() == 'y': created, user = self.User.find_or_create( email=d.strip()) if created: print("New user created") user.domain = domain_obj except IOError: pass except Exception as e: display_error("Error processing pymeta_{}.csv: {}".format(cmd["target"], e)) self.User.commit()
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name)
class Module(ToolTemplate): name = "Whois" binary_name = "whois" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to query") self.options.add_argument("-c", "--cidr", help="CIDR to query") self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) self.options.add_argument( "-a", "--all_data", help="Scan all data in database, regardless of scope", action="store_true", ) self.options.add_argument( "-i", "--import_database", help="Run WHOIS on all domains and CIDRs in database", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: targets.append({"domain": args.domain, "cidr": ""}) elif args.cidr: targets.append({"domain": "", "cidr": args.cidr.split("/")[0]}) elif args.import_database: if args.all_data: scope_type = "" else: scope_type = "passive" if args.rescan: domains = self.BaseDomain.all(scope_type=scope_type) cidrs = self.ScopeCidr.all() else: domains = self.BaseDomain.all(scope_type=scope_type, tool=self.name) cidrs = self.ScopeCidr.all(tool=self.name) for domain in domains: targets.append({"domain": domain.domain, "cidr": ""}) for cidr in cidrs: targets.append({"domain": "", "cidr": cidr.cidr.split("/")[0]}) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t["output"] = os.path.join(output_path, t["domain"] + t["cidr"]) return targets def build_cmd(self, args): if not args.tool_args: args.tool_args = "" cmd = ( 'bash -c "' + self.binary # noqa: W503 + " {domain}{cidr} " # noqa: W503 + args.tool_args # noqa: W503 + '> {output}" ' # noqa: W503 ) return cmd def process_output(self, cmds): display("Importing data to database") for cmd in cmds: if cmd["cidr"]: _, cidr = self.ScopeCidr.find_or_create(cidr=cmd["cidr"]) cidr.meta["whois"] = open(cmd["output"]).read() display(cidr.meta["whois"]) cidr.update() elif cmd["domain"]: _, domain = self.BaseDomain.find_or_create( domain=cmd["domain"]) domain.meta["whois"] = open(cmd["output"]).read() display(domain.meta["whois"]) domain.update() self.BaseDomain.commit()
def __init__(self, db): self.BaseDomain = BaseDomainRepository(db) self.Domain = DomainRepository(db) self.IPAddress = IPRepository(db) self.CIDR = CIDRRepository(db)
class Module(ToolTemplate): ''' This module uses the Ruby version of Aquatone. You can usually install it with "gem install aquatone" ''' name = "aquatone-discover" binary_name = "aquatone-discover" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) self.BaseDomain = BaseDomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Target domain for aquatone") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-r", "--rescan", help="Run aquatone on hosts that have already been processed.", action="store_true", ) self.options.set_defaults(timeout=None) def get_targets(self, args): """ This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [(target, output), (target, output), etc, etc] """ targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domainsFile = open(args.file).read().split("\n") for d in domainsFile: if d: created, domain = self.BaseDomain.find_or_create(domain=d) targets.append(domain.domain) elif args.import_database: if args.rescan: all_domains = self.BaseDomain.all(scope_type="passive") else: all_domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in all_domains: targets.append(d.domain) else: print("You need to supply domain(s).") output_path = os.path.join(self.base_config["PROJECT"]["base_path"], "output", "aquatone") if not os.path.exists(output_path): os.makedirs(output_path) res = [] for t in targets: res.append({ "target": t, "output": "{}/{}/hosts.json".format(output_path, t) }) return res def build_cmd(self, args): """ Create the actual command that will be executed. Use {target} and {output} as placeholders. """ cmd = self.binary + " -d {target} " if args.tool_args: cmd += args.tool_args return cmd def pre_run(self, args): output_path = os.path.join(self.base_config["PROJECT"]["base_path"], "output") self.orig_home = os.environ["HOME"] os.environ["HOME"] = output_path def process_output(self, cmds): """ Process the output generated by the earlier commands. """ for cmd in cmds: try: data2 = json.loads(open(cmd["output"]).read()) for sub, ip in data2.items(): created = False new_domain = sub.lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) except Exception as e: display_error("Couldn't find file: {}".format(cmd["output"])) self.Domain.commit() def post_run(self, args): os.environ["HOME"] = self.orig_home
class Module(ModuleTemplate): """ Unfortunately, this is a Windows only deal. So we output a list of commands to be run, and have a way of ingesting the output. Since metadata is more junk then good data, items have to be manually verified """ name = "PowerMeta" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-c", "--command", help="Print out a list of commands to run", action="store_true", ) self.options.add_argument( "-ps", "--powershell", help="Filename for a .ps1 command output" ) self.options.add_argument( "-i", "--import_file", help="import CSV file generated by PowerMeta" ) self.options.add_argument( "-f", "--import_folder", help="import CSV files generated by PowerMeta" ) def run(self, args): # pdb.set_trace() if args.command: res = [] res.append("Import-Module PowerMeta.ps1") res += [ "Invoke-PowerMeta -TargetDomain %s -Download -Extract -ExtractAllToCsv %s.csv" % (b.domain, b.domain) for b in self.BaseDomain.all() ] if args.powershell: open(args.powershell, "w").write("\r\n".join(res)) print("Command saved as %s" % args.powershell) else: print("\n".join(res)) if args.import_file: self.process_domain(args.import_file, args) if args.import_folder: if args.import_folder[-1] == "/": args.import_folder = args.import_folder[:-1] files = glob.glob(args.import_folder + "/*.csv") for f in files: print("Processing %s" % f) self.process_domain(f, args) def process_domain(self, csvfile, args): domain = csvfile.split("/")[-1].split(".csv")[0] domain_obj = self.BaseDomain.find(domain=domain) if not domain_obj: print("Error: Domain not found: %s" % domain) return csvdata = open(csvfile).read().decode("utf-16").encode("utf-8") csvreader = csv.reader(BytesIO(csvdata), delimiter=",", quotechar='"') headers = csvreader.next() searchable_headers = ["Author", "Creator"] indexes = [headers.index(s) for s in searchable_headers if s in headers] data = [] for row in csvreader: for i in indexes: data.append(row[i]) data = list(set(data)) # Dedup for d in data: if d.strip(): res = six.input("Is %s a valid name? [y/N] " % d) if res and res[0].lower() == "y": if " " in d: if ", " in d: first_name = d.split(", ")[1] last_name = d.split(", ")[0] else: first_name = d.split(" ")[0] last_name = " ".join(d.split(" ")[1:]) created, user = self.User.find_or_create( first_name=first_name, last_name=last_name ) if created: print("New user created") user.domain = domain_obj # else: # created, user = self.User.find_or_create(username=d) # if created: # print("New username created") # user.domain = domain_obj user.update() self.User.commit()
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name)
class Module(ToolTemplate): ''' This module uses Fierce, a Perl based domain brute forcing tool. It can be installed from https://github.com/davidpepper/fierce-domain-scanner ''' name = "Fierce" binary_name = "fierce" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Target domain for Fierce") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "--rescan", help="Force rescan of already scanned domains", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) if args.file: domains = open(args.file).read().split("\n") for d in domains: if d: targets.append(d) if args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for domain in domains: targets.append(domain.domain) if args.output_path[0] == "/": self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) res = [] for t in targets: res.append({ "target": t, "output": os.path.join(self.path, t + ".txt") }) return res def build_cmd(self, args): command = self.binary command += " -dns {target} -file {output} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): for c in cmds: target = c["target"] output_path = c["output"] try: fierceOutput = open(output_path).read() except IOError: display_error( "The output file for {} was not found. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file." .format(target)) continue domains = [] if "Now performing" in fierceOutput: hosts = re.findall( r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$", fierceOutput, re.MULTILINE, ) if hosts: for host in hosts: # print host domain = (host.split("\t")[1].lower().replace( "www.", "").rstrip(".")) if domain not in domains: domains.append(domain) elif "Whoah, it worked" in fierceOutput: print("Zone transfer found!") hosts = re.findall( r".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", fierceOutput, re.MULTILINE, ) if hosts: for host in hosts: domain = (host.split("\t")[0].lower().replace( "www.", "").rstrip(".")) if domain not in domains: domains.append(domain) else: display_error( "No results found in {}. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file." .format(output_path)) if domains: for _domain in domains: created, domain = self.Domain.find_or_create( domain=_domain)
def __init__(self, db): self.BaseDomain = BaseDomainRepository(db) self.User = UserRepository(db) self.Cred = CredRepository(db)
class Report(ReportTemplate): """ This report displays data related to discovered user accounts. """ name = "UserReport" markdown = ["-", "--"] def __init__(self, db): self.BaseDomain = BaseDomainRepository(db) self.User = UserRepository(db) self.Cred = CredRepository(db) def set_options(self): super(Report, self).set_options() self.options.add_argument( "-u1", "--usernames_passwords", help="Prints out username/password pairs", action="store_true", ) self.options.add_argument( "-u2", "--emails_passwords", help="Prints out email/password pairs", action="store_true", ) self.options.add_argument("-u3", "--emails", help="Prints out e-mail addresses", action="store_true") self.options.add_argument("-u4", "--accounts", help="Prints out user accounts", action="store_true") self.options.add_argument("-u5", "--full", help="Prints out full user data", action="store_true") self.options.add_argument('-t', '--title', help="Add title to the sections of results", action="store_true") def run(self, args): results = [] qry, model = self.BaseDomain.get_query() if args.scope == 'active': domains = qry.filter_by(in_scope=True).order_by(model.domain).all() elif args.scope == 'passive': domains = qry.filter_by(passive_scope=True).order_by( model.domain).all() else: domains = qry.order_by(model.domain).all() for d in domains: if args.title and d.users: results.append('{}'.format(d.domain)) if args.emails: emails = sorted([u.email.lower() for u in d.users if u.email]) if args.title: results += ['\t{}'.format(e) for e in emails] else: results += ['{}'.format(e) for e in emails] elif args.accounts: emails = sorted([u.email.lower() for u in d.users if u.email]) if args.title: results += ['\t{}'.format(e.split("@")[0]) for e in emails] else: results += ['{}'.format(e.split("@")[0]) for e in emails] elif args.full: for user in d.users: results.append("{}|{}|{}|{}".format( user.first_name, user.last_name, user.email, user.job_title)) else: for user in d.users: for cred in user.creds: if cred.password and cred.password != "None": if args.emails_passwords: txt = "%s:%s" % (user.email, cred.password) elif args.usernames_passwords: txt = "%s:%s" % ( user.email.split("@")[0], cred.password, ) if txt not in results: results.append(txt) self.process_output(results, args)
class Module(ToolTemplate): name = "TheHarvester" binary_name = "theharvester" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to harvest") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: targets.append({"target": args.domain}) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) targets.append({"target": domain.domain}) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append({"target": d.domain}) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t["output"] = os.path.join( output_path, "%s-theharvester" % t["target"].replace(".", "_")) return targets def build_cmd(self, args): cmd = self.binary + " -f {output} -b default -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: try: data = xmltodict.parse(open(cmd["output"] + ".xml").read()) except Exception as e: # display_error("Error with {}: {}".format(cmd["output"], e)) data = None if data: if data["theHarvester"].get("email", False): if type(data["theHarvester"]["email"]) == list: emails = data["theHarvester"]["email"] else: emails = [data["theHarvester"]["email"]] for e in emails: display("Processing E-mail: {}".format(e)) created, user = self.User.find_or_create(email=e) _, domain = self.BaseDomain.find_or_create( domain=e.split("@")[1]) user.domain = domain user.update() if created: display_new("New email: %s" % e) if data["theHarvester"].get("host", False): if type(data["theHarvester"]["host"]) == list: hosts = data["theHarvester"]["host"] else: hosts = [data["theHarvester"]["host"]] for d in hosts: created, domain = self.Domain.find_or_create( domain=d["hostname"]) if data["theHarvester"].get("vhost", False): if type(data["theHarvester"]["vhost"]) == list: hosts = data["theHarvester"]["vhost"] else: hosts = [data["theHarvester"]["vhost"]] for d in hosts: created, domain = self.Domain.find_or_create( domain=d["hostname"]) self.BaseDomain.commit()
class Module(ToolTemplate): name = "Sublist3r" binary_name = "sublist3r" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to brute force") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: targets.append(d) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output = os.path.join(output_path, "%s-sublist3r.txt" % t) res.append({"target": t, "output": output}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd["output"] if os.path.isfile(output_path): data = open(output_path).read().split("\n") for d in data: new_domain = d.split(":")[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) else: display_error("{} not found.".format(output_path)) next self.Domain.commit()
class Module(ToolTemplate): name = "Subfinder" binary_name = "subfinder" def __init__(self, db): self.db = db self.BaseDomains = BaseDomainRepository(db, self.name) self.Domains = DomainRepository(db, self.name) self.IPs = IPRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to run subfinder against.") self.options.add_argument( "-dL", "--domain_list", help="Read in a list of domains within the given file.", ) self.options.add_argument( "-i", "--db_domains", help="Import the domains from the database.", action="store_true", ) self.options.add_argument("--rescan", help="Overwrite files without asking", action="store_true") def get_targets(self, args): targets = [] outpath = "" if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) if args.domain: out_file = os.path.join(outpath, "{}.subfinder".format(args.domain)) targets.append({ "target": args.domain, "output": os.path.join(output_path, out_file) }) if args.db_domains: if args.rescan: domains = self.BaseDomains.all(scope_type="passive") else: domains = self.BaseDomains.all(tool=self.name, scope_type="passive") for d in domains: out_file = os.path.join(outpath, "{}.subfinder".format(d.domain)) targets.append({ "target": d.domain, "output": os.path.join(output_path, out_file) }) elif args.domain_list: domains = io.open(args.domain_list, encoding="utf-8").read().split("\n") for d in domains: if d: targets.append({ "target": d, "output": os.path.join(output_path, "{}.subfinder".format(d)), }) return targets def build_cmd(self, args): if args.binary: cmd = "{} ".format(args.binary) else: cmd = "{} ".format(self.binary_name) cmd = "{} -o {} -d {}".format(cmd, "{output}", "{target}") return cmd def process_output(self, targets): for target in targets: try: with io.open(target["output"], encoding="utf-8") as fd: for line in fd: domain = line.strip() if domain[0] == '.': domain = domain[1:] ips = get_domain_ip.run(domain) ip_obj = None _, dom = self.Domains.find_or_create(domain=domain) if ips: for ip in ips: _, ip_obj = self.IPs.find_or_create( ip_address=ip) if ip_obj: dom.ip_addresses.append(ip_obj) dom.save() except FileNotFoundError: display_error("File doesn't exist for {}".format( target["output"])) self.BaseDomains.commit() self.IPs.commit() def post_run(self, args): # Remove the temporary db file if it was created. if getattr(self, "db_domain_file", None): try: os.unlink(self.db_domain_file) except IOError as e: print("Failed to remove the Subfinder db temp file: '{}'.". format(e)) def __get_tempfile(self, domain=None, args=None): # Create a temporary file and place all of the current database domains within the file. from tempfile import NamedTemporaryFile with NamedTemporaryFile(delete=False) as fd: if domain: fd.write("{}\n".format(domain).encode("utf-8")) else: # Go through the database and grab the domains adding them to the file. if args.rescan: domains = self.BaseDomains.all(passive_scope=True) else: domains = self.BaseDomains.all(tool=self.name, passive_scope=True) if domains: for domain in domains: fd.write("{}\n".format(domain.domain).encode("utf-8")) else: return None return fd.name
def __init__(self, db): self.BaseDomain = BaseDomainRepository(db) self.ScopeCIDR = ScopeCIDRRepository(db)
class Module(ToolTemplate): ''' This module uses Gobuster in the DNS brute forcing mode. Gobuster can be installed from: https://github.com/OJ/gobuster ''' name = "GobusterDNS" binary_name = "gobuster" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to brute force") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been brute forced", action="store_true", ) self.options.set_defaults( timeout=600) # Kick the default timeout to 10 minutes def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) if args.file: domains = open(args.file).read().split("\n") for d in domains: if d: targets.append(d) if args.import_database: if args.rescan: targets += [ b.domain for b in self.BaseDomain.all(scope_type="passive") ] else: targets += [ b.domain for b in self.BaseDomain.all(scope_type="passive", tool=self.name) ] if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) res = [] for t in targets: res.append({ "target": t, "output": os.path.join(output_path, t.replace("/", "_") + "-dns.txt"), }) return res def build_cmd(self, args): cmd = self.binary + " dns " cmd += " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for c in cmds: output_path = c["output"] if os.path.isfile(output_path): data = open(output_path).read().split("\n") for d in data: if "Found: " in d: new_domain = d.split(" ")[1].lower() created, subdomain = self.Domain.find_or_create( domain=new_domain) else: display_error("{} not found.".format(output_path)) created, bd = self.BaseDomain.find_or_create(domain=c['target']) bd.set_tool(self.name) self.Domain.commit()
class Report(ReportTemplate): """ Displays WHOIS records for domains and IPs. """ name = "WhoisReport" markdown = ["##", "###", "`"] def __init__(self, db): self.BaseDomain = BaseDomainRepository(db) self.ScopeCIDR = ScopeCIDRRepository(db) def run(self, args): # Cidrs = self.CIDR. results = [] if args.scope != "all": domains = self.BaseDomain.all(scope_type=args.scope) else: domains = self.BaseDomain.all() domain_data = {} for d in domains: if d.meta.get("whois", False): domain_data[d.domain] = d.meta["whois"] cidr_data = {} CIDRs = self.ScopeCIDR.all() for c in CIDRs: if c.meta.get("whois", False): cidr_data[c.cidr] = c.meta["whois"] # pdb.set_trace() domain_blacklist = [ "Please note: ", "URL of the ICANN WHOIS", ">>>", "Notes:", "whitelisting here:", "NOTICE: ", "TERMS OF USE: ", "by the following", "to: (1) allow", ] results.append("ARIN Registration") for c in sorted(cidr_data.keys()): results.append("\t" + c) for l in cidr_data[c].split("\n"): if ": " in l and l[0] != "#": results.append("\t\t" + l) results.append("Domain Registration") for d in sorted(domain_data.keys()): results.append("\t" + d.upper()) for l in domain_data[d].split("\n"): if ": " in l: clean = True for b in domain_blacklist: if b in l: clean = False if clean: results.append("\t\t" + l) self.process_output(results, args)
class Module(ToolTemplate): name = "DNSRecon" binary_name = "dnsrecon" """ This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries. It can also run over IP ranges, looking for additional domains in the PTR records. DNSRecon can be installed from https://github.com/darkoperator/dnsrecon """ def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Target domain for dnsRecon") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument("-r", "--range", help="Range to scan for PTR records") self.options.add_argument( "-R", "--import_range", help="Import CIDRs from in-scope ranges in database", action="store_true", ) self.options.add_argument("--rescan", help="Rescan domains already scanned", action="store_true") # self.options.add_argument('--import_output_xml', help="Import XML file") # self.options.add_argument('--import_output_json', help="Import json file") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain, passive_scope=True) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: created, domain = self.BaseDomain.find_or_create( domain=d, passive_scope=True) targets.append(domain.domain) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(scope_type="passive", tool=self.name) for domain in domains: targets.append(domain.domain) elif args.range: targets.append(args.range) elif args.import_range: if args.rescan: cidrs = self.ScopeCIDR.all() else: cidrs = self.ScopeCIDR.all(tool=self.name) for cidr in cidrs: targets.append(cidr.cidr) if args.output_path[0] == "/": self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) res = [] for t in targets: res.append({ "target": t, "output": os.path.join(self.path, t.replace("/", "_") + ".json"), }) return res def build_cmd(self, args): command = self.binary if args.domain or args.file or args.import_database: command += " -d {target} -j {output} " else: command += " -s -r {target} -j {output} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): for c in cmds: target = c["target"] output_path = c["output"] try: res = json.loads(open(output_path).read()) except IOError: display_error("DnsRecon failed for {}".format(target)) continue if " -d " in res[0]["arguments"]: created, dbrec = self.Domain.find_or_create(domain=target) dbrec.dns = res dbrec.save() for record in res: domain = None ip = None if record.get("type") == "A" or record.get("type") == "PTR": domain = record.get("name").lower().replace("www.", "") ip = record.get("address") elif record.get("type") == "MX": domain = record.get("exchange").lower().replace("www.", "") elif record.get("type") == "SRV" or record.get("type" == "NS"): domain = record.get("target").lower().replace("www.", "") elif record.get("type") == "SOA": domain = record.get("mname").lower().replace("www.", "") if domain: created, domain_obj = self.Domain.find_or_create( domain=domain) if ip: created, ip_obj = self.IPAddress.find_or_create( ip_address=ip) domain_obj.ip_addresses.append(ip_obj) domain_obj.save() if '/' in target: created, bd = self.ScopeCIDR.find_or_create(cidr=target) else: created, bd = self.BaseDomain.find_or_create(domain=target) bd.set_tool(self.name) self.Domain.commit()
class Module(ModuleTemplate): """ Ingests domains and IPs. Domains get ip info and cidr info, and IPs get CIDR info. """ name = "Ingestor" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-d", "--import_domains", help="Either domain to import or file containing domains to import. One per line", ) self.options.add_argument( "-i", "--import_ips", help="Either IP/range to import or file containing IPs and ranges, one per line.", ) self.options.add_argument( "-a", "--active", help="Set scoping on imported data as active", action="store_true", ) self.options.add_argument( "-p", "--passive", help="Set scoping on imported data as passive", action="store_true", ) self.options.add_argument( "-sc", "--scope_cidrs", help="Cycle through out of scope networks and decide if you want to add them in scope", action="store_true", ) self.options.add_argument( "-sb", "--scope_base_domains", help="Cycle through out of scope base domains and decide if you want to add them in scope", action="store_true", ) self.options.add_argument("--descope", help="Descope an IP, domain, or CIDR") self.options.add_argument( "-Ii", "--import_database_ips", help="Import IPs from database", action="store_true", ) self.options.add_argument( "--force", help="Force processing again, even if already processed", action="store_true", ) def run(self, args): self.in_scope = args.active self.passive_scope = args.passive if args.descope: if "/" in args.descope: self.descope_cidr(args.descope) elif check_string(args.descope): pass else: self.descope_ip(args.descope) # Check if in ScopeCIDR and remove if found if args.import_ips: try: ips = open(args.import_ips) for line in ips: if line.strip(): if "/" in line or "-" in line: self.process_cidr(line) else: self.process_ip(line.strip(), force_scope=True) self.Domain.commit() except IOError: if "/" in args.import_ips or "-" in args.import_ips: self.process_cidr(args.import_ips) else: self.process_ip(args.import_ips.strip(), force_scope=True) self.Domain.commit() if args.import_domains: try: domains = open(args.import_domains) for line in domains: if line.strip(): self.process_domain(line.strip()) self.Domain.commit() except IOError: self.process_domain(args.import_domains.strip()) self.Domain.commit() if args.scope_base_domains: base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False) for bd in base_domains: self.reclassify_domain(bd) self.BaseDomain.commit() def get_domain_ips(self, domain): ips = [] try: answers = dns.resolver.query(domain, "A") for a in answers: ips.append(a.address) return ips except Exception: return [] def process_domain(self, domain_str): created, domain = self.Domain.find_or_create( only_tool=True, domain=domain_str, in_scope=self.in_scope, passive_scope=self.passive_scope, ) if not created: if ( domain.in_scope != self.in_scope or domain.passive_scope != self.passive_scope # noqa: W503 ): display( "Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (domain_str, self.in_scope, self.passive_scope) ) domain.in_scope = self.in_scope domain.passive_scope = self.passive_scope domain.update() if domain.base_domain.domain == domain.domain: display("Name also matches a base domain. Updating that as well.") domain.base_domain.in_scope = self.in_scope domain.base_domain.passive_scope = self.passive_scope domain.base_domain.update() def process_ip(self, ip_str, force_scope=True): created, ip = self.IPAddress.find_or_create( only_tool=True, ip_address=ip_str, in_scope=self.in_scope, passive_scope=self.passive_scope, ) if not created: if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope: display( "IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (ip_str, self.in_scope, self.passive_scope) ) ip.in_scope = self.in_scope ip.passive_scope = self.passive_scope ip.update() return ip def process_cidr(self, line): display("Processing %s" % line) if "/" in line: created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip()) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() elif "-" in line: start_ip, end_ip = line.strip().replace(" ", "").split("-") if "." not in end_ip: end_ip = ".".join(start_ip.split(".")[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c)) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() def reclassify_domain(self, bd): if bd.meta.get("whois", False): display_new("Whois data found for {}".format(bd.domain)) print(bd.meta["whois"]) res = six.input( "Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] " ) if res.lower() == "a": bd.in_scope = True bd.passive_scope = True elif res.lower() == "p": bd.in_scope = False bd.passive_scope = True else: bd.in_scope = False bd.passive_scope = False bd.save() else: display_error( "Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format( bd.domain ) ) def descope_ip(self, ip): ip = self.IPAddress.all(ip_address=ip) if ip: for i in ip: display("Removing IP {} from scope".format(i.ip_address)) i.in_scope = False i.passive_scope = False i.update() for d in i.domains: in_scope_ips = [ ipa for ipa in d.ip_addresses if ipa.in_scope or ipa.passive_scope ] if not in_scope_ips: display( "Domain {} has no more scoped IPs. Removing from scope.".format( d.domain ) ) d.in_scope = False d.passive_scope = False self.IPAddress.commit() def descope_cidr(self, cidr): CIDR = self.ScopeCIDR.all(cidr=cidr) if CIDR: for c in CIDR: display("Removing {} from ScopeCIDRs".format(c.cidr)) c.delete() cnet = IPNetwork(cidr) for ip in self.IPAddress.all(): if IPAddress(ip.ip_address) in cnet: self.descope_ip(ip.ip_address)