def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name)
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name)
class Report(ReportTemplate): """ This lists who owns the CIDR for each of the domains. """ name = "DomainOwner" def __init__(self, db): self.Domain = DomainRepository(db) self.IPAddress = IPRepository(db) self.CIDR = CIDRRepository(db) def run(self, args): # Cidrs = self.CIDR. results = [] domains = self.Domain.all() for d in domains: if len(d.ip_addresses) > 0: owner = d.ip_addresses[0].cidr.org_name results.append("%s,%s" % (owner, d.domain)) res = sorted(results) self.process_output(res, args)
class Report(ReportTemplate): """ This is a domain summary report. It shows base domain, then all of the subdomains, along with any resolved IPs. """ name = "DomainSummaryReport" markdown = ["###", "-"] def __init__(self, db): self.Domain = DomainRepository(db, self.name) def run(self, args): # Cidrs = self.CIDR. results = [] if args.scope == 'all': args.scope == None domains = self.Domain.all(scope_type=args.scope) domain_data = {} for d in domains: if not domain_data.get(d.base_domain.domain, False): domain_data[d.base_domain.domain] = {} domain_data[d.base_domain.domain][d.domain] = [ i.ip_address for i in d.ip_addresses if (i.in_scope == True and args.scope == 'active') or (i.passive_scope and args.scope == 'passive') or not args.scope ] for b in sorted(domain_data.keys()): results.append(b) for d in sorted(domain_data[b].keys()): results.append('\t{} ({})'.format(d, ', '.join( domain_data[b][d]))) self.process_output(results, args)
class Module(ModuleTemplate): name = "Nmap" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( '--hosts', help= "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS", nargs='+') self.options.add_argument('--hosts_file', help="File containing hosts") self.options.add_argument('--hosts_database', help="Use unscanned hosts from the database", action="store_true") self.options.add_argument('--import_file', help="Import nmap XML file") self.options.add_argument('-A', help="OS and service info", action="store_true") self.options.add_argument( '-o', '--output_path', help="Relative directory to store Nmap XML output name", default="nmap") self.options.add_argument('-nf', '--nFile', help="Nmap XML output name") self.options.add_argument( '-T', '--timing', help="Set timing template (higher is faster)", default="4", type=str) self.options.add_argument( '--scripts', help="Nmap scripts", default= 'ssl-cert,http-headers,http-methods,http-auth,http-title,http-robots.txt,banner' ) self.options.add_argument( '-p', help="Comma separate ports to scan", default= "21,22,23,25,80,110,443,467,587,8000,8080,8081,8082,8443,8008,1099,5005,9080,8880,8887,7001,7002,16200" ) self.options.add_argument('-Pn', help="Disable ping", action="store_true", default=False) self.options.add_argument('-sS', help='Syn scan (default)', action="store_true", default=True) self.options.add_argument('-sT', help='TCP scan', action="store_true", default=False) self.options.add_argument('-sU', help='UDP scan', action="store_true", default=False) self.options.add_argument('-OS', help='Enable OS detection', action="store_true", default=False) self.options.add_argument("--open", help="Only show open ports", action="store_true", default=False) self.options.add_argument("--top_ports", help="Only check X top ports") self.options.add_argument("--force", help="Overwrite files without asking", action="store_true") self.options.add_argument( '--interactive', help="Prompt to store domains not in Base Domains already", default=False, action="store_true") self.options.add_argument( '--internal', help="Store domains not in Base Domains already", action="store_true") def run(self, args): if args.binary: if os.path.exists(args.binary): command = args.binary + " " else: exit("Specified binary doesn't exist. Quitting now") else: if not which.run('nmap'): exit("Nmap is not globally accessible. Quitting now") else: command = "nmap " if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) file_name = "" if args.hosts: if type(args.hosts) == list: hosts = args.hosts else: hosts = [args.hosts] _, file_name = tempfile.mkstemp() open(file_name, 'w').write('\n'.join(hosts)) elif args.hosts_database: hosts = [h.ip_address for h in self.IPAddress.all(tool=self.name)] hosts += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)] _, file_name = tempfile.mkstemp() open(file_name, 'w').write('\n'.join(hosts)) elif args.hosts_file: file_name = args.hosts_file if file_name: self.execute_nmap(args, file_name, command) elif args.import_file: self.import_nmap(args.import_file, args) def execute_nmap(self, args, host_file, command): if args.nFile: nFile = os.path.join(self.path, args.nFile) else: nFile = os.path.join(self.path, "nmap-scan.xml") # if type(hosts) == list: # nFile = os.path.join(self.path,"nmaped_"+",".join(hosts).replace("/","_")) # else: # nFile = os.path.join(self.path,"nmaped_"+hosts.replace("/","_").replace(" ",",")) if os.path.isfile(nFile) and not args.force: print(nFile, "exists.") answered = False while answered == False: rerun = raw_input( "Would you like to [r]un nmap again and overwrite the file, [p]arse the file, or change the file [n]ame? " ) if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return nFile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists(path + newFile): nFile = path + newFile answered = True new = True else: print("That file exists as well") else: "Please enter \'r\' to run nmap, \'p\' to parse the file" if args.sS and args.sT: technique = "-sT " else: technique = "-sS " if args.sU: technique += "-sU " if args.A: technique += "-A " command += technique command += "-T" + str(args.timing) + " " if args.Pn: command += "-Pn " if args.open: command += "--open " if args.top_ports: command += "--top-ports %s " % args.top_ports if args.OS: command += "-O " command += "--script " + args.scripts + " " command += "-p " + args.p + " " command += " -iL %s " % host_file command += "-oX " + nFile if (args.sS and not args.sT) or args.OS: if os.geteuid() != 0: #exit("You need to have root privileges to run a Syn scan and OS detection.\nPlease try again, this time using 'sudo'. Exiting.") command = "sudo " + command scan = subprocess.Popen(command, shell=True).wait() self.import_nmap(nFile, args) return nFile def parseHeaders(self, httpHeaders): bsHeaders = [ 'Pragma', 'Expires', 'Date', 'Transfer-Encoding', 'Connection', 'X-Content-Type-Options', 'Cache-Control', 'X-Frame-Options', 'Content-Type', 'Content-Length', '(Request type' ] keepHeaders = {} for i in range(0, len(httpHeaders)): if httpHeaders[i].strip() != '' and httpHeaders[i].split( ":")[0].strip() not in " ".join(bsHeaders): hName = httpHeaders[i].split(":")[0].strip() hValue = "".join(httpHeaders[i].split(":")[1:]).strip() keepHeaders[hName] = hValue if keepHeaders == {}: keepHeaders = "" return keepHeaders def import_nmap(self, filename, args): #domains={}, ips={}, rejects=[] == temp while no db nFile = filename try: tree = ET.parse(nFile) root = tree.getroot() hosts = root.findall("host") except: print(nFile + " doesn't exist somehow...skipping") return domains, ips, rejects #needs to be changed for db tmpNames = [] tmpIPs = { } #tmpIps = {'127.0.0.1':['domain.com']} -- not really used; decided to just let nslookup take care of IP info skip = [] for host in hosts: hostIP = host.find("address").get("addr") created, ip = self.IPAddress.find_or_create(ip_address=hostIP) for hostname in host.findall("hostnames/hostname"): hostname = hostname.get("name") hostname = hostname.lower().replace("www.", "") reHostname = re.search( r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname) #attempt to not get PTR record if not reHostname and not args.internal: created, domain = self.Domain.find_or_create( domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.save() elif not reHostname and args.internal: created, domain = self.Domain.find_or_create( domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.save() #else: # print("IP hostname found? %s" % hostname) for port in host.findall("ports/port"): if port.find("state").get("state"): portState = port.find("state").get("state") hostPort = port.get("portid") portProto = port.get("protocol") created, db_port = self.Port.find_or_create( port_number=hostPort, status=portState, proto=portProto, ip_address=ip) if port.find("service") != None: portName = port.find("service").get("name") if portName == "http" and hostPort == "443": portName = "https" else: portName = "Unknown" if created: db_port.service_name = portName info = db_port.info if not info: info = {} for script in port.findall( "script"): #just getting commonName from cert if script.get("id") == "ssl-cert": db_port.cert = script.get('output') cert_domains = self.get_domains_from_cert( script.get('output')) for hostname in cert_domains: hostname = hostname.lower().replace("www.", "") created, domain = self.Domain.find_or_create( domain=hostname) if created: print("New domain found: %s" % hostname) elif script.get("id") == "vulners": print( "Gathering vuln info for {} : {}/{}\n".format( hostIP, portProto, hostPort)) self.parseVulners(script.get("output"), db_port) elif script.get("id") == "banner": info["banner"] = script.get("output") elif script.get("id") == "http-headers": httpHeaders = script.get("output") httpHeaders = httpHeaders.strip().split("\n") keepHeaders = self.parseHeaders(httpHeaders) info["http-headers"] = keepHeaders elif script.get("id") == "http-auth": info['http-auth'] = script.get("output") elif script.get("id") == "http-title": info['http-title'] = script.get("output") db_port.info = info db_port.save() self.IPAddress.commit() def parseVulners(self, scriptOutput, db_port): urls = re.findall('(https://vulners.com/cve/CVE-\d*-\d*)', scriptOutput) for url in urls: vuln_refs = [] exploitable = False cve = url.split("/cve/")[1] vulners = requests.get("https://vulners.com/cve/%s" % cve).text exploitdb = re.findall( 'https://www.exploit-db.com/exploits/\d{,7}', vulners) for edb in exploitdb: exploitable = True if edb.split("/exploits/")[1] not in vuln_refs: vuln_refs.append(edb.split("/exploits/")[1]) if not self.CVE.find(name=cve): #print "Gathering CVE info for", cve try: res = json.loads( requests.get('http://cve.circl.lu/api/cve/%s' % cve).text) cveDescription = res['summary'] cvss = float(res['cvss']) findingName = res['oval'][0]['title'] if int(cvss) <= 3: severity = 1 elif (int(cvss) / 2) == 5: severity = 4 else: severity = int(cvss) / 2 if not self.Vulnerability.find(name=findingName): #print "Creating", findingName created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=cveDescription) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploit_reference = {'edb-id': vuln_refs} db_vuln.save() else: #print "modifying",findingName db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploitable = exploitable if db_vuln.exploit_reference is not None: if 'edb-id' in db_vuln.exploit_reference: for ref in vuln_refs: if ref not in db_vuln.exploit_reference[ 'edb-id']: db_vuln.exploit_reference[ 'edb-id'].append(ref) else: db_vuln.exploit_reference[ 'edb-id'] = vuln_refs else: db_vuln.exploit_reference = { 'edb-id': vuln_refs } db_vuln.save() if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss) db_cve.vulnerabilities.append(db_vuln) db_cve.save() else: db_cve = self.CVE.find(name=cve) db_cve.vulnerabilities.append(db_vuln) db_cve.save() self.Vulnerability.commit() self.CVE.commit() except: print( "something went wrong with the vuln/cve info gathering" ) if vulners: print( "Vulners report was found but no exploit-db was discovered" ) #"Affected vulners items" #print vulners print("Affected CVE") print(cve) pass else: db_cve = self.CVE.find(name=cve) for db_vulns in db_cve.vulnerabilities: if db_port not in db_vulns.ports: db_vulns.ports.append(db_port) return def get_domains_from_cert(self, cert): # Shamelessly lifted regex from stack overflow regex = r'(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}' domains = list( set([d for d in re.findall(regex, cert) if '*' not in d])) return domains
class Module(ModuleTemplate): name = "GobusterDNS" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-fw', '--force_wildcard', help="Continues after hitting wildcard cert", action="store_true") self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-w', '--wordlist', help="Wordlist to use") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default="gobuster") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been brute forced", action="store_true") def run(self, args): if not args.wordlist: print("Wordlist is required!") return if not args.binary: self.binary = which.run('gobuster') else: self.binary = which.run(args.binary) if not self.binary: print( "Gobuster binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.brute_force_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.brute_force_domain(domain, args) self.BaseDomain.commit() elif args.import_database: if args.rescan: domains = self.BaseDomain.all() else: domains = self.BaseDomain.all(tool=self.name) for domain in domains: self.brute_force_domain(domain, args) domain.set_tool(self.name) self.BaseDomain.commit() def brute_force_domain(self, domain_obj, args): domain = domain_obj.domain command_args = " -m dns" if args.force_wildcard: command_args += " -fw" if args.threads: command_args += " -t " + args.threads command_args += " -w " + args.wordlist cmd = shlex.split(self.binary + command_args + " -u " + domain) print("Executing: %s" % ' '.join(cmd)) res = check_output(cmd) if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-dns.txt" % domain) open(output_path, 'w').write(res) data = res.split('\n') for d in data: if 'Found: ' in d: new_domain = d.split(' ')[1].lower() created, subdomain = self.Domain.find_or_create( domain=new_domain) if created: print("New subdomain found: %s" % new_domain)
class Module(ModuleTemplate): name = "Nessus" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--import_file", help="Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument( "--interactive", help="Prompt to store domains not in Base Domains already", action="store_true", ) self.options.add_argument( "--internal", help="Store domains not in Base Domains already", action="store_true", ) self.options.add_argument( "--launch", help="Launch Nessus scan using Actively scoped IPs and domains in the database", action="store_true", ) self.options.add_argument( "--job_name", help="Job name inside Nessus", default="Armory Job" ) self.options.add_argument("--username", help="Nessus Username") self.options.add_argument("--password", help="Nessus Password") self.options.add_argument( "--host", help="Hostname:Port of Nessus web interface (ie localhost:8835" ) self.options.add_argument("--uuid", help="UUID of Nessus Policy to run") self.options.add_argument("--policy_id", help="Policy ID to use") self.options.add_argument("--folder_id", help="ID for folder to store job in") self.options.add_argument( "--download", help="Download Nessus job from server and import", action="store_true", ) self.options.add_argument("--job_id", help="Job ID to download and import") self.options.add_argument( "--output_path", help="Path to store downloaded file (Default: Nessus)", default=self.name, ) def run(self, args): if args.import_file: for nFile in args.import_file: self.process_data(nFile, args) elif args.launch: if ( not args.username and not args.password and not args.host and not args.uuid and not args.policy_id and not args.folder_id ): display_error( "You must supply a username, password, and host to launch a Nessus job" ) else: n = NessusRequest( args.username, args.password, args.host, uuid=args.uuid, policy_id=args.policy_id, folder_id=args.folder_id, ) ips = [ ip.ip_address for ip in self.IPAddress.all(scope_type="active", tool=self.name) ] cidrs = [cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)] domains = [ domain.domain for domain in self.Domain.all(scope_type="active", tool=self.name) ] targets = ", ".join(merge_ranges(ips + cidrs) + domains) res = n.launch_job(targets, args.job_name) display("New Nessus job launched with ID {}".format(res)) display( "Remember this number! You'll need it to download the job once it is done." ) elif args.download: if ( not args.username and not args.password and not args.host and not args.job_id ): display_error( "You must supply host, username, password and job_id to download a report to import" ) else: n = NessusRequest( args.username, args.password, args.host, proxies={"https": "127.0.0.1:8080"}, ) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:] ) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path ) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join( output_path, "Nessus-export-{}.nessus".format(int(time.time())) ) n.export_file(args.job_id, output_path) self.process_data(output_path, args) def nessCheckPlugin(self, tag): nessPlugins = [ "10759", "77026", "20089", "56984", "71049", "70658", "40984", "11411", ] pluginID = tag.get("pluginID") if pluginID in nessPlugins: # print pluginID + " is in the list" if pluginID == "10759": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split("\n\n")[3].strip() ) # returns IP for Web Server HTTP Header INternal IP disclosure else: return "" if pluginID == "77026": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split("\n\n")[3].strip() ) # Microsoft Exchange Client Access Server Information Disclosure (IP addy) else: return "" if pluginID == "71049" or pluginID == "70658": output = "" if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split(":")[ 1 ] # SSH Weak MAC & CBC Algorithms Enabled # print "#"*5 tmp = tmp.split("\n\n")[1].replace(" ", "") # print "#"*5 output = tmp.split("\n") # print ", ".join(output) return ", ".join(output) if pluginID == "56984": if tag.find("plugin_output") is not None: tmp = ( tag.find("plugin_output") .text.split("This port supports ")[1] .strip() ) # SSL / TLS Versions Supported tmp = tmp.split("/") bad = [] for i in tmp: # print i if "SSLv" in i: bad.append(i) elif "TLSv1.0" in i: bad.append(i) if bad != []: return ", ".join(bad).rstrip(".") else: return "" else: return "" if pluginID == "40984": # browsable web dirs if tag.find("plugin_output") is not None: tmp = ( tag.find("plugin_output") .text.split("The following directories are browsable :")[1] .strip() ) directories = tmp.split("\n") return "\n".join(directories) if pluginID == "11411": # Backup Files Disclosure if tag.find("plugin_output") is not None: urls = [] tmp = ( tag.find("plugin_output") .text.split( "It is possible to read the following backup files :" )[1] .strip() ) tmpUrls = tmp.split("\n") for url in tmpUrls: if "URL" in url: urls.append(url.split(":")[1].lstrip()) if urls: return "\n".join(urls) else: return "" if pluginID == "20089": # F5 cookie if tag.find("plugin_output") is not None: f5Output = [] cookieVal = [] output = tag.find("plugin_output").text.strip().split("\n") for line in output: # print line line = line.split(":") for i, item in enumerate(line): item = item.strip() if "Cookie" in item: cabbage = line.pop(i) tmp = line.pop(i) tmp.strip() cookieVal.append(tmp) else: item = "".join(item) f5Output.append(item) f5Output = " : ".join(f5Output) f5Output = f5Output.replace(" : : ", ", ") f5Output += " [" + ", ".join(cookieVal) + "]" c = 0 tmpF5Output = f5Output.split() for i, letter in enumerate(tmpF5Output): if letter == ":": c += 1 if (c % 2) == 0: tmpF5Output[i] = " " return "".join(tmpF5Output).replace("[", " [") else: return "" else: return False def getVulns(self, ip, ReportHost): """Gets vulns and associated services""" for tag in ReportHost.iter("ReportItem"): exploitable = False cves = [] vuln_refs = {} proto = tag.get("protocol") port = tag.get("port") svc_name = tag.get("svc_name").replace("?", "") tmpPort = proto + "/" + port if tmpPort.lower() == "tcp/443": portName = "https" elif tmpPort.lower() == "tcp/80": portName = "http" elif svc_name == "www": plugin_name = tag.get("pluginName") if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(): portName = "https" else: portName = "http" else: portName = svc_name if "general" not in portName: created, db_port = self.Port.find_or_create( port_number=port, status="open", proto=proto, ip_address_id=ip.id ) if db_port.service_name == "http": if portName == "https": db_port.service_name = portName elif db_port.service_name == "https": pass else: db_port.service_name = portName db_port.save() if tag.get("pluginID") == "56984": severity = 1 elif tag.get("pluginID") == "11411": severity = 3 else: severity = int(tag.get("severity")) findingName = tag.get("pluginName") description = tag.find("description").text if tag.find("solution") is not None and tag.find("solution") != "n/a": solution = tag.find("solution").text else: solution = "No Remediation From Nessus" nessCheck = self.nessCheckPlugin(tag) if nessCheck: if not db_port.info: db_port.info = {findingName: nessCheck} else: db_port.info[findingName] = nessCheck db_port.save() if tag.find("exploit_available") is not None: # print "\nexploit avalable for", findingName exploitable = True metasploits = tag.findall("metasploit_name") if metasploits: vuln_refs["metasploit"] = [] for tmp in metasploits: vuln_refs["metasploit"].append(tmp.text) edb_id = tag.findall("edb-id") if edb_id: vuln_refs["edb-id"] = [] for tmp in edb_id: vuln_refs["edb-id"].append(tmp.text) tmpcves = tag.findall("cve") for c in tmpcves: if c.text not in cves: cves.append(c.text) if not self.Vulnerability.find(name=findingName): created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=description, remediation=solution, ) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if exploitable == True: print("\nexploit avalable for", findingName) print() if vuln_refs: db_vuln.exploit_reference = vuln_refs else: db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: if db_vuln.exploit_reference is not None: for key in vuln_refs.keys(): if key not in db_vuln.exploit_reference.keys(): db_vuln.exploit_reference[key] = vuln_refs[key] else: for ref in vuln_refs[key]: if ref not in db_vuln.exploit_reference[key]: db_vuln.exploit_reference[key].append(ref) else: db_vuln.exploit_reference = vuln_refs for cve in cves: if not self.CVE.find(name=cve): # print "Gathering CVE information for", cve try: res = json.loads( requests.get( "http://cve.circl.lu/api/cve/%s" % cve ).text ) cveDescription = res["summary"] cvss = float(res["cvss"]) except: cveDescription = None cvss = None if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss, ) db_cve.vulnerabilities.append(db_vuln) else: db_cve = self.CVE.find(name=cve) if ( db_cve.description is None and cveDescription is not None ): db_cve.description = cveDescription if db_cve.temporal_score is None and cvss is not None: db_cve.temporal_score = cvss db_cve.vulnerabilities.append(db_vuln) def process_data(self, nFile, args): print("Reading", nFile) tree = ET.parse(nFile) root = tree.getroot() skip = [] for ReportHost in root.iter("ReportHost"): os = [] hostname = "" hostIP = "" for HostProperties in ReportHost.iter("HostProperties"): for tag in HostProperties: if tag.get("name") == "host-ip": hostIP = tag.text if tag.get("name") == "host-fqdn": hostname = tag.text.lower() hostname = hostname.replace("www.", "") if tag.get("name") == "operating-system": os = tag.text.split("\n") if hostIP: # apparently nessus doesn't always have an IP to work with... if hostname: display( "Gathering Nessus info for {} ( {} )".format(hostIP, hostname) ) else: display("Gathering Nessus info for {}".format(hostIP)) created, ip = self.IPAddress.find_or_create(ip_address=hostIP) if hostname: if not args.internal: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: ip.save() domain.ip_addresses.append(ip) domain.save() else: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.update() if os: for o in os: if not ip.OS: ip.OS = o else: if o not in ip.OS.split(" OR "): ip.OS += " OR " + o self.getVulns(ip, ReportHost) self.IPAddress.commit() return
def __init__(self, db): self.db = db self.IPAddress = IPRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Url = UrlRepository(db, self.name)
class Module(ModuleTemplate): ''' Ingests domains and IPs. Domains get ip info and cidr info, and IPs get CIDR info. ''' name = "Ingestor" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( '-f', '--import_file', help="File containing domains to import. One per line") self.options.add_argument('-d', '--domain', help="Single domain to import") self.options.add_argument( '-i', '--import_ips', help="File containing IPs and ranges, one per line.") self.options.add_argument('-Id', '--import_database_domains', help='Import domains from database', action="store_true") self.options.add_argument('-Ii', '--import_database_ips', help='Import IPs from database', action="store_true") self.options.add_argument( '--force', help="Force processing again, even if already processed", action="store_true") def run(self, args): if args.import_file: domains = open(args.import_file) for line in domains: if line.strip(): self.process_domain(line.strip(), force_scope=True) self.Domain.commit() if args.domain: self.process_domain(args.domain, force_scope=True) self.Domain.commit() if args.import_database_domains: if args.force: domains = self.Domain.all() else: domains = self.Domain.all(tool=self.name) for d in domains: # pdb.set_trace() self.process_domain(d.domain) self.Domain.commit() if args.import_ips: try: ips = open(args.import_ips) for line in ips: if line.strip(): if '/' in line or '-' in line: self.process_cidr(line) else: self.process_ip(line.strip(), force_scope=True) self.Domain.commit() except IOError: if '/' in args.import_ips or '-' in args.import_ips: self.process_cidr(args.import_ips) else: self.process_ip(args.import_ips.strip(), force_scope=True) self.Domain.commit() if args.import_database_ips: for ip in self.IPAddress.all(): self.process_ip(ip.ip_address) self.Domain.commit() def get_domain_ips(self, domain): ips = [] try: answers = dns.resolver.query(domain, 'A') for a in answers: ips.append(a.address) return ips except: return [] def process_domain(self, domain_str, force_scope=False): # First check if the root domain exists, and if it doesn't, add it created, domain = self.Domain.find_or_create( only_tool=True, domain=domain_str, force_in_scope=force_scope) # if not created: # # print("%s already processed, skipping." % domain_str) # return print("Processing %s" % domain_str) # Next get ip addresses of domain ips = self.get_domain_ips(domain_str) for i in ips: ip = self.process_ip(i, force_scope=force_scope) domain.ip_addresses.append(ip) domain.save() def process_ip(self, ip_str, force_scope=False): created, ip = self.IPAddress.find_or_create(only_tool=True, ip_address=ip_str, force_in_scope=force_scope) if created: print(" - Found New IP: %s" % ip_str) res = self.check_private_subnets(ip_str) if res: cidr_data = res else: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except: res = IPWhois(ip_str).lookup_whois() cidr_data = [] for n in res['nets']: if ',' in n['cidr']: for cidr_str in n['cidr'].split(', '): cidr_data.append([cidr_str, n['description']]) else: cidr_data.append([n['cidr'], n['description']]) try: cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] except: pdb.set_trace() cidr_len = len(IPNetwork(cidr_data[0][0])) matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c print("New CIDR found: %s - %s" % (matching_cidr[1], matching_cidr[0])) cidr = self.CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0], org_name=matching_cidr[1])[1] ip.cidr = cidr ip.save() # else: # print(" - IP Already processed: %s" % ip_str) return ip def check_private_subnets(self, ip_str): for cidr in private_subnets: if IPAddress(ip_str) in cidr: return ([str(cidr), 'Non-Public Subnet'], ) return False def process_cidr(self, line): if '/' in line: print("Adding %s to scoped CIDRs" % line.strip()) self.ScopeCIDR.find_or_create(cidr=line.strip()) elif '-' in line: start_ip, end_ip = line.strip().replace(' ', '').split('-') if '.' not in end_ip: end_ip = '.'.join(start_ip.split('.')[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: print("Adding %s to scoped CIDRs" % str(c)) self.ScopeCIDR.find_or_create(cidr=str(c))
class Module(ToolTemplate): name = "Sublist3r" binary_name = 'sublist3r' def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output_path = os.path.join(output_path, "%s-sublist3r.txt" % t) res.append({'target': t, 'output': output_path}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd['output'] data = open(output_path).read().split('\n') for d in data: new_domain = d.split(':')[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit()
class Module(ModuleTemplate): name = "Fierce" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Target domain for Fierce") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help="Relative path (to the base directory) to store Fierce output", default="fierceFiles") self.options.add_argument('-x', '--fiercefile', help="Fierce output name") self.options.add_argument('--threads', help="Number of threads to use", default="10") self.options.add_argument('--force', help="Force overwrite", action="store_true") self.options.add_argument('--import_file', help="Import Fierce file") def run(self, args): if not args.binary: self.binary = which.run('fierce') else: self.binary = which.run(args.binary) if not self.binary: print( "Fierce binary not found. Please explicitly provide path with --binary" ) if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all(tool=self.name) for domain in domains: self.process_domain(domain, args) domain.set_tool(self.name) self.BaseDomain.commit() elif args.import_file: fiercefile = args.import_file self.process_data(fiercefile, args) self.BaseDomain.commit() else: print("You need to supply some options to do something.") def process_domain(self, domain_obj, args): domain = domain_obj.domain print("Processing %s" % domain) if args.fiercefile: fiercefile = os.path.join(self.path, args.fiercefile) else: fiercefile = os.path.join(self.path, domain + ".txt") if os.path.isfile(fiercefile): if not args.force: answered = False while answered == False: rerun = raw_input( "Would you like to [r]un Fierce again and overwrite the file, [p]arse the file, or change the file [n]ame? " ) if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return fiercefile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists( os.path.join(self.path, newFile)): fiercefile = os.path.join(path, newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " -threads %s " % args.threads command += " -dns {} -file {} ".format(domain, fiercefile) subprocess.Popen(command, shell=True).wait() try: self.process_data(fiercefile, args) except IOError: print("Fierce for %s failed." % domain) return None def process_data(self, fiercefile, args): self.fiercefile = fiercefile print "Reading", self.fiercefile fierceOutput = '' with open(self.fiercefile, 'r') as fiercefile: for line in fiercefile: fierceOutput += line domains = [] if "Now performing" in fierceOutput: hosts = re.findall("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: #print host domain = host.split("\t")[1].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) elif "Whoah, it worked" in fierceOutput: print "Zone transfer found!" hosts = re.findall(".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: domain = host.split("\t")[0].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) else: print "Unable to process {}".format(fiercefile) if domains: for _domain in domains: created, domain = self.Domain.find_or_create(domain=_domain) return
class Module(ToolTemplate): ''' This tool will check various subdomains for domain takeover vulnerabilities. ''' name = "Tko-subs" binary_name = "tko-subs" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('--data', help="Path to the providers_data.csv file") self.options.add_argument('-d', '--domain', help="Domain to run the tool against.") self.options.add_argument('-i', '--importdb', help="Import subdomains from the database.", action="store_true") self.options.add_argument('--rescan', help="Rescan already processed entries", action="store_true") def get_targets(self, args): ''' This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [{'target':target, 'output':output}, {'target':target, 'output':output}, etc, etc] ''' # Create an empty list to add targets to domains = [] # Check if a domain has been explicitly passed, and if so add it to targets if args.domain: domains.append(args.domain) # Check if the --import option was passed and if so get data from the domain. # The scoping is set to "active" since the tool could potentially make a # request from the server. if args.importdb: if args.rescan: all_domains = self.Domain.all(scope_type="active") else: all_domains = self.Domain.all(scope_type="active", tool=self.name) for d in all_domains: domains.append(d.domain) # Set the output_path base, as a junction of the base_path and the path name supplied if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) # Create the path if it doesn't already exist if not os.path.exists(output_path): os.makedirs(output_path) res = [] # Create the final targets list, with output paths added. for d in domains: res.append({'target': d, 'output': os.path.join(output_path, d)}) return res def build_cmd(self, args): ''' Create the actual command that will be executed. Use {target} and {output} as placeholders. ''' cmd = self.binary + " -domain {target} -output {output} " # Add in any extra arguments passed in the extra_args parameter if args.tool_args: cmd += args.tool_args # Add that data parameter in there cmd += " -data " + args.data return cmd def process_output(self, cmds): ''' Process the output generated by the earlier commands. ''' # Cycle through all of the targets we ran earlier for c in cmds: output_file = c['output'] target = c['target'] # Read file data = open(output_file).read().split('\n') # Quick and dirty way to filter out headers and blank lines, as well # as duplicates res = list( set([ d for d in data if 'Domain,Cname,Provider' not in d and d ])) if res: # Load up the DB entry. created, subdomain = self.Domain.find_or_create(domain=target) # Process results for d in res: results = d.split(',') if results[3] == "false": display_warning( "Hosting found at {} for {}, not vulnerable.". format(target, results[2])) elif results[3] == "true": display_new("{} vulnerable to {}!".format( target, results[2])) if not subdomain.meta[self.name].get( 'vulnerable', False): subdomain.meta[self.name]['vulnerable'] = [] subdomain.meta[self.name]['vulnerable'].append(d) else: display_warning("Not sure of result: {}".format(data)) # This is a little hackish, but needed to get data to save t = dict(subdomain.meta) self.Domain.commit() subdomain.meta = t self.Domain.commit()
def __init__(self, db): self.Domain = DomainRepository(db) self.IPAddress = IPRepository(db) self.CIDR = CIDRRepository(db)
def __init__(self, db): self.Domain = DomainRepository(db, self.name)
def run(hosts, db, proto="tcp", svc="ssl", lookup_domains=False): IPAddress = IPRepository(db) Domain = DomainRepository(db) ips = {} for h in hosts: if h.count(":") == 2: host, port, svc = h.split(":") else: host, port = h.split(":") try: int(host.replace(".", "")) if not ips.get(host, False): ips[host] = {"domains": [], "ports": []} if (port, svc) not in ips[host]["ports"]: ips[host]["ports"].append((port, svc)) except: domains = Domain.all(domain=host) if domains: domain = domains[0] for ip in domain.ip_addresses: if not ips.get(ip.ip_address, False): ips[ip.ip_address] = {"domains": [], "ports": []} if host not in ips[ip.ip_address]["domains"]: ips[ip.ip_address]["domains"].append(host) if (port, svc) not in ips[ip.ip_address]["ports"]: ips[ip.ip_address]["ports"].append((port, svc)) else: # domain is not in the database. domain_ips = get_ip(host) for ip in domain_ips: ips[ip] = {"domains": [host], "ports": []} results = [] if lookup_domains: for ip in sorted(ips.keys()): # print("Checking %s" % ip) try: ip_obj = IPAddress.all(ip_address=ip)[0] domains = [d.domain for d in ip_obj.domains] if domains: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(domains)), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) except: if ips[ip]["domains"]: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(ips[ip]["domains"])), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: for ip in sorted(ips.keys()): if ips[ip]["domains"]: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(ips[ip]["domains"])), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) return results
class Module(ToolTemplate): name = "GobusterDNS" binary_name = "gobuster" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been brute forced", action="store_true") self.options.set_defaults( timeout=600) # Kick the default timeout to 10 minutes def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) if args.file: domains = open(args.file).read().split('\n') for d in domains: if d: targets.append(d) if args.import_database: if args.rescan: targets += [ b.domain for b in self.BaseDomain.all(scope_type="passive") ] else: targets += [ b.domain for b in self.BaseDomain.all(scope_type="passive", tool=self.name) ] if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) res = [] for t in targets: res.append({ 'target': t, 'output': os.path.join(output_path, t.replace('/', '_') + "-dns.txt") }) return res def build_cmd(self, args): cmd = self.binary + " -m dns " cmd += " -o {output} -u {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for c in cmds: target = c['target'] output_path = c['output'] if os.path.isfile(output_path): data = open(output_path).read().split('\n') else: display_error("{} not found.".format(output_path)) return for d in data: if 'Found: ' in d: new_domain = d.split(' ')[1].lower() created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit()
class Module(ToolTemplate): name = "TheHarvester" binary_name = "theharvester" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Domain to harvest") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) targets.append({'target':domain.domain}) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) targets.append({'target':domain.domain}) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append({'target':d.domain}) if args.output_path[0] == "/": output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: t['output'] = os.path.join(output_path, "%s-theharvester" % t['target'].replace('.', '_') ) return targets def build_cmd(self, args): cmd = self.binary + " -f {output} -b all -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: try: data = xmltodict.parse(open(cmd['output'] + '.xml').read()) except: data = None if data: if data['theHarvester'].get('email', False): if type(data['theHarvester']['email']) == list: emails = data['theHarvester']['email'] else: emails = [data['theHarvester']['email']] for e in emails: created, user = self.User.find_or_create(email=e) _, domain = self.BaseDomain.find_or_create(e.split('@')[1]) user.domain = domain user.update() if created: display_new("New email: %s" % e) if data['theHarvester'].get('host', False): if type(data['theHarvester']['host']) == list: hosts = data['theHarvester']['host'] else: hosts = [data['theHarvester']['host']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if data['theHarvester'].get('vhost', False): if type(data['theHarvester']['vhost']) == list: hosts = data['theHarvester']['vhost'] else: hosts = [data['theHarvester']['vhost']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) self.BaseDomain.commit()
def __init__(self, db): self.IPAddress = IPRepository(db) self.Domains = DomainRepository(db) self.BaseDomains = BaseDomainRepository(db) self.CIDRs = CIDRRepository(db)
class Module(ModuleTemplate): """ Ingests domains and IPs. Domains get ip info and cidr info, and IPs get CIDR info. """ name = "Ingestor" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "-d", "--import_domains", help= "Either domain to import or file containing domains to import. One per line", ) self.options.add_argument( "-i", "--import_ips", help= "Either IP/range to import or file containing IPs and ranges, one per line.", ) self.options.add_argument( "-a", "--active", help="Set scoping on imported data as active", action="store_true", ) self.options.add_argument( "-p", "--passive", help="Set scoping on imported data as passive", action="store_true", ) self.options.add_argument( "-sc", "--scope_cidrs", help= "Cycle through out of scope networks and decide if you want to add them in scope", action="store_true", ) self.options.add_argument( "-sb", "--scope_base_domains", help= "Cycle through out of scope base domains and decide if you want to add them in scope", action="store_true", ) self.options.add_argument("--descope", help="Descope an IP, domain, or CIDR") self.options.add_argument( "-Ii", "--import_database_ips", help="Import IPs from database", action="store_true", ) self.options.add_argument( "--force", help="Force processing again, even if already processed", action="store_true", ) def run(self, args): self.in_scope = args.active self.passive_scope = args.passive if args.descope: if "/" in args.descope: self.descope_cidr(args.descope) elif check_string(args.descope): pass else: self.descope_ip(args.descope) # Check if in ScopeCIDR and remove if found if args.import_ips: try: ips = open(args.import_ips) for line in ips: if line.strip(): if "/" in line or "-" in line: self.process_cidr(line) else: self.process_ip(line.strip(), force_scope=True) self.Domain.commit() except IOError: if "/" in args.import_ips or "-" in args.import_ips: self.process_cidr(args.import_ips) else: self.process_ip(args.import_ips.strip(), force_scope=True) self.Domain.commit() if args.import_domains: try: domains = open(args.import_domains) for line in domains: if line.strip(): self.process_domain(line.strip()) self.Domain.commit() except IOError: self.process_domain(args.import_domains.strip()) self.Domain.commit() if args.scope_base_domains: base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False) for bd in base_domains: self.reclassify_domain(bd) self.BaseDomain.commit() def get_domain_ips(self, domain): ips = [] try: answers = dns.resolver.query(domain, "A") for a in answers: ips.append(a.address) return ips except: return [] def process_domain(self, domain_str): created, domain = self.Domain.find_or_create( only_tool=True, domain=domain_str, in_scope=self.in_scope, passive_scope=self.passive_scope, ) if not created: if (domain.in_scope != self.in_scope or domain.passive_scope != self.passive_scope): display( "Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (domain_str, self.in_scope, self.passive_scope)) domain.in_scope = self.in_scope domain.passive_scope = self.passive_scope domain.update() def process_ip(self, ip_str, force_scope=True): created, ip = self.IPAddress.find_or_create( only_tool=True, ip_address=ip_str, in_scope=self.in_scope, passive_scope=self.passive_scope, ) if not created: if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope: display( "IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (ip_str, self.in_scope, self.passive_scope)) ip.in_scope = self.in_scope ip.passive_scope = self.passive_scope ip.update() return ip def process_cidr(self, line): display("Processing %s" % line) if "/" in line: created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip()) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() elif "-" in line: start_ip, end_ip = line.strip().replace(" ", "").split("-") if "." not in end_ip: end_ip = ".".join(start_ip.split(".")[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c)) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() def scope_ips(self): IPAddresses = self.IPAddress.all() def reclassify_domain(self, bd): if bd.meta.get("whois", False): display_new("Whois data found for {}".format(bd.domain)) print(bd.meta["whois"]) res = raw_input( "Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] " ) if res.lower() == "a": bd.in_scope = True bd.passive_scope = True elif res.lower() == "p": bd.in_scope = False bd.passive_scope = True else: bd.in_scope = False bd.passive_scope = False bd.save() else: display_error( "Unfortunately, there is no whois information for {}. Please populate it using the Whois module" .format(bd.domain)) def descope_ip(self, ip): ip = self.IPAddress.all(ip_address=ip) if ip: for i in ip: display("Removing IP {} from scope".format(i.ip_address)) i.in_scope = False i.passive_scope = False i.update() for d in i.domains: in_scope_ips = [ ipa for ipa in d.ip_addresses if ipa.in_scope or ipa.passive_scope ] if not in_scope_ips: display( "Domain {} has no more scoped IPs. Removing from scope." .format(d.domain)) d.in_scope = False d.passive_scope = False self.IPAddress.commit() def descope_cidr(self, cidr): CIDR = self.ScopeCIDR.all(cidr=cidr) if CIDR: for c in CIDR: display("Removing {} from ScopeCIDRs".format(c.cidr)) c.delete() cnet = IPNetwork(cidr) for ip in self.IPAddress.all(): if IPAddress(ip.ip_address) in cnet: self.descope_ip(ip.ip_address)
class Module(ToolTemplate): name = "Sublist3r" binary_name = "sublist3r" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Domain to brute force") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-s", "--rescan", help="Rescan domains that have already been scanned", action="store_true", ) def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split("\n") for d in domains: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in domains: targets.append(d.domain) res = [] if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) for t in targets: output = os.path.join(output_path, "%s-sublist3r.txt" % t) res.append({"target": t, "output": output}) return res def build_cmd(self, args): cmd = self.binary + " -o {output} -d {target} " if args.tool_args: cmd += args.tool_args return cmd def process_output(self, cmds): for cmd in cmds: output_path = cmd["output"] if os.path.isfile(output_path): data = open(output_path).read().split("\n") else: display_error("{} not found.".format(output_path)) next for d in data: new_domain = d.split(":")[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit()
class Module(ToolTemplate): name = "DNSRecon" binary_name = "dnsrecon" """ This module runs DNSRecon on a domain or set of domains. This will extract found DNS entries. It can also run over IP ranges, looking for additional domains in the PTR records. """ def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d','--domain', help="Target domain for dnsRecon") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-r', '--range', help="Range to scan for PTR records") self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true") self.options.add_argument('--rescan', help="Rescan domains already scanned", action="store_true") # self.options.add_argument('--import_output_xml', help="Import XML file") # self.options.add_argument('--import_output_json', help="Import json file") def get_targets(self, args): targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain, passive_scope=True) targets.append(domain.domain) elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d, passive_scope=True) targets.append(domain.domain) elif args.import_database: domains = self.BaseDomain.all(scope_type="passive", tool=self.name) for domain in domains: targets.append(domain.domain) elif args.range: targets.append(args.range) elif args.import_range: cidrs = self.ScopeCIDR.all(tool=self.name) for cidr in cidrs: targets.append(cidr.cidr) if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] ) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) res = [] for t in targets: res.append({'target':t, 'output':os.path.join(self.path, t.replace("/", "_") + ".json")}) return res def build_cmd(self, args): command = self.binary if args.domain or args.file or args.import_database: command += " -d {target} -j {output} " else: command += " -s -r {target} -j {output} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): for c in cmds: target = c['target'] output_path = c['output'] try: res = json.loads(open(output_path).read()) except IOError: display_error("DnsRecon failed for {}".format(target)) if " -d " in res[0]['arguments']: created, dbrec = self.Domain.find_or_create(domain=target) dbrec.dns = res dbrec.save() for record in res: domain = None if record.get("type") == "A" or record.get("type") == "PTR": domain = record.get("name").lower().replace("www.","") elif record.get("type") == "MX": domain = record.get("exchange").lower().replace("www.","") elif record.get("type") == "SRV" or record.get("type" == "NS"): domain = record.get("target").lower().replace("www.","") elif record.get("type") == "SOA": domain = record.get("mname").lower().replace("www.","") if domain: created, domain_obj = self.Domain.find_or_create(domain=domain) self.Domain.commit()
class Module(ToolTemplate): name = "aquatone-discover" binary_name = "aquatone-discover" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) self.BaseDomain = BaseDomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-d", "--domain", help="Target domain for aquatone") self.options.add_argument("-f", "--file", help="Import domains from file") self.options.add_argument( "-i", "--import_database", help="Import domains from database", action="store_true", ) self.options.add_argument( "-r", "--rescan", help="Run aquatone on hosts that have already been processed.", action="store_true", ) self.options.set_defaults(timeout=None) def get_targets(self, args): """ This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [(target, output), (target, output), etc, etc] """ targets = [] if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.file: domainsFile = open(args.file).read().split("\n") for d in domainsFile: if d: created, domain = self.BaseDomain.find_or_create( domain=args.domain) targets.append(domain.domain) elif args.import_database: if args.rescan: all_domains = self.BaseDomain.all(scope_type="passive") else: all_domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for d in all_domains: targets.append(d.domain) else: print("You need to supply domain(s).") output_path = os.path.join(self.base_config["PROJECT"]["base_path"], "aquatone") if not os.path.exists(output_path): os.makedirs(output_path) res = [] for t in targets: res.append({ "target": t, "output": "{}/{}/hosts.json".format(output_path, t) }) return res def build_cmd(self, args): """ Create the actual command that will be executed. Use {target} and {output} as placeholders. """ cmd = self.binary + " -d {target} " if args.tool_args: cmd += args.tool_args return cmd def pre_run(self, args): output_path = self.base_config["PROJECT"]["base_path"] self.orig_home = os.environ["HOME"] os.environ["HOME"] = output_path def process_output(self, cmds): """ Process the output generated by the earlier commands. """ for cmd in cmds: data2 = json.loads(open(cmd["output"]).read()) for sub, ip in data2.items(): created = False new_domain = sub.lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) self.Domain.commit() def post_run(self, args): os.environ["HOME"] = self.orig_home
class Module(ToolTemplate): """ Module for running nmap. Make sure to pass all nmap-specific arguments at the end, after --tool_args """ name = "Nmap" binary_name = "nmap" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--hosts", help= "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument("--hosts_file", help="File containing hosts") self.options.add_argument( "-i", "--hosts_database", help="Use unscanned hosts from the database", action="store_true", ) self.options.add_argument("--rescan", help="Overwrite files without asking", action="store_true") self.options.add_argument( "--filename", help="Output filename. By default will use the current timestamp.", ) self.options.set_defaults(timeout=None) def get_targets(self, args): targets = [] if args.hosts: if type(args.hosts) == list: targets += args.hosts else: targets += [args.hosts] if args.hosts_database: if args.rescan: targets += [ h.ip_address for h in self.IPAddress.all(scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all()] else: targets += [ h.ip_address for h in self.IPAddress.all(tool=self.name, scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)] if args.hosts_file: targets += [ l for l in open(args.hosts_file).read().split("\n") if l ] # Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs data = [] for t in targets: ips = [str(i) for i in list(IPNetwork(t))] data += ips _, file_name = tempfile.mkstemp() open(file_name, "w").write("\n".join(list(set(data)))) if args.output_path[0] == "/": self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.filename: output_path = os.path.join(self.path, args.filename) else: output_path = os.path.join( self.path, "nmap-scan-%s.xml" % datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"), ) return [{"target": file_name, "output": output_path}] def build_cmd(self, args): command = "sudo " + self.binary + " -oX {output} -iL {target} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): self.import_nmap(cmds[0]["output"]) os.unlink(cmds[0]["target"]) def parseHeaders(self, httpHeaders): bsHeaders = [ "Pragma", "Expires", "Date", "Transfer-Encoding", "Connection", "X-Content-Type-Options", "Cache-Control", "X-Frame-Options", "Content-Type", "Content-Length", "(Request type", ] keepHeaders = {} for i in range(0, len(httpHeaders)): if httpHeaders[i].strip() != "" and httpHeaders[i].split( ":")[0].strip() not in " ".join(bsHeaders): hName = httpHeaders[i].split(":")[0].strip() hValue = "".join(httpHeaders[i].split(":")[1:]).strip() keepHeaders[hName] = hValue if keepHeaders == {}: keepHeaders = "" return keepHeaders def import_nmap( self, filename): # domains={}, ips={}, rejects=[] == temp while no db nFile = filename try: tree = ET.parse(nFile) root = tree.getroot() hosts = root.findall("host") except: print(nFile + " doesn't exist somehow...skipping") return tmpNames = [] tmpIPs = ( {} ) # tmpIps = {'127.0.0.1':['domain.com']} -- not really used; decided to just let nslookup take care of IP info skip = [] for host in hosts: hostIP = host.find("address").get("addr") created, ip = self.IPAddress.find_or_create(ip_address=hostIP) for hostname in host.findall("hostnames/hostname"): hostname = hostname.get("name") hostname = hostname.lower().replace("www.", "") reHostname = re.search( r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname) # attempt to not get PTR record if not reHostname: created, domain = self.Domain.find_or_create( domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.save() for port in host.findall("ports/port"): if port.find("state").get("state"): portState = port.find("state").get("state") hostPort = port.get("portid") portProto = port.get("protocol") created, db_port = self.Port.find_or_create( port_number=hostPort, status=portState, proto=portProto, ip_address=ip, ) if port.find("service") != None: portName = port.find("service").get("name") if portName == "http" and hostPort == "443": portName = "https" else: portName = "Unknown" if created: db_port.service_name = portName info = db_port.info if not info: info = {} for script in port.findall( "script"): # just getting commonName from cert if script.get("id") == "ssl-cert": db_port.cert = script.get("output") cert_domains = self.get_domains_from_cert( script.get("output")) for hostname in cert_domains: hostname = hostname.lower().replace("www.", "") created, domain = self.Domain.find_or_create( domain=hostname) if created: print("New domain found: %s" % hostname) elif script.get("id") == "vulners": print( "Gathering vuln info for {} : {}/{}\n".format( hostIP, portProto, hostPort)) self.parseVulners(script.get("output"), db_port) elif script.get("id") == "banner": info["banner"] = script.get("output") elif script.get("id") == "http-headers": httpHeaders = script.get("output") httpHeaders = httpHeaders.strip().split("\n") keepHeaders = self.parseHeaders(httpHeaders) info["http-headers"] = keepHeaders elif script.get("id") == "http-auth": info["http-auth"] = script.get("output") elif script.get("id") == "http-title": info["http-title"] = script.get("output") db_port.info = info db_port.save() self.IPAddress.commit() def parseVulners(self, scriptOutput, db_port): urls = re.findall("(https://vulners.com/cve/CVE-\d*-\d*)", scriptOutput) for url in urls: vuln_refs = [] exploitable = False cve = url.split("/cve/")[1] vulners = requests.get("https://vulners.com/cve/%s" % cve).text exploitdb = re.findall( "https://www.exploit-db.com/exploits/\d{,7}", vulners) for edb in exploitdb: exploitable = True if edb.split("/exploits/")[1] not in vuln_refs: vuln_refs.append(edb.split("/exploits/")[1]) if not self.CVE.find(name=cve): # print "Gathering CVE info for", cve try: res = json.loads( requests.get("http://cve.circl.lu/api/cve/%s" % cve).text) cveDescription = res["summary"] cvss = float(res["cvss"]) findingName = res["oval"][0]["title"] if int(cvss) <= 3: severity = 1 elif (int(cvss) / 2) == 5: severity = 4 else: severity = int(cvss) / 2 if not self.Vulnerability.find(name=findingName): # print "Creating", findingName created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=cveDescription, ) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploit_reference = {"edb-id": vuln_refs} db_vuln.save() else: # print "modifying",findingName db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploitable = exploitable if db_vuln.exploit_reference is not None: if "edb-id" in db_vuln.exploit_reference: for ref in vuln_refs: if (ref not in db_vuln. exploit_reference["edb-id"]): db_vuln.exploit_reference[ "edb-id"].append(ref) else: db_vuln.exploit_reference[ "edb-id"] = vuln_refs else: db_vuln.exploit_reference = { "edb-id": vuln_refs } db_vuln.save() if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss) db_cve.vulnerabilities.append(db_vuln) db_cve.save() else: db_cve = self.CVE.find(name=cve) db_cve.vulnerabilities.append(db_vuln) db_cve.save() self.Vulnerability.commit() self.CVE.commit() except: print( "something went wrong with the vuln/cve info gathering" ) if vulners: print( "Vulners report was found but no exploit-db was discovered" ) # "Affected vulners items" # print vulners print("Affected CVE") print(cve) pass else: db_cve = self.CVE.find(name=cve) for db_vulns in db_cve.vulnerabilities: if db_port not in db_vulns.ports: db_vulns.ports.append(db_port) return def get_domains_from_cert(self, cert): # Shamelessly lifted regex from stack overflow regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}" domains = list( set([d for d in re.findall(regex, cert) if "*" not in d])) return domains
class Module(ModuleTemplate): name = "TheHarvester" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.User = UserRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-o', '--output_path', help="Path which will contain program output (relative to base_path in config", default=self.name) self.options.add_argument('-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def run(self, args): if not args.binary: self.binary = which.run('theharvester') else: self.binary = which.run(args.binary) if not self.binary: print("TheHarvester binary not found. Please explicitly provide path with --binary") if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: if args.rescan: domains = self.BaseDomain.all() else: domains = self.BaseDomain.all(tool=self.name) for d in domains: self.process_domain(d, args) d.set_tool(self.name) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-theharvester" % domain.replace('.', '_') ) command_args = " -f %s" % output_path command_args += " -b all " # if args.threads: # command_args += " -t " + args.threads cmd = shlex.split(self.binary + command_args + " -d " + domain) print("Executing: %s" % ' '.join(cmd)) res = subprocess.Popen(cmd).wait() try: data = xmltodict.parse(open(output_path + '.xml').read()) except: data = None if data: if data['theHarvester'].get('email', False): if type(data['theHarvester']['email']) == list: emails = data['theHarvester']['email'] else: emails = [data['theHarvester']['email']] for e in emails: created, user = self.User.find_or_create(email=e) user.domain = domain_obj user.update() if created: print("New email: %s" % e) if data['theHarvester'].get('host', False): if type(data['theHarvester']['host']) == list: hosts = data['theHarvester']['host'] else: hosts = [data['theHarvester']['host']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if created: print("New domain: %s" % d['hostname']) if data['theHarvester'].get('vhost', False): if type(data['theHarvester']['vhost']) == list: hosts = data['theHarvester']['vhost'] else: hosts = [data['theHarvester']['vhost']] for d in hosts: created, domain = self.Domain.find_or_create(domain=d['hostname']) if created: print("New domain: %s" % d['hostname'])
class Module(ModuleTemplate): name = "Nessus" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('--import_file', help="Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS", nargs='+') self.options.add_argument('--interactive', help="Prompt to store domains not in Base Domains already", action="store_true") self.options.add_argument('--internal', help="Store domains not in Base Domains already", action="store_true") def run(self, args): if not args.import_file: print("You need to supply some options to do something.") else: for nFile in args.import_file: self.process_data(nFile, args) def nessCheckPlugin(self, tag): nessPlugins = ["10759","77026", "20089", "56984", "71049", "70658", "40984", "11411"] pluginID = tag.get("pluginID") if pluginID in nessPlugins: #print pluginID + " is in the list" if pluginID == "10759": if tag.find("plugin_output") is not None: return tag.find("plugin_output").text.split("\n\n")[3].strip() #returns IP for Web Server HTTP Header INternal IP disclosure else: return "" if pluginID == "77026": if tag.find("plugin_output") is not None: return tag.find("plugin_output").text.split("\n\n")[3].strip() #Microsoft Exchange Client Access Server Information Disclosure (IP addy) else: return "" if pluginID == "71049" or pluginID == "70658": output = "" if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split(":")[1] #SSH Weak MAC & CBC Algorithms Enabled #print "#"*5 tmp = tmp.split("\n\n")[1].replace(" ","") #print "#"*5 output = tmp.split("\n") #print ", ".join(output) return ", ".join(output) if pluginID == "56984": if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split("This port supports ")[1].strip() # SSL / TLS Versions Supported tmp = tmp.split("/") bad = [] for i in tmp: #print i if "SSLv" in i: bad.append(i) elif "TLSv1.0" in i: bad.append(i) if bad != []: return ", ".join(bad).rstrip(".") else: return "" else: return "" if pluginID == "40984": #broswable web dirs if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split("The following directories are browsable :")[1].strip() directories = tmp.split("\n") return "\n".join(directories) if pluginID == "11411": #Backup Files Disclosure if tag.find("plugin_output") is not None: urls = [] tmp = tag.find("plugin_output").text.split("It is possible to read the following backup files :")[1].strip() tmpUrls = tmp.split("\n") for url in tmpUrls: if "URL" in url: urls.append(url.split(":")[1].lstrip()) if urls: return "\n".join(urls) else: return "" if pluginID == "20089": #F5 cookie if tag.find("plugin_output") is not None: f5Output = [] cookieVal = [] output = tag.find("plugin_output").text.strip().split("\n") for line in output: #print line line = line.split(":") for i, item in enumerate(line): item = item.strip() if "Cookie" in item: cabbage = line.pop(i) tmp = line.pop(i) tmp.strip() cookieVal.append(tmp) else: item = "".join(item) f5Output.append(item) f5Output = " : ".join(f5Output) f5Output = f5Output.replace(" : : ", ", ") f5Output += " [" + ", ".join(cookieVal) + "]" c = 0 tmpF5Output = f5Output.split() for i, letter in enumerate(tmpF5Output): if letter == ":": c += 1 if (c%2) == 0: tmpF5Output[i] = " " return "".join(tmpF5Output).replace("["," [") else: return "" else: return False def getVulns(self, ip, ReportHost): '''Gets vulns and associated services''' for tag in ReportHost.iter("ReportItem"): exploitable = False cves = [] vuln_refs = {} proto = tag.get("protocol") port = tag.get("port") svc_name = tag.get("svc_name").replace("?","") tmpPort = proto+"/"+port if tmpPort.lower() == "tcp/443": portName = "https" elif tmpPort.lower() == "tcp/80": portName = "http" elif svc_name == "www": plugin_name = tag.get("pluginName") if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(): portName = "https" else: portName = "http" else: portName = svc_name if "general" not in portName: created, db_port = self.Port.find_or_create(port_number=port, status='open', proto=proto, ip_address_id=ip.id) if db_port.service_name == "http": if portName == "https": db_port.service_name = portName elif db_port.service_name == "https": pass else: db_port.service_name = portName db_port.save() if tag.get("pluginID") == "56984": severity = 1 elif tag.get("pluginID") == "11411": severity = 3 else: severity = int(tag.get("severity")) findingName = tag.get("pluginName") description = tag.find("description").text if tag.find("solution") is not None and tag.find("solution") != "n/a": solution = tag.find("solution").text else: solution = "No Remediation From Nessus" nessCheck = self.nessCheckPlugin(tag) if nessCheck: if not db_port.info: db_port.info = {findingName:nessCheck} else: db_port.info[findingName] = nessCheck db_port.save() if tag.find("exploit_available") is not None: #print "\nexploit avalable for", findingName exploitable = True metasploits = tag.findall("metasploit_name") if metasploits: vuln_refs['metasploit'] = [] for tmp in metasploits: vuln_refs['metasploit'].append(tmp.text) edb_id = tag.findall("edb-id") if edb_id: vuln_refs['edb-id'] = [] for tmp in edb_id: vuln_refs['edb-id'].append(tmp.text) tmpcves = tag.findall("cve") for c in tmpcves: if c.text not in cves: cves.append(c.text) if not self.Vulnerability.find(name=findingName): created, db_vuln = self.Vulnerability.find_or_create(name=findingName, severity=severity, description=description, remediation=solution) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if exploitable == True: print "\nexploit avalable for", findingName print if vuln_refs: db_vuln.exploit_reference = vuln_refs else: db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: if db_vuln.exploit_reference is not None: for key in vuln_refs.keys(): if key not in db_vuln.exploit_reference.keys(): db_vuln.exploit_reference[key] = vuln_refs[key] else: for ref in vuln_refs[key]: if ref not in db_vuln.exploit_reference[key]: db_vuln.exploit_reference[key].append(ref) else: db_vuln.exploit_reference = vuln_refs for cve in cves: if not self.CVE.find(name=cve): #print "Gathering CVE information for", cve try: res = json.loads(requests.get('http://cve.circl.lu/api/cve/%s' % cve).text) cveDescription = res['summary'] cvss = float(res['cvss']) except: cveDescription = None cvss = None if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create(name=cve, description=cveDescription, temporal_score=cvss) db_cve.vulnerabilities.append(db_vuln) else: db_cve = self.CVE.find(name=cve) if db_cve.description is None and cveDescription is not None: db_cve.description = cveDescription if db_cve.temporal_score is None and cvss is not None: db_cve.temporal_score = cvss db_cve.vulnerabilities.append(db_vuln) def process_data(self, nFile, args): print "Reading",nFile tree = ET.parse(nFile) root = tree.getroot() skip = [] for ReportHost in root.iter('ReportHost'): os = [] hostname = "" hostIP = "" for HostProperties in ReportHost.iter("HostProperties"): for tag in HostProperties: if tag.get('name') == "host-ip": hostIP = tag.text if tag.get('name') == "host-fqdn": hostname = tag.text.lower() hostname = hostname.replace("www.","") if tag.get('name') == "operating-system": os = tag.text.split("\n") if hostIP: #apparently nessus doesn't always have an IP to work with... if hostname: print "Gathering Nessus info for {} ( {} )".format(hostIP,hostname) else: print "Gathering Nessus info for",hostIP created, ip = self.IPAddress.find_or_create(ip_address=hostIP) if hostname: if not args.internal: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: ip.save() domain.ip_addresses.append(ip) domain.save() else: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.update() if os: for o in os: if not ip.OS: ip.OS = o else: if o not in ip.OS.split(" OR "): ip.OS += " OR "+o self.getVulns(ip, ReportHost) self.IPAddress.commit() return
class Module(ModuleTemplate): name = "DNSRecon" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d','--domain', help="Target domain for dnsRecon") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument('-o', '--output_path', help="Relative path (to the base directory) to store dnsRecon XML output", default="dnsReconFiles") self.options.add_argument('-x', '--dns_recon_file', help="dnsRecon XML output name") self.options.add_argument('-r', '--range', help="Range to scan for PTR records") self.options.add_argument('-R', '--import_range', help="Import CIDRs from in-scope ranges in database", action="store_true") self.options.add_argument('--threads', help="Number of threads to use", default="10") self.options.add_argument('--force', help="Force overwrite", action="store_true") self.options.add_argument('--import_output_xml', help="Import XML file") self.options.add_argument('--import_output_json', help="Import json file") def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run('dnsrecon') else: self.binary = which.run(args.binary) if not self.binary: print("Dnsrecon binary not found. Please explicitly provide path with --binary") if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:] ) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all() for domain in domains: self.process_domain(domain, args) self.BaseDomain.commit() elif args.range: self.process_range(args.range, args) self.BaseDomain.commit() elif args.import_range: cidrs = self.ScopeCIDR.all() for cidr in cidrs: self.process_range(cidr.cidr, args) self.BaseDomain.commit() elif args.import_output_json: data = open(args.import_output_json).read() self.process_data(data, args, data_type="json") self.BaseDomain.commit() elif args.import_output_xml: data = open(args.import_output_xml).read() self.process_data(data, args, data_type="json") self.BaseDomain.commit() else: print("You need to supply some options to do something.") def process_domain(self, domain_obj, args): domain = domain_obj.domain print("Processing %s" % domain) if args.dns_recon_file: dns_recon_file = os.path.join(self.path,args.dnsReconFile) else: dns_recon_file = os.path.join(self.path,domain+".json") if os.path.isfile(dns_recon_file): if not args.force: answered = False while answered == False: rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ") if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return dnsReconFile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists(os.path.join(self.path,newFile)): dns_recon_file = os.path.join(path,newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " --threads %s " % args.threads command += " -d {} -j {} ".format(domain, dns_recon_file) subprocess.Popen(command, shell=True).wait() try: res = json.loads(open(dns_recon_file).read()) domain_obj.dns = {'data':res} domain_obj.save() self.process_data(res, args, data_type='json') except IOError: print("DnsRecon for %s failed." % domain) return None def process_data(self, data, args, data_type="json"): if data_type == "xml": tree = ET.fromstring(data) root = tree.getroot() records = root.findall("record") else: records = data for record in data: created = False domain = "" if record.get("type") == "A": created, domain = self.Domain.find_or_create(domain=record.get("name").lower().replace("www.","")) if record.get("type") == "A" : domain = record.get("name").lower().replace("www.","") #ip = record.get("address") #take what nslookup(get_domain_ip) says instead elif record.get("type") == "MX": domain = record.get("exchange").lower().replace("www.","") elif record.get("type") == "SRV": domain = record.get("target").lower().replace("www.","") if domain: created, domain_obj = self.Domain.find_or_create(domain=domain) if created: print("New domain found: " + domain) def process_range(self, cidr, args): print("Processing %s" % cidr) if args.dns_recon_file: dns_recon_file = os.path.join(self.path,args.dnsReconFile) else: dns_recon_file = os.path.join(self.path,cidr.replace('/', '_')+".json") if os.path.isfile(dns_recon_file): if not args.force: answered = False while answered == False: rerun = raw_input("Would you like to [r]un dnsRecon again and overwrite the file, [p]arse the file, or change the file [n]ame? ") if rerun.lower() == 'r': answered = True elif rerun.lower() == 'p': answered = True return dnsReconFile elif rerun.lower() == 'n': new = False while new == False: newFile = raw_input("enter a new file name: ") if not os.path.exists(os.path.join(self.path,newFile)): dns_recon_file = os.path.join(path,newFile) answered = True new = True else: print "That file exists as well" command = self.binary if args.threads: command += " --threads %s " % args.threads command += " -s -r {} -j {} ".format(cidr, dns_recon_file) subprocess.Popen(command, shell=True).wait() try: res = json.loads(open(dns_recon_file).read()) for r in res[1:]: domain = r['name'] created, domain_obj = self.Domain.find_or_create(domain=domain) if created: print("New domain found: %s" % domain) self.process_data(res, args, data_type='json') except IOError: print("DnsRecon for %s failed." % domain) return None
class Module(ToolTemplate): name = "Fierce" binary_name = "fierce" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-d', '--domain', help="Target domain for Fierce") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '--rescan', help="Force rescan of already scanned domains", action="store_true") def get_targets(self, args): targets = [] if args.domain: targets.append(args.domain) if args.file: domains = open(args.file).read().split('\n') for d in domains: if d: targets.append(args.domain) if args.import_database: if args.rescan: domains = self.BaseDomain.all(scope_type="passive") else: domains = self.BaseDomain.all(tool=self.name, scope_type="passive") for domain in domains: targets.append(domain.domain) if args.output_path[0] == "/": self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: self.path = os.path.join(self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) res = [] for t in targets: res.append({ 'target': t, 'output': os.path.join(self.path, t + ".txt") }) return res def build_cmd(self, args): command = self.binary command += " -dns {target} -file {output} " if args.tool_args: command += args.tool_args pdb.set_trace() return command def process_output(self, cmds): for c in cmds: target = c['target'] output_path = c['output'] try: fierceOutput = open(output_path).read() except IOError: display_error( "The output file for {} was not found. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file." .format(target)) continue domains = [] if "Now performing" in fierceOutput: hosts = re.findall("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\t.*$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: #print host domain = host.split("\t")[1].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) elif "Whoah, it worked" in fierceOutput: print "Zone transfer found!" hosts = re.findall( ".*\tA\t\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", fierceOutput, re.MULTILINE) if hosts: for host in hosts: domain = host.split("\t")[0].lower().replace( "www.", "").rstrip(".") if domain not in domains: domains.append(domain) else: display_error( "Unable to process {}. If fierce timed out, but is still running, you can run this tool again with the --no_binary flag to just grab the file." .format(output_path)) if domains: for _domain in domains: created, domain = self.Domain.find_or_create( domain=_domain)
class Module(ModuleTemplate): name = "Sublist3r" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('-t', '--threads', help="Number of threads") self.options.add_argument('-d', '--domain', help="Domain to brute force") self.options.add_argument('-f', '--file', help="Import domains from file") self.options.add_argument('-i', '--import_database', help="Import domains from database", action="store_true") self.options.add_argument( '-o', '--output_path', help= "Path which will contain program output (relative to base_path in config", default="sublist3r") self.options.add_argument( '-s', '--rescan', help="Rescan domains that have already been scanned", action="store_true") def run(self, args): if not args.binary: self.binary = which.run('sublist3r') else: self.binary = which.run(args.binary) if not self.binary: print( "Sublist3r binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create( domain=args.domain) self.process_domain(domain, args) self.BaseDomain.commit() elif args.file: domains = open(args.file).read().split('\n') for d in domains: if d: created, domain = self.BaseDomain.find_or_create(domain=d) self.process_domain(domain, args) self.BaseDomain.commit() elif args.import_database: domains = self.BaseDomain.all(tool=self.name) for d in domains: self.process_domain(d, args) d.set_tool(self.name) self.BaseDomain.commit() def process_domain(self, domain_obj, args): domain = domain_obj.domain if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join(output_path, "%s-sublist3r.txt" % domain) command_args = " -o %s" % output_path if args.threads: command_args += " -t " + args.threads cmd = shlex.split(self.binary + command_args + " -d " + domain) print("Executing: %s" % ' '.join(cmd)) res = subprocess.Popen(cmd).wait() try: data = open(output_path).read().split('\n') for d in data: new_domain = d.split(':')[0].lower() if new_domain: created, subdomain = self.Domain.find_or_create( domain=new_domain) if created: print("New subdomain found: %s" % new_domain) except IOError: print("No results found.")
def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name)
#!/usr/bin/python from armory import initialize_database from armory import get_config_options from database.repositories import BaseDomainRepository, DomainRepository, IPRepository, CIDRRepository, UserRepository, CredRepository, VulnRepository, PortRepository, UrlRepository, ScopeCIDRRepository config = get_config_options() db = initialize_database(config) Domains = DomainRepository(db, "Shell Client") BaseDomains = BaseDomainRepository(db, "Shell Client") IPAddresses = IPRepository(db, "Shell Client") CIDRs = CIDRRepository(db, "Shell Client") Users = UserRepository(db, "Shell Client") Creds = CredRepository(db, "Shell Client") Vulns = VulnRepository(db, "Shell Client") Ports = PortRepository(db, "Shell Client") Urls = UrlRepository(db, "Shell Client") ScopeCIDRs = ScopeCIDRRepository(db, "Shell Client") print("Make sure to use this script with ipython and -i") print(" ipython -i shell.py") print("Available database modules: Domains, BaseDomains, IPAddresses,") print(" CIDRs, Users, Creds, Vulns, Services, Ports, Urls, ScopeCIDRs")