class Report(ReportTemplate): """ This lists who owns the CIDR for each of the domains. """ name = "DomainOwner" def __init__(self, db): self.Domain = DomainRepository(db) self.IPAddress = IPRepository(db) self.CIDR = CIDRRepository(db) def run(self, args): # Cidrs = self.CIDR. results = [] domains = self.Domain.all() for d in domains: if len(d.ip_addresses) > 0: owner = d.ip_addresses[0].cidr.org_name results.append("%s,%s" % (owner, d.domain)) res = sorted(results) self.process_output(res, args)
class Report(ReportTemplate): """ This is a domain summary report. It shows base domain, then all of the subdomains, along with any resolved IPs. """ name = "DomainSummaryReport" markdown = ["###", "-"] def __init__(self, db): self.Domain = DomainRepository(db, self.name) def run(self, args): # Cidrs = self.CIDR. results = [] if args.scope == 'all': args.scope == None domains = self.Domain.all(scope_type=args.scope) domain_data = {} for d in domains: if not domain_data.get(d.base_domain.domain, False): domain_data[d.base_domain.domain] = {} domain_data[d.base_domain.domain][d.domain] = [ i.ip_address for i in d.ip_addresses if (i.in_scope == True and args.scope == 'active') or (i.passive_scope and args.scope == 'passive') or not args.scope ] for b in sorted(domain_data.keys()): results.append(b) for d in sorted(domain_data[b].keys()): results.append('\t{} ({})'.format(d, ', '.join( domain_data[b][d]))) self.process_output(results, args)
class Module(ModuleTemplate): name = "Nessus" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--import_file", help="Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument( "--interactive", help="Prompt to store domains not in Base Domains already", action="store_true", ) self.options.add_argument( "--internal", help="Store domains not in Base Domains already", action="store_true", ) self.options.add_argument( "--launch", help="Launch Nessus scan using Actively scoped IPs and domains in the database", action="store_true", ) self.options.add_argument( "--job_name", help="Job name inside Nessus", default="Armory Job" ) self.options.add_argument("--username", help="Nessus Username") self.options.add_argument("--password", help="Nessus Password") self.options.add_argument( "--host", help="Hostname:Port of Nessus web interface (ie localhost:8835" ) self.options.add_argument("--uuid", help="UUID of Nessus Policy to run") self.options.add_argument("--policy_id", help="Policy ID to use") self.options.add_argument("--folder_id", help="ID for folder to store job in") self.options.add_argument( "--download", help="Download Nessus job from server and import", action="store_true", ) self.options.add_argument("--job_id", help="Job ID to download and import") self.options.add_argument( "--output_path", help="Path to store downloaded file (Default: Nessus)", default=self.name, ) def run(self, args): if args.import_file: for nFile in args.import_file: self.process_data(nFile, args) elif args.launch: if ( not args.username and not args.password and not args.host and not args.uuid and not args.policy_id and not args.folder_id ): display_error( "You must supply a username, password, and host to launch a Nessus job" ) else: n = NessusRequest( args.username, args.password, args.host, uuid=args.uuid, policy_id=args.policy_id, folder_id=args.folder_id, ) ips = [ ip.ip_address for ip in self.IPAddress.all(scope_type="active", tool=self.name) ] cidrs = [cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)] domains = [ domain.domain for domain in self.Domain.all(scope_type="active", tool=self.name) ] targets = ", ".join(merge_ranges(ips + cidrs) + domains) res = n.launch_job(targets, args.job_name) display("New Nessus job launched with ID {}".format(res)) display( "Remember this number! You'll need it to download the job once it is done." ) elif args.download: if ( not args.username and not args.password and not args.host and not args.job_id ): display_error( "You must supply host, username, password and job_id to download a report to import" ) else: n = NessusRequest( args.username, args.password, args.host, proxies={"https": "127.0.0.1:8080"}, ) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:] ) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path ) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join( output_path, "Nessus-export-{}.nessus".format(int(time.time())) ) n.export_file(args.job_id, output_path) self.process_data(output_path, args) def nessCheckPlugin(self, tag): nessPlugins = [ "10759", "77026", "20089", "56984", "71049", "70658", "40984", "11411", ] pluginID = tag.get("pluginID") if pluginID in nessPlugins: # print pluginID + " is in the list" if pluginID == "10759": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split("\n\n")[3].strip() ) # returns IP for Web Server HTTP Header INternal IP disclosure else: return "" if pluginID == "77026": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split("\n\n")[3].strip() ) # Microsoft Exchange Client Access Server Information Disclosure (IP addy) else: return "" if pluginID == "71049" or pluginID == "70658": output = "" if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split(":")[ 1 ] # SSH Weak MAC & CBC Algorithms Enabled # print "#"*5 tmp = tmp.split("\n\n")[1].replace(" ", "") # print "#"*5 output = tmp.split("\n") # print ", ".join(output) return ", ".join(output) if pluginID == "56984": if tag.find("plugin_output") is not None: tmp = ( tag.find("plugin_output") .text.split("This port supports ")[1] .strip() ) # SSL / TLS Versions Supported tmp = tmp.split("/") bad = [] for i in tmp: # print i if "SSLv" in i: bad.append(i) elif "TLSv1.0" in i: bad.append(i) if bad != []: return ", ".join(bad).rstrip(".") else: return "" else: return "" if pluginID == "40984": # browsable web dirs if tag.find("plugin_output") is not None: tmp = ( tag.find("plugin_output") .text.split("The following directories are browsable :")[1] .strip() ) directories = tmp.split("\n") return "\n".join(directories) if pluginID == "11411": # Backup Files Disclosure if tag.find("plugin_output") is not None: urls = [] tmp = ( tag.find("plugin_output") .text.split( "It is possible to read the following backup files :" )[1] .strip() ) tmpUrls = tmp.split("\n") for url in tmpUrls: if "URL" in url: urls.append(url.split(":")[1].lstrip()) if urls: return "\n".join(urls) else: return "" if pluginID == "20089": # F5 cookie if tag.find("plugin_output") is not None: f5Output = [] cookieVal = [] output = tag.find("plugin_output").text.strip().split("\n") for line in output: # print line line = line.split(":") for i, item in enumerate(line): item = item.strip() if "Cookie" in item: cabbage = line.pop(i) tmp = line.pop(i) tmp.strip() cookieVal.append(tmp) else: item = "".join(item) f5Output.append(item) f5Output = " : ".join(f5Output) f5Output = f5Output.replace(" : : ", ", ") f5Output += " [" + ", ".join(cookieVal) + "]" c = 0 tmpF5Output = f5Output.split() for i, letter in enumerate(tmpF5Output): if letter == ":": c += 1 if (c % 2) == 0: tmpF5Output[i] = " " return "".join(tmpF5Output).replace("[", " [") else: return "" else: return False def getVulns(self, ip, ReportHost): """Gets vulns and associated services""" for tag in ReportHost.iter("ReportItem"): exploitable = False cves = [] vuln_refs = {} proto = tag.get("protocol") port = tag.get("port") svc_name = tag.get("svc_name").replace("?", "") tmpPort = proto + "/" + port if tmpPort.lower() == "tcp/443": portName = "https" elif tmpPort.lower() == "tcp/80": portName = "http" elif svc_name == "www": plugin_name = tag.get("pluginName") if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(): portName = "https" else: portName = "http" else: portName = svc_name if "general" not in portName: created, db_port = self.Port.find_or_create( port_number=port, status="open", proto=proto, ip_address_id=ip.id ) if db_port.service_name == "http": if portName == "https": db_port.service_name = portName elif db_port.service_name == "https": pass else: db_port.service_name = portName db_port.save() if tag.get("pluginID") == "56984": severity = 1 elif tag.get("pluginID") == "11411": severity = 3 else: severity = int(tag.get("severity")) findingName = tag.get("pluginName") description = tag.find("description").text if tag.find("solution") is not None and tag.find("solution") != "n/a": solution = tag.find("solution").text else: solution = "No Remediation From Nessus" nessCheck = self.nessCheckPlugin(tag) if nessCheck: if not db_port.info: db_port.info = {findingName: nessCheck} else: db_port.info[findingName] = nessCheck db_port.save() if tag.find("exploit_available") is not None: # print "\nexploit avalable for", findingName exploitable = True metasploits = tag.findall("metasploit_name") if metasploits: vuln_refs["metasploit"] = [] for tmp in metasploits: vuln_refs["metasploit"].append(tmp.text) edb_id = tag.findall("edb-id") if edb_id: vuln_refs["edb-id"] = [] for tmp in edb_id: vuln_refs["edb-id"].append(tmp.text) tmpcves = tag.findall("cve") for c in tmpcves: if c.text not in cves: cves.append(c.text) if not self.Vulnerability.find(name=findingName): created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=description, remediation=solution, ) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if exploitable == True: print("\nexploit avalable for", findingName) print() if vuln_refs: db_vuln.exploit_reference = vuln_refs else: db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: if db_vuln.exploit_reference is not None: for key in vuln_refs.keys(): if key not in db_vuln.exploit_reference.keys(): db_vuln.exploit_reference[key] = vuln_refs[key] else: for ref in vuln_refs[key]: if ref not in db_vuln.exploit_reference[key]: db_vuln.exploit_reference[key].append(ref) else: db_vuln.exploit_reference = vuln_refs for cve in cves: if not self.CVE.find(name=cve): # print "Gathering CVE information for", cve try: res = json.loads( requests.get( "http://cve.circl.lu/api/cve/%s" % cve ).text ) cveDescription = res["summary"] cvss = float(res["cvss"]) except: cveDescription = None cvss = None if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss, ) db_cve.vulnerabilities.append(db_vuln) else: db_cve = self.CVE.find(name=cve) if ( db_cve.description is None and cveDescription is not None ): db_cve.description = cveDescription if db_cve.temporal_score is None and cvss is not None: db_cve.temporal_score = cvss db_cve.vulnerabilities.append(db_vuln) def process_data(self, nFile, args): print("Reading", nFile) tree = ET.parse(nFile) root = tree.getroot() skip = [] for ReportHost in root.iter("ReportHost"): os = [] hostname = "" hostIP = "" for HostProperties in ReportHost.iter("HostProperties"): for tag in HostProperties: if tag.get("name") == "host-ip": hostIP = tag.text if tag.get("name") == "host-fqdn": hostname = tag.text.lower() hostname = hostname.replace("www.", "") if tag.get("name") == "operating-system": os = tag.text.split("\n") if hostIP: # apparently nessus doesn't always have an IP to work with... if hostname: display( "Gathering Nessus info for {} ( {} )".format(hostIP, hostname) ) else: display("Gathering Nessus info for {}".format(hostIP)) created, ip = self.IPAddress.find_or_create(ip_address=hostIP) if hostname: if not args.internal: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: ip.save() domain.ip_addresses.append(ip) domain.save() else: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.update() if os: for o in os: if not ip.OS: ip.OS = o else: if o not in ip.OS.split(" OR "): ip.OS += " OR " + o self.getVulns(ip, ReportHost) self.IPAddress.commit() return
class Module(ModuleTemplate): ''' Ingests domains and IPs. Domains get ip info and cidr info, and IPs get CIDR info. ''' name = "Ingestor" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.CIDR = CIDRRepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( '-f', '--import_file', help="File containing domains to import. One per line") self.options.add_argument('-d', '--domain', help="Single domain to import") self.options.add_argument( '-i', '--import_ips', help="File containing IPs and ranges, one per line.") self.options.add_argument('-Id', '--import_database_domains', help='Import domains from database', action="store_true") self.options.add_argument('-Ii', '--import_database_ips', help='Import IPs from database', action="store_true") self.options.add_argument( '--force', help="Force processing again, even if already processed", action="store_true") def run(self, args): if args.import_file: domains = open(args.import_file) for line in domains: if line.strip(): self.process_domain(line.strip(), force_scope=True) self.Domain.commit() if args.domain: self.process_domain(args.domain, force_scope=True) self.Domain.commit() if args.import_database_domains: if args.force: domains = self.Domain.all() else: domains = self.Domain.all(tool=self.name) for d in domains: # pdb.set_trace() self.process_domain(d.domain) self.Domain.commit() if args.import_ips: try: ips = open(args.import_ips) for line in ips: if line.strip(): if '/' in line or '-' in line: self.process_cidr(line) else: self.process_ip(line.strip(), force_scope=True) self.Domain.commit() except IOError: if '/' in args.import_ips or '-' in args.import_ips: self.process_cidr(args.import_ips) else: self.process_ip(args.import_ips.strip(), force_scope=True) self.Domain.commit() if args.import_database_ips: for ip in self.IPAddress.all(): self.process_ip(ip.ip_address) self.Domain.commit() def get_domain_ips(self, domain): ips = [] try: answers = dns.resolver.query(domain, 'A') for a in answers: ips.append(a.address) return ips except: return [] def process_domain(self, domain_str, force_scope=False): # First check if the root domain exists, and if it doesn't, add it created, domain = self.Domain.find_or_create( only_tool=True, domain=domain_str, force_in_scope=force_scope) # if not created: # # print("%s already processed, skipping." % domain_str) # return print("Processing %s" % domain_str) # Next get ip addresses of domain ips = self.get_domain_ips(domain_str) for i in ips: ip = self.process_ip(i, force_scope=force_scope) domain.ip_addresses.append(ip) domain.save() def process_ip(self, ip_str, force_scope=False): created, ip = self.IPAddress.find_or_create(only_tool=True, ip_address=ip_str, force_in_scope=force_scope) if created: print(" - Found New IP: %s" % ip_str) res = self.check_private_subnets(ip_str) if res: cidr_data = res else: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except: res = IPWhois(ip_str).lookup_whois() cidr_data = [] for n in res['nets']: if ',' in n['cidr']: for cidr_str in n['cidr'].split(', '): cidr_data.append([cidr_str, n['description']]) else: cidr_data.append([n['cidr'], n['description']]) try: cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] except: pdb.set_trace() cidr_len = len(IPNetwork(cidr_data[0][0])) matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c print("New CIDR found: %s - %s" % (matching_cidr[1], matching_cidr[0])) cidr = self.CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0], org_name=matching_cidr[1])[1] ip.cidr = cidr ip.save() # else: # print(" - IP Already processed: %s" % ip_str) return ip def check_private_subnets(self, ip_str): for cidr in private_subnets: if IPAddress(ip_str) in cidr: return ([str(cidr), 'Non-Public Subnet'], ) return False def process_cidr(self, line): if '/' in line: print("Adding %s to scoped CIDRs" % line.strip()) self.ScopeCIDR.find_or_create(cidr=line.strip()) elif '-' in line: start_ip, end_ip = line.strip().replace(' ', '').split('-') if '.' not in end_ip: end_ip = '.'.join(start_ip.split('.')[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: print("Adding %s to scoped CIDRs" % str(c)) self.ScopeCIDR.find_or_create(cidr=str(c))
def run(hosts, db, proto="tcp", svc="ssl", lookup_domains=False): IPAddress = IPRepository(db) Domain = DomainRepository(db) ips = {} for h in hosts: if h.count(":") == 2: host, port, svc = h.split(":") else: host, port = h.split(":") try: int(host.replace(".", "")) if not ips.get(host, False): ips[host] = {"domains": [], "ports": []} if (port, svc) not in ips[host]["ports"]: ips[host]["ports"].append((port, svc)) except: domains = Domain.all(domain=host) if domains: domain = domains[0] for ip in domain.ip_addresses: if not ips.get(ip.ip_address, False): ips[ip.ip_address] = {"domains": [], "ports": []} if host not in ips[ip.ip_address]["domains"]: ips[ip.ip_address]["domains"].append(host) if (port, svc) not in ips[ip.ip_address]["ports"]: ips[ip.ip_address]["ports"].append((port, svc)) else: # domain is not in the database. domain_ips = get_ip(host) for ip in domain_ips: ips[ip] = {"domains": [host], "ports": []} results = [] if lookup_domains: for ip in sorted(ips.keys()): # print("Checking %s" % ip) try: ip_obj = IPAddress.all(ip_address=ip)[0] domains = [d.domain for d in ip_obj.domains] if domains: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(domains)), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) except: if ips[ip]["domains"]: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(ips[ip]["domains"])), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: for ip in sorted(ips.keys()): if ips[ip]["domains"]: results.append("%s / %s: %s" % ( ip, ", ".join(sorted(ips[ip]["domains"])), ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) else: results.append("%s / No Hostname Registered: %s" % ( ip, ", ".join([ "%s/%s/%s" % (proto, p, svc) for p, svc in sorted(ips[ip]["ports"]) ]), )) return results
class Module(ToolTemplate): ''' This tool will check various subdomains for domain takeover vulnerabilities. ''' name = "Tko-subs" binary_name = "tko-subs" def __init__(self, db): self.db = db self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument('--data', help="Path to the providers_data.csv file") self.options.add_argument('-d', '--domain', help="Domain to run the tool against.") self.options.add_argument('-i', '--importdb', help="Import subdomains from the database.", action="store_true") self.options.add_argument('--rescan', help="Rescan already processed entries", action="store_true") def get_targets(self, args): ''' This module is used to build out a target list and output file list, depending on the arguments. Should return a list in the format [{'target':target, 'output':output}, {'target':target, 'output':output}, etc, etc] ''' # Create an empty list to add targets to domains = [] # Check if a domain has been explicitly passed, and if so add it to targets if args.domain: domains.append(args.domain) # Check if the --import option was passed and if so get data from the domain. # The scoping is set to "active" since the tool could potentially make a # request from the server. if args.importdb: if args.rescan: all_domains = self.Domain.all(scope_type="active") else: all_domains = self.Domain.all(scope_type="active", tool=self.name) for d in all_domains: domains.append(d.domain) # Set the output_path base, as a junction of the base_path and the path name supplied if args.output_path[0] == "/": output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path[1:]) else: output_path = os.path.join( self.base_config['PROJECT']['base_path'], args.output_path) # Create the path if it doesn't already exist if not os.path.exists(output_path): os.makedirs(output_path) res = [] # Create the final targets list, with output paths added. for d in domains: res.append({'target': d, 'output': os.path.join(output_path, d)}) return res def build_cmd(self, args): ''' Create the actual command that will be executed. Use {target} and {output} as placeholders. ''' cmd = self.binary + " -domain {target} -output {output} " # Add in any extra arguments passed in the extra_args parameter if args.tool_args: cmd += args.tool_args # Add that data parameter in there cmd += " -data " + args.data return cmd def process_output(self, cmds): ''' Process the output generated by the earlier commands. ''' # Cycle through all of the targets we ran earlier for c in cmds: output_file = c['output'] target = c['target'] # Read file data = open(output_file).read().split('\n') # Quick and dirty way to filter out headers and blank lines, as well # as duplicates res = list( set([ d for d in data if 'Domain,Cname,Provider' not in d and d ])) if res: # Load up the DB entry. created, subdomain = self.Domain.find_or_create(domain=target) # Process results for d in res: results = d.split(',') if results[3] == "false": display_warning( "Hosting found at {} for {}, not vulnerable.". format(target, results[2])) elif results[3] == "true": display_new("{} vulnerable to {}!".format( target, results[2])) if not subdomain.meta[self.name].get( 'vulnerable', False): subdomain.meta[self.name]['vulnerable'] = [] subdomain.meta[self.name]['vulnerable'].append(d) else: display_warning("Not sure of result: {}".format(data)) # This is a little hackish, but needed to get data to save t = dict(subdomain.meta) self.Domain.commit() subdomain.meta = t self.Domain.commit()