class Module(ModuleTemplate): """ The Shodan module will either iterate through Shodan search results from net:<cidr> for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be added to the database, along with a dictionary object of the API results. """ name = "ShodanImport" def __init__(self, db): self.db = db self.Port = PortRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) self.Domain = DomainRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-k", "--api_key", help="API Key for accessing Shodan") self.options.add_argument( "-s", "--search", help="Custom search string (will use credits)") self.options.add_argument( "-i", "--import_db", help="Import scoped IPs from the database", action="store_true", ) self.options.add_argument("--rescan", help="Rescan CIDRs already processed", action="store_true") self.options.add_argument("--fast", help="Use 'net' filter. (May use credits)", action="store_true") self.options.add_argument( "--cidr_only", help="Import only CIDRs from database (not individual IPs)", action="store_true", ) self.options.add_argument("--target", "-t", help="Scan a specific CIDR/IP") def run(self, args): ranges = [] cidrs = [] ips = [] search = [] if not args.api_key: display_error("You must supply an API key to use shodan!") return if args.search: search = [args.search] if args.import_db: if args.rescan: if args.fast: search += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all() ] else: cidrs += [c.cidr for c in self.ScopeCidr.all()] if not args.cidr_only: ips += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active") ] else: if args.fast: search += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all(tool=self.name) ] else: cidrs += [ c.cidr for c in self.ScopeCidr.all(tool=self.name) ] if not args.cidr_only: ips += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active", tool=self.name) ] if args.target: if '/' not in args.target: ips += [args.target] elif args.fast: cidrs += ["net:{}".format(args.target)] else: cidrs += [args.target] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] ranges += ips ranges += search display( "Doing a total of {} queries. Estimated time: {} days, {} hours, {} minutes and {} seconds." .format(len(ranges), int(len(ranges) / 24.0 / 60.0 / 60.0), int(len(ranges) / 60.0 / 60.0) % 60, int(len(ranges) / 60.0) % 60, len(ranges) % 60)) for c in cidrs: ranges = [str(i) for i in IPNetwork(c)] display( "Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds." .format(c, int(len(ranges) / 24.0 / 60.0 / 60.0), int(len(ranges) / 60.0 / 60.0) % 60, int(len(ranges) / 60.0) % 60, len(ranges) % 60)) for r in ranges: self.get_shodan(r, args) created, cd = self.ScopeCidr.find_or_create(cidr=c) if created: cd.delete() else: cd.set_tool(self.name) self.ScopeCidr.commit() display( "Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds." .format(len(ips), int(len(ranges) / 24.0 / 60.0 / 60.0), int(len(ranges) / 60.0 / 60.0) % 60, int(len(ranges) / 60.0) % 60, len(ranges) % 60)) for i in ips: self.get_shodan(i, args) created, ip = self.IPAddress.find_or_create(ip_address=i) if created: ip.delete() else: ip.set_tool(self.name) self.IPAddress.commit() for s in search: self.get_shodan(s, args) if s[:4] == "net:": created, cd = self.ScopeCidr.find_or_create(cidr=s[4:]) if created: cd.delete() else: cd.set_tool(self.name) self.ScopeCidr.commit() def get_shodan(self, r, args): api_host_url = "https://api.shodan.io/shodan/host/{}?key={}" api_search_url = ( "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}") time.sleep(1) if ":" in r: display("Doing Shodan search: {}".format(r)) try: results = json.loads( requests.get(api_search_url.format(args.api_key, r, 1)).text) if results.get( "error") and "request timed out" in results["error"]: display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get(api_search_url.format(args.api_key, r, 1)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next total = len(results["matches"]) matches = [] i = 1 # pdb.set_trace() while total > 0: display("Adding {} results from page {}".format(total, i)) matches += results["matches"] i += 1 try: time.sleep(1) results = json.loads( requests.get(api_search_url.format(args.api_key, r, i)).text) if (results.get("error") and "request timed out" in results["error"] # noqa: W503 ): display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) total = len(results["matches"]) except Exception as e: display_error("Something went wrong: {}".format(e)) total = 0 pdb.set_trace() domains = [] for res in matches: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create(ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create(ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.status = "open" port.meta["shodan_data"] = res port.save() if res.get("ssl", {}).get('cert', {}).get('extensions'): for d in res['ssl']['cert']['extensions']: if d['name'] == 'subjectAltName': domains += get_domains_from_data(d['name']) if res.get("ssl", {}).get('cert', {}).get( 'subject', {} ).get('CN') and '*' not in res['ssl']['cert']['subject']['CN']: domains.append(res['ssl']['cert']['subject']['CN']) if res.get('hostnames'): domains += res['hostnames'] for d in list(set(domains)): display("Adding discovered domain {}".format(only_valid(d))) created, domain = self.Domain.find_or_create( domain=only_valid(d)) else: display("Searching for {}".format(r)) try: results = json.loads( requests.get(api_host_url.format(r, args.api_key)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next # pdb.set_trace() if results.get("data", False): display("{} results found for: {}".format( len(results["data"]), r)) domains = [] for res in results["data"]: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.status = "open" port.meta["shodan_data"] = res port.save() if res.get("ssl", {}).get('cert', {}).get('extensions'): for d in res['ssl']['cert']['extensions']: if d['name'] == 'subjectAltName': domains += get_domains_from_data(d['data']) display( "Domains discovered in subjectAltName: {}". format(", ".join( get_domains_from_data(d['data'])))) if res.get("ssl", {}).get('cert', {}).get( 'subject', {}).get('CN') and '*' not in res['ssl'][ 'cert']['subject']['CN']: domains.append(res['ssl']['cert']['subject']['CN']) if res.get('hostnames'): domains += res['hostnames'] for d in list(set(domains)): display("Adding discovered domain {}".format(d)) created, domain = self.Domain.find_or_create(domain=d)
class Module(ModuleTemplate): name = "Nessus" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--import_file", help= "Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument( "--launch", help= "Launch Nessus scan using Actively scoped IPs and domains in the database", action="store_true", ) self.options.add_argument("--job_name", help="Job name inside Nessus", default="Armory Job") self.options.add_argument("--username", help="Nessus Username") self.options.add_argument("--password", help="Nessus Password") self.options.add_argument( "--host", help="Hostname:Port of Nessus web interface (ie localhost:8835") self.options.add_argument("--uuid", help="UUID of Nessus Policy to run") self.options.add_argument("--policy_id", help="Policy ID to use") self.options.add_argument("--folder_id", help="ID for folder to store job in") self.options.add_argument( "--download", help="Download Nessus job from server and import", action="store_true", ) self.options.add_argument("--job_id", help="Job ID to download and import") self.options.add_argument( "--output_path", help="Path to store downloaded file (Default: Nessus)", default=self.name, ) self.options.add_argument( "--disable_mitre", help="Disable mitre CVE data gathering.", action="store_true", ) def run(self, args): if args.import_file: for nFile in args.import_file: self.process_data(nFile, args) elif args.launch: if (not args.username # noqa: W503 and not args.password # noqa: W503 and not args.host # noqa: W503 and not args.uuid # noqa: W503 and not args.policy_id # noqa: W503 and not args.folder_id # noqa: W503 ): display_error( "You must supply a username, password, and host to launch a Nessus job" ) else: n = NessusRequest( args.username, args.password, args.host, uuid=args.uuid, policy_id=args.policy_id, folder_id=args.folder_id, ) ips = [ ip.ip_address for ip in self.IPAddress.all(scope_type="active", tool=self.name) ] cidrs = [ cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name) ] domains = [ domain.domain for domain in self.Domain.all(scope_type="active", tool=self.name) ] targets = ", ".join(merge_ranges(ips + cidrs) + domains) res = n.launch_job(targets, args.job_name) display("New Nessus job launched with ID {}".format(res)) display( "Remember this number! You'll need it to download the job once it is done." ) elif args.download: if (not args.username # noqa: W503 and not args.password # noqa: W503 and not args.host # noqa: W503 and not args.job_id # noqa: W503 ): display_error( "You must supply host, username, password and job_id to download a report to import" ) else: n = NessusRequest( args.username, args.password, args.host, ) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join( output_path, "Nessus-export-{}.nessus".format(int(time.time()))) n.export_file(args.job_id, output_path) self.process_data(output_path, args) def nessCheckPlugin(self, tag): nessPlugins = [ "10759", "77026", "20089", "56984", "71049", "70658", "40984", "11411", ] pluginID = tag.get("pluginID") if pluginID in nessPlugins: # print pluginID + " is in the list" if pluginID == "10759": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split( "\n\n")[3].strip() ) # returns IP for Web Server HTTP Header INternal IP disclosure else: return "" if pluginID == "77026": if tag.find("plugin_output") is not None: return ( tag.find("plugin_output").text.split( "\n\n")[3].strip() ) # Microsoft Exchange Client Access Server Information Disclosure (IP addy) else: return "" if pluginID == "71049" or pluginID == "70658": output = "" if tag.find("plugin_output") is not None: tmp = tag.find("plugin_output").text.split(":")[ 1] # SSH Weak MAC & CBC Algorithms Enabled # print "#"*5 tmp = tmp.split("\n\n")[1].replace(" ", "") # print "#"*5 output = tmp.split("\n") # print ", ".join(output) return ", ".join(output) if pluginID == "56984": if tag.find("plugin_output") is not None: tmp = (tag.find("plugin_output").text.split( "This port supports ")[1].strip() ) # SSL / TLS Versions Supported tmp = tmp.split("/") bad = [] for i in tmp: # print i if "SSLv" in i: bad.append(i) elif "TLSv1.0" in i: bad.append(i) if bad != []: return ", ".join(bad).rstrip(".") else: return "" else: return "" if pluginID == "40984": # browsable web dirs if tag.find("plugin_output") is not None: tmp = (tag.find("plugin_output").text.split( "The following directories are browsable :") [1].strip()) directories = tmp.split("\n") return "\n".join(directories) if pluginID == "11411": # Backup Files Disclosure if tag.find("plugin_output") is not None: urls = [] tmp = (tag.find("plugin_output").text.split( "It is possible to read the following backup file") [1].strip()) tmpUrls = tmp.split("\n") for url in tmpUrls: if "URL" in url: urls.append(url.split(":")[1].lstrip()) if urls: return "\n".join(urls) else: return "" if pluginID == "20089": # F5 cookie if tag.find("plugin_output") is not None: f5Output = [] cookieVal = [] output = tag.find("plugin_output").text.strip().split("\n") for line in output: # print line line = line.split(":") for i, item in enumerate(line): item = item.strip() if "Cookie" in item: line.pop(i) # Pop to remove the first? tmp = line.pop(i) tmp.strip() cookieVal.append(tmp) else: item = "".join(item) f5Output.append(item) f5Output = " : ".join(f5Output) f5Output = f5Output.replace(" : : ", ", ") f5Output += " [" + ", ".join(cookieVal) + "]" c = 0 tmpF5Output = f5Output.split() for i, letter in enumerate(tmpF5Output): if letter == ":": c += 1 if (c % 2) == 0: tmpF5Output[i] = " " return "".join(tmpF5Output).replace("[", " [") else: return "" else: return False def getVulns(self, ip, ReportHost): """Gets vulns and associated services""" for tag in ReportHost.iter("ReportItem"): exploitable = False cves = [] vuln_refs = {} proto = tag.get("protocol") port = tag.get("port") svc_name = tag.get("svc_name").replace("?", "") plugin_output = [] tmpPort = proto + "/" + port if tmpPort.lower() == "tcp/443": portName = "https" elif tmpPort.lower() == "tcp/80": portName = "http" elif svc_name == "www": plugin_name = tag.get("pluginName") if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower( ): portName = "https" else: portName = "http" else: portName = svc_name created, db_port = self.Port.find_or_create(port_number=port, status="open", proto=proto, ip_address_id=ip.id) if db_port.service_name == "http": if portName == "https": db_port.service_name = portName elif db_port.service_name == "https": pass else: db_port.service_name = portName db_port.save() if tag.get("pluginID") == "56984": severity = 1 elif tag.get("pluginID") == "11411": severity = 3 else: severity = int(tag.get("severity")) findingName = tag.get("pluginName") description = tag.find("description").text if tag.find( "solution") is not None and tag.find("solution") != "n/a": solution = tag.find("solution").text else: solution = "No Remediation From Nessus" nessCheck = self.nessCheckPlugin(tag) if nessCheck: if not db_port.info: db_port.info = {findingName: nessCheck} else: db_port.info[findingName] = nessCheck db_port.save() if tag.find("exploit_available") is not None: exploitable = True metasploits = tag.findall("metasploit_name") if metasploits: vuln_refs["metasploit"] = [] for tmp in metasploits: vuln_refs["metasploit"].append(tmp.text) edb_id = tag.findall("edb-id") if edb_id: vuln_refs["edb-id"] = [] for tmp in edb_id: vuln_refs["edb-id"].append(tmp.text) tmpcves = tag.findall("cve") for c in tmpcves: if c.text not in cves: cves.append(c.text) cwe_ids = [c.text for c in tag.findall("cwe")] references = [c.text for c in tag.findall("see_also")] if not self.Vulnerability.find(name=findingName): created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=description, remediation=solution, ) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if exploitable: display_new("exploit avalable for " + findingName) if vuln_refs: db_vuln.exploit_reference = vuln_refs else: db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: if db_vuln.exploit_reference is not None: for key in vuln_refs.keys(): if key not in db_vuln.exploit_reference.keys(): db_vuln.exploit_reference[key] = vuln_refs[key] else: for ref in vuln_refs[key]: if ref not in db_vuln.exploit_reference[ key]: db_vuln.exploit_reference[key].append( ref) else: db_vuln.exploit_reference = vuln_refs db_vuln.meta['CWEs'] = cwe_ids db_vuln.meta['Refs'] = references if tag.find("plugin_output") is not None: plugin_output = tag.find("plugin_output").text if not db_vuln.meta.get('plugin_output', False): db_vuln.meta['plugin_output'] = {} if not db_vuln.meta['plugin_output'].get(ip.ip_address, False): db_vuln.meta['plugin_output'][ip.ip_address] = {} if not db_vuln.meta['plugin_output'][ip.ip_address].get( port, False): db_vuln.meta['plugin_output'][ip.ip_address][port] = [] if plugin_output not in db_vuln.meta['plugin_output'][ ip.ip_address][port]: db_vuln.meta['plugin_output'][ip.ip_address][port].append( plugin_output) if not self.args.disable_mitre: for cve in cves: if not self.CVE.find(name=cve): try: url = 'https://nvd.nist.gov/vuln/detail//{}' res = requests.get(url.format(cve)).text cveDescription = res.split( '<p data-testid="vuln-description">')[1].split( '</p>')[0] if 'vuln-cvssv3-base-score' in res: cvss = float( res.split( '<span data-testid="vuln-cvssv3-base-score">' )[1].split('</span>')[0].strip()) else: cvss = float( res.split( '<span data-testid="vuln-cvssv2-base-score">' )[1].split('</span>')[0].strip()) cveDescription = res["summary"] cvss = float(res["cvss"]) except Exception: cveDescription = None cvss = None if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss, ) db_cve.vulnerabilities.append(db_vuln) else: db_cve = self.CVE.find(name=cve) if (db_cve.description is None and cveDescription is not None # noqa: W503 ): db_cve.description = cveDescription if db_cve.temporal_score is None and cvss is not None: db_cve.temporal_score = cvss db_cve.vulnerabilities.append(db_vuln) def process_data(self, nFile, args): display("Reading " + nFile) tree = ET.parse(nFile) root = tree.getroot() self.args = args for ReportHost in root.iter("ReportHost"): os = [] hostname = "" hostIP = "" for HostProperties in ReportHost.iter("HostProperties"): for tag in HostProperties: if tag.get("name") == "host-ip": hostIP = tag.text if tag.get("name") == "host-fqdn": hostname = tag.text.lower() hostname = hostname.replace("www.", "") if tag.get("name") == "operating-system": os = tag.text.split("\n") if hostIP: # apparently nessus doesn't always have an IP to work with... if hostname: display("Gathering Nessus info for {} ( {} )".format( hostIP, hostname)) else: display("Gathering Nessus info for {}".format(hostIP)) created, ip = self.IPAddress.find_or_create(ip_address=hostIP) if hostname: created, domain = self.Domain.find_or_create( domain=hostname) if ip not in domain.ip_addresses: ip.save() domain.ip_addresses.append(ip) domain.save() if os: for o in os: if not ip.OS: ip.OS = o else: if o not in ip.OS.split(" OR "): ip.OS += " OR " + o self.getVulns(ip, ReportHost) self.IPAddress.commit() return
class Module(ToolTemplate): """ Module for running nmap. Make sure to pass all nmap-specific arguments at the end, after --tool_args """ name = "Nmap" binary_name = "nmap" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--hosts", help= "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument("--hosts_file", help="File containing hosts") self.options.add_argument( "-i", "--hosts_database", help="Use unscanned hosts from the database", action="store_true", ) self.options.add_argument("--rescan", help="Overwrite files without asking", action="store_true") self.options.add_argument( "--filename", help="Output filename. By default will use the current timestamp.", ) self.options.add_argument( "--ssl_cert_mode", help= "Scan only SSL enabled hosts to collect SSL certs (and domain names)", action="store_true", ) self.options.add_argument( "--filter_ports", help= "Comma separated list of protoPort to filter out of results. Useful if firewall returns specific ports open on every host. Ex: t80,u5060" ) self.options.set_defaults(timeout=None) self.options.add_argument("--import_file", help="Import results from an Nmap XML file.") def get_targets(self, args): self.args = args if args.import_file: args.no_binary = True return [{"target": "", "output": args.import_file}] targets = [] if args.hosts: if type(args.hosts) == list: for h in args.hosts: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] else: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] if args.hosts_database: if args.rescan: targets += [ h.ip_address for h in self.IPAddress.all(scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all()] else: targets += [ h.ip_address for h in self.IPAddress.all(tool=self.name, scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)] if args.hosts_file: for h in [ l for l in open(args.hosts_file).read().split("\n") if l ]: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] data = [] if args.ssl_cert_mode: ports = self.Port.all(service_name='https') data = list(set([i.ip_address.ip_address for i in ports])) port_numbers = list(set([str(i.port_number) for i in ports])) args.tool_args += " -sV -p {} --script ssl-cert ".format( ','.join(port_numbers)) else: # Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs for t in targets: ips = [str(i) for i in list(IPNetwork(t))] data += ips _, file_name = tempfile.mkstemp() open(file_name, "w").write("\n".join(list(set(data)))) if args.output_path[0] == "/": self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.filename: output_path = os.path.join(self.path, args.filename) else: output_path = os.path.join( self.path, "nmap-scan-%s.xml" % datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"), ) return [{"target": file_name, "output": output_path}] def build_cmd(self, args): command = "sudo " + self.binary + " -oX {output} -iL {target} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): self.import_nmap(cmds[0]["output"]) if cmds[0]["target"]: os.unlink(cmds[0]["target"]) def parseHeaders(self, httpHeaders): bsHeaders = [ "Pragma", "Expires", "Date", "Transfer-Encoding", "Connection", "X-Content-Type-Options", "Cache-Control", "X-Frame-Options", "Content-Type", "Content-Length", "(Request type", ] keepHeaders = {} for i in range(0, len(httpHeaders)): if httpHeaders[i].strip() != "" and httpHeaders[i].split( ":")[0].strip() not in " ".join(bsHeaders): hName = httpHeaders[i].split(":")[0].strip() hValue = "".join(httpHeaders[i].split(":")[1:]).strip() keepHeaders[hName] = hValue if keepHeaders == {}: keepHeaders = "" return keepHeaders def import_nmap( self, filename): # domains={}, ips={}, rejects=[] == temp while no db nFile = filename # pdb.set_trace() try: tree = ET.parse(nFile) root = tree.getroot() hosts = root.findall("host") except Exception as e: print("Error: {}" % e) print(nFile + " doesn't exist somehow...skipping") return for host in hosts: hostIP = host.find("address").get("addr") created, ip = self.IPAddress.find_or_create(ip_address=hostIP) for hostname in host.findall("hostnames/hostname"): hostname = hostname.get("name") hostname = hostname.lower().replace("www.", "") # reHostname = re.search( # r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname # ) # attempt to not get PTR record # if not reHostname: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.save() for port in host.findall("ports/port"): if port.find("state").get("state"): portState = port.find("state").get("state") hostPort = port.get("portid") portProto = port.get("protocol") # pdb.set_trace() if not self.args.filter_ports or int(hostPort) not in [ int(p) for p in self.args.filter_ports.split(',') ]: created, db_port = self.Port.find_or_create( port_number=hostPort, status=portState, proto=portProto, ip_address=ip, ) if port.find("service") is not None: portName = port.find("service").get("name") if portName == "http" and hostPort == "443": portName = "https" else: portName = "Unknown" if created: db_port.service_name = portName info = db_port.info if not info: info = {} for script in port.findall( "script"): # just getting commonName from cert if script.get("id") == "ssl-cert": db_port.cert = script.get("output") cert_domains = self.get_domains_from_cert( script.get("output")) for hostname in cert_domains: hostname = hostname.lower().replace( "www.", "") created, domain = self.Domain.find_or_create( domain=hostname) if created: print("New domain found: %s" % hostname) elif script.get("id") == "vulners": print("Gathering vuln info for {} : {}/{}\n". format(hostIP, portProto, hostPort)) self.parseVulners(script.get("output"), db_port) elif script.get("id") == "banner": info["banner"] = script.get("output") elif script.get("id") == "http-headers": httpHeaders = script.get("output") httpHeaders = httpHeaders.strip().split("\n") keepHeaders = self.parseHeaders(httpHeaders) info["http-headers"] = keepHeaders elif script.get("id") == "http-auth": info["http-auth"] = script.get("output") elif script.get("id") == "http-title": info["http-title"] = script.get("output") db_port.info = info db_port.save() self.IPAddress.commit() def parseVulners(self, scriptOutput, db_port): urls = re.findall(r"(https://vulners.com/cve/CVE-\d*-\d*)", scriptOutput) for url in urls: vuln_refs = [] exploitable = False cve = url.split("/cve/")[1] vulners = requests.get("https://vulners.com/cve/%s" % cve).text exploitdb = re.findall( r"https://www.exploit-db.com/exploits/\d{,7}", vulners) for edb in exploitdb: exploitable = True if edb.split("/exploits/")[1] not in vuln_refs: vuln_refs.append(edb.split("/exploits/")[1]) if not self.CVE.find(name=cve): # print "Gathering CVE info for", cve try: res = json.loads( requests.get("http://cve.circl.lu/api/cve/%s" % cve).text) cveDescription = res["summary"] cvss = float(res["cvss"]) findingName = res["oval"][0]["title"] if int(cvss) <= 3: severity = 1 elif (int(cvss) / 2) == 5: severity = 4 else: severity = int(cvss) / 2 if not self.Vulnerability.find(name=findingName): # print "Creating", findingName created, db_vuln = self.Vulnerability.find_or_create( name=findingName, severity=severity, description=cveDescription, ) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploit_reference = {"edb-id": vuln_refs} db_vuln.save() else: # print "modifying",findingName db_vuln = self.Vulnerability.find(name=findingName) db_vuln.ports.append(db_port) db_vuln.exploitable = exploitable if vuln_refs: db_vuln.exploitable = exploitable if db_vuln.exploit_reference is not None: if "edb-id" in db_vuln.exploit_reference: for ref in vuln_refs: if (ref not in db_vuln. exploit_reference["edb-id"]): db_vuln.exploit_reference[ "edb-id"].append(ref) else: db_vuln.exploit_reference[ "edb-id"] = vuln_refs else: db_vuln.exploit_reference = { "edb-id": vuln_refs } db_vuln.save() if not self.CVE.find(name=cve): created, db_cve = self.CVE.find_or_create( name=cve, description=cveDescription, temporal_score=cvss) db_cve.vulnerabilities.append(db_vuln) db_cve.save() else: db_cve = self.CVE.find(name=cve) db_cve.vulnerabilities.append(db_vuln) db_cve.save() self.Vulnerability.commit() self.CVE.commit() except Exception: print( "something went wrong with the vuln/cve info gathering" ) if vulners: print( "Vulners report was found but no exploit-db was discovered" ) # "Affected vulners items" # print vulners print("Affected CVE") print(cve) pass else: db_cve = self.CVE.find(name=cve) for db_vulns in db_cve.vulnerabilities: if db_port not in db_vulns.ports: db_vulns.ports.append(db_port) return def get_domains_from_cert(self, cert): # Shamelessly lifted regex from stack overflow regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}" domains = list( set([d for d in re.findall(regex, cert) if "*" not in d])) return domains
class Module(ToolTemplate): """ Module for running masscan. Make sure to pass all masscan-specific arguments at the end, after --tool_args """ name = "Masscan" binary_name = "masscan" def __init__(self, db): self.db = db self.BaseDomain = BaseDomainRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.Port = PortRepository(db, self.name) self.Vulnerability = VulnRepository(db, self.name) self.CVE = CVERepository(db, self.name) self.ScopeCIDR = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument( "--hosts", help= "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS", nargs="+", ) self.options.add_argument("--hosts_file", help="File containing hosts") self.options.add_argument( "-i", "--hosts_database", help="Use unscanned hosts from the database", action="store_true", ) self.options.add_argument("--rescan", help="Overwrite files without asking", action="store_true") self.options.add_argument( "--filename", help="Output filename. By default will use the current timestamp.", ) self.options.set_defaults(timeout=None) self.options.add_argument( "--import_file", help="Import results from an Masscan/Nmap XML file.") def get_targets(self, args): if args.import_file: args.no_binary = True return [{"target": "", "output": args.import_file}] targets = [] if args.hosts: if type(args.hosts) == list: for h in args.hosts: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] else: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] if args.hosts_database: if args.rescan: targets += [ h.ip_address for h in self.IPAddress.all(scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all()] else: targets += [ h.ip_address for h in self.IPAddress.all(tool=self.name, scope_type="active") ] targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)] if args.hosts_file: for h in [ l for l in open(args.hosts_file).read().split("\n") if l ]: if check_if_ip(h): targets.append(h) else: created, domain = self.Domain.find_or_create(domain=h) targets += [i.ip_address for i in domain.ip_addresses] # Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs data = [] for t in targets: ips = [str(i) for i in list(IPNetwork(t))] data += ips _, file_name = tempfile.mkstemp() open(file_name, "w").write("\n".join(set(data))) if args.output_path[0] == "/": self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path[1:]) else: self.path = os.path.join(self.base_config["PROJECT"]["base_path"], args.output_path) if not os.path.exists(self.path): os.makedirs(self.path) if args.filename: output_path = os.path.join(self.path, args.filename) else: output_path = os.path.join( self.path, "masscan-%s.xml" % datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"), ) return [{"target": file_name, "output": output_path}] def build_cmd(self, args): command = "sudo " + self.binary + " -oX {output} -iL {target} " if args.tool_args: command += args.tool_args return command def process_output(self, cmds): self.import_masscan(cmds[0]["output"]) if cmds[0]["target"]: os.unlink(cmds[0]["target"]) def import_masscan( self, filename): # domains={}, ips={}, rejects=[] == temp while no db nFile = filename try: tree = ET.parse(nFile) root = tree.getroot() hosts = root.findall("host") except Exception: print(nFile + " doesn't exist somehow...skipping") return for host in hosts: hostIP = host.find("address").get("addr") created, ip = self.IPAddress.find_or_create(ip_address=hostIP) for hostname in host.findall("hostnames/hostname"): hostname = hostname.get("name") hostname = hostname.lower().replace("www.", "") # reHostname = re.search( # r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname # ) # attempt to not get PTR record # if not reHostname: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.save() for port in host.findall("ports/port"): if port.find("state").get("state"): portState = port.find("state").get("state") hostPort = port.get("portid") portProto = port.get("protocol") created, db_port = self.Port.find_or_create( port_number=hostPort, status=portState, proto=portProto, ip_address=ip, ) info = db_port.info if not info: info = {} if port.find("service") is not None: service = port.find("service") portName = service.get("name") if portName == "http" and hostPort == "443": portName = "https" banner = service.get("banner", None) if banner: print("Found banner: {}".format(banner)) info["banner"] = banner else: portName = "Unknown" if created: db_port.service_name = portName db_port.info = info db_port.save() self.IPAddress.commit() def get_domains_from_cert(self, cert): # Shamelessly lifted regex from stack overflow regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}" domains = list( set([d for d in re.findall(regex, cert) if "*" not in d])) return domains
class Module(ModuleTemplate): """ The Shodan module will either iterate through Shodan search results from net:<cidr> for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be added to the database, along with a dictionary object of the API results. """ name = "ShodanImport" def __init__(self, db): self.db = db self.Port = PortRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) self.ScopeCidr = ScopeCIDRRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-k", "--api_key", help="API Key for accessing Shodan") self.options.add_argument( "-s", "--search", help="Custom search string (will use credits)") self.options.add_argument( "-i", "--import_db", help="Import scoped IPs from the database", action="store_true", ) self.options.add_argument("--rescan", help="Rescan CIDRs already processed", action="store_true") self.options.add_argument("--fast", help="Use 'net' filter. (May use credits)", action="store_true") self.options.add_argument( "--cidr_only", help="Import only CIDRs from database (not individual IPs)", action="store_true", ) def run(self, args): if not args.api_key: display_error("You must supply an API key to use shodan!") return if args.search: ranges = [args.search] if args.import_db: ranges = [] if args.rescan: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all() ] else: cidrs = [c.cidr for c in self.ScopeCidr.all()] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active") ] else: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all(tool=self.name) ] else: cidrs = [ c.cidr for c in self.ScopeCidr.all(tool=self.name) ] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active", tool=self.name) ] api_host_url = "https://api.shodan.io/shodan/host/{}?key={}" api_search_url = ( "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}") for r in ranges: time.sleep(1) if ":" in r: display("Doing Shodan search: {}".format(r)) try: results = json.loads( requests.get(api_search_url.format(args.api_key, r, 1)).text) if results.get("error") and "request timed out" in results[ "error"]: display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next total = len(results["matches"]) matches = [] i = 1 while total > 0: display("Adding {} results from page {}".format(total, i)) matches += results["matches"] i += 1 try: time.sleep(1) results = json.loads( requests.get( api_search_url.format(args.api_key, r, i)).text) if (results.get("error") and "request timed out" in results["error"] # noqa: W503 ): display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) total = len(results["matches"]) except Exception as e: display_error("Something went wrong: {}".format(e)) total = 0 pdb.set_trace() for res in matches: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.status = "open" port.meta["shodan_data"] = res port.save() else: try: results = json.loads( requests.get(api_host_url.format(r, args.api_key)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next # pdb.set_trace() if results.get("data", False): display("{} results found for: {}".format( len(results["data"]), r)) for res in results["data"]: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.status = "open" port.meta["shodan_data"] = res port.save() self.IPAddress.commit()
class Module(ModuleTemplate): name = "HeaderScanner" def __init__(self, db): self.db = db self.Port = PortRepository(db, self.name) self.Domain = DomainRepository(db, self.name) self.IPAddress = IPRepository(db, self.name) def set_options(self): super(Module, self).set_options() self.options.add_argument("-t", "--timeout", help="Connection timeout (default 5)", default="5") self.options.add_argument("-u", "--url", help="URL to get headers") self.options.add_argument("--file", help="Import URLs from file") self.options.add_argument( "-i", "--import_db", help="Import URLs from the database", action="store_true", ) self.options.add_argument("-th", "--threads", help="Number of threads to run", default="10") self.options.add_argument("--rescan", help="Rescan URLs already processed", action="store_true") def run(self, args): data = [] if args.url: service = args.url.split(":")[0] host = args.url.split("/")[2] if args.url.count(":") == 2: port = args.url.split(":")[2].split("/")[0] elif service == "http": port = "80" elif service == "https": port = "443" else: display_error( "Could not figure out port number for url: {}".format( args.url)) sys.exit(1) if check_if_ip(host): created, ip = self.IPAddress.find_or_create(ip_address=host) else: created, domain = self.Domain.find_or_create(domain=host) ip = domain.ip_addresses[0] created, service_id = self.Port.find_or_create(ip_address=ip, port_number=port) service_id.service_name = service data.append([service_id.id, [args.url], args.timeout]) if args.file: url = open(args.file).read().split("\n") for u in url: if u: service = u.split(":")[0] host = u.split("/")[2] if u.count(":") == 2: port = u.split(":")[2].split("/")[0] elif service == "http": port = "80" elif service == "https": port = "443" else: display_error( "Could not figure out port number for url: {}". format(args.url)) sys.exit(1) if check_if_ip(host): created, ip = self.IPAddress.find_or_create( ip_address=host) else: created, domain = self.Domain.find_or_create( domain=host) ip = domain.ip_addresses[0] created, service_id = self.Port.find_or_create( ip_address=ip, port_number=port) service_id.service_name = service data.append([service_id.id, [u], args.timeout]) if args.import_db: if args.rescan: svc = self.Port.all(service_name="http") svc += self.Port.all(service_name="https") else: svc = self.Port.all(service_name="http", tool=self.name) svc += self.Port.all(service_name="https", tool=self.name) for s in svc: if s.ip_address.in_scope: urls = [ "%s://%s:%s" % (s.service_name, s.ip_address.ip_address, s.port_number) ] for d in s.ip_address.domains: urls.append("%s://%s:%s" % (s.service_name, d.domain, s.port_number)) data.append([s.id, urls, args.timeout]) if data: pool = ThreadPool(int(args.threads)) results = pool.map(process_urls, data) display_new("Adding headers to the database") for i, headers, cookies in results: created, svc = self.Port.find_or_create(id=i) svc.meta["headers"] = headers svc.meta["cookies"] = cookies svc.update() self.Port.commit()