def process_urls(data): i, urls, timeout = data blacklist = [ 'Date', 'Connection', 'Content-Type', 'Content-Length', 'Keep-Alive', 'Content-Encoding', 'Vary' ] new_headers = {} for u in urls: display("Processing %s" % u) try: res = requests.get(u, timeout=int(timeout), verify=False) for k in res.headers.keys(): if k not in blacklist: if not new_headers.get(u, False): new_headers[u] = [] new_headers[u].append("%s: %s" % (k, res.headers[k])) except KeyboardInterrupt: display_warning("Got Ctrl+C, exiting") sys.exit(1) except Exception as e: display_error("%s no good, skipping: %s" % (u, e)) return (i, new_headers)
def process_output(self, cmds): ''' Process the output generated by the earlier commands. ''' # Cycle through all of the targets we ran earlier for c in cmds: output_file = c['output'] target = c['target'] # Read file data = open(output_file).read().split('\n') # Quick and dirty way to filter out headers and blank lines, as well # as duplicates res = list( set([ d for d in data if 'Domain,Cname,Provider' not in d and d ])) if res: # Load up the DB entry. created, subdomain = self.Domain.find_or_create(domain=target) # Process results for d in res: results = d.split(',') if results[3] == "false": display_warning( "Hosting found at {} for {}, not vulnerable.". format(target, results[2])) elif results[3] == "true": display_new("{} vulnerable to {}!".format( target, results[2])) if not subdomain.meta[self.name].get( 'vulnerable', False): subdomain.meta[self.name]['vulnerable'] = [] subdomain.meta[self.name]['vulnerable'].append(d) else: display_warning("Not sure of result: {}".format(data)) # This is a little hackish, but needed to get data to save t = dict(subdomain.meta) self.Domain.commit() subdomain.meta = t self.Domain.commit()
def process_output(self, cmds): display_warning( "There is currently no post-processing for this module. For the juicy results, refer to the output file paths." )
def run(self, args): if not args.api_key: display_error("You must supply an API key to use shodan!") return if args.search: ranges = [args.search] if args.import_db: ranges = [] if args.rescan: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all() ] else: cidrs = [c.cidr for c in self.ScopeCidr.all()] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active") ] else: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all(tool=self.name) ] else: cidrs = [ c.cidr for c in self.ScopeCidr.all(tool=self.name) ] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active", tool=self.name) ] api_host_url = "https://api.shodan.io/shodan/host/{}?key={}" api_search_url = ( "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}") for r in ranges: time.sleep(1) if ":" in r: display("Doing Shodan search: {}".format(r)) try: results = json.loads( requests.get(api_search_url.format(args.api_key, r, 1)).text) if results.get("error") and "request timed out" in results[ "error"]: display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next total = len(results["matches"]) matches = [] i = 1 while total > 0: display("Adding {} results from page {}".format(total, i)) matches += results["matches"] i += 1 try: time.sleep(1) results = json.loads( requests.get( api_search_url.format(args.api_key, r, i)).text) if (results.get("error") and "request timed out" in results["error"]): display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) total = len(results["matches"]) except Exception as e: display_error("Something went wrong: {}".format(e)) total = 0 pdb.set_trace() for res in matches: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.meta["shodan_data"] = res port.save() else: try: results = json.loads( requests.get(api_host_url.format(r, args.api_key)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next # pdb.set_trace() if results.get("data", False): display("{} results found for: {}".format( len(results["data"]), r)) for res in results["data"]: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.meta["shodan_data"] = res port.save() self.IPAddress.commit()
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=False, **kwargs): created, d = super(DomainRepository, self).find_or_create(only_tool, **kwargs) display("Processing %s" % d.domain) if created: # If this is a new subdomain, set scoping info based on what is passed to the function initially. d.in_scope = in_scope d.passive_scope = passive_scope base_domain = '.'.join( [t for t in tldextract.extract(d.domain)[1:] if t]) BaseDomains = BaseDomainRepository(self.db, "") # If the base domain is new, it'll inherit the same scoping permissions. created, bd = BaseDomains.find_or_create( only_tool, passive_scope=d.passive_scope, in_scope=in_scope, domain=base_domain) if created: display_new( "The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s" % (base_domain, bd.in_scope, bd.passive_scope)) else: # If the base domain already exists, then the subdomain inherits the scope info from the base domain. d.passive_scope = bd.passive_scope d.in_scope = bd.in_scope d.base_domain = bd # Get all IPs that this domain resolves to. ips = [] try: answers = dns.resolver.query(d.domain, 'A') for a in answers: ips.append(a.address) except: # If something goes wrong with DNS, we end up here pass if not ips: display_warning("No IPs discovered for %s" % d.domain) for i in ips: IPAddresses = IPRepository(self.db, "") display("Processing IP address %s" % i) created, ip = IPAddresses.find_or_create( only_tool, in_scope=d.in_scope, passive_scope=d.passive_scope, ip_address=i) # If the IP is in scope, then the domain should be if ip.in_scope: d.in_scope = ip.in_scope ip.passive_scope = True d.passive_scope = True # display("%s marked active scope due to IP being marked active." % d.domain) elif ip.passive_scope: d.passive_scope = ip.passive_scope d.ip_addresses.append(ip) display_new( "%s is being added to the database. Active Scope: %s Passive Scope: %s" % (d.domain, d.in_scope, d.passive_scope)) # Final sanity check - if a domain is active scoped, it should also be passively scoped. if d.in_scope: d.passive_scope = True return created, d
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=True, **kwargs): created, ip = super(IPRepository, self).find_or_create(only_tool, **kwargs) if created: # If newly created then will determine scoping based on parent options and if in a scoped cidr. ip_str = ip.ip_address ip.passive_scope = passive_scope # If the parent domain is active scope, then this also is. if in_scope: ip.in_scope = in_scope else: # Go through ScopeCIDR table and see if this IP is in a CIDR in scope ScopeCidrs = ScopeCIDRRepository(self.db, "") addr = IPAddress(ip.ip_address) cidrs = ScopeCidrs.all() # pdb.set_trace() for c in cidrs: if addr in IPNetwork(c.cidr): ip.in_scope = True # Final sanity check - if an IP is active scoped, it should also be passive scoped. if ip.in_scope: ip.passive_scope = True ip.update() # Build CIDR info - mainly for reporting res = False for cidr in private_subnets: if IPAddress(ip_str) in cidr: res = ([str(cidr), "Non-Public Subnet"], ) if res: cidr_data = res else: while True: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except: res = IPWhois(ip_str).lookup_whois() if res["nets"]: break else: display_warning( "The networks didn't populate from whois. Usually retrying after a couple of seconds resolves this. Sleeping for 5 seconds and trying again." ) time.sleep(5) cidr_data = [] for n in res["nets"]: if "," in n["cidr"]: for cidr_str in n["cidr"].split(", "): cidr_data.append([cidr_str, n["description"]]) else: cidr_data.append([n["cidr"], n["description"]]) cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] try: cidr_len = len(IPNetwork(cidr_data[0][0])) except: pdb.set_trace() matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c display("Processing CIDR from whois: %s - %s" % (matching_cidr[1], matching_cidr[0])) CIDR = CIDRRepository(self.db, "") created, cidr = CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0]) if created: display_new("CIDR %s added to database" % cidr.cidr) cidr.org_name = matching_cidr[1] cidr.update() ip.cidr = cidr ip.update() display_new( "IP address %s added to database. Active Scope: %s Passive Scope: %s" % (ip.ip_address, ip.in_scope, ip.passive_scope)) return created, ip