def process_cidr(self, line): display("Processing %s" % line) if "/" in line: created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip()) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() elif "-" in line: start_ip, end_ip = line.strip().replace(" ", "").split("-") if "." not in end_ip: end_ip = ".".join(start_ip.split(".")[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c)) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update()
def process_urls(data): i, urls, timeout = data blacklist = [ 'Date', 'Connection', 'Content-Type', 'Content-Length', 'Keep-Alive', 'Content-Encoding', 'Vary' ] new_headers = {} for u in urls: display("Processing %s" % u) try: res = requests.get(u, timeout=int(timeout), verify=False) for k in res.headers.keys(): if k not in blacklist: if not new_headers.get(u, False): new_headers[u] = [] new_headers[u].append("%s: %s" % (k, res.headers[k])) except KeyboardInterrupt: display_warning("Got Ctrl+C, exiting") sys.exit(1) except Exception as e: display_error("%s no good, skipping: %s" % (u, e)) return (i, new_headers)
def process_domain(self, domain_str): created, domain = self.Domain.find_or_create(only_tool=True, domain=domain_str, in_scope=self.in_scope, passive_scope=self.passive_scope) if not created: if domain.in_scope != self.in_scope or domain.passive_scope != self.passive_scope: display("Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (domain_str, self.in_scope, self.passive_scope)) domain.in_scope = self.in_scope domain.passive_scope = self.passive_scope domain.update()
def process_ip(self, ip_str, force_scope=True): created, ip = self.IPAddress.find_or_create(only_tool=True, ip_address=ip_str, in_scope=in_scope, passive_scope=self.passive_scope) if not created: if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope: display("IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s" % (ip_str, self.in_scope, self.passive_scope)) ip.in_scope = self.in_scope ip.passive_scope = self.passive_scope ip.update() return ip
def run_cmd(cmd): c = cmd[:-1] timeout = cmd[-1] display("Executing command: %s" % ' '.join(c)) try: Popen(c).wait(timeout=timeout) except: display_error( "Timeout of %s reached. Aborting thread for command: %s" % (timeout, ' '.join(c)))
def descope_cidr(self, cidr): CIDR = self.ScopeCIDR.all(cidr=cidr) if CIDR: for c in CIDR: display("Removing {} from ScopeCIDRs".format(c.cidr)) c.delete() cnet = IPNetwork(cidr) for ip in self.IPAddress.all(): if IPAddress(ip.ip_address) in cnet: self.descope_ip(ip.ip_address)
def run(self, args): # pdb.set_trace() if not args.binary: self.binary = which.run("LinkedInt.py") else: self.binary = which.run(args.binary) if not self.binary: display_error( "LinkedInt binary not found. Please explicitly provide path with --binary" ) if args.domain: created, domain = self.BaseDomain.find_or_create(domain=args.domain) if args.top: titles = [ user.job_title.split(" at ")[0] for user in domain.users if user.job_title ] words = [] for t in titles: words += [w.lower() for w in t.split(" ")] word_count = Counter(words).most_common() display("Using the top %s words:" % args.top) res = [] for w in word_count[: int(args.top)]: display("\t{}\t{}".format(w[0], w[1])) res.append(w[0]) # pdb.set_trace() args.smart_shuffle = ",".join(res) if args.smart_shuffle: args.keywords = " OR ".join( ['"{}"'.format(i) for i in args.smart_shuffle.split(",")] ) self.process_domain(domain, args) args.keywords = " AND ".join( ['-"{}"'.format(i) for i in args.smart_shuffle.split(",")] ) self.process_domain(domain, args) else: self.process_domain(domain, args) self.BaseDomain.commit()
def run(self, args): if not args.api_key: display_error("You must supply an API key to use shodan!") return if args.search: ranges = [args.search] if args.import_db: ranges = [] if args.rescan: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all() ] else: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all(tool=self.name) ] api = shodan_api.Shodan(args.api_key) for r in ranges: time.sleep(1) # pdb.set_trace() results = api.search(r) display("{} results found for: {}".format(results['total'], r)) for res in results['matches']: ip_address_str = res['ip_str'] port_str = res['port'] transport = res['transport'] display("Processing IP: {} Port: {}/{}".format( ip_address_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_address_str) created, port = self.Port.find_or_create(ip_address=IP, port_number=port_str, proto=transport) port.meta['shodan_data'] = res port.save() self.IPAddress.commit()
def run_cmd(cmd): c = cmd[:-1] timeout = cmd[-1] display("Executing command: %s" % " ".join(c)) current_time = time.time() if timeout: process = Popen(c) while time.time() < current_time + timeout and process.poll() is None: time.sleep(5) if process.poll() is None: display_error( "Timeout of %s reached. Aborting thread for command: %s" % (timeout, " ".join(c))) process.terminate() else: Popen(c).wait()
def descope_ip(self, ip): ip = self.IPAddress.all(ip_address=ip) if ip: for i in ip: display("Removing IP {} from scope".format(i.ip_address)) i.in_scope = False i.passive_scope = False i.update() for d in i.domains: in_scope_ips = [ ipa for ipa in d.ip_addresses if ipa.in_scope or ipa.passive_scope ] if not in_scope_ips: display( "Domain {} has no more scoped IPs. Removing from scope." .format(d.domain)) d.in_scope = False d.passive_scope = False self.IPAddress.commit()
def process_output(self, cmds): display("Importing data to database") for cmd in cmds: if cmd['cidr']: _, cidr = self.ScopeCidr.find_or_create(cidr=cmd['cidr']) cidr.meta['whois'] = open(cmd['output']).read() display(cidr.meta['whois']) cidr.update() elif cmd['domain']: _, domain = self.BaseDomain.find_or_create(domain=cmd['domain']) domain.meta['whois'] = open(cmd['output']).read() display(domain.meta['whois']) domain.update() self.BaseDomain.commit()
def run(self, args): if args.import_file: for nFile in args.import_file: self.process_data(nFile, args) elif args.launch: if ( not args.username and not args.password and not args.host and not args.uuid and not args.policy_id and not args.folder_id ): display_error( "You must supply a username, password, and host to launch a Nessus job" ) else: n = NessusRequest( args.username, args.password, args.host, uuid=args.uuid, policy_id=args.policy_id, folder_id=args.folder_id, ) ips = [ ip.ip_address for ip in self.IPAddress.all(scope_type="active", tool=self.name) ] cidrs = [cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)] domains = [ domain.domain for domain in self.Domain.all(scope_type="active", tool=self.name) ] targets = ", ".join(merge_ranges(ips + cidrs) + domains) res = n.launch_job(targets, args.job_name) display("New Nessus job launched with ID {}".format(res)) display( "Remember this number! You'll need it to download the job once it is done." ) elif args.download: if ( not args.username and not args.password and not args.host and not args.job_id ): display_error( "You must supply host, username, password and job_id to download a report to import" ) else: n = NessusRequest( args.username, args.password, args.host, proxies={"https": "127.0.0.1:8080"}, ) if args.output_path[0] == "/": output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path[1:] ) else: output_path = os.path.join( self.base_config["PROJECT"]["base_path"], args.output_path ) if not os.path.exists(output_path): os.makedirs(output_path) output_path = os.path.join( output_path, "Nessus-export-{}.nessus".format(int(time.time())) ) n.export_file(args.job_id, output_path) self.process_data(output_path, args)
def process_data(self, nFile, args): print("Reading", nFile) tree = ET.parse(nFile) root = tree.getroot() skip = [] for ReportHost in root.iter("ReportHost"): os = [] hostname = "" hostIP = "" for HostProperties in ReportHost.iter("HostProperties"): for tag in HostProperties: if tag.get("name") == "host-ip": hostIP = tag.text if tag.get("name") == "host-fqdn": hostname = tag.text.lower() hostname = hostname.replace("www.", "") if tag.get("name") == "operating-system": os = tag.text.split("\n") if hostIP: # apparently nessus doesn't always have an IP to work with... if hostname: display( "Gathering Nessus info for {} ( {} )".format(hostIP, hostname) ) else: display("Gathering Nessus info for {}".format(hostIP)) created, ip = self.IPAddress.find_or_create(ip_address=hostIP) if hostname: if not args.internal: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: ip.save() domain.ip_addresses.append(ip) domain.save() else: created, domain = self.Domain.find_or_create(domain=hostname) if ip not in domain.ip_addresses: domain.ip_addresses.append(ip) domain.update() if os: for o in os: if not ip.OS: ip.OS = o else: if o not in ip.OS.split(" OR "): ip.OS += " OR " + o self.getVulns(ip, ReportHost) self.IPAddress.commit() return
def post_run(self, args): display("Potential takeovers are stored in {}".format(os.environ['HOME'])) os.environ['HOME'] = self.orig_home
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=False, **kwargs): created, d = super(DomainRepository, self).find_or_create(only_tool, **kwargs) display("Processing %s" % d.domain) if created: # If this is a new subdomain, set scoping info based on what is passed to the function initially. d.in_scope = in_scope d.passive_scope = passive_scope base_domain = '.'.join( [t for t in tldextract.extract(d.domain)[1:] if t]) BaseDomains = BaseDomainRepository(self.db, "") # If the base domain is new, it'll inherit the same scoping permissions. created, bd = BaseDomains.find_or_create( only_tool, passive_scope=d.passive_scope, in_scope=in_scope, domain=base_domain) if created: display_new( "The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s" % (base_domain, bd.in_scope, bd.passive_scope)) else: # If the base domain already exists, then the subdomain inherits the scope info from the base domain. d.passive_scope = bd.passive_scope d.in_scope = bd.in_scope d.base_domain = bd # Get all IPs that this domain resolves to. ips = [] try: answers = dns.resolver.query(d.domain, 'A') for a in answers: ips.append(a.address) except: # If something goes wrong with DNS, we end up here pass if not ips: display_warning("No IPs discovered for %s" % d.domain) for i in ips: IPAddresses = IPRepository(self.db, "") display("Processing IP address %s" % i) created, ip = IPAddresses.find_or_create( only_tool, in_scope=d.in_scope, passive_scope=d.passive_scope, ip_address=i) # If the IP is in scope, then the domain should be if ip.in_scope: d.in_scope = ip.in_scope ip.passive_scope = True d.passive_scope = True # display("%s marked active scope due to IP being marked active." % d.domain) elif ip.passive_scope: d.passive_scope = ip.passive_scope d.ip_addresses.append(ip) display_new( "%s is being added to the database. Active Scope: %s Passive Scope: %s" % (d.domain, d.in_scope, d.passive_scope)) # Final sanity check - if a domain is active scoped, it should also be passively scoped. if d.in_scope: d.passive_scope = True return created, d
def run(self, args): if not args.api_key: display_error("You must supply an API key to use shodan!") return if args.search: ranges = [args.search] if args.import_db: ranges = [] if args.rescan: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all() ] else: cidrs = [c.cidr for c in self.ScopeCidr.all()] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active") ] else: if args.fast: ranges += [ "net:{}".format(c.cidr) for c in self.ScopeCidr.all(tool=self.name) ] else: cidrs = [ c.cidr for c in self.ScopeCidr.all(tool=self.name) ] for c in cidrs: ranges += [str(i) for i in IPNetwork(c)] if not args.cidr_only: ranges += [ "{}".format(i.ip_address) for i in self.IPAddress.all(scope_type="active", tool=self.name) ] api_host_url = "https://api.shodan.io/shodan/host/{}?key={}" api_search_url = ( "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}") for r in ranges: time.sleep(1) if ":" in r: display("Doing Shodan search: {}".format(r)) try: results = json.loads( requests.get(api_search_url.format(args.api_key, r, 1)).text) if results.get("error") and "request timed out" in results[ "error"]: display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next total = len(results["matches"]) matches = [] i = 1 while total > 0: display("Adding {} results from page {}".format(total, i)) matches += results["matches"] i += 1 try: time.sleep(1) results = json.loads( requests.get( api_search_url.format(args.api_key, r, i)).text) if (results.get("error") and "request timed out" in results["error"]): display_warning( "Timeout occurred on Shodan's side.. trying again in 5 seconds." ) results = json.loads( requests.get( api_search_url.format(args.api_key, r, 1)).text) total = len(results["matches"]) except Exception as e: display_error("Something went wrong: {}".format(e)) total = 0 pdb.set_trace() for res in matches: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.meta["shodan_data"] = res port.save() else: try: results = json.loads( requests.get(api_host_url.format(r, args.api_key)).text) except Exception as e: display_error("Something went wrong: {}".format(e)) next # pdb.set_trace() if results.get("data", False): display("{} results found for: {}".format( len(results["data"]), r)) for res in results["data"]: ip_str = res["ip_str"] port_str = res["port"] transport = res["transport"] display("Processing IP: {} Port: {}/{}".format( ip_str, port_str, transport)) created, IP = self.IPAddress.find_or_create( ip_address=ip_str) IP.meta["shodan_data"] = results created, port = self.Port.find_or_create( ip_address=IP, port_number=port_str, proto=transport) if created: svc = "" if res.get("ssl", False): svc = "https" elif res.get("http", False): svc = "http" else: svc = "" port.service_name = svc port.meta["shodan_data"] = res port.save() self.IPAddress.commit()
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=True, **kwargs): created, ip = super(IPRepository, self).find_or_create(only_tool, **kwargs) if created: # If newly created then will determine scoping based on parent options and if in a scoped cidr. ip_str = ip.ip_address ip.passive_scope = passive_scope # If the parent domain is active scope, then this also is. if in_scope: ip.in_scope = in_scope else: # Go through ScopeCIDR table and see if this IP is in a CIDR in scope ScopeCidrs = ScopeCIDRRepository(self.db, "") addr = IPAddress(ip.ip_address) cidrs = ScopeCidrs.all() # pdb.set_trace() for c in cidrs: if addr in IPNetwork(c.cidr): ip.in_scope = True # Final sanity check - if an IP is active scoped, it should also be passive scoped. if ip.in_scope: ip.passive_scope = True ip.update() # Build CIDR info - mainly for reporting res = False for cidr in private_subnets: if IPAddress(ip_str) in cidr: res = ([str(cidr), 'Non-Public Subnet'], ) if res: cidr_data = res else: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except: res = IPWhois(ip_str).lookup_whois() cidr_data = [] for n in res['nets']: if ',' in n['cidr']: for cidr_str in n['cidr'].split(', '): cidr_data.append([cidr_str, n['description']]) else: cidr_data.append([n['cidr'], n['description']]) cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] cidr_len = len(IPNetwork(cidr_data[0][0])) matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c display("Processing CIDR from whois: %s - %s" % (matching_cidr[1], matching_cidr[0])) CIDR = CIDRRepository(self.db, "") created, cidr = CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0]) if created: display_new("CIDR %s added to database" % cidr.cidr) cidr.org_name = matching_cidr[1] cidr.update() ip.cidr = cidr ip.update() display_new( "IP address %s added to database. Active Scope: %s Passive Scope: %s" % (ip.ip_address, ip.in_scope, ip.passive_scope)) return created, ip