def run(self, args): if args.import_db: if args.rescan: svc = self.Port.all(service_name='http') svc += self.Port.all(service_name='https') else: svc = self.Port.all(service_name='http', tool=self.name) svc += self.Port.all(service_name='https', tool=self.name) data = [] for s in svc: if s.ip_address.in_scope: urls = [ "%s://%s:%s" % (s.service_name, s.ip_address.ip_address, s.port_number) ] for d in s.ip_address.domains: urls.append("%s://%s:%s" % (s.service_name, d.domain, s.port_number)) data.append([s.id, urls, args.timeout]) pool = ThreadPool(int(args.threads)) results = pool.map(process_urls, data) display_new("Adding headers to the database") for i, headers in results: created, svc = self.Port.find_or_create(id=i) svc.meta['headers'] = headers svc.update() self.Port.commit()
def process_cidr(self, line): display("Processing %s" % line) if "/" in line: created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip()) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update() elif "-" in line: start_ip, end_ip = line.strip().replace(" ", "").split("-") if "." not in end_ip: end_ip = ".".join(start_ip.split(".")[:3] + [end_ip]) cidrs = iprange_to_cidrs(start_ip, end_ip) for c in cidrs: created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c)) if created: display_new("Adding %s to scoped CIDRs in database" % line.strip()) cidr.in_scope = True cidr.update()
def find_or_create(self, only_tool=False, passive_scope=True, in_scope=False, **kwargs): created, port = super(PortRepository, self).find_or_create(only_tool, **kwargs) if created: display_new("Port {} added to database for IP {}".format( port.port_number, port.ip_address.ip_address)) return created, port
def process_output(self, cmds): ''' Process the output generated by the earlier commands. ''' # Cycle through all of the targets we ran earlier for c in cmds: output_file = c['output'] target = c['target'] # Read file data = open(output_file).read().split('\n') # Quick and dirty way to filter out headers and blank lines, as well # as duplicates res = list( set([ d for d in data if 'Domain,Cname,Provider' not in d and d ])) if res: # Load up the DB entry. created, subdomain = self.Domain.find_or_create(domain=target) # Process results for d in res: results = d.split(',') if results[3] == "false": display_warning( "Hosting found at {} for {}, not vulnerable.". format(target, results[2])) elif results[3] == "true": display_new("{} vulnerable to {}!".format( target, results[2])) if not subdomain.meta[self.name].get( 'vulnerable', False): subdomain.meta[self.name]['vulnerable'] = [] subdomain.meta[self.name]['vulnerable'].append(d) else: display_warning("Not sure of result: {}".format(data)) # This is a little hackish, but needed to get data to save t = dict(subdomain.meta) self.Domain.commit() subdomain.meta = t self.Domain.commit()
def reclassify_domain(self, bd): if bd.meta.get('whois', False): display_new("Whois data found for {}".format(bd.domain)) print(bd.meta['whois']) res = raw_input("Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] ") if res.lower() == 'a': bd.in_scope = True bd.passive_scope = True elif res.lower() == 'p': bd.in_scope = False bd.passive_scope = True else: bd.in_scope = False bd.passive_scope = False bd.save() else: display_error("Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(bd.domain))
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=True, **kwargs): created, ip = super(IPRepository, self).find_or_create(only_tool, **kwargs) if created: # If newly created then will determine scoping based on parent options and if in a scoped cidr. ip_str = ip.ip_address ip.passive_scope = passive_scope # If the parent domain is active scope, then this also is. if in_scope: ip.in_scope = in_scope else: # Go through ScopeCIDR table and see if this IP is in a CIDR in scope ScopeCidrs = ScopeCIDRRepository(self.db, "") addr = IPAddress(ip.ip_address) cidrs = ScopeCidrs.all() # pdb.set_trace() for c in cidrs: if addr in IPNetwork(c.cidr): ip.in_scope = True # Final sanity check - if an IP is active scoped, it should also be passive scoped. if ip.in_scope: ip.passive_scope = True ip.update() # Build CIDR info - mainly for reporting res = False for cidr in private_subnets: if IPAddress(ip_str) in cidr: res = ([str(cidr), 'Non-Public Subnet'], ) if res: cidr_data = res else: try: res = IPWhois(ip_str).lookup_whois(get_referral=True) except: res = IPWhois(ip_str).lookup_whois() cidr_data = [] for n in res['nets']: if ',' in n['cidr']: for cidr_str in n['cidr'].split(', '): cidr_data.append([cidr_str, n['description']]) else: cidr_data.append([n['cidr'], n['description']]) cidr_data = [ cidr_d for cidr_d in cidr_data if IPAddress(ip_str) in IPNetwork(cidr_d[0]) ] cidr_len = len(IPNetwork(cidr_data[0][0])) matching_cidr = cidr_data[0] for c in cidr_data: if len(IPNetwork(c[0])) < cidr_len: matching_cidr = c display("Processing CIDR from whois: %s - %s" % (matching_cidr[1], matching_cidr[0])) CIDR = CIDRRepository(self.db, "") created, cidr = CIDR.find_or_create(only_tool=True, cidr=matching_cidr[0]) if created: display_new("CIDR %s added to database" % cidr.cidr) cidr.org_name = matching_cidr[1] cidr.update() ip.cidr = cidr ip.update() display_new( "IP address %s added to database. Active Scope: %s Passive Scope: %s" % (ip.ip_address, ip.in_scope, ip.passive_scope)) return created, ip
def find_or_create(self, only_tool=False, in_scope=False, passive_scope=False, **kwargs): created, d = super(DomainRepository, self).find_or_create(only_tool, **kwargs) display("Processing %s" % d.domain) if created: # If this is a new subdomain, set scoping info based on what is passed to the function initially. d.in_scope = in_scope d.passive_scope = passive_scope base_domain = '.'.join( [t for t in tldextract.extract(d.domain)[1:] if t]) BaseDomains = BaseDomainRepository(self.db, "") # If the base domain is new, it'll inherit the same scoping permissions. created, bd = BaseDomains.find_or_create( only_tool, passive_scope=d.passive_scope, in_scope=in_scope, domain=base_domain) if created: display_new( "The base domain %s is being added to the database. Active Scope: %s Passive Scope: %s" % (base_domain, bd.in_scope, bd.passive_scope)) else: # If the base domain already exists, then the subdomain inherits the scope info from the base domain. d.passive_scope = bd.passive_scope d.in_scope = bd.in_scope d.base_domain = bd # Get all IPs that this domain resolves to. ips = [] try: answers = dns.resolver.query(d.domain, 'A') for a in answers: ips.append(a.address) except: # If something goes wrong with DNS, we end up here pass if not ips: display_warning("No IPs discovered for %s" % d.domain) for i in ips: IPAddresses = IPRepository(self.db, "") display("Processing IP address %s" % i) created, ip = IPAddresses.find_or_create( only_tool, in_scope=d.in_scope, passive_scope=d.passive_scope, ip_address=i) # If the IP is in scope, then the domain should be if ip.in_scope: d.in_scope = ip.in_scope ip.passive_scope = True d.passive_scope = True # display("%s marked active scope due to IP being marked active." % d.domain) elif ip.passive_scope: d.passive_scope = ip.passive_scope d.ip_addresses.append(ip) display_new( "%s is being added to the database. Active Scope: %s Passive Scope: %s" % (d.domain, d.in_scope, d.passive_scope)) # Final sanity check - if a domain is active scoped, it should also be passively scoped. if d.in_scope: d.passive_scope = True return created, d