def exec_stage1_sites(self): ## exec stage 1 sites for site in self.stage1_sites.keys(): print "*************************************************" print "Harvesting domains from: {0}".format(site.upper()) print "*************************************************" s = self.stage1_sites[site] s.run() urls_file = "{0}{1}_urls.txt".format(self.outfile_base_s1,site.lower()) print "Storing URLs at: {0}".format(urls_file) self.store_file(urls_file,s.urls) print "Processing domains..." s.get_domains(validate=True) domains_file = "{0}{1}_domains.txt".format(self.outfile_base_s1,site.lower()) print "Storing found domains at: {0}".format(domains_file) self.store_file(domains_file,s.domains) print "Processing hosts and getting IPs..." s.get_ips(validate=True) ips_file = "{0}{1}_hosts_ip.txt".format(self.outfile_base_s1,site.lower()) print "Storing IPs at: {0}\n".format(ips_file) self.store_file(ips_file,s.hosts_and_ips) networks = [] tv = TargetValidator(s.keywords,s.urls,s.domains) i = IpUtils() for item in s.hosts_and_ips: ip = item.strip().split(":")[1] if not i.checkIfIPinNetworks(ip,networks): w = i.ipWhois(ip) asn_cidr = w["asn_cidr"] for reg_info in w["nets"]: if tv.check_registrant_info(reg_info): if reg_info.has_key("cidr"): tmp = reg_info["cidr"].split(",") if len(tmp) > 1: networks += tmp else: asn_cidr = tmp[0] networks.append(asn_cidr) else: networks.append(asn_cidr) networks_file = "{0}{1}_networks.txt".format(self.outfile_base_s1,site.lower()) print "Storing IPs at: {0}\n".format(networks_file) self.store_file(networks_file,networks) return True
def get_domains(self, validate=False): u_utils = UrlUtils() for url in self.urls: domain = u_utils.getDomain(url) if domain not in self.domains: if validate: tv = TargetValidator(self.keywords,self.urls,self.domains) if tv.keywords_at_URL_and_domain(domain,url): self.domains.append(domain) else: self.domains.append(domain) return True
def get_ips(self, validate=False): hosts = [] u_utils = UrlUtils() for url in self.urls: host = u_utils.getHost(url) ip = u_utils.getIP(url) domain = u_utils.getDomain(url) if host not in hosts: if validate: tv = TargetValidator(self.keywords,self.urls,self.domains) if tv.keywords_at_URL_and_domain(domain,url): hosts.append(host) self.hosts_and_ips.append("{0}:{1}".format(host,ip)) else: hosts.append(host) self.hosts_and_ips.append("{0}:{1}".format(host,ip)) return True