def exec_stage1_sites(self):
## exec stage 1 sites
for site in self.stage1_sites.keys():
print "*************************************************"
print "Harvesting domains from: {0}".format(site.upper())
print "*************************************************"
s = self.stage1_sites[site]
s.run()
urls_file = "{0}{1}_urls.txt".format(self.outfile_base_s1,site.lower())
print "Storing URLs at: {0}".format(urls_file)
self.store_file(urls_file,s.urls)
print "Processing domains..."
s.get_domains(validate=True)
domains_file = "{0}{1}_domains.txt".format(self.outfile_base_s1,site.lower())
print "Storing found domains at: {0}".format(domains_file)
self.store_file(domains_file,s.domains)
print "Processing hosts and getting IPs..."
s.get_ips(validate=True)
ips_file = "{0}{1}_hosts_ip.txt".format(self.outfile_base_s1,site.lower())
print "Storing IPs at: {0}\n".format(ips_file)
self.store_file(ips_file,s.hosts_and_ips)
networks = []
tv = TargetValidator(s.keywords,s.urls,s.domains)
i = IpUtils()
for item in s.hosts_and_ips:
ip = item.strip().split(":")[1]
if not i.checkIfIPinNetworks(ip,networks):
w = i.ipWhois(ip)
asn_cidr = w["asn_cidr"]
for reg_info in w["nets"]:
if tv.check_registrant_info(reg_info):
if reg_info.has_key("cidr"):
tmp = reg_info["cidr"].split(",")
if len(tmp) > 1:
networks += tmp
else:
asn_cidr = tmp[0]
networks.append(asn_cidr)
else:
networks.append(asn_cidr)
networks_file = "{0}{1}_networks.txt".format(self.outfile_base_s1,site.lower())
print "Storing IPs at: {0}\n".format(networks_file)
self.store_file(networks_file,networks)
return True
def get_domains(self, validate=False):
u_utils = UrlUtils()
for url in self.urls:
domain = u_utils.getDomain(url)
if domain not in self.domains:
if validate:
tv = TargetValidator(self.keywords,self.urls,self.domains)
if tv.keywords_at_URL_and_domain(domain,url):
self.domains.append(domain)
else:
self.domains.append(domain)
return True
def get_ips(self, validate=False):
hosts = []
u_utils = UrlUtils()
for url in self.urls:
host = u_utils.getHost(url)
ip = u_utils.getIP(url)
domain = u_utils.getDomain(url)
if host not in hosts:
if validate:
tv = TargetValidator(self.keywords,self.urls,self.domains)
if tv.keywords_at_URL_and_domain(domain,url):
hosts.append(host)
self.hosts_and_ips.append("{0}:{1}".format(host,ip))
else:
hosts.append(host)
self.hosts_and_ips.append("{0}:{1}".format(host,ip))
return True