Ejemplo n.º 1
0
    def exec_stage1_sites(self):

        ## exec stage 1 sites
        for site in self.stage1_sites.keys():

            print "*************************************************"
            print "Harvesting domains from: {0}".format(site.upper())
            print "*************************************************"
            s = self.stage1_sites[site]
            s.run()

            urls_file = "{0}{1}_urls.txt".format(self.outfile_base_s1,site.lower())
            print "Storing URLs at: {0}".format(urls_file)
            self.store_file(urls_file,s.urls)

            print "Processing domains..."
            s.get_domains(validate=True)
            domains_file = "{0}{1}_domains.txt".format(self.outfile_base_s1,site.lower())
            print "Storing found domains at: {0}".format(domains_file)
            self.store_file(domains_file,s.domains)

            print "Processing hosts and getting IPs..."
            s.get_ips(validate=True)
            ips_file = "{0}{1}_hosts_ip.txt".format(self.outfile_base_s1,site.lower())
            print "Storing IPs at: {0}\n".format(ips_file)
            self.store_file(ips_file,s.hosts_and_ips)

            networks = []
            tv = TargetValidator(s.keywords,s.urls,s.domains)
            i = IpUtils()
            for item in s.hosts_and_ips:
                ip = item.strip().split(":")[1]

                if not i.checkIfIPinNetworks(ip,networks):
                    w = i.ipWhois(ip)
                    asn_cidr = w["asn_cidr"]
                    for reg_info in w["nets"]:
                        if tv.check_registrant_info(reg_info):
                            if reg_info.has_key("cidr"):
                                tmp = reg_info["cidr"].split(",")
                                if len(tmp) > 1:
                                    networks += tmp
                                else:
                                    asn_cidr = tmp[0]
                                    networks.append(asn_cidr)
                            else:
                                networks.append(asn_cidr)

            networks_file = "{0}{1}_networks.txt".format(self.outfile_base_s1,site.lower())
            print "Storing IPs at: {0}\n".format(networks_file)
            self.store_file(networks_file,networks)

        return True
Ejemplo n.º 2
0
    def get_domains(self, validate=False):

        u_utils = UrlUtils()
        for url in self.urls:
            domain = u_utils.getDomain(url)
            if domain not in self.domains:
                if validate:
                    tv = TargetValidator(self.keywords,self.urls,self.domains)
                    if tv.keywords_at_URL_and_domain(domain,url):
                        self.domains.append(domain)
                else:
                    self.domains.append(domain)

        return True
Ejemplo n.º 3
0
    def get_ips(self, validate=False):

        hosts = []
        u_utils = UrlUtils()
        for url in self.urls:
            host = u_utils.getHost(url)
            ip = u_utils.getIP(url)
            domain = u_utils.getDomain(url)
            if host not in hosts:
                if validate:
                    tv = TargetValidator(self.keywords,self.urls,self.domains)
                    if tv.keywords_at_URL_and_domain(domain,url):
                        hosts.append(host)
                        self.hosts_and_ips.append("{0}:{1}".format(host,ip))
                else:
                    hosts.append(host)
                    self.hosts_and_ips.append("{0}:{1}".format(host,ip))

        return True