def sub(self): """ This method is used to find all the sub-domain from base-domain """ logger.info("Start finding sub-domains of %s" % self.profile.get_target()) self.kb.set_sub_domains(self.targets) to_walk = self.targets sub_plugins = self.plugins["search"] + self.plugins["brute"] while len(to_walk): sub_domain_list = [] new_domain = [] for plugin in sub_plugins: for domain in to_walk: engine_result = plugin.discover(domain) for sd in engine_result: sub_domain_list.append(sd) for domain in sub_domain_list: if self.kb.add_sub_domain(domain): new_domain.append(domain) to_walk = new_domain return self.kb.get_sub_domains()
def extract(self, url): site = self.base_domain.domain_name header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0", "Cookie": "csrftoken=xW6DIMrTX9qge6cQuCE1OmZgmGunVinw", "Referer": "https://dnsdumpster.com/" } payload = { "csrfmiddlewaretoken": "xW6DIMrTX9qge6cQuCE1OmZgmGunVinw", "targetip": site } content = self.requester.post(url, payload, header).text soup = BeautifulSoup(content, "lxml") search_tags = soup.find_all('td', attrs={"class": "col-md-4"}) if not search_tags: logger.info("Plug-in DNsDumpster: can't get any result for: %s" % self.base_domain.domain_name) return for tag in search_tags: domain = tag.contents[0].strip('.') if site in domain: self.add(domain)
def add_sub_domain(self, sub_domain): """ Add the sub-domain to the result list If add successfully, return True. Else, return False """ if not self._is_existed_domain(sub_domain): logger.info('New domain was found by %s: %s' % (sub_domain.meta_data['domain_found_by'], sub_domain.domain_name)) self._sub_domains.append(sub_domain) return True return False
def port(self): """ This method is used to find all open ports of a domain """ logger.info("Start finding opened ports") for plugin in self.plugins['port']: already_scanned = [] for domain in self.kb.get_sub_domains(): if domain.ip not in already_scanned: open_ports = plugin.scan(domain) self.kb.add_open_ports(domain, open_ports) logger.info( "%s (%s) open ports: %s" % ( domain.domain_name, domain.ip, ", ".join([str(port) for port in domain.get_open_ports()])))
def post(self, url, data=None, headers=None): if headers is None: headers = self._headers for i in range(3): while True: try: return requests.post(url, headers=headers, proxies=self._proxies, data=data, timeout=60) except requests.exceptions.Timeout: logger.error("It takes a request so long so I must kill it.") logger.info("Trying to reconnect...") continue except: logger.error("I don't know why this error occurred") logger.info("Trying to reconnect...") continue return None
def extract(self, url): try: content = self.requester.get(url).text if self.has_error(content): logger.error("Can not find any result for: %s" % self.base_domain.domain_name) return None _soup = BeautifulSoup(content, "html5lib") if not _soup.find_all("em"): logger.error("This site seem to blocked your requests") return _last = "" _from = "" total = int(_soup.find_all("em")[0].string.split()[1]) logger.info("Total of results: %d for: %s " % (total, self.base_domain.domain_name)) if total > 20: count = total / 20 for tem in range(count + 1): url_temp = "" r = self.requester.get(url + _last + _from) soup = BeautifulSoup(r.text, "html5lib") search_region = BeautifulSoup( str(soup.find_all("table", attrs={"class": "TBtable"})), "lxml") for item in search_region.find_all('a', attrs={"rel": True}): url_temp = self.parse_domain_name(item['href']) self.add(url_temp) _last = "&last=" + url_temp _from = "&from=" + str((tem + 1) * 20 + 1) else: search_region = BeautifulSoup( str(_soup.find_all("table", attrs={"class": "TBtable"})), "lxml") for item in search_region.find_all('a', attrs={"rel": True}): url_temp = self.parse_domain_name(item['href']) self.add(url_temp) except: raise
def discover(self, domain): """ This is the main method, it pass domain_name parameter and returns a list of the sub-domain found by the search engine """ try: logger.info("Plugin %s has been activated" % self.get_name()) self.base_domain = domain _dict = self.dictionary() pool = ThreadPool(self.max_worker) pool.map(self.worker, _dict) pool.close() pool.join() return self.sub_domains except: msg = "Error occurred with plugin %s " % self.get_name() raise PluginException(msg)
def get_total_page(self): max_page_temp = self.max_page while max_page_temp >= 0: url = self.base_url.format(query=self.get_query(), page=max_page_temp) header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" } content = self.requester.get(url, header).text if self.has_error(content): logger.error("To much requests and Yahoo knew") return 0 if (("We did not find results for" not in content) or ("Check spelling or type a new query" not in content)): list_seed = [] soup = BeautifulSoup(content, "html5lib") search_page = soup.find_all("a", href=True, title=True, attrs={'class': None}) for i in search_page: list_seed.append(int(i.string)) current = soup.find("strong") try: list_seed.append(int(current.string)) except: logger.error( "Yahoo Plug-in: Failed to get current seed but that means there are " "no more sub-domain for this base_domain but Yahoo could block your requests also" ) if not list_seed: return 0 else: return max(list_seed) else: logger.info( "Yahoo Plug-in: max_page down to %d since bot can not get any info about total_page" % max_page_temp) max_page_temp -= 10 return 0
def get_total_page(self): max_page_temp = self.max_page while max_page_temp >= 0: url = self.base_url.format(query=self.get_query(), page=max_page_temp) r = self.requester.get(url) if r is None: return 0 content = r.text if self.was_blocked(content): logger.error("Ask blocked the request") return 0 elif self.was_not_found(content): logger.info( "Ask Plug-in: max_page down to %d for domain: %s" % (max_page_temp, self.base_domain.domain_name)) max_page_temp -= 5 else: soup = BeautifulSoup(content, "html5lib") search_pages = soup.find_all("a", attrs={"ul-attr-accn": "pagination"}) list_no_page = [] for tag in search_pages: try: no_page = int(tag.string) list_no_page.append(no_page) except: continue if not list_no_page: logger.debug(soup) return 1 else: return max(list_no_page) return 0