def detect_wildcard(domain): is_enable = to_detect_wildcard(domain) if is_enable: logger.log('ALERT', f'The domain {domain} enables wildcard') else: logger.log('ALERT', f'The domain {domain} disables wildcard') return is_enable
def match_subdomains(domain, html, distinct=True, fuzzy=True): """ Use regexp to match subdomains :param str domain: main domain :param str html: response html text :param bool distinct: deduplicate results or not (default True) :param bool fuzzy: fuzzy match subdomain or not (default True) :return set/list: result set or list """ logger.log('TRACE', f'Use regexp to match subdomains in the response body') if fuzzy: regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') result = re.findall(regexp, html, re.I) if not result: return set() deal = map(lambda s: s.lower(), result) if distinct: return set(deal) else: return list(deal) else: regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') result = re.findall(regexp, html, re.I) if not result: return set() regexp = r'(?:http://|https://)' deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) if distinct: return set(deal) else: return list(deal)
def query(self): """ 向接口查询子域并做子域匹配 """ page = 0 while True: self.header = self.get_header() self.proxy = self.get_proxy(self.source) params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page} resp = self.get(self.addr, params) if not resp: return if resp.status_code != 200: break # 请求不正常通常网络是有问题,不再继续请求下去 try: json = resp.json() except Exception as e: logger.log('DEBUG', e.args) break subdomains = self.match_subdomains(str(json)) if not subdomains: break self.subdomains.update(subdomains) # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名 subdomains = json.get('Subdomains') if subdomains and len(subdomains) < 300: # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页 break page += 1 if page >= 50: # ipv4info子域查询接口最多允许查询50页 break
def deal_output(output_path): logger.log('INFOR', f'Processing resolved results') infos = dict() # 用来记录所有域名有关信息 with open(output_path) as fd: for line in fd: line = line.strip() try: items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) logger.log('ERROR', f'Error resolve line {line}, skip this line') continue info = dict() info['resolver'] = items.get('resolver') qname = items.get('name')[:-1] # 去除最右边的`.`点号 status = items.get('status') if status != 'NOERROR': logger.log('DEBUG', f'Resolving {qname}: {status}') continue data = items.get('data') if 'answers' not in data: logger.log('DEBUG', f'Resolving {qname} have not any answers') info['alive'] = 0 info['resolve'] = 0 info['reason'] = 'NoAnswer' infos[qname] = info continue infos = gen_infos(data, qname, info, infos) return infos
def query(self): """ 向接口查询子域并做子域匹配 """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) data = { 'query': f'parsed.names: {self.domain}', 'page': 1, 'fields': ['parsed.subject_dn', 'parsed.names'], 'flatten': True } resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return json = resp.json() status = json.get('status') if status != 'ok': logger.log('ALERT', f'{self.source} module {status}') return subdomains = self.match_subdomains(resp.text) self.subdomains.update(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): data['page'] = page resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) self.subdomains = self.collect_subdomains(resp)
def gen_infos(data, qname, info, infos): flag = False cnames = list() ips = list() ttl = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': flag = True name = answer.get('name') cname = name[:-1].lower() # 去除最右边的`.`点号 cnames.append(cname) ip = answer.get('data') ips.append(ip) ttl.append(str(answer.get('ttl'))) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cnames) info['ip'] = ','.join(ips) info['ttl'] = ','.join(ttl) infos[qname] = info if not flag: logger.log('DEBUG', f'Resolving {qname} have not a record') info['alive'] = 0 info['resolve'] = 0 info['reason'] = 'NoARecord' infos[qname] = info return infos
def begin(self): """ begin log """ logger.log( 'DEBUG', f'Start {self.source} module to ' f'collect subdomains of {self.domain}')
def save_json(self): """ Save the results of each module as a json file :return bool: whether saved successfully """ if not settings.save_module_result: return False logger.log( 'TRACE', f'Save the subdomain results found by ' f'{self.source} module as a json file') path = settings.result_save_dir.joinpath(self.domain, self.module) path.mkdir(parents=True, exist_ok=True) name = self.source + '.json' path = path.joinpath(name) with open(path, mode='w', errors='ignore') as file: result = { 'domain': self.domain, 'name': self.module, 'source': self.source, 'elapse': self.elapse, 'find': len(self.subdomains), 'subdomains': list(self.subdomains), 'infos': self.infos } json.dump(result, file, ensure_ascii=False, indent=4) return True
def check_path(path, name, fmt): """ 检查结果输出目录路径 :param path: 保存路径 :param name: 导出名字 :param fmt: 保存格式 :return: 保存路径 """ filename = f'{name}.{fmt}' default_path = settings.result_save_dir.joinpath(filename) if isinstance(path, str): path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠 path = path.replace('\'', '') # 去除多余的转义 else: path = default_path path = Path(path) if not path.suffix: # 输入是目录的情况 path = path.joinpath(filename) parent_dir = path.parent if not parent_dir.exists(): logger.log('ALERT', f'{parent_dir} does not exist, directory will be created') parent_dir.mkdir(parents=True, exist_ok=True) if path.exists(): logger.log('ALERT', f'The {path} exists and will be overwritten') return path
def list_dns(self, zone_id): page = 1 list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', params={'page': page, 'per_page': 10}) if not list_dns_resp: logger.log('DEBUG', f'{list_dns_resp.status_code} {list_dns_resp.text}') return subdomains = self.match_subdomains(list_dns_resp.text) self.subdomains.update(subdomains) if not self.subdomains: # waiting for cloudflare enumerate subdomains sleep(5) self.list_dns(zone_id) else: while True: list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', params={'page': page, 'per_page': 10}) if not list_dns_resp: logger.log('DEBUG', f'{list_dns_resp.status_code} {list_dns_resp.text}') return total_pages = list_dns_resp.json()['result_info']['total_pages'] subdomains = (self.match_subdomains(list_dns_resp.text)) self.subdomains.update(subdomains) page += 1 if page > total_pages: break return
def query(self): """ 向接口查询子域并做子域匹配 """ base_addr = 'http://114.55.181.28/check_web/' \ 'databaseInfo_mainSearch.action' page_num = 1 while True: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) params = {'isSearch': 'true', 'searchType': 'url', 'term': self.domain, 'pageNo': page_num} try: resp = self.get(base_addr, params) except Exception as e: logger.log('ERROR', e.args) break if not resp: break subdomains = self.match_subdomains(resp.text) if not subdomains: # 没有发现子域名则停止查询 break self.subdomains.update(subdomains) if not subdomains: break if page_num > 10: break page_num += 1
def gen_subdomains(expression, path): """ Generate subdomains :param str expression: generate subdomains expression :param str path: path of wordlist :return set subdomains: list of subdomains """ subdomains = set() with open(path, encoding='utf-8', errors='ignore') as fd: for line in fd: word = line.strip().lower() if len(word) == 0: continue if not utils.is_subname(word): continue if word.startswith('.'): word = word[1:] if word.endswith('.'): word = word[:-1] subdomain = expression.replace('*', word) subdomains.add(subdomain) size = len(subdomains) logger.log('DEBUG', f'The size of the dictionary generated by {path} is {size}') if size == 0: logger.log('ALERT', 'Please check the dictionary content!') else: utils.check_random_subdomain(subdomains) return subdomains
def gen_fuzz_subdomains(expression, rule, fuzzlist): """ Generate subdomains based on fuzz mode :param str expression: generate subdomains expression :param str rule: regexp rule :param str fuzzlist: fuzz dictionary :return set subdomains: list of subdomains """ subdomains = set() if fuzzlist: fuzz_domain = gen_subdomains(expression, fuzzlist) subdomains.update(fuzz_domain) if rule: fuzz_count = exrex.count(rule) if fuzz_count > 10000000: logger.log( 'ALERT', f'The dictionary generated by this rule is too large: ' f'{fuzz_count} > 10000000') for fuzz_string in exrex.generate(rule): fuzz_string = fuzz_string.lower() if not fuzz_string.isalnum(): continue fuzz_domain = expression.replace('*', fuzz_string) subdomains.add(fuzz_domain) utils.check_random_subdomain(subdomains) logger.log('DEBUG', f'Dictionary size based on fuzz mode: {len(subdomains)}') return subdomains
def axfr(self, server): """ Perform domain transfer :param server: domain server """ logger.log( 'DEBUG', f'Trying to perform domain transfer in {server} ' f'of {self.domain}') try: xfr = dns.query.xfr(where=server, zone=self.domain, timeout=5.0, lifetime=10.0) zone = dns.zone.from_xfr(xfr) except Exception as e: logger.log('DEBUG', e.args) logger.log( 'DEBUG', f'Domain transfer to server {server} of ' f'{self.domain} failed') return names = zone.nodes.keys() for name in names: full_domain = str(name) + '.' + self.domain subdomain = self.match_subdomains(full_domain) self.subdomains.update(subdomain) record = zone[name].to_text(name) self.results.append(record) if self.results: logger.log( 'DEBUG', f'Found the domain transfer record of ' f'{self.domain} on {server}') logger.log('DEBUG', '\n'.join(self.results)) self.results = []
def find_subdomains(domain, data): subdomains = set() js_urls = set() db = Database() for infos in data: jump_history = infos.get('history') req_url = infos.get('url') subdomains.update(find_in_history(domain, req_url, jump_history)) rsp_html = db.get_resp_by_url(domain, req_url) if not rsp_html: logger.log( 'DEBUG', f'an abnormal response occurred in the request {req_url}') continue subdomains.update(find_in_resp(domain, req_url, rsp_html)) js_urls.update(find_js_urls(domain, req_url, rsp_html)) req_data = convert_to_dict(js_urls) resp_data = request.bulk_request(domain, req_data, ret=True) while not resp_data.empty(): _, resp = resp_data.get() if not isinstance(resp, Response): continue text = utils.decode_resp_text(resp) subdomains.update(find_in_resp(domain, resp.url, text)) return subdomains
def save(self): logger.log('DEBUG', 'Saving results') if self.fmt == 'txt': data = str(self.results) else: data = self.results.export(self.fmt) utils.save_to_file(self.path, data)
def head(self, url, params=None, check=True, **kwargs): """ Custom head request :param str url: request url :param dict params: request parameters :param bool check: check response :param kwargs: other params :return: response object """ session = requests.Session() session.trust_env = False try: resp = session.head(url, params=params, cookies=self.cookie, headers=self.header, proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs) except Exception as e: logger.log('ERROR', e.args[0]) return None if not check: return resp if utils.check_response('HEAD', resp): return resp return None
def match_subdomains(domain, text): if isinstance(text, str): subdomains = utils.match_subdomains(domain, text, fuzzy=False) else: logger.log('DEBUG', f'abnormal object: {type(text)}') subdomains = set() logger.log('TRACE', f'matched subdomains: {subdomains}') return subdomains
def req_thread_count(): count = settings.request_thread_count if isinstance(count, int): count = max(16, count) else: count = utils.get_request_count() logger.log('DEBUG', f'Number of request threads {count}') return count
def check_param(self): """ Check parameter """ if self.target is None and self.targets is None: logger.log('FATAL', 'You must provide either target or targets parameter') exit(1)
def save_db(name, data): """ Save resolved results to database :param str name: table name :param list data: data to be saved """ logger.log('INFOR', f'Saving resolved results') utils.save_to_db(name, data, 'resolve')
def ip_to_int(ip): if isinstance(ip, int): return ip try: ipv4 = IPv4Address(ip) except Exception as e: logger.log('ERROR', e.args) return 0 return int(ipv4)
def check_random_subdomain(subdomains): if not subdomains: logger.log('ALERT', f'The generated dictionary is empty') return for subdomain in subdomains: if subdomain: logger.log('ALERT', f'Please check whether {subdomain} is correct or not') return
def get_cname(subdomain): resolver = utils.dns_resolver() try: answers = resolver.query(subdomain, 'CNAME') except Exception as e: logger.log('TRACE', e.args) return None for answer in answers: return answer.to_text() # 一个子域只有一个CNAME记录
def do_export(fmt, path, rows, show, domain, target): fmt = utils.check_format(fmt) path = utils.check_path(path, target, fmt) if show: print(rows.dataset) data = rows.export(fmt) utils.save_to_file(path, data) logger.log('ALERT', f'The subdomain result for {domain}: {path}') data = rows.as_dict() return data, fmt, path
def have_api(self, *apis): """ Simply check whether the api information configure or not :param apis: apis set :return bool: check result """ if not all(apis): logger.log('DEBUG', f'{self.source} module is not configured') return False return True
def compare(self, subdomain, cname, responses): domain_resp = self.get('http://' + subdomain, check=False, ignore=True) cname_resp = self.get('http://' + cname, check=False, ignore=True) if domain_resp is None or cname_resp is None: return for resp in responses: if resp in domain_resp.text and resp in cname_resp.text: logger.log('ALERT', f'{subdomain} takeover threat found') self.results.append([subdomain, cname]) break
def save_db(self): """ Save module results into the database """ logger.log('DEBUG', f'Saving results to database') lock.acquire() db = Database() db.create_table(self.domain) db.save_db(self.domain, self.results, self.source) db.close() lock.release()
def deal_wildcard(data): new_data = list() appear_times = stat_times(data) for info in data: subdomain = info.get('subdomain') isvalid, reason = check_valid_subdomain(appear_times, info) logger.log( 'DEBUG', f'{subdomain} is {isvalid} subdomain reason because {reason}') if isvalid: new_data.append(info) return new_data
def grab_loop(self, csp_header, urls): for url in urls: self.header = self.get_header() self.proxy = self.get_proxy(self.source) try: response = self.get(url, check=False, ignore=True, raise_error=True) except requests.exceptions.ConnectTimeout: logger.log('DEBUG', f'Connection to {url} timed out, so break check') break if response: return response.headers return csp_header