def detect_wildcard(domain):
is_enable = to_detect_wildcard(domain)
if is_enable:
logger.log('ALERT', f'The domain {domain} enables wildcard')
else:
logger.log('ALERT', f'The domain {domain} disables wildcard')
return is_enable
def match_subdomains(domain, html, distinct=True, fuzzy=True):
"""
Use regexp to match subdomains
:param str domain: main domain
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:param bool fuzzy: fuzzy match subdomain or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
if fuzzy:
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
else:
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)
def query(self):
"""
向接口查询子域并做子域匹配
"""
page = 0
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'type': 'SUBDOMAINS', 'key': self.api,
'value': self.domain, 'page': page}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code != 200:
break # 请求不正常通常网络是有问题，不再继续请求下去
try:
json = resp.json()
except Exception as e:
logger.log('DEBUG', e.args)
break
subdomains = self.match_subdomains(str(json))
if not subdomains:
break
self.subdomains.update(subdomains)
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
subdomains = json.get('Subdomains')
if subdomains and len(subdomains) < 300:
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
break
page += 1
if page >= 50: # ipv4info子域查询接口最多允许查询50页
break
def deal_output(output_path):
logger.log('INFOR', f'Processing resolved results')
infos = dict() # 用来记录所有域名有关信息
with open(output_path) as fd:
for line in fd:
line = line.strip()
try:
items = json.loads(line)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR', f'Error resolve line {line}, skip this line')
continue
info = dict()
info['resolver'] = items.get('resolver')
qname = items.get('name')[:-1] # 去除最右边的`.`点号
status = items.get('status')
if status != 'NOERROR':
logger.log('DEBUG', f'Resolving {qname}: {status}')
continue
data = items.get('data')
if 'answers' not in data:
logger.log('DEBUG', f'Resolving {qname} have not any answers')
info['alive'] = 0
info['resolve'] = 0
info['reason'] = 'NoAnswer'
infos[qname] = info
continue
infos = gen_infos(data, qname, info, infos)
return infos
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
data = {
'query': f'parsed.names: {self.domain}',
'page': 1,
'fields': ['parsed.subject_dn', 'parsed.names'],
'flatten': True
}
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
json = resp.json()
status = json.get('status')
if status != 'ok':
logger.log('ALERT', f'{self.source} module {status}')
return
subdomains = self.match_subdomains(resp.text)
self.subdomains.update(subdomains)
pages = json.get('metadata').get('pages')
for page in range(2, pages + 1):
data['page'] = page
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
self.subdomains = self.collect_subdomains(resp)
def gen_infos(data, qname, info, infos):
flag = False
cnames = list()
ips = list()
ttl = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
flag = True
name = answer.get('name')
cname = name[:-1].lower() # 去除最右边的`.`点号
cnames.append(cname)
ip = answer.get('data')
ips.append(ip)
ttl.append(str(answer.get('ttl')))
info['resolve'] = 1
info['reason'] = 'OK'
info['cname'] = ','.join(cnames)
info['ip'] = ','.join(ips)
info['ttl'] = ','.join(ttl)
infos[qname] = info
if not flag:
logger.log('DEBUG', f'Resolving {qname} have not a record')
info['alive'] = 0
info['resolve'] = 0
info['reason'] = 'NoARecord'
infos[qname] = info
return infos
def begin(self):
"""
begin log
"""
logger.log(
'DEBUG', f'Start {self.source} module to '
f'collect subdomains of {self.domain}')
def save_json(self):
"""
Save the results of each module as a json file
:return bool: whether saved successfully
"""
if not settings.save_module_result:
return False
logger.log(
'TRACE', f'Save the subdomain results found by '
f'{self.source} module as a json file')
path = settings.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
path = path.joinpath(name)
with open(path, mode='w', errors='ignore') as file:
result = {
'domain': self.domain,
'name': self.module,
'source': self.source,
'elapse': self.elapse,
'find': len(self.subdomains),
'subdomains': list(self.subdomains),
'infos': self.infos
}
json.dump(result, file, ensure_ascii=False, indent=4)
return True
def check_path(path, name, fmt):
"""
检查结果输出目录路径
:param path: 保存路径
:param name: 导出名字
:param fmt: 保存格式
:return: 保存路径
"""
filename = f'{name}.{fmt}'
default_path = settings.result_save_dir.joinpath(filename)
if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
path = path.replace('\'', '') # 去除多余的转义
else:
path = default_path
path = Path(path)
if not path.suffix: # 输入是目录的情况
path = path.joinpath(filename)
parent_dir = path.parent
if not parent_dir.exists():
logger.log('ALERT',
f'{parent_dir} does not exist, directory will be created')
parent_dir.mkdir(parents=True, exist_ok=True)
if path.exists():
logger.log('ALERT', f'The {path} exists and will be overwritten')
return path
def list_dns(self, zone_id):
page = 1
list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records',
params={'page': page, 'per_page': 10})
if not list_dns_resp:
logger.log('DEBUG',
f'{list_dns_resp.status_code} {list_dns_resp.text}')
return
subdomains = self.match_subdomains(list_dns_resp.text)
self.subdomains.update(subdomains)
if not self.subdomains:
# waiting for cloudflare enumerate subdomains
sleep(5)
self.list_dns(zone_id)
else:
while True:
list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records',
params={'page': page, 'per_page': 10})
if not list_dns_resp:
logger.log('DEBUG',
f'{list_dns_resp.status_code} {list_dns_resp.text}')
return
total_pages = list_dns_resp.json()['result_info']['total_pages']
subdomains = (self.match_subdomains(list_dns_resp.text))
self.subdomains.update(subdomains)
page += 1
if page > total_pages:
break
return
def query(self):
"""
向接口查询子域并做子域匹配
"""
base_addr = 'http://114.55.181.28/check_web/' \
'databaseInfo_mainSearch.action'
page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'isSearch': 'true', 'searchType': 'url',
'term': self.domain, 'pageNo': page_num}
try:
resp = self.get(base_addr, params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
subdomains = self.match_subdomains(resp.text)
if not subdomains: # 没有发现子域名则停止查询
break
self.subdomains.update(subdomains)
if not subdomains:
break
if page_num > 10:
break
page_num += 1
def gen_subdomains(expression, path):
"""
Generate subdomains
:param str expression: generate subdomains expression
:param str path: path of wordlist
:return set subdomains: list of subdomains
"""
subdomains = set()
with open(path, encoding='utf-8', errors='ignore') as fd:
for line in fd:
word = line.strip().lower()
if len(word) == 0:
continue
if not utils.is_subname(word):
continue
if word.startswith('.'):
word = word[1:]
if word.endswith('.'):
word = word[:-1]
subdomain = expression.replace('*', word)
subdomains.add(subdomain)
size = len(subdomains)
logger.log('DEBUG',
f'The size of the dictionary generated by {path} is {size}')
if size == 0:
logger.log('ALERT', 'Please check the dictionary content!')
else:
utils.check_random_subdomain(subdomains)
return subdomains
def gen_fuzz_subdomains(expression, rule, fuzzlist):
"""
Generate subdomains based on fuzz mode
:param str expression: generate subdomains expression
:param str rule: regexp rule
:param str fuzzlist: fuzz dictionary
:return set subdomains: list of subdomains
"""
subdomains = set()
if fuzzlist:
fuzz_domain = gen_subdomains(expression, fuzzlist)
subdomains.update(fuzz_domain)
if rule:
fuzz_count = exrex.count(rule)
if fuzz_count > 10000000:
logger.log(
'ALERT',
f'The dictionary generated by this rule is too large: '
f'{fuzz_count} > 10000000')
for fuzz_string in exrex.generate(rule):
fuzz_string = fuzz_string.lower()
if not fuzz_string.isalnum():
continue
fuzz_domain = expression.replace('*', fuzz_string)
subdomains.add(fuzz_domain)
utils.check_random_subdomain(subdomains)
logger.log('DEBUG',
f'Dictionary size based on fuzz mode: {len(subdomains)}')
return subdomains
def axfr(self, server):
"""
Perform domain transfer
:param server: domain server
"""
logger.log(
'DEBUG', f'Trying to perform domain transfer in {server} '
f'of {self.domain}')
try:
xfr = dns.query.xfr(where=server,
zone=self.domain,
timeout=5.0,
lifetime=10.0)
zone = dns.zone.from_xfr(xfr)
except Exception as e:
logger.log('DEBUG', e.args)
logger.log(
'DEBUG', f'Domain transfer to server {server} of '
f'{self.domain} failed')
return
names = zone.nodes.keys()
for name in names:
full_domain = str(name) + '.' + self.domain
subdomain = self.match_subdomains(full_domain)
self.subdomains.update(subdomain)
record = zone[name].to_text(name)
self.results.append(record)
if self.results:
logger.log(
'DEBUG', f'Found the domain transfer record of '
f'{self.domain} on {server}')
logger.log('DEBUG', '\n'.join(self.results))
self.results = []
def find_subdomains(domain, data):
subdomains = set()
js_urls = set()
db = Database()
for infos in data:
jump_history = infos.get('history')
req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = db.get_resp_by_url(domain, req_url)
if not rsp_html:
logger.log(
'DEBUG',
f'an abnormal response occurred in the request {req_url}')
continue
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
req_data = convert_to_dict(js_urls)
resp_data = request.bulk_request(domain, req_data, ret=True)
while not resp_data.empty():
_, resp = resp_data.get()
if not isinstance(resp, Response):
continue
text = utils.decode_resp_text(resp)
subdomains.update(find_in_resp(domain, resp.url, text))
return subdomains
def save(self):
logger.log('DEBUG', 'Saving results')
if self.fmt == 'txt':
data = str(self.results)
else:
data = self.results.export(self.fmt)
utils.save_to_file(self.path, data)
def head(self, url, params=None, check=True, **kwargs):
"""
Custom head request
:param str url: request url
:param dict params: request parameters
:param bool check: check response
:param kwargs: other params
:return: response object
"""
session = requests.Session()
session.trust_env = False
try:
resp = session.head(url,
params=params,
cookies=self.cookie,
headers=self.header,
proxies=self.proxy,
timeout=self.timeout,
verify=self.verify,
**kwargs)
except Exception as e:
logger.log('ERROR', e.args[0])
return None
if not check:
return resp
if utils.check_response('HEAD', resp):
return resp
return None
def match_subdomains(domain, text):
if isinstance(text, str):
subdomains = utils.match_subdomains(domain, text, fuzzy=False)
else:
logger.log('DEBUG', f'abnormal object: {type(text)}')
subdomains = set()
logger.log('TRACE', f'matched subdomains: {subdomains}')
return subdomains
def req_thread_count():
count = settings.request_thread_count
if isinstance(count, int):
count = max(16, count)
else:
count = utils.get_request_count()
logger.log('DEBUG', f'Number of request threads {count}')
return count
def check_param(self):
"""
Check parameter
"""
if self.target is None and self.targets is None:
logger.log('FATAL',
'You must provide either target or targets parameter')
exit(1)
def save_db(name, data):
"""
Save resolved results to database
:param str name: table name
:param list data: data to be saved
"""
logger.log('INFOR', f'Saving resolved results')
utils.save_to_db(name, data, 'resolve')
def ip_to_int(ip):
if isinstance(ip, int):
return ip
try:
ipv4 = IPv4Address(ip)
except Exception as e:
logger.log('ERROR', e.args)
return 0
return int(ipv4)
def check_random_subdomain(subdomains):
if not subdomains:
logger.log('ALERT', f'The generated dictionary is empty')
return
for subdomain in subdomains:
if subdomain:
logger.log('ALERT',
f'Please check whether {subdomain} is correct or not')
return
def get_cname(subdomain):
resolver = utils.dns_resolver()
try:
answers = resolver.query(subdomain, 'CNAME')
except Exception as e:
logger.log('TRACE', e.args)
return None
for answer in answers:
return answer.to_text() # 一个子域只有一个CNAME记录
def do_export(fmt, path, rows, show, domain, target):
fmt = utils.check_format(fmt)
path = utils.check_path(path, target, fmt)
if show:
print(rows.dataset)
data = rows.export(fmt)
utils.save_to_file(path, data)
logger.log('ALERT', f'The subdomain result for {domain}: {path}')
data = rows.as_dict()
return data, fmt, path
def have_api(self, *apis):
"""
Simply check whether the api information configure or not
:param apis: apis set
:return bool: check result
"""
if not all(apis):
logger.log('DEBUG', f'{self.source} module is not configured')
return False
return True
def compare(self, subdomain, cname, responses):
domain_resp = self.get('http://' + subdomain, check=False, ignore=True)
cname_resp = self.get('http://' + cname, check=False, ignore=True)
if domain_resp is None or cname_resp is None:
return
for resp in responses:
if resp in domain_resp.text and resp in cname_resp.text:
logger.log('ALERT', f'{subdomain} takeover threat found')
self.results.append([subdomain, cname])
break
def save_db(self):
"""
Save module results into the database
"""
logger.log('DEBUG', f'Saving results to database')
lock.acquire()
db = Database()
db.create_table(self.domain)
db.save_db(self.domain, self.results, self.source)
db.close()
lock.release()
def deal_wildcard(data):
new_data = list()
appear_times = stat_times(data)
for info in data:
subdomain = info.get('subdomain')
isvalid, reason = check_valid_subdomain(appear_times, info)
logger.log(
'DEBUG',
f'{subdomain} is {isvalid} subdomain reason because {reason}')
if isvalid:
new_data.append(info)
return new_data
def grab_loop(self, csp_header, urls):
for url in urls:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
try:
response = self.get(url, check=False, ignore=True, raise_error=True)
except requests.exceptions.ConnectTimeout:
logger.log('DEBUG', f'Connection to {url} timed out, so break check')
break
if response:
return response.headers
return csp_header
