def head(self, url, params=None, check=True, **kwargs):
"""
Custom head request
:param str url: request url
:param dict params: request parameters
:param bool check: check response
:param kwargs: other params
:return: response object
"""
session = requests.Session()
session.trust_env = False
try:
resp = session.head(url,
params=params,
cookies=self.cookie,
headers=self.header,
proxies=self.proxy,
timeout=self.timeout,
verify=self.verify,
**kwargs)
except Exception as e:
logger.log('ERROR', e.args)
return None
if not check:
return resp
if utils.check_response('HEAD', resp):
return resp
return None
def save_db(self, table_name, results, module_name=None):
"""
Save the results of each module in the database
:param str table_name: table name
:param list results: results list
:param str module_name: module
"""
logger.log(
'TRACE', f'Saving the subdomain results of {table_name} '
f'found by module {module_name} into database')
table_name = table_name.replace('.', '_')
if results:
try:
self.conn.bulk_query(
f'insert into "{table_name}" '
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
f'cname, ip, public, cdn, status, reason, title, banner, header,'
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
f'module, source, elapse, find) '
f'values (:id, :alive, :resolve, :request, :new, :url,'
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
f':status, :reason, :title, :banner, :header, :history, :response,'
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
f':source, :elapse, :find)', results)
except Exception as e:
logger.log('ERROR', e)
def search(self):
"""
向接口查询子域并做子域匹配
"""
page_num = 1
while True:
time.sleep(self.delay)
params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
try:
resp = self.get(self.addr, params=params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
if resp.status_code != 200:
logger.log('ERROR', f'{self.source} module query failed')
break
if 'class="empty-box"' in resp.text:
break
soup = BeautifulSoup(resp.text, 'html.parser')
subdomains = self.match_subdomains(soup, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains.update(subdomains)
if '<li class="disabled"><a href="###">' in resp.text:
break
page_num += 1
if page_num >= 100:
break
def check_cdn(data):
logger.log('DEBUG', f'Start cdn check module')
for index, item in enumerate(data):
cname = item.get('cname')
if cname:
if check_cname_keyword(cname):
data[index]['cdn'] = 1
continue
header = item.get('header')
if header:
header = json.loads(header)
if check_header_key(header):
data[index]['cdn'] = 1
continue
ip = item.get('ip')
if ip:
if check_cdn_cidr(ip):
data[index]['cdn'] = 1
continue
asn = item.get('asn')
if asn:
asn = asn[2:] # 去除AS
if check_cdn_asn(asn):
data[index]['cdn'] = 1
continue
data[index]['cdn'] = 0
return data
def check_param(self):
"""
Check parameter
"""
if self.target is None and self.targets is None:
logger.log('FATAL', 'You must provide either target or targets parameter')
exit(1)
def run(self):
"""
OneForAll running entrance
:return: All subdomain results
:rtype: list
"""
print(oneforall_banner)
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f'[*] Starting OneForAll @ {dt}\n')
utils.check_env()
utils.auto_select_nameserver()
if settings.enable_check_version:
utils.check_version(version)
logger.log('DEBUG', 'Python ' + utils.python_version())
logger.log('DEBUG', 'OneForAll ' + version)
logger.log('INFOR', 'Start running OneForAll')
self.config_param()
self.check_param()
self.domains = utils.get_domains(self.target, self.targets)
if self.domains:
for domain in self.domains:
self.domain = utils.get_main_domain(domain)
self.main()
return self.datas
#utils.export_all(self.alive, self.format, self.path, self.datas)
else:
logger.log('FATAL', 'Failed to obtain domain')
return None
logger.log('INFOR', 'Finished OneForAll')
def gen_fuzz_subdomains(expression, rule, fuzzlist):
"""
Generate subdomains based on fuzz mode
:param str expression: generate subdomains's expression
:param str rule: regexp rule
:param str fuzzlist: fuzz dictionary
:return set subdomains: list of subdomains
"""
subdomains = set()
if fuzzlist:
fuzz_domain = gen_subdomains(expression, fuzzlist)
subdomains.update(fuzz_domain)
if rule:
fuzz_count = exrex.count(rule)
if fuzz_count > 10000000:
logger.log(
'ALERT',
f'The dictionary generated by this rule is too large: '
f'{fuzz_count} > 10000000')
for fuzz_string in exrex.generate(rule):
fuzz_string = fuzz_string.lower()
if not fuzz_string.isalnum():
continue
fuzz_domain = expression.replace('*', fuzz_string)
subdomains.add(fuzz_domain)
utils.check_random_subdomain(subdomains)
logger.log('DEBUG',
f'Dictionary size based on fuzz mode: {len(subdomains)}')
return subdomains
def gen_subdomains(expression, path):
"""
Generate subdomains
:param str expression: generate subdomains's expression
:param str path: path of wordlist
:return set subdomains: list of subdomains
"""
subdomains = set()
with open(path, encoding='utf-8', errors='ignore') as fd:
for line in fd:
word = line.strip().lower()
if len(word) == 0:
continue
if not utils.is_subname(word):
continue
if word.startswith('.'):
word = word[1:]
if word.endswith('.'):
word = word[:-1]
subdomain = expression.replace('*', word)
subdomains.add(subdomain)
size = len(subdomains)
logger.log('DEBUG',
f'The size of the dictionary generated by {path} is {size}')
if size == 0:
logger.log('ALERT', 'Please check the dictionary content!')
else:
utils.check_random_subdomain(subdomains)
return subdomains
def is_enable_wildcard(domain):
is_enable = detect_wildcard(domain)
if is_enable:
logger.log('ALERT', f'The domain {domain} enables wildcard')
else:
logger.log('ALERT', f'The domain {domain} disables wildcard')
return is_enable
def save(self):
logger.log('DEBUG', 'Saving results')
if self.format == 'txt':
data = str(self.results)
else:
data = self.results.export(self.format)
utils.save_data(self.path, data)
def axfr(self, server):
"""
Perform domain transfer
:param server: domain server
"""
logger.log('DEBUG', f'Trying to perform domain transfer in {server} '
f'of {self.domain}')
try:
xfr = dns.query.xfr(where=server, zone=self.domain,
timeout=5.0, lifetime=10.0)
zone = dns.zone.from_xfr(xfr)
except Exception as e:
logger.log('DEBUG', e.args)
logger.log('DEBUG', f'Domain transfer to server {server} of '
f'{self.domain} failed')
return
names = zone.nodes.keys()
for name in names:
full_domain = str(name) + '.' + self.domain
subdomain = self.match_subdomains(full_domain)
self.subdomains.update(subdomain)
record = zone[name].to_text(name)
self.results.append(record)
if self.results:
logger.log('DEBUG', f'Found the domain transfer record of '
f'{self.domain} on {server}')
logger.log('DEBUG', '\n'.join(self.results))
self.results = []
def save_json(self):
"""
Save the results of each module as a json file
:return bool: whether saved successfully
"""
if not settings.save_module_result:
return False
logger.log(
'TRACE', f'Save the subdomain results found by '
f'{self.source} module as a json file')
path = settings.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
path = path.joinpath(name)
with open(path, mode='w', errors='ignore') as file:
result = {
'domain': self.domain,
'name': self.module,
'source': self.source,
'elapse': self.elapse,
'find': len(self.subdomains),
'subdomains': list(self.subdomains),
'infos': self.infos
}
json.dump(result, file, ensure_ascii=False, indent=4)
return True
def match_subdomains(domain, html, distinct=True, fuzzy=True):
"""
Use regexp to match subdomains
:param str domain: main domain
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:param bool fuzzy: fuzzy match subdomain or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
if fuzzy:
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
else:
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)
def bulk_request(urls):
logger.log('INFOR', 'Requesting urls in bulk')
resp_list = list()
urls_queue = Queue()
for url in urls:
urls_queue.put(url)
total = len(urls)
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
progress_thread = Thread(target=progress,
name='ProgressThread',
args=(bar, total, urls_queue),
daemon=True)
progress_thread.start()
for i in range(thread_count):
request_thread = Thread(target=request,
name=f'RequestThread-{i}',
args=(urls_queue, resp_list, session),
daemon=True)
request_thread.start()
urls_queue.join()
return resp_list
def load_rules(self):
new_rules = {}
new_rule_types = set()
for rule_type in os.listdir(self.rule_dir):
rule_type_dir = os.path.join(self.rule_dir, rule_type)
if not os.path.isdir(rule_type_dir):
continue
new_rule_types.add(rule_type)
for i in os.listdir(rule_type_dir):
if not i.endswith('.json'):
continue
with open(os.path.join(rule_type_dir, i), encoding='utf-8') as fd:
try:
data = json.load(fd)
for match in data['matches']:
if 'regexp' in match: # 默认 大小写不敏感 可信度100%
match['regexp'] = re.compile(
match['regexp'], re.I)
if 'certainty' not in match:
match['certainty'] = 100
data['origin'] = rule_type
key = '%s_%s' % (rule_type, data['name'])
new_rules[key] = data
except Exception as e:
logger.log('ERROR', f'Parse {i} failed, error: {e}')
RULES = new_rules
RULE_TYPES = new_rule_types
return len(RULES), RULES, RULE_TYPES
def run(self):
"""
Class entrance
"""
logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
self.get_mod()
self.import_func()
threads = []
# Create subdomain collection threads
for collect_func in self.collect_funcs:
func_obj, func_name = collect_func
thread = threading.Thread(target=func_obj,
name=func_name,
args=(self.domain, ),
daemon=True)
threads.append(thread)
# Start all threads
for thread in threads:
thread.start()
# Wait for all threads to finish
for thread in threads:
# 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
# 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
thread.join(settings.module_thread_timeout)
for thread in threads:
if thread.is_alive():
logger.log('ALERT', f'{thread.name} module thread timed out')
def update_data(data, infos):
"""
更新解析结果
:param list data: 待更新的数据列表
:param dict infos: 子域有关结果信息
:return: 更新后的数据列表
"""
logger.log('DEBUG', f'Updating resolved results')
if not infos:
logger.log('ALERT', f'No valid resolved result')
return data
for index, items in enumerate(data):
if items.get('ip'):
continue
subdomain = items.get('subdomain')
record = infos.get(subdomain)
if record:
items.update(record)
else:
items['resolve'] = 0
items['alive'] = 0
items['reason'] = 'NoResult'
data[index] = items
return data
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
data = {
'query': f'parsed.names: {self.domain}',
'page': 1,
'fields': ['parsed.subject_dn', 'parsed.names'],
'flatten': True
}
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
json = resp.json()
status = json.get('status')
if status != 'ok':
logger.log('ALERT', f'{self.source} module {status}')
return
subdomains = self.match_subdomains(resp.text)
self.subdomains.update(subdomains)
pages = json.get('metadata').get('pages')
for page in range(2, pages + 1):
data['page'] = page
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
self.subdomains = self.collect_subdomains(resp)
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.header.update({
'Referer': 'https://phonebook.cz/',
'Origin': 'https://phonebook.cz'
})
addr = 'https://public.intelx.io/phonebook/search'
key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab'
url = f'{addr}?k={key}'
data = {
"term": self.domain,
"maxresults": 10000,
"media": 0,
"target": 1,
"terminate": [],
"timeout": 20
}
resp = self.post(url, json=data)
if not resp:
return
json = resp.json()
ids = json.get('id')
if not ids:
logger.log('ALERT', f'Get PhoneBook id fail')
return
url = f'{addr}/result?k={key}&id={ids}&limit=10000'
resp = self.get(url)
self.subdomains = self.collect_subdomains(resp)
def query(self):
"""
向接口查询子域并做子域匹配
"""
base_addr = 'http://114.55.181.28/check_web/' \
'databaseInfo_mainSearch.action'
page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {
'isSearch': 'true',
'searchType': 'url',
'term': self.domain,
'pageNo': page_num
}
try:
resp = self.get(base_addr, params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
subdomains = self.match_subdomains(resp.text)
if not subdomains: # 没有发现子域名则停止查询
break
self.subdomains.update(subdomains)
if not subdomains:
break
if page_num > 10:
break
page_num += 1
def begin(self):
"""
begin log
"""
logger.log(
'DEBUG', f'Start {self.source} module to '
f'collect subdomains of {self.domain}')
def query(self, sql):
try:
results = self.conn.query(sql)
except Exception as e:
logger.log('ERROR', e.args)
return None
return results
def check_path(path, name, format):
"""
检查结果输出目录路径
:param path: 保存路径
:param name: 导出名字
:param format: 保存格式
:return: 保存路径
"""
filename = f'{name}.{format}'
default_path = settings.result_save_dir.joinpath(filename)
if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
path = path.replace('\'', '') # 去除多余的转义
else:
path = default_path
path = Path(path)
if not path.suffix: # 输入是目录的情况
path = path.joinpath(filename)
parent_dir = path.parent
if not parent_dir.exists():
logger.log('ALERT', f'{parent_dir} does not exist, directory will be created')
parent_dir.mkdir(parents=True, exist_ok=True)
if path.exists():
logger.log('ALERT', f'The {path} exists and will be overwritten')
return path
def check_random_subdomain(subdomains):
if not subdomains:
logger.log('ALERT', f'The generated dictionary is empty')
return
for subdomain in subdomains:
if subdomain:
logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
return
def match_subdomains(domain, text):
if isinstance(text, str):
subdomains = utils.match_subdomains(domain, text, fuzzy=False)
else:
logger.log('DEBUG', f'abnormal object: {type(text)}')
subdomains = set()
logger.log('TRACE', f'matched subdomains: {subdomains}')
return subdomains
def req_thread_count():
count = settings.request_thread_count
if isinstance(count, int):
count = max(16, count)
else:
count = utils.get_request_count()
logger.log('DEBUG', f'Number of request threads {count}')
return count
def drop_table(self, table_name):
"""
Delete table
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Deleting {table_name} table')
self.query(f'drop table if exists "{table_name}"')
def save_db(name, data):
"""
Save request results to database
:param str name: table name
:param list data: data to be saved
"""
logger.log('INFOR', f'Saving requested results')
utils.save_db(name, data, 'request')
def get_data(self, table_name):
"""
Get all the data in the table
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Get all the data from {table_name} table')
return self.query(f'select * from "{table_name}"')
def ip_to_int(ip):
if isinstance(ip, int):
return ip
try:
ipv4 = IPv4Address(ip)
except Exception as e:
logger.log('ERROR', e.args)
return 0
return int(ipv4)
