def export(table, db=None, alive=False, limit=None, path=None, format='csv', show=False):
"""
OneForAll数据库导出模块
Example:
python3 dbexport.py --table name --format csv --dir= ./result.csv
python3 dbexport.py --db result.db --table name --show False
Note:
参数alive可选值True，False分别表示导出存活，全部子域结果
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录自动生成路径
:param str table: 要导出的表
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
:param bool alive: 只导出存活的子域结果(默认False)
:param str limit: 导出限制条件(默认None)
:param str format: 导出文件格式(默认csv)
:param str path: 导出文件路径(默认None)
:param bool show: 终端显示导出数据(默认False)
"""
database = Database(db)
rows = database.export_data(table, alive, limit)
format = utils.check_format(format, len(rows))
path = utils.check_path(path, table, format)
if show:
print(rows.dataset)
data = rows.export(format)
database.close()
utils.save_data(path, data)
logger.log('INFOR', f'{table}主域的子域结果 {path}')
data_dict = rows.as_dict()
return data_dict
def search(self, full_search=False):
"""
向接口查询子域并做子域匹配
"""
page_num = 1
while True:
time.sleep(self.delay)
params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
try:
resp = self.get(self.addr, params=params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
if resp.status_code != 200:
logger.log('ERROR', f'{self.source}模块搜索出错')
break
if 'class="empty-box"' in resp.text:
break
soup = BeautifulSoup(resp.text, 'html.parser')
subdomains = self.match(self.domain, soup.text)
self.subdomains = self.subdomains.union(subdomains)
if not subdomains:
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
if '<li class="disabled"><a href="###">' in resp.text:
break
if page_num > 100:
break
page_num += 1
def gen_req_data(data, ports):
logger.log('INFOR', f'正在生成请求地址')
new_data = []
for data in data:
resolve = data.get('resolve')
# 解析失败(0)的子域不进行http请求探测
if resolve == 0:
continue
subdomain = data.get('subdomain')
for port in ports:
if str(port).endswith('443'):
url = f'https://{subdomain}:{port}'
if port == 443:
url = f'https://{subdomain}'
data['id'] = None
data['url'] = url
data['port'] = port
new_data.append(data)
data = dict(data) # 需要生成一个新的字典对象
else:
url = f'http://{subdomain}:{port}'
if port == 80:
url = f'http://{subdomain}'
data['id'] = None
data['url'] = url
data['port'] = port
new_data.append(data)
data = dict(data) # 需要生成一个新的字典对象
return new_data
def query(self):
"""
向接口查询子域并做子域匹配
"""
page = 0
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'type': 'SUBDOMAINS', 'key': self.api,
'value': self.domain, 'page': page}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code != 200:
break # 请求不正常通常网络是有问题，不再继续请求下去
try:
json = resp.json()
except Exception as e:
logger.log('DEBUG', e.args)
break
subdomains = self.match(self.domain, str(json))
if not subdomains:
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
subdomains = json.get('Subdomains')
if subdomains:
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
if len(subdomains) < 300:
break
page += 1
if page >= 50: # ipv4info子域查询接口最多允许查询50页
break
def axfr(self, server):
"""
执行域传送
:param server: 域名服务器
"""
logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
try:
xfr = dns.query.xfr(where=server,
zone=self.domain,
timeout=5.0,
lifetime=10.0)
zone = dns.zone.from_xfr(xfr)
except Exception as e:
logger.log('DEBUG', e.args)
logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
return
names = zone.nodes.keys()
for name in names:
full_domain = str(name) + '.' + self.domain
subdomain = utils.match_subdomain(self.domain, full_domain)
self.subdomains = self.subdomains.union(subdomain)
record = zone[name].to_text(name)
self.results.append(record)
if self.results:
logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
logger.log('DEBUG', '\n'.join(self.results))
self.results = []
def save(self):
logger.log('DEBUG', '正在保存检查结果')
if self.format == 'txt':
data = str(self.results)
else:
data = self.results.export(self.format)
utils.save_data(self.path, data)
def request(self, url):
try:
r = requests.get(url,
timeout=config.timeout,
headers=self.get_headers(),
verify=config.verify_ssl,
allow_redirects=config.allow_redirects)
text = r.content.decode(
encoding=chardet.detect(r.content)['encoding'])
title = self.get_title(text).strip().replace('\r',
'').replace('\n', '')
banner = self.get_banner(r.headers)
size = self.sizeHuman(len(r.text))
status = r.status_code
subdomain = (url.split('//')[-1]).split(':')[0]
if status in config.ignore_status_code:
return
result = {
'url': r.url,
'title': title,
'status': status,
'size': size,
'fingerprint': banner
}
self.data.append(result)
logger.log('INFOR', f'AliveWeb:[{url}]')
return r, text
except Exception as e:
# print(e)
return e
def save_json(self):
"""
将各模块结果保存为json文件
:return： 是否保存成功
"""
if not config.save_module_result:
return False
logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件')
path = config.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
path = path.joinpath(name)
with open(path, mode='w', encoding='utf-8', errors='ignore') as file:
result = {
'domain': self.domain,
'name': self.module,
'source': self.source,
'elapse': self.elapse,
'find': len(self.subdomains),
'subdomains': list(self.subdomains),
'records': self.records
}
json.dump(result, file, ensure_ascii=False, indent=4)
return True
def query(self):
"""
向接口查询子域并做子域匹配
"""
base_addr = 'http://114.55.181.28/check_web/' \
'databaseInfo_mainSearch.action'
page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'isSearch': 'true', 'searchType': 'url',
'term': self.domain, 'pageNo': page_num}
try:
resp = self.get(base_addr, params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
subdomains = self.match(self.domain, resp.text)
self.subdomains = self.subdomains.union(subdomains)
if not subdomains:
break
if page_num > 10:
break
page_num += 1
def export_all(format, path, datas):
"""
将所有结果数据导出到一个文件
:param str format: 导出文件格式
:param str path: 导出文件路径
:param list datas: 待导出的结果数据
"""
format = check_format(format, len(datas))
timestamp = get_timestring()
name = f'all_subdomain_result_{timestamp}'
path = check_path(path, name, format)
logger.log('INFOR', f'所有主域的子域结果 {path}')
row_list = list()
for row in datas:
if 'header' in row:
row.pop('header')
if 'response' in row:
row.pop('response')
keys = row.keys()
values = row.values()
if format in {'xls', 'xlsx'}:
values = check_value(values)
row_list.append(Record(keys, values))
rows = RecordCollection(iter(row_list))
content = rows.export(format)
save_data(path, content)
async def bulk_request(data, port):
ports = get_ports(port)
no_req_data = utils.get_filtered_data(data)
to_req_data = gen_req_data(data, ports)
method = config.request_method
logger.log('INFOR', f'请求使用{method}方法')
logger.log('INFOR', f'正在进行异步子域请求')
connector = get_connector()
header = get_header()
async with ClientSession(connector=connector, headers=header) as session:
tasks = []
for i, data in enumerate(to_req_data):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url))
task.add_done_callback(
functools.partial(request_callback, index=i,
datas=to_req_data))
tasks.append(task)
# 任务列表里有任务不空时才进行解析
if tasks:
# 等待所有task完成 错误聚合到结果列表里
futures = asyncio.as_completed(tasks)
for future in tqdm.tqdm(futures,
total=len(tasks),
desc='Request Progress',
ncols=80):
await future
return to_req_data + no_req_data
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
data = {
'query': f'parsed.names: {self.domain}',
'page': 1,
'fields': ['parsed.subject_dn', 'parsed.names'],
'flatten': True}
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
json = resp.json()
status = json.get('status')
if status != 'ok':
logger.log('ALERT', status)
return
subdomains = self.match(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains)
pages = json.get('metadata').get('pages')
for page in range(2, pages + 1):
data['page'] = page
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def check_path(path, name, format):
"""
检查结果输出目录路径
:param path: 保存路径
:param name: 导出名字
:param format: 保存格式
:return: 保存路径
"""
filename = f'{name}.{format}'
default_path = config.result_save_dir.joinpath(filename)
if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
path = path.replace('\'', '') # 去除多余的转义
else:
path = default_path
path = Path(path)
if not path.suffix: # 输入是目录的情况
path = path.joinpath(filename)
parent_dir = path.parent
if not parent_dir.exists():
logger.log('ALERT', f'不存在{parent_dir}目录将会新建')
parent_dir.mkdir(parents=True, exist_ok=True)
if path.exists():
logger.log('ALERT', f'存在{path}文件将会覆盖')
return path
def query(self, sql):
try:
results = self.conn.query(sql)
except Exception as e:
logger.log('ERROR', e.args)
else:
return results
def save_db(self, table_name, results, module_name=None):
"""
将各模块结果存入数据库
:param str table_name: 表名
:param list results: 结果列表
:param str module_name: 模块名
"""
logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域' '结果存入数据库')
table_name = table_name.replace('.', '_')
if results:
try:
self.conn.bulk_query(
f'insert into "{table_name}" ('
f'id, type, alive, resolve, request, new, url, subdomain,'
f'port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, resolver,'
f'module, source, elapse, find, brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new,'
f':url, :subdomain, :port, :level, :cname, :content,'
f':public, :status, :reason, :title, :banner, :header,'
f':response, :times, :ttl, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e)
def deal_output(output_path, ip_times, wildcard_ips, wildcard_ttl):
logger.log('INFOR', f'正在处理解析结果')
records = dict() # 用来记录所有域名解析数据
subdomains = list() # 用来保存所有通过有效性检查的子域
with open(output_path) as fd:
for line in fd:
line = line.strip()
try:
items = json.loads(line)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR', f'解析行{line}出错跳过解析该行')
continue
qname = items.get('name')[:-1] # 去出最右边的`.`点号
status = items.get('status')
if status != 'NOERROR':
logger.log('TRACE', f'处理{line}时发现{qname}查询结果状态{status}')
continue
data = items.get('data')
if 'answers' not in data:
logger.log('TRACE', f'处理{line}时发现{qname}返回的结果无应答')
continue
records, subdomains = gen_records(items, records, subdomains,
ip_times, wildcard_ips,
wildcard_ttl)
return records, subdomains
def get(self, url, params=None, check=True, **kwargs):
"""
自定义get请求
:param str url: 请求地址
:param dict params: 请求参数
:param bool check: 检查响应
:param kwargs: 其他参数
:return: requests响应对象
"""
try:
resp = requests.get(url,
params=params,
cookies=self.cookie,
headers=self.header,
proxies=self.proxy,
timeout=self.timeout,
verify=self.verify,
**kwargs)
except Exception as e:
logger.log('ERROR', e.args)
return None
if not check:
return resp
if utils.check_response('GET', resp):
return resp
return None
def request_callback(future, index, datas):
result = future.result()
if isinstance(result, BaseException):
logger.log('TRACE', result.args)
name = utils.get_classname(result)
datas[index]['reason'] = name + ' ' + str(result)
datas[index]['request'] = 0
datas[index]['alive'] = 0
elif isinstance(result, tuple):
resp, text = result
datas[index]['reason'] = resp.reason
datas[index]['status'] = resp.status
if resp.status == 400 or resp.status >= 500:
datas[index]['request'] = 0
datas[index]['alive'] = 0
else:
datas[index]['request'] = 1
datas[index]['alive'] = 1
headers = resp.headers
datas[index]['banner'] = utils.get_sample_banner(headers)
datas[index]['header'] = str(dict(headers))[1:-1]
if isinstance(text, str):
title = get_title(text).strip()
datas[index]['title'] = utils.remove_invalid_string(title)
datas[index]['response'] = utils.remove_invalid_string(text)
def init_shodan(self):
api = shodan.Shodan(config.shodan_api)
try:
api.info()
return api
except Exception as e:
logger.log('ALERT', f'shodan api异常:{e}')
return None
def drop_table(self, table_name):
"""
删除表
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在删除{table_name}表')
self.query(f'drop table if exists "{table_name}"')
def get_data(self, table_name):
"""
获取表中的所有数据
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'获取{table_name}表中的所有数据')
return self.query(f'select * from "{table_name}"')
def clear_table(self, table_name):
"""
清空表中数据
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在清空{table_name}表中的数据')
self.query(f'delete from "{table_name}"')
def init_fofa(self):
try:
url = f'https://fofa.so/api/v1/info/my?email={config.fofa_email}&key={config.fofa_key}'
r = requests.get(url)
r.json()
return f'https://fofa.so/api/v1/search/all?email={config.fofa_email}&key={config.fofa_key}'
except Exception as e:
logger.log('ALERT', f'fofa连接异常:{e}')
return None
def get_cname(subdomain):
resolver = utils.dns_resolver()
try:
answers = resolver.query(subdomain, 'CNAME')
except Exception as e:
logger.log('TRACE', e.args)
return None
for answer in answers:
return answer.to_text() # 一个子域只有一个CNAME记录
def remove_invalid(self, table_name):
"""
去除表中的空值或无效子域
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
self.query(f'delete from "{table_name}" where '
f'subdomain is null or resolve == 0')
def run(self):
logger.log('INFOR', f'WebAlive探测进程启动:WebAliveScan')
domain_list = [self.subdomain]
url_list = self.gen_url_list(domain_list)
gevent_pool = pool.Pool(config.threads)
while url_list:
for task in [
gevent_pool.spawn(self.request, url_list.pop())
for i in range(len(url_list[:config.threads]))
]:
task.join()
def deduplicate_subdomain(self, table_name):
"""
去重表中的子域
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去重{table_name}表中的子域')
self.query(f'delete from "{table_name}" where '
f'id not in (select min(id) '
f'from "{table_name}" group by subdomain)')
def compare(self, subdomain, cname, responses):
domain_resp = self.get('http://' + subdomain, check=False)
cname_resp = self.get('http://' + cname, check=False)
if domain_resp is None or cname_resp is None:
return
for resp in responses:
if resp in domain_resp.text and resp in cname_resp.text:
logger.log('ALERT', f'{subdomain}存在子域接管风险')
self.results.append([subdomain, cname])
break
def check(self, *apis):
"""
简单检查是否配置了api信息
:param apis: api信息元组
:return: 检查结果
"""
if not all(apis):
logger.log('ALERT', f'{self.source}模块没有配置API跳过执行')
return False
return True
def save_db(self):
"""
将模块结果存入数据库中
"""
logger.log('DEBUG', f'正在将结果存入到数据库')
lock.acquire()
db = Database()
db.create_table(self.domain)
db.save_db(self.domain, self.results, self.source)
db.close()
lock.release()
