def run(self):
print(banner)
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f'[*] Starting OneForAll @ {dt}\n')
logger.log('INFOR', f'开始运行OneForAll')
self.domains = utils.get_domains(self.target)
if self.domains:
for self.domain in self.domains:
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
brute = AIOBrute(self.domain)
brute.run()
table_name = self.domain.replace('.', '_')
db_conn = database.connect_db()
self.datas = database.get_data(db_conn, table_name).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
loop.run_until_complete(asyncio.sleep(0.25)) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.close()
database.clear_table(db_conn, table_name)
database.save_db(db_conn, table_name, self.datas)
# 数据库导出
if not self.path:
self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
db_conn.close()
else:
logger.log('FATAL', f'获取域名失败')
logger.log('INFOR', f'结束运行OneForAll')
def run(self):
"""
类运行入口
"""
start = time.time()
logger.log('INFOR', f'开始收集{self.domain}的子域')
self.get_mod()
self.import_func()
threads = []
# 创建多个子域收集线程
for collect_func in self.collect_func:
thread = threading.Thread(target=collect_func,
args=(self.domain, ),
daemon=True)
threads.append(thread)
# 启动所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 数据库导出
if self.export:
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = config.result_save_path.joinpath(name)
dbexport.export(self.domain, path=self.path, format=self.format)
end = time.time()
self.elapsed = round(end - start, 1)
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.verify is None:
self.verify = config.enable_verify_subdomain
rename_table = self.domain + '_last'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain + '_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
# 不验证子域的情况
if not self.verify:
# 数据库导出
self.valid = None
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
return
# 开始验证子域工作
self.datas = db.get_data(self.domain).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.datas)
self.datas = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.datas, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.datas, self.port)
self.datas = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
db.clear_table(self.domain)
db.save_db(self.domain, self.datas)
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
db.close()
def run(self, rx_queue=None):
self.domains = utils.get_domains(self.target)
while self.domains:
self.domain = self.domains.pop()
start = time.time()
db = Database()
db.create_table(self.domain)
if not rx_queue:
rx_queue = queue.Queue()
logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程')
# fuzz模式不使用递归爆破
if self.recursive_brute and not self.fuzz:
logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self.main(self.domain, rx_queue))
# 递归爆破下一层的子域
# fuzz模式不使用递归爆破
if self.recursive_brute and not self.fuzz:
for layer_num in range(1, self.recursive_depth):
# 之前已经做过1层子域爆破 当前实际递归层数是layer+1
logger.log('INFOR', f'开始递归爆破{self.domain}的'
f'第{layer_num + 1}层子域')
for subdomain in self.subdomains.copy():
# 进行下一层子域爆破的限制条件
if subdomain.count('.') - self.domain.count(
'.') == layer_num:
loop.run_until_complete(
self.main(subdomain, rx_queue))
# 队列不空就一直取数据存数据库
while not rx_queue.empty():
source, results = rx_queue.get()
# 将结果存入数据库中
db.save_db(self.domain, results, source)
end = time.time()
self.elapsed = round(end - start, 1)
logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
length = len(self.subdomains)
logger.log(
'INFOR', f'{self.source}模块耗时{self.elapsed}秒'
f'发现{self.domain}的域名{length}个')
logger.log(
'DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
f'{self.subdomains}')
# 数据库导出
if self.export:
if not self.path:
name = f'{self.domain}_brute.{self.format}'
self.path = config.result_save_path.joinpath(name)
dbexport.export(self.domain,
valid=self.valid,
path=self.path,
format=self.format,
show=self.show)
def run(self):
logger.log('INFOR', f'Start running {self.source} module')
if self.check_env:
utils.check_env()
self.domains = utils.get_domains(self.target, self.targets)
for self.domain in self.domains:
self.results = list() # 置空
all_subdomains = list()
self.check_brute_params()
if self.recursive_brute:
logger.log(
'INFOR', f'Start recursively brute the 1 layer subdomain'
f' of {self.domain}')
valid_subdomains = self.main(self.domain)
all_subdomains.extend(valid_subdomains)
# 递归爆破下一层的子域
# fuzz模式不使用递归爆破
if self.recursive_brute:
for layer_num in range(1, self.recursive_depth):
# 之前已经做过1层子域爆破 当前实际递归层数是layer+1
logger.log(
'INFOR',
f'Start recursively brute the {layer_num + 1} '
f'layer subdomain of {self.domain}')
for subdomain in all_subdomains:
self.place = '*.' + subdomain
# 进行下一层子域爆破的限制条件
num = subdomain.count('.') - self.domain.count('.')
if num == layer_num:
valid_subdomains = self.main(subdomain)
all_subdomains.extend(valid_subdomains)
logger.log(
'INFOR',
f'Finished {self.source} module to brute {self.domain}')
if not self.path:
name = f'{self.domain}_brute_result.{self.format}'
self.path = settings.result_save_dir.joinpath(name)
# 数据库导出
if self.export:
dbexport.export(self.domain,
type='table',
alive=self.alive,
limit='resolve',
path=self.path,
format=self.format)
def main(self):
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain+'_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
self.datas = db.get_data(self.domain).as_dict()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.datas)
self.datas = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.datas, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.datas, self.port)
self.datas = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
loop.close()
db.clear_table(self.domain)
db.save_db(self.domain, self.datas)
# 数据库导出
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = config.result_save_path.joinpath(name)
dbexport.export(self.domain, db.conn, self.valid, self.path,
self.format, self.output)
rename_table = self.domain + '_last'
db.drop_table(rename_table)
db.rename_table(self.domain, rename_table)
def run(self):
"""
Class entrance
"""
start = time.time()
logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
self.get_mod()
self.import_func()
threads = []
# Create subdomain collection threads
for collect_func in self.collect_funcs:
func_obj, func_name = collect_func
thread = threading.Thread(target=func_obj,
name=func_name,
args=(self.domain, ),
daemon=True)
threads.append(thread)
# Start all threads
for thread in threads:
thread.start()
# Wait for all threads to finish
for thread in threads:
# 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
# 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
thread.join(settings.module_thread_timeout)
for thread in threads:
if thread.is_alive():
logger.log('ALERT', f'{thread.name} module thread timed out')
# Export
if self.export:
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = settings.result_save_dir.joinpath(name)
dbexport.export(self.domain,
type='table',
path=self.path,
format=self.format)
end = time.time()
self.elapse = round(end - start, 1)
def run(self):
logger.log('INFOR', f'开始执行{self.source}模块')
if self.check_env:
utils.check_env()
self.domains = utils.get_domains(self.target)
all_subdomains = list()
for self.domain in self.domains:
self.check_brute_params()
if self.recursive_brute:
logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
valid_subdomains = self.main(self.domain)
all_subdomains.extend(valid_subdomains)
# 递归爆破下一层的子域
# fuzz模式不使用递归爆破
if self.recursive_brute:
for layer_num in range(1, self.recursive_depth):
# 之前已经做过1层子域爆破 当前实际递归层数是layer+1
logger.log('INFOR', f'开始递归爆破{self.domain}的'
f'第{layer_num + 1}层子域')
for subdomain in all_subdomains:
self.place = '*.' + subdomain
# 进行下一层子域爆破的限制条件
num = subdomain.count('.') - self.domain.count('.')
if num == layer_num:
valid_subdomains = self.main(subdomain)
all_subdomains.extend(valid_subdomains)
logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
if not self.path:
name = f'{self.domain}_brute_result.{self.format}'
self.path = config.result_save_dir.joinpath(name)
# 数据库导出
if self.export:
dbexport.export(self.domain,
alive=self.alive,
limit='resolve',
path=self.path,
format=self.format)
def run(self):
"""
类运行入口
"""
start = time.time()
logger.log('INFOR', f'开始收集{self.domain}的子域')
self.get_mod()
self.import_func()
threads = []
# 创建多个子域收集线程
for collect in self.collect_funcs:
func_obj, func_name = collect
thread = threading.Thread(target=func_obj,
name=func_name,
args=(self.domain,),
daemon=True)
threads.append(thread)
# 启动所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
# 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
# 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
thread.join(config.module_thread_timeout)
for thread in threads:
if thread.is_alive():
logger.log('ALERT', f'{thread.name}模块线程发生超时')
# 数据库导出
if self.export:
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = config.result_save_path.joinpath(name)
dbexport.export(self.domain, path=self.path, format=self.format)
end = time.time()
self.elapsed = round(end - start, 1)
def run(self, rx_queue=None):
"""
类运行入口
"""
start = time.time()
self.get_mod()
self.import_func()
if not rx_queue:
rx_queue = queue.Queue(maxsize=len(self.collect_func)) # 结果集队列
threads = []
# 创建多个子域收集线程
for collect_func in self.collect_func:
thread = threading.Thread(target=collect_func,
args=(self.domain, rx_queue),
daemon=True)
threads.append(thread)
# 启动所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
db_conn = database.connect_db()
table_name = self.domain.replace('.', '_')
database.create_table(db_conn, table_name)
database.copy_table(db_conn, table_name)
database.deduplicate_subdomain(db_conn, table_name)
database.remove_invalid(db_conn, table_name)
db_conn.close()
# 数据库导出
if self.export:
if not self.path:
self.path = config.result_save_path.joinpath(
f'{self.domain}.{self.format}')
dbexport.export(table_name, path=self.path, format=self.format)
end = time.time()
self.elapsed = round(end - start, 1)
def export(self, table):
"""
从数据库中导出数据并做一些后续数据库善后处理
:param table: 要导出的表名
:return: 导出的数据
:rtype: list
"""
db = Database()
data = dbexport.export(table, alive=self.alive, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def export(self, table):
"""
Export data from the database and do some follow-up processing
:param table: table name
:return: export data
:rtype: list
"""
db = Database()
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.dns is None:
self.dns = config.enable_dns_resolve
if self.req is None:
self.req = config.enable_http_request
old_table = self.domain + '_last_result'
new_table = self.domain + '_now_result'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
original_table = self.domain + '_original_result'
db.copy_table(self.domain, original_table)
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 不解析子域直接导出结果
if not self.dns:
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
# 获取事件循环
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析子域
task = resolve.bulk_resolve(self.data)
self.data = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_resolve_result'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
# 不请求子域直接导出结果
if not self.req:
# 数据库导出
dbexport.export(resolve_table,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
# 请求子域
task = request.bulk_request(self.data, self.port)
self.data = loop.run_until_complete(task)
self.datas.extend(self.data)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_valid(self.data)
logger.log('INFOR', f'经验证{self.domain}有效子域{count}个')
# 保存请求结果
db.clear_table(self.domain)
db.save_db(self.domain, self.data, 'request')
# 数据库导出
dbexport.export(self.domain,
valid=self.valid,
format=self.format,
show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
takeover = Takeover(subdomains)
takeover.run()
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.verify is None:
self.verify = config.enable_verify_subdomain
old_table = self.domain + '_last'
new_table = self.domain + '_now'
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
# 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
db.copy_table(self.domain, self.domain+'_ori')
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 不验证子域的情况
if not self.verify:
# 数据库导出
self.valid = None
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
return
# 开始验证子域工作
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
# 解析域名地址
task = resolve.bulk_query_a(self.data)
self.data = loop.run_until_complete(task)
# 保存解析结果
resolve_table = self.domain + '_res'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
# 请求域名地址
task = request.bulk_get_request(self.data, self.port)
self.data = loop.run_until_complete(task)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
db.clear_table(self.domain)
db.save_db(self.domain, self.data)
# 数据库导出
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
takeover = Takeover(subdomains)
takeover.run()
#!/usr/bin/env python3
# coding=utf-8
"""
Example
"""
from oneforall import OneForAll
from dbexport import export
def oneforall(target):
test = OneForAll(target=target)
test.brute = True
test.req = True
test.takeover = True
test.run()
if __name__ == '__main__':
TARGET = 'freebuf.com'
oneforall(target=TARGET)
export(target=TARGET)
