async def bulk_get_request(datas, port):
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
logger.log('INFOR', f'正在异步进行子域的GET请求')
limit_open_conn = config.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
limit_open_conn = utils.get_semaphore()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(limit_open_conn)
header = None
if config.fake_header:
header = utils.gen_fake_header()
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
# 等待所有task完成 错误聚合到结果列表里
results = await asyncio.gather(*tasks, return_exceptions=True)
new_datas = deal_results(new_datas, results)
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
def get_limit_conn():
limit_open_conn = config.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
limit_open_conn = utils.get_semaphore()
return limit_open_conn
async def main(self, domain, rx_queue):
if not self.fuzz: # fuzz模式不探测域名是否使用泛解析
self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(
domain)
tasks = self.gen_tasks(domain)
logger.log('INFOR', f'正在爆破{domain}的域名')
sem = asyncio.Semaphore(utils.get_semaphore())
for task in tqdm.tqdm(tasks,
desc='Progress',
smoothing=1.0,
ncols=True):
async with aiomultiprocess.Pool(
processes=self.processes,
initializer=init_worker,
childconcurrency=self.coroutine) as pool:
try:
results = await pool.map(resolve.aiodns_query_a, task)
except KeyboardInterrupt:
logger.log('ALERT', '爆破终止正在退出')
pool.terminate() # 关闭pool，结束工作进程，不在处理未完成的任务。
self.save_json()
self.gen_result()
rx_queue.put(self.results)
return
else:
self.deal_results(results)
self.save_json()
self.gen_result()
rx_queue.put(self.results)
async def bulk_get_request(datas, port):
logger.log('INFOR', f'正在异步进行子域的GET请求')
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
header = None
if config.fake_header:
header = utils.gen_fake_header()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=config.limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(utils.get_semaphore())
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
task.add_done_callback(
functools.partial(request_callback, index=i, datas=new_datas))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
await asyncio.wait(tasks) # 等待所有task完成
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
async def bulk_get_request(datas, port):
ports = get_ports(port)
new_datas = gen_new_datas(datas, ports)
logger.log('INFOR', f'正在异步进行子域的GET请求')
limit_open_conn = config.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
limit_open_conn = utils.get_semaphore()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(limit_open_conn)
header = None
if config.fake_header:
header = utils.gen_fake_header()
async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore))
task.add_done_callback(functools.partial(request_callback,
index=i,
datas=new_datas))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
# 等待所有task完成 错误聚合到结果列表里
futures = asyncio.as_completed(tasks)
for future in tqdm.tqdm(futures,
total=len(tasks),
desc='Progress',
smoothing=1.0,
ncols=True):
try:
await future
except:
pass
logger.log('INFOR', f'完成异步进行子域的GET请求')
return new_datas
async def aiodns_query_a(hostname, semaphore=None):
"""
异步查询A记录
:param str hostname: 主机名
:param semaphore: 并发查询数量
:return: 主机名或查询结果或查询异常
"""
if semaphore is None:
semaphore = utils.get_semaphore()
async with semaphore:
resolver = aiodns_resolver()
answers = await resolver.query(hostname, 'A')
return hostname, answers
async def bulk_query_a(datas):
"""
批量查询A记录
:param datas: 待查的数据集
:return: 查询过得到的结果集
"""
logger.log('INFOR', '正在异步查询子域的A记录')
tasks = []
semaphore = asyncio.Semaphore(utils.get_semaphore())
for i, data in enumerate(datas):
if not data.get('ips'):
subdomain = data.get('subdomain')
task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
task.add_done_callback(
functools.partial(resolve_callback, index=i,
datas=datas)) # 回调
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
await asyncio.wait(tasks) # 等待所有task完成
logger.log('INFOR', '完成异步查询子域的A记录')
return datas
def get_limit_conn():
limit_open_conn = settings.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
return limit_open_conn
