def main(self): if self.brute is None: self.brute = config.enable_brute_module if self.verify is None: self.verify = config.enable_verify_subdomain rename_table = self.domain + '_last' collect = Collect(self.domain, export=False) collect.run() if self.brute: # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常 brute = AIOBrute(self.domain, export=False) brute.run() db = Database() db.copy_table(self.domain, self.domain + '_ori') db.remove_invalid(self.domain) db.deduplicate_subdomain(self.domain) # 不验证子域的情况 if not self.verify: # 数据库导出 self.valid = None dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(rename_table) db.rename_table(self.domain, rename_table) return # 开始验证子域工作 self.datas = db.get_data(self.domain).as_dict() loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) # 解析域名地址 task = resolve.bulk_query_a(self.datas) self.datas = loop.run_until_complete(task) # 保存解析结果 resolve_table = self.domain + '_res' db.drop_table(resolve_table) db.create_table(resolve_table) db.save_db(resolve_table, self.datas, 'resolve') # 请求域名地址 task = request.bulk_get_request(self.datas, self.port) self.datas = loop.run_until_complete(task) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 loop.run_until_complete(asyncio.sleep(0.25)) db.clear_table(self.domain) db.save_db(self.domain, self.datas) # 数据库导出 dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(rename_table) db.rename_table(self.domain, rename_table) db.close()
def save_db(self): lock.acquire() db = Database() db.create_table(self.domain) source, results = self.results # 将结果存入数据库中 db.save_db(self.domain, results, source) lock.release()
def run(self, rx_queue=None): self.domains = utils.get_domains(self.target) while self.domains: self.domain = self.domains.pop() start = time.time() db = Database() db.create_table(self.domain) if not rx_queue: rx_queue = queue.Queue() logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}') logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程') # fuzz模式不使用递归爆破 if self.recursive_brute and not self.fuzz: logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域') loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(self.main(self.domain, rx_queue)) # 递归爆破下一层的子域 # fuzz模式不使用递归爆破 if self.recursive_brute and not self.fuzz: for layer_num in range(1, self.recursive_depth): # 之前已经做过1层子域爆破 当前实际递归层数是layer+1 logger.log('INFOR', f'开始递归爆破{self.domain}的' f'第{layer_num + 1}层子域') for subdomain in self.subdomains.copy(): # 进行下一层子域爆破的限制条件 if subdomain.count('.') - self.domain.count( '.') == layer_num: loop.run_until_complete( self.main(subdomain, rx_queue)) # 队列不空就一直取数据存数据库 while not rx_queue.empty(): source, results = rx_queue.get() # 将结果存入数据库中 db.save_db(self.domain, results, source) end = time.time() self.elapsed = round(end - start, 1) logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}') length = len(self.subdomains) logger.log( 'INFOR', f'{self.source}模块耗时{self.elapsed}秒' f'发现{self.domain}的域名{length}个') logger.log( 'DEBUG', f'{self.source}模块发现{self.domain}的域名:\n' f'{self.subdomains}') # 数据库导出 if self.export: if not self.path: name = f'{self.domain}_brute.{self.format}' self.path = config.result_save_path.joinpath(name) dbexport.export(self.domain, valid=self.valid, path=self.path, format=self.format, show=self.show)
def save_db(self): """ Save module results into the database """ logger.log('DEBUG', f'Saving results to database') lock.acquire() db = Database() db.create_table(self.domain) db.save_db(self.domain, self.results, self.source) db.close() lock.release()
def save_db(self): """ 将模块结果存入数据库中 """ lock.acquire() db = Database() db.create_table(self.domain) db.save_db(self.domain, self.results, self.source) db.close() lock.release()
def save_data(name, data): """ 保存解析结果到数据库 :param str name: 保存表名 :param list data: 待保存的数据 """ db = Database() db.drop_table(name) db.create_table(name) db.save_db(name, data, 'resolve') db.close()
def save_data(name, data): """ Save request results to database :param str name: table name :param list data: data to be saved """ db = Database() db.drop_table(name) db.create_table(name) db.save_db(name, data, 'request') db.close()
def save_db(self): """ 将模块结果存入数据库中 """ logger.log('DEBUG', f'正在将结果存入到数据库') lock.acquire() db = Database() db.create_table(self.domain) db.save_db(self.domain, self.results, self.source) db.close() lock.release()
def save_db(name, data, module): """ Save request results to database :param str name: table name :param list data: data to be saved :param str module: module name """ db = Database() db.drop_table(name) db.create_table(name) db.save_db(name, data, module) db.close()
def save_data(name, data): """ 保存解析结果到数据库 :param str name: 保存表名 :param list data: 待保存的数据 """ logger.log('INFOR', f'Saving resolved results') db = Database() db.drop_table(name) db.create_table(name) db.save_db(name, data, 'resolve') db.close()
def save(name, total, req_data, resp_queue): db = Database() db.create_table(name) i = 0 while True: if not resp_queue.empty(): i += 1 index, resp = resp_queue.get() old_info = req_data[index] new_info = gen_new_info(old_info, resp) db.insert_table(name, new_info) resp_queue.task_done() if i >= total: # You have to save all the request results to end break db.close()
def save(name, total, req_data, resp_queue): db = Database() db.create_table(name) i = 0 while True: if not resp_queue.empty(): i += 1 index, resp = resp_queue.get() old_info = req_data[index] new_info = gen_new_info(old_info, resp) db.insert_table(name, new_info) resp_queue.task_done() if i >= total: # 得存入完所有请求结果才能结束 break db.close()
def main(self): collect = Collect(self.domain, export=False) collect.run() if self.brute: # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常 brute = AIOBrute(self.domain, export=False) brute.run() db = Database() db.copy_table(self.domain, self.domain+'_ori') db.remove_invalid(self.domain) db.deduplicate_subdomain(self.domain) self.datas = db.get_data(self.domain).as_dict() loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) # 解析域名地址 task = resolve.bulk_query_a(self.datas) self.datas = loop.run_until_complete(task) # 保存解析结果 resolve_table = self.domain + '_res' db.drop_table(resolve_table) db.create_table(resolve_table) db.save_db(resolve_table, self.datas, 'resolve') # 请求域名地址 task = request.bulk_get_request(self.datas, self.port) self.datas = loop.run_until_complete(task) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 loop.run_until_complete(asyncio.sleep(0.25)) loop.close() db.clear_table(self.domain) db.save_db(self.domain, self.datas) # 数据库导出 if not self.path: name = f'{self.domain}.{self.format}' self.path = config.result_save_path.joinpath(name) dbexport.export(self.domain, db.conn, self.valid, self.path, self.format, self.output) rename_table = self.domain + '_last' db.drop_table(rename_table) db.rename_table(self.domain, rename_table)
def main(self): if self.brute is None: self.brute = config.enable_brute_module if self.dns is None: self.dns = config.enable_dns_resolve if self.req is None: self.req = config.enable_http_request old_table = self.domain + '_last_result' new_table = self.domain + '_now_result' collect = Collect(self.domain, export=False) collect.run() if self.brute: # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常 brute = AIOBrute(self.domain, export=False) brute.run() db = Database() original_table = self.domain + '_original_result' db.copy_table(self.domain, original_table) db.remove_invalid(self.domain) db.deduplicate_subdomain(self.domain) old_data = [] # 非第一次收集子域的情况时数据库预处理 if db.exist_table(new_table): db.drop_table(old_table) # 如果存在上次收集结果表就先删除 db.rename_table(new_table, old_table) # 新表重命名为旧表 old_data = db.get_data(old_table).as_dict() # 不解析子域直接导出结果 if not self.dns: # 数据库导出 dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(new_table) db.rename_table(self.domain, new_table) db.close() return self.data = db.get_data(self.domain).as_dict() # 标记新发现子域 self.data = utils.mark_subdomain(old_data, self.data) # 获取事件循环 loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) # 解析子域 task = resolve.bulk_resolve(self.data) self.data = loop.run_until_complete(task) # 保存解析结果 resolve_table = self.domain + '_resolve_result' db.drop_table(resolve_table) db.create_table(resolve_table) db.save_db(resolve_table, self.data, 'resolve') # 不请求子域直接导出结果 if not self.req: # 数据库导出 dbexport.export(resolve_table, valid=self.valid, format=self.format, show=self.show) db.drop_table(new_table) db.rename_table(self.domain, new_table) db.close() return # 请求子域 task = request.bulk_request(self.data, self.port) self.data = loop.run_until_complete(task) self.datas.extend(self.data) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 loop.run_until_complete(asyncio.sleep(0.25)) count = utils.count_valid(self.data) logger.log('INFOR', f'经验证{self.domain}有效子域{count}个') # 保存请求结果 db.clear_table(self.domain) db.save_db(self.domain, self.data, 'request') # 数据库导出 dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(new_table) db.rename_table(self.domain, new_table) db.close() # 子域接管检查 if self.takeover: subdomains = set(map(lambda x: x.get('subdomain'), self.data)) takeover = Takeover(subdomains) takeover.run()
def init_table(domain): db = Database() db.drop_table(domain) db.create_table(domain) db.close()
def main(self): if self.brute is None: self.brute = config.enable_brute_module if self.verify is None: self.verify = config.enable_verify_subdomain old_table = self.domain + '_last' new_table = self.domain + '_now' collect = Collect(self.domain, export=False) collect.run() if self.brute: # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常 brute = AIOBrute(self.domain, export=False) brute.run() db = Database() db.copy_table(self.domain, self.domain+'_ori') db.remove_invalid(self.domain) db.deduplicate_subdomain(self.domain) old_data = [] # 非第一次收集子域的情况时数据库预处理 if db.exist_table(new_table): db.drop_table(old_table) # 如果存在上次收集结果表就先删除 db.rename_table(new_table, old_table) # 新表重命名为旧表 old_data = db.get_data(old_table).as_dict() # 不验证子域的情况 if not self.verify: # 数据库导出 self.valid = None dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(new_table) db.rename_table(self.domain, new_table) return # 开始验证子域工作 self.data = db.get_data(self.domain).as_dict() # 标记新发现子域 self.data = utils.mark_subdomain(old_data, self.data) loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) # 解析域名地址 task = resolve.bulk_query_a(self.data) self.data = loop.run_until_complete(task) # 保存解析结果 resolve_table = self.domain + '_res' db.drop_table(resolve_table) db.create_table(resolve_table) db.save_db(resolve_table, self.data, 'resolve') # 请求域名地址 task = request.bulk_get_request(self.data, self.port) self.data = loop.run_until_complete(task) # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 loop.run_until_complete(asyncio.sleep(0.25)) db.clear_table(self.domain) db.save_db(self.domain, self.data) # 数据库导出 dbexport.export(self.domain, valid=self.valid, format=self.format, show=self.show) db.drop_table(new_table) db.rename_table(self.domain, new_table) db.close() # 子域接管检查 if self.takeover: subdomains = set(map(lambda x: x.get('subdomain'), self.data)) takeover = Takeover(subdomains) takeover.run()