def export(table, db=None, alive=False, limit=None, path=None, format='csv', show=False): """ OneForAll数据库导出模块 Example: python3 dbexport.py --table name --format csv --dir= ./result.csv python3 dbexport.py --db result.db --table name --show False Note: 参数alive可选值True,False分别表示导出存活,全部子域结果 参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods' 参数path默认None使用OneForAll结果目录自动生成路径 :param str table: 要导出的表 :param str db: 要导出的数据库路径(默认为results/result.sqlite3) :param bool alive: 只导出存活的子域结果(默认False) :param str limit: 导出限制条件(默认None) :param str format: 导出文件格式(默认csv) :param str path: 导出文件路径(默认None) :param bool show: 终端显示导出数据(默认False) """ database = Database(db) rows = database.export_data(table, alive, limit) format = utils.check_format(format, len(rows)) path = utils.check_path(path, table, format) if show: print(rows.dataset) data = rows.export(format) database.close() utils.save_data(path, data) logger.log('INFOR', f'{table}主域的子域结果 {path}') data_dict = rows.as_dict() return data_dict
def search(self, full_search=False): """ 向接口查询子域并做子域匹配 """ page_num = 1 while True: time.sleep(self.delay) params = {'pageno': page_num, 'q': self.domain, 'type': 'code'} try: resp = self.get(self.addr, params=params) except Exception as e: logger.log('ERROR', e.args) break if not resp: break if resp.status_code != 200: logger.log('ERROR', f'{self.source}模块搜索出错') break if 'class="empty-box"' in resp.text: break soup = BeautifulSoup(resp.text, 'html.parser') subdomains = self.match(self.domain, soup.text) self.subdomains = self.subdomains.union(subdomains) if not subdomains: break if not full_search: # 搜索中发现搜索出的结果有完全重复的结果就停止搜索 if subdomains.issubset(self.subdomains): break if '<li class="disabled"><a href="###">' in resp.text: break if page_num > 100: break page_num += 1
def gen_req_data(data, ports): logger.log('INFOR', f'正在生成请求地址') new_data = [] for data in data: resolve = data.get('resolve') # 解析失败(0)的子域不进行http请求探测 if resolve == 0: continue subdomain = data.get('subdomain') for port in ports: if str(port).endswith('443'): url = f'https://{subdomain}:{port}' if port == 443: url = f'https://{subdomain}' data['id'] = None data['url'] = url data['port'] = port new_data.append(data) data = dict(data) # 需要生成一个新的字典对象 else: url = f'http://{subdomain}:{port}' if port == 80: url = f'http://{subdomain}' data['id'] = None data['url'] = url data['port'] = port new_data.append(data) data = dict(data) # 需要生成一个新的字典对象 return new_data
def query(self): """ 向接口查询子域并做子域匹配 """ page = 0 while True: self.header = self.get_header() self.proxy = self.get_proxy(self.source) params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page} resp = self.get(self.addr, params) if not resp: return if resp.status_code != 200: break # 请求不正常通常网络是有问题,不再继续请求下去 try: json = resp.json() except Exception as e: logger.log('DEBUG', e.args) break subdomains = self.match(self.domain, str(json)) if not subdomains: break # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名 subdomains = json.get('Subdomains') if subdomains: # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页 if len(subdomains) < 300: break page += 1 if page >= 50: # ipv4info子域查询接口最多允许查询50页 break
def axfr(self, server): """ 执行域传送 :param server: 域名服务器 """ logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送') try: xfr = dns.query.xfr(where=server, zone=self.domain, timeout=5.0, lifetime=10.0) zone = dns.zone.from_xfr(xfr) except Exception as e: logger.log('DEBUG', e.args) logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败') return names = zone.nodes.keys() for name in names: full_domain = str(name) + '.' + self.domain subdomain = utils.match_subdomain(self.domain, full_domain) self.subdomains = self.subdomains.union(subdomain) record = zone[name].to_text(name) self.results.append(record) if self.results: logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录') logger.log('DEBUG', '\n'.join(self.results)) self.results = []
def save(self): logger.log('DEBUG', '正在保存检查结果') if self.format == 'txt': data = str(self.results) else: data = self.results.export(self.format) utils.save_data(self.path, data)
def request(self, url): try: r = requests.get(url, timeout=config.timeout, headers=self.get_headers(), verify=config.verify_ssl, allow_redirects=config.allow_redirects) text = r.content.decode( encoding=chardet.detect(r.content)['encoding']) title = self.get_title(text).strip().replace('\r', '').replace('\n', '') banner = self.get_banner(r.headers) size = self.sizeHuman(len(r.text)) status = r.status_code subdomain = (url.split('//')[-1]).split(':')[0] if status in config.ignore_status_code: return result = { 'url': r.url, 'title': title, 'status': status, 'size': size, 'fingerprint': banner } self.data.append(result) logger.log('INFOR', f'AliveWeb:[{url}]') return r, text except Exception as e: # print(e) return e
def save_json(self): """ 将各模块结果保存为json文件 :return: 是否保存成功 """ if not config.save_module_result: return False logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件') path = config.result_save_dir.joinpath(self.domain, self.module) path.mkdir(parents=True, exist_ok=True) name = self.source + '.json' path = path.joinpath(name) with open(path, mode='w', encoding='utf-8', errors='ignore') as file: result = { 'domain': self.domain, 'name': self.module, 'source': self.source, 'elapse': self.elapse, 'find': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records } json.dump(result, file, ensure_ascii=False, indent=4) return True
def query(self): """ 向接口查询子域并做子域匹配 """ base_addr = 'http://114.55.181.28/check_web/' \ 'databaseInfo_mainSearch.action' page_num = 1 while True: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) params = {'isSearch': 'true', 'searchType': 'url', 'term': self.domain, 'pageNo': page_num} try: resp = self.get(base_addr, params) except Exception as e: logger.log('ERROR', e.args) break if not resp: break subdomains = self.match(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) if not subdomains: break if page_num > 10: break page_num += 1
def export_all(format, path, datas): """ 将所有结果数据导出到一个文件 :param str format: 导出文件格式 :param str path: 导出文件路径 :param list datas: 待导出的结果数据 """ format = check_format(format, len(datas)) timestamp = get_timestring() name = f'all_subdomain_result_{timestamp}' path = check_path(path, name, format) logger.log('INFOR', f'所有主域的子域结果 {path}') row_list = list() for row in datas: if 'header' in row: row.pop('header') if 'response' in row: row.pop('response') keys = row.keys() values = row.values() if format in {'xls', 'xlsx'}: values = check_value(values) row_list.append(Record(keys, values)) rows = RecordCollection(iter(row_list)) content = rows.export(format) save_data(path, content)
async def bulk_request(data, port): ports = get_ports(port) no_req_data = utils.get_filtered_data(data) to_req_data = gen_req_data(data, ports) method = config.request_method logger.log('INFOR', f'请求使用{method}方法') logger.log('INFOR', f'正在进行异步子域请求') connector = get_connector() header = get_header() async with ClientSession(connector=connector, headers=header) as session: tasks = [] for i, data in enumerate(to_req_data): url = data.get('url') task = asyncio.ensure_future(fetch(session, url)) task.add_done_callback( functools.partial(request_callback, index=i, datas=to_req_data)) tasks.append(task) # 任务列表里有任务不空时才进行解析 if tasks: # 等待所有task完成 错误聚合到结果列表里 futures = asyncio.as_completed(tasks) for future in tqdm.tqdm(futures, total=len(tasks), desc='Request Progress', ncols=80): await future return to_req_data + no_req_data
def query(self): """ 向接口查询子域并做子域匹配 """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) data = { 'query': f'parsed.names: {self.domain}', 'page': 1, 'fields': ['parsed.subject_dn', 'parsed.names'], 'flatten': True} resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return json = resp.json() status = json.get('status') if status != 'ok': logger.log('ALERT', status) return subdomains = self.match(self.domain, str(json)) self.subdomains = self.subdomains.union(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): data['page'] = page resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return subdomains = self.match(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains)
def check_path(path, name, format): """ 检查结果输出目录路径 :param path: 保存路径 :param name: 导出名字 :param format: 保存格式 :return: 保存路径 """ filename = f'{name}.{format}' default_path = config.result_save_dir.joinpath(filename) if isinstance(path, str): path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠 path = path.replace('\'', '') # 去除多余的转义 else: path = default_path path = Path(path) if not path.suffix: # 输入是目录的情况 path = path.joinpath(filename) parent_dir = path.parent if not parent_dir.exists(): logger.log('ALERT', f'不存在{parent_dir}目录将会新建') parent_dir.mkdir(parents=True, exist_ok=True) if path.exists(): logger.log('ALERT', f'存在{path}文件将会覆盖') return path
def query(self, sql): try: results = self.conn.query(sql) except Exception as e: logger.log('ERROR', e.args) else: return results
def save_db(self, table_name, results, module_name=None): """ 将各模块结果存入数据库 :param str table_name: 表名 :param list results: 结果列表 :param str module_name: 模块名 """ logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域' '结果存入数据库') table_name = table_name.replace('.', '_') if results: try: self.conn.bulk_query( f'insert into "{table_name}" (' f'id, type, alive, resolve, request, new, url, subdomain,' f'port, level, cname, content, public, status, reason,' f'title, banner, header, response, times, ttl, resolver,' f'module, source, elapse, find, brute, valid) ' f'values (:id, :type, :alive, :resolve, :request, :new,' f':url, :subdomain, :port, :level, :cname, :content,' f':public, :status, :reason, :title, :banner, :header,' f':response, :times, :ttl, :resolver, :module, :source,' f':elapse, :find, :brute, :valid)', results) except Exception as e: logger.log('ERROR', e)
def deal_output(output_path, ip_times, wildcard_ips, wildcard_ttl): logger.log('INFOR', f'正在处理解析结果') records = dict() # 用来记录所有域名解析数据 subdomains = list() # 用来保存所有通过有效性检查的子域 with open(output_path) as fd: for line in fd: line = line.strip() try: items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) logger.log('ERROR', f'解析行{line}出错跳过解析该行') continue qname = items.get('name')[:-1] # 去出最右边的`.`点号 status = items.get('status') if status != 'NOERROR': logger.log('TRACE', f'处理{line}时发现{qname}查询结果状态{status}') continue data = items.get('data') if 'answers' not in data: logger.log('TRACE', f'处理{line}时发现{qname}返回的结果无应答') continue records, subdomains = gen_records(items, records, subdomains, ip_times, wildcard_ips, wildcard_ttl) return records, subdomains
def get(self, url, params=None, check=True, **kwargs): """ 自定义get请求 :param str url: 请求地址 :param dict params: 请求参数 :param bool check: 检查响应 :param kwargs: 其他参数 :return: requests响应对象 """ try: resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header, proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs) except Exception as e: logger.log('ERROR', e.args) return None if not check: return resp if utils.check_response('GET', resp): return resp return None
def request_callback(future, index, datas): result = future.result() if isinstance(result, BaseException): logger.log('TRACE', result.args) name = utils.get_classname(result) datas[index]['reason'] = name + ' ' + str(result) datas[index]['request'] = 0 datas[index]['alive'] = 0 elif isinstance(result, tuple): resp, text = result datas[index]['reason'] = resp.reason datas[index]['status'] = resp.status if resp.status == 400 or resp.status >= 500: datas[index]['request'] = 0 datas[index]['alive'] = 0 else: datas[index]['request'] = 1 datas[index]['alive'] = 1 headers = resp.headers datas[index]['banner'] = utils.get_sample_banner(headers) datas[index]['header'] = str(dict(headers))[1:-1] if isinstance(text, str): title = get_title(text).strip() datas[index]['title'] = utils.remove_invalid_string(title) datas[index]['response'] = utils.remove_invalid_string(text)
def init_shodan(self): api = shodan.Shodan(config.shodan_api) try: api.info() return api except Exception as e: logger.log('ALERT', f'shodan api异常:{e}') return None
def drop_table(self, table_name): """ 删除表 :param str table_name: 表名 """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'正在删除{table_name}表') self.query(f'drop table if exists "{table_name}"')
def get_data(self, table_name): """ 获取表中的所有数据 :param str table_name: 表名 """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'获取{table_name}表中的所有数据') return self.query(f'select * from "{table_name}"')
def clear_table(self, table_name): """ 清空表中数据 :param str table_name: 表名 """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'正在清空{table_name}表中的数据') self.query(f'delete from "{table_name}"')
def init_fofa(self): try: url = f'https://fofa.so/api/v1/info/my?email={config.fofa_email}&key={config.fofa_key}' r = requests.get(url) r.json() return f'https://fofa.so/api/v1/search/all?email={config.fofa_email}&key={config.fofa_key}' except Exception as e: logger.log('ALERT', f'fofa连接异常:{e}') return None
def get_cname(subdomain): resolver = utils.dns_resolver() try: answers = resolver.query(subdomain, 'CNAME') except Exception as e: logger.log('TRACE', e.args) return None for answer in answers: return answer.to_text() # 一个子域只有一个CNAME记录
def remove_invalid(self, table_name): """ 去除表中的空值或无效子域 :param str table_name: 表名 """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'正在去除{table_name}表中的无效子域') self.query(f'delete from "{table_name}" where ' f'subdomain is null or resolve == 0')
def run(self): logger.log('INFOR', f'WebAlive探测进程启动:WebAliveScan') domain_list = [self.subdomain] url_list = self.gen_url_list(domain_list) gevent_pool = pool.Pool(config.threads) while url_list: for task in [ gevent_pool.spawn(self.request, url_list.pop()) for i in range(len(url_list[:config.threads])) ]: task.join()
def deduplicate_subdomain(self, table_name): """ 去重表中的子域 :param str table_name: 表名 """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'正在去重{table_name}表中的子域') self.query(f'delete from "{table_name}" where ' f'id not in (select min(id) ' f'from "{table_name}" group by subdomain)')
def compare(self, subdomain, cname, responses): domain_resp = self.get('http://' + subdomain, check=False) cname_resp = self.get('http://' + cname, check=False) if domain_resp is None or cname_resp is None: return for resp in responses: if resp in domain_resp.text and resp in cname_resp.text: logger.log('ALERT', f'{subdomain}存在子域接管风险') self.results.append([subdomain, cname]) break
def check(self, *apis): """ 简单检查是否配置了api信息 :param apis: api信息元组 :return: 检查结果 """ if not all(apis): logger.log('ALERT', f'{self.source}模块没有配置API跳过执行') return False return True
def save_db(self): """ 将模块结果存入数据库中 """ logger.log('DEBUG', f'正在将结果存入到数据库') lock.acquire() db = Database() db.create_table(self.domain) db.save_db(self.domain, self.results, self.source) db.close() lock.release()