if not len(new_subdomains): self.finish() # 未发现新的子域就直接返回 return data self.subdomains = new_subdomains self.finish() self.gen_result() temp_data = resolve.run_resolve(domain, self.results) fina_data = request.run_request(domain, temp_data, port) data = data + fina_data logger.log('INFOR', f'Saving finder results') utils.save_db(domain, data, 'finder') return data file_path = settings.data_storage_dir.joinpath('common_js_library.json') black_name = utils.load_json(file_path) # Regular expression comes from https://github.com/GerbenJavado/LinkFinder expression = r""" (?:"|') # Start newline delimiter ( ((?:[a-zA-Z]{1,10}://|//) # Match a scheme [a-Z]*1-10 or // [^"'/]{1,}\. # Match a domain name (any character + dot) [a-zA-Z]{2,}[^"']{0,}) # The domain extension and/or path | ((?:/|\.\./|\./) # Start with /,../,./ [^"'><,;| *()(%%$^/\\\[\]] # Next character can't be... [^"'><,;|()]{1,}) # Rest of the characters can't be | ([a-zA-Z0-9_\-/]{1,}/ # Relative endpoint with / [a-zA-Z0-9_\-/]{1,} # Resource name \.(?:[a-zA-Z]{1,4}|action) # Rest + extension (length 1-4 or action)
import json import ipaddress from config import settings from common import utils from common.database import Database from config.log import logger data_dir = settings.data_storage_dir # from https://github.com/al0ne/Vxscan/blob/master/lib/iscdn.py cdn_ip_cidr = utils.load_json(data_dir.joinpath('cdn_ip_cidr.json')) cdn_asn_list = utils.load_json(data_dir.joinpath('cdn_asn_list.json')) # from https://github.com/Qclover/CDNCheck/blob/master/checkCDN/cdn3_check.py cdn_cname_keyword = utils.load_json(data_dir.joinpath('cdn_cname_keywords.json')) cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json')) def check_cname_keyword(cname): if not cname: return False names = cname.lower().split(',') for name in names: for keyword in cdn_cname_keyword.keys(): if keyword in name: return True def check_header_key(header):