def enrich_info(data): ip_asn = IPAsnInfo() ip_reg = IpRegData() for index, info in enumerate(data): ips = get_ips(info) if not ips: continue public = list() cidr = list() asn = list() org = list() addr = list() isp = list() for ip in ips: public.append(str(utils.ip_is_public(ip))) asn_info = ip_asn.find(ip) cidr.append(asn_info.get('cidr')) asn.append(asn_info.get('asn')) org.append(asn_info.get('org')) ip_info = ip_reg.query(ip) addr.append(ip_info.get('addr')) isp.append(ip_info.get('isp')) data[index]['public'] = ','.join(public) data[index]['cidr'] = ','.join(cidr) data[index]['asn'] = ','.join(asn) data[index]['org'] = ','.join(org) data[index]['addr'] = ','.join(addr) data[index]['isp'] = ','.join(isp) return data
def deal_output(output_path): logger.log('INFOR', f'Processing resolved results') records = dict() # 用来记录所有域名解析数据 with open(output_path) as fd: for line in fd: line = line.strip() try: items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) logger.log('ERROR', f'Error resolve line {line}, skip this line') continue record = dict() record['resolver'] = items.get('resolver') qname = items.get('name')[:-1] # 去除最右边的`.`点号 status = items.get('status') if status != 'NOERROR': record['alive'] = 0 record['resolve'] = 0 record['reason'] = status records[qname] = record continue data = items.get('data') if 'answers' not in data: record['alive'] = 0 record['resolve'] = 0 record['reason'] = 'NOANSWER' records[qname] = record continue flag = False cname = list() ips = list() public = list() ttls = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': flag = True cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) ttl = answer.get('ttl') ttls.append(str(ttl)) is_public = utils.ip_is_public(ip) public.append(str(is_public)) record['resolve'] = 1 record['reason'] = status record['cname'] = ','.join(cname) record['content'] = ','.join(ips) record['public'] = ','.join(public) record['ttl'] = ','.join(ttls) records[qname] = record if not flag: record['alive'] = 0 record['resolve'] = 0 record['reason'] = 'NOARECORD' records[qname] = record return records
def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl, bk_cname): qname = items.get('name')[:-1] # 去除最右边的`.`点号 reason = items.get('status') resolver = items.get('resolver') data = items.get('data') answers = data.get('answers') info = dict() cname = list() ips = list() public = list() times = list() ttls = list() is_valid_flags = list() have_a_record = False for answer in answers: if answer.get('type') != 'A': logger.log( 'TRACE', f'The query result of {qname} has no A record\n{answer}') continue logger.log('TRACE', f'The query result of {qname} no A record\n{answer}') have_a_record = True ttl = answer.get('ttl') ttls.append(ttl) cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) public.append(utils.ip_is_public(ip)) num = ip_times.get(ip) times.append(num) isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname, bk_cname) logger.log('TRACE', f'{ip} effective: {isvalid} reason: {reason}') is_valid_flags.append(isvalid) if not have_a_record: logger.log('TRACE', f'All query result of {qname} no A record{answers}') # 为了优化内存 只添加有A记录且通过判断的子域到记录中 if have_a_record and all(is_valid_flags): info['resolve'] = 1 info['reason'] = reason info['ttl'] = ttls info['cname'] = cname info['ip'] = ips info['public'] = public info['times'] = times info['resolver'] = resolver infos[qname] = info subdomains.append(qname) return infos, subdomains
def deal_result(result_list): logger.log('INFOR', f'正在处理解析结果') records = dict() # 用来记录域名解析数据 times = dict() # 用来统计IP出现次数 for items in result_list: record = dict() qname = items.get('name')[:-1] # 去出最右边的`.`点号 record['resolver'] = items.get('resolver') status = items.get('status') record['reason'] = status records[qname] = record if status != 'NOERROR': record['reason'] = status record['resolve'] = 0 record['alive'] = 0 records[qname] = record continue data = items.get('data') if 'answers' not in data: record['reason'] = 'NOANSWER' record['resolve'] = 0 record['alive'] = 0 records[qname] = record continue answers = data.get('answers') flag = False cname = list() ips = list() public = list() ttl = list() for answer in answers: if answer.get('type') == 'A': flag = True ttl.append(answer.get('ttl')) cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号 ip = answer.get('data') ips.append(ip) public.append(utils.ip_is_public(ip)) record['ttl'] = ttl record['cname'] = cname record['content'] = ips record['public'] = public records[qname] = record # 取值 如果是首次出现的IP集合 出现次数先赋值0 value = times.setdefault(ip, 0) times[ip] = value + 1 if not flag: record['reason'] = 'NOA' record['resolve'] = 0 record['alive'] = 0 records[qname] = record return records, times
def gen_infos(data, qname, info, infos): flag = False cname = list() ips = list() public = list() ttls = list() cidrs = list() asns = list() orgs = list() locs = list() regs = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': flag = True cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) ttl = answer.get('ttl') ttls.append(str(ttl)) is_public = utils.ip_is_public(ip) public.append(str(is_public)) asn_info = ip_asn.find(ip) cidrs.append(asn_info.get('cidr')) asns.append(asn_info.get('asn')) orgs.append(asn_info.get('org')) loc = f'{ip_geo.get_country_long(ip)} ' \ f'{ip_geo.get_region(ip)} ' \ f'{ip_geo.get_city(ip)}' locs.append(loc) reg = ip_reg.memory_search(ip).get('region').decode('utf-8') regs.append(reg) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cname) info['content'] = ','.join(ips) info['public'] = ','.join(public) info['ttl'] = ','.join(ttls) info['cidr'] = ','.join(cidrs) info['asn'] = ','.join(asns) info['org'] = ','.join(orgs) info['ip2location'] = ','.join(locs) info['ip2region'] = ','.join(regs) infos[qname] = info if not flag: info['alive'] = 0 info['resolve'] = 0 info['reason'] = 'NoARecord' infos[qname] = info return infos
def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl): qname = items.get('name')[:-1] # 去出最右边的`.`点号 reason = items.get('status') resolver = items.get('resolver') data = items.get('data') answers = data.get('answers') record = dict() cname = list() ips = list() public = list() times = list() ttls = list() have_a_record = False is_valid_flag = True for answer in answers: if answer.get('type') != 'A': logger.log('TRACE', f'查询{qname}返回的应答没有A记录\n{answer}') continue logger.log('TRACE', f'查询{qname}返回的应答具有A记录\n{answer}') have_a_record = True ttl = answer.get('ttl') ttls.append(ttl) cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号 ip = answer.get('data') ips.append(ip) public.append(utils.ip_is_public(ip)) num = ip_times.get(ip) times.append(num) isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl) logger.log('TRACE', f'{ip}是否有效:{isvalid} 原因:{reason}') if isvalid == 0: is_valid_flag = False # 只要有一条A记录判断通不过就认为改子域为无效子域 break if not have_a_record: logger.log('TRACE', f'查询{qname}返回的所有应答都中没有A记录{answers}') # 为了优化内存 判断通不过的子域暂时不添加到记录里 if is_valid_flag: record['resolve'] = 1 record['reason'] = reason record['ttl'] = ttls record['cname'] = cname record['content'] = ips record['public'] = public record['times'] = times record['resolver'] = resolver records[qname] = record subdomains.append(qname) return records, subdomains
def gen_infos(data, qname, info, infos): flag = False cname = list() ips = list() public = list() ttl = list() cidr = list() asn = list() org = list() addr = list() isp = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': flag = True cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) ttl.append(str(answer.get('ttl'))) public.append(str(utils.ip_is_public(ip))) asn_info = ip_asn.find(ip) cidr.append(asn_info.get('cidr')) asn.append(asn_info.get('asn')) org.append(asn_info.get('org')) ip_info = ip_reg.query(ip) addr.append(ip_info.get('addr')) isp.append(ip_info.get('isp')) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cname) info['content'] = ','.join(ips) info['public'] = ','.join(public) info['ttl'] = ','.join(ttl) info['cidr'] = ','.join(cidr) info['asn'] = ','.join(asn) info['org'] = ','.join(org) info['addr'] = ','.join(addr) info['isp'] = ','.join(isp) infos[qname] = info if not flag: info['alive'] = 0 info['resolve'] = 0 info['reason'] = 'NoARecord' infos[qname] = info return infos
def deal_output(output_path): logger.log('INFOR', f'Processing resolved results') records = dict() # 用来记录所有域名解析数据 ip_asn = IPAsnInfo() ip_geo = IpGeoInfo db_path = setting.data_storage_dir.joinpath('ip2region.db') ip_reg = IpRegInfo(db_path) with open(output_path) as fd: for line in fd: line = line.strip() try: items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) logger.log('ERROR', f'Error resolve line {line}, skip this line') continue record = dict() record['resolver'] = items.get('resolver') qname = items.get('name')[:-1] # 去除最右边的`.`点号 status = items.get('status') if status != 'NOERROR': record['alive'] = 0 record['resolve'] = 0 record['reason'] = status records[qname] = record continue data = items.get('data') if 'answers' not in data: record['alive'] = 0 record['resolve'] = 0 record['reason'] = 'NOANSWER' records[qname] = record continue flag = False cname = list() ips = list() public = list() ttls = list() cidrs = list() asns = list() locs = list() regs = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': flag = True cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) ttl = answer.get('ttl') ttls.append(str(ttl)) is_public = utils.ip_is_public(ip) public.append(str(is_public)) asn_info = ip_asn.find(ip) cidrs.append(asn_info.get('cidr')) asns.append(asn_info.get('asn')) loc = f'{ip_geo.get_country_long(ip)} ' \ f'{ip_geo.get_region(ip)} ' \ f'{ip_geo.get_city(ip)}' locs.append(loc) reg = ip_reg.memory_search(ip).get('region').decode( 'utf-8') regs.append(reg) record['resolve'] = 1 record['reason'] = status record['cname'] = ','.join(cname) record['content'] = ','.join(ips) record['public'] = ','.join(public) record['ttl'] = ','.join(ttls) record['cidr'] = ','.join(cidrs) record['asn'] = ','.join(asns) record['ip2location'] = ','.join(locs) record['ip2region'] = ','.join(regs) records[qname] = record if not flag: record['alive'] = 0 record['resolve'] = 0 record['reason'] = 'NOARECORD' records[qname] = record return records