def enrich_info(data):
ip_asn = IPAsnInfo()
ip_reg = IpRegData()
for index, info in enumerate(data):
ips = get_ips(info)
if not ips:
continue
public = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
for ip in ips:
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
data[index]['public'] = ','.join(public)
data[index]['cidr'] = ','.join(cidr)
data[index]['asn'] = ','.join(asn)
data[index]['org'] = ','.join(org)
data[index]['addr'] = ','.join(addr)
data[index]['isp'] = ','.join(isp)
return data
def deal_output(output_path):
logger.log('INFOR', f'Processing resolved results')
records = dict() # 用来记录所有域名解析数据
with open(output_path) as fd:
for line in fd:
line = line.strip()
try:
items = json.loads(line)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR',
f'Error resolve line {line}, skip this line')
continue
record = dict()
record['resolver'] = items.get('resolver')
qname = items.get('name')[:-1] # 去除最右边的`.`点号
status = items.get('status')
if status != 'NOERROR':
record['alive'] = 0
record['resolve'] = 0
record['reason'] = status
records[qname] = record
continue
data = items.get('data')
if 'answers' not in data:
record['alive'] = 0
record['resolve'] = 0
record['reason'] = 'NOANSWER'
records[qname] = record
continue
flag = False
cname = list()
ips = list()
public = list()
ttls = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
flag = True
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
ttl = answer.get('ttl')
ttls.append(str(ttl))
is_public = utils.ip_is_public(ip)
public.append(str(is_public))
record['resolve'] = 1
record['reason'] = status
record['cname'] = ','.join(cname)
record['content'] = ','.join(ips)
record['public'] = ','.join(public)
record['ttl'] = ','.join(ttls)
records[qname] = record
if not flag:
record['alive'] = 0
record['resolve'] = 0
record['reason'] = 'NOARECORD'
records[qname] = record
return records
def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl,
bk_cname):
qname = items.get('name')[:-1] # 去除最右边的`.`点号
reason = items.get('status')
resolver = items.get('resolver')
data = items.get('data')
answers = data.get('answers')
info = dict()
cname = list()
ips = list()
public = list()
times = list()
ttls = list()
is_valid_flags = list()
have_a_record = False
for answer in answers:
if answer.get('type') != 'A':
logger.log(
'TRACE',
f'The query result of {qname} has no A record\n{answer}')
continue
logger.log('TRACE',
f'The query result of {qname} no A record\n{answer}')
have_a_record = True
ttl = answer.get('ttl')
ttls.append(ttl)
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
public.append(utils.ip_is_public(ip))
num = ip_times.get(ip)
times.append(num)
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl,
cname, bk_cname)
logger.log('TRACE', f'{ip} effective: {isvalid} reason: {reason}')
is_valid_flags.append(isvalid)
if not have_a_record:
logger.log('TRACE',
f'All query result of {qname} no A record{answers}')
# 为了优化内存 只添加有A记录且通过判断的子域到记录中
if have_a_record and all(is_valid_flags):
info['resolve'] = 1
info['reason'] = reason
info['ttl'] = ttls
info['cname'] = cname
info['ip'] = ips
info['public'] = public
info['times'] = times
info['resolver'] = resolver
infos[qname] = info
subdomains.append(qname)
return infos, subdomains
def deal_result(result_list):
logger.log('INFOR', f'正在处理解析结果')
records = dict() # 用来记录域名解析数据
times = dict() # 用来统计IP出现次数
for items in result_list:
record = dict()
qname = items.get('name')[:-1] # 去出最右边的`.`点号
record['resolver'] = items.get('resolver')
status = items.get('status')
record['reason'] = status
records[qname] = record
if status != 'NOERROR':
record['reason'] = status
record['resolve'] = 0
record['alive'] = 0
records[qname] = record
continue
data = items.get('data')
if 'answers' not in data:
record['reason'] = 'NOANSWER'
record['resolve'] = 0
record['alive'] = 0
records[qname] = record
continue
answers = data.get('answers')
flag = False
cname = list()
ips = list()
public = list()
ttl = list()
for answer in answers:
if answer.get('type') == 'A':
flag = True
ttl.append(answer.get('ttl'))
cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
public.append(utils.ip_is_public(ip))
record['ttl'] = ttl
record['cname'] = cname
record['content'] = ips
record['public'] = public
records[qname] = record
# 取值 如果是首次出现的IP集合 出现次数先赋值0
value = times.setdefault(ip, 0)
times[ip] = value + 1
if not flag:
record['reason'] = 'NOA'
record['resolve'] = 0
record['alive'] = 0
records[qname] = record
return records, times
def gen_infos(data, qname, info, infos):
flag = False
cname = list()
ips = list()
public = list()
ttls = list()
cidrs = list()
asns = list()
orgs = list()
locs = list()
regs = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
flag = True
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
ttl = answer.get('ttl')
ttls.append(str(ttl))
is_public = utils.ip_is_public(ip)
public.append(str(is_public))
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
orgs.append(asn_info.get('org'))
loc = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
locs.append(loc)
reg = ip_reg.memory_search(ip).get('region').decode('utf-8')
regs.append(reg)
info['resolve'] = 1
info['reason'] = 'OK'
info['cname'] = ','.join(cname)
info['content'] = ','.join(ips)
info['public'] = ','.join(public)
info['ttl'] = ','.join(ttls)
info['cidr'] = ','.join(cidrs)
info['asn'] = ','.join(asns)
info['org'] = ','.join(orgs)
info['ip2location'] = ','.join(locs)
info['ip2region'] = ','.join(regs)
infos[qname] = info
if not flag:
info['alive'] = 0
info['resolve'] = 0
info['reason'] = 'NoARecord'
infos[qname] = info
return infos
def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl):
qname = items.get('name')[:-1] # 去出最右边的`.`点号
reason = items.get('status')
resolver = items.get('resolver')
data = items.get('data')
answers = data.get('answers')
record = dict()
cname = list()
ips = list()
public = list()
times = list()
ttls = list()
have_a_record = False
is_valid_flag = True
for answer in answers:
if answer.get('type') != 'A':
logger.log('TRACE', f'查询{qname}返回的应答没有A记录\n{answer}')
continue
logger.log('TRACE', f'查询{qname}返回的应答具有A记录\n{answer}')
have_a_record = True
ttl = answer.get('ttl')
ttls.append(ttl)
cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
public.append(utils.ip_is_public(ip))
num = ip_times.get(ip)
times.append(num)
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl)
logger.log('TRACE', f'{ip}是否有效:{isvalid} 原因:{reason}')
if isvalid == 0:
is_valid_flag = False # 只要有一条A记录判断通不过就认为改子域为无效子域
break
if not have_a_record:
logger.log('TRACE', f'查询{qname}返回的所有应答都中没有A记录{answers}')
# 为了优化内存 判断通不过的子域暂时不添加到记录里
if is_valid_flag:
record['resolve'] = 1
record['reason'] = reason
record['ttl'] = ttls
record['cname'] = cname
record['content'] = ips
record['public'] = public
record['times'] = times
record['resolver'] = resolver
records[qname] = record
subdomains.append(qname)
return records, subdomains
def gen_infos(data, qname, info, infos):
flag = False
cname = list()
ips = list()
public = list()
ttl = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
flag = True
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
ttl.append(str(answer.get('ttl')))
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
info['resolve'] = 1
info['reason'] = 'OK'
info['cname'] = ','.join(cname)
info['content'] = ','.join(ips)
info['public'] = ','.join(public)
info['ttl'] = ','.join(ttl)
info['cidr'] = ','.join(cidr)
info['asn'] = ','.join(asn)
info['org'] = ','.join(org)
info['addr'] = ','.join(addr)
info['isp'] = ','.join(isp)
infos[qname] = info
if not flag:
info['alive'] = 0
info['resolve'] = 0
info['reason'] = 'NoARecord'
infos[qname] = info
return infos
def deal_output(output_path):
logger.log('INFOR', f'Processing resolved results')
records = dict() # 用来记录所有域名解析数据
ip_asn = IPAsnInfo()
ip_geo = IpGeoInfo
db_path = setting.data_storage_dir.joinpath('ip2region.db')
ip_reg = IpRegInfo(db_path)
with open(output_path) as fd:
for line in fd:
line = line.strip()
try:
items = json.loads(line)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR',
f'Error resolve line {line}, skip this line')
continue
record = dict()
record['resolver'] = items.get('resolver')
qname = items.get('name')[:-1] # 去除最右边的`.`点号
status = items.get('status')
if status != 'NOERROR':
record['alive'] = 0
record['resolve'] = 0
record['reason'] = status
records[qname] = record
continue
data = items.get('data')
if 'answers' not in data:
record['alive'] = 0
record['resolve'] = 0
record['reason'] = 'NOANSWER'
records[qname] = record
continue
flag = False
cname = list()
ips = list()
public = list()
ttls = list()
cidrs = list()
asns = list()
locs = list()
regs = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
flag = True
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
ttl = answer.get('ttl')
ttls.append(str(ttl))
is_public = utils.ip_is_public(ip)
public.append(str(is_public))
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
loc = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
locs.append(loc)
reg = ip_reg.memory_search(ip).get('region').decode(
'utf-8')
regs.append(reg)
record['resolve'] = 1
record['reason'] = status
record['cname'] = ','.join(cname)
record['content'] = ','.join(ips)
record['public'] = ','.join(public)
record['ttl'] = ','.join(ttls)
record['cidr'] = ','.join(cidrs)
record['asn'] = ','.join(asns)
record['ip2location'] = ','.join(locs)
record['ip2region'] = ','.join(regs)
records[qname] = record
if not flag:
record['alive'] = 0
record['resolve'] = 0
record['reason'] = 'NOARECORD'
records[qname] = record
return records
