def get_ns_history_record(self):
try:
db = MySQL(SOURCE_CONFIG)
except:
pass
full_list = []
sql = 'SELECT domain,ns,ns_ip FROM domain_ns WHERE ns!=""'
try:
db.query(sql)
data = db.fetch_all_rows()
except Exception as e:
return "read tld Error" + str(e)
for i in data:
tld = str(i['domain'])
ns = str(i['ns'])
ns_list = ns.split(';')
ns = ','.join(ns_list)
ns_ip = str(i['ns_ip'])
flag = '0'
full_list.append((tld,ns,ns_ip,flag))
taskid = str(random.randint(10000000, 99999999))
self.write_tld(taskid)
new_file_name = 'full_' + taskid
with open("domain_dns_data/sec_compared/"+new_file_name, "w") as f:
for t in full_list:
f.write(t[0]+'\t'+'NS'+'\t'+t[1]+'\t'+t[3]+'\n')
ns_list = t[1].split(',')
ip_list = t[2].split(';')
for i in range(0,len(ns_list)):
f.write(ns_list[i]+'\t'+'A'+'\t'+ip_list[i]+'\t'+'0'+'\n')
self.produce(new_file_name)
self.produce('tld_'+taskid)
def main():
result = []
tld_result1 = extract_ns_ip_by_root() # 通过询问根域名服务器获得数据, todo 不能有这种命名
tld_result2 = get_tld_ns_by_zone() # 通过下载zone文件获得数据
# 取两个结果的并集
for i in tld_result1:
for t in tld_result2:
if i[0] == t[0]:
ns_list1 = i[1].split(';') # 将字符串变为列表然后取并集
ns_list = list(set(ns_list1) | set(t[1]))
ns_str = ';'.join(ns_list) # 列表变为字符串
ipv4_list1 = i[2].split(';') # 将字符串变为列表然后取并集
ipv4_list = list(set(ipv4_list1) | set(t[2]))
ipv4_str = ';'.join(ipv4_list) # 列表变为字符串
ipv6_list1 = i[3].split(';') # 将字符串变为列表然后取并集
ipv6_list = list(set(ipv6_list1) | set(t[3]))
ipv6_str = ';'.join(ipv6_list) # 列表变为字符串
result.append((i[0], ns_str, ipv4_str, ipv6_str))
# for i in result:
# print i
try:
db = MySQL(SOURCE_CONFIG)
except:
logger.logger.error("数据库异常：获取域名失败")
return
insert_tld_ns_db(db, tld_result1)
root_ns,root_ipv4,root_ipv6 = get_root()
insert_root_ns_db(db, root_ns, root_ipv4, root_ipv6)
db.close()
def fetch_tld_ns():
"""
获取顶级域名的权威服务器（ns）IP地址
"""
tld_ns = defaultdict(set)
try:
db = MySQL(SOURCE_CONFIG)
sql = 'SELECT tld,server_ipv4 from tld_ns_zone'
db.query(sql)
tld_ns_query = db.fetch_all_rows() # 获取已存储的顶级域名的权威服务器信息
except Exception, e:
logger.logger.error("获取顶级域名异常：", e)
return tld_ns
def insert_domains_db(domains):
"""将域名插入到数据库中"""
try:
db = MySQL(SOURCE_CONFIG)
except Exception,e:
logger.logger.error(e)
return False
def save_sec_domains(domains):
"""将sec的域名存入到数据库中"""
try:
db = MySQL(SOURCE_CONFIG)
except Exception, e:
logger.logger.error(e)
return False
def start(self):
try:
db = MySQL(SOURCE_CONFIG)
except:
logger.logger.error("数据库异常：获取域名失败")
return
tld_list = self.read_tld(db)
mysql_data = self.read_domain_ns(db)
file_list = self.get_file_name(unres_address, res_address)
for i in file_list:
update_list = []
res_list = []
file_address = unres_address + '/' + i
file_data = self.read_file(file_address)
for k, v in file_data.items():
if k not in mysql_data.keys():
update_list.append((k, v[0], v[1]))
res_list.append((k, v[0], v[1]))
else:
file_flag = int(v[1])
mysql = mysql_data[k]
ns = mysql[0]
flag = mysql[1]
mysql_flag = int(mysql[1])
if file_flag > mysql_flag:
print k
update_list.append((k, v[0], v[1]))
res_list.append((k, v[0], v[1]))
else:
res_list.append((k, ns, flag))
self.update_mysql(db, update_list)
update_file_address = res_address + '/' + i + '_update'
with open(update_file_address, 'w') as fw:
for s in update_list:
if s[2] == '0' or s[2] == '-1':
pass
else:
fw.write(s[0] + '\t' + 'NS' + '\t' + s[1] + '\t' +
s[2] + '\n')
self.produce(update_file_address)
res_file_address = res_address + '/' + i + '_res'
with open(res_file_address, 'w') as fw:
for i in res_list:
if i[2] == '0' or i[2] == '-1':
pass
else:
fw.write(i[0] + '\t' + 'NS' + '\t' + i[1] + '\t' +
i[2] + '\n')
for t in tld_list:
if t[2] == '0' or t[2] == '-1':
pass
else:
fw.write(t[0] + '\t' + 'NS' + '\t' + t[1] + '\t' +
t[2] + '\n')
self.produce(res_file_address)
def write_tld(self, task_id):
try:
db = MySQL(SOURCE_CONFIG)
except:
pass
tld_list = self.read_tld(db)
with open('./domain_dns_data/sec_compared/tld_'+task_id, 'w') as fw:
for t in tld_list:
fw.write(t[0]+'\t'+'NS'+'\t'+t[1]+'\t'+t[3]+'\n')
ns_list = t[1].split(',')
ip_list = t[2].split(';')
for i in range(0,len(ns_list)):
fw.write(ns_list[i]+'\t'+'A'+'\t'+ip_list[i]+'\t'+'0'+'\n')
def read_domains():
"""
从数据库中读取要探测的域名，并解析出其主域名和顶级域名（一级）
注意：若是不符合规范的域名，则丢弃
"""
domains = []
main_domains = []
tlds = []
no_fetch_extract = tldextract.TLDExtract(suffix_list_urls=None)
try:
db = MySQL(SOURCE_CONFIG)
except Exception, e:
logger.logger.error(e)
return False
def start(self):
try:
db = MySQL(SOURCE_CONFIG)
except:
logger.logger.error("数据库异常：获取域名失败")
return
tld_list = self.read_tld(db)
mysql_data = self.read_domain_ns(db)
file_list = self.get_file_name(unres_address, res_address)
for file_name in file_list:
update_list = []
a_list = []
res_list = []
file_address = unres_address + '/' + str(file_name)
file_data, file_a = self.read_file(file_address)
for k, v in file_data.items():
if v[1] == '1':
ip = []
ns = v[0]
ns_list = ns.split(',')
for i in ns_list:
ns_ip = list(file_a[i])
ns_a = ','.join(ns_ip)
ip.append(ns_a)
a_list.append((i, ns_a, '0'))
data_ip = ';'.join(ip)
update_list.append((k, v[0], data_ip, '0'))
if v[0] != '':
res_list.append((k, v[0], '0'))
print len(update_list)
update_file_address = res_address + '/' + 'update_' + str(
file_name)
with open(update_file_address, 'w') as fw:
for s in update_list:
if s[1] != '':
fw.write(s[0] + '\t' + 'NS' + '\t' + s[1] + '\t' +
'0' + '\n')
for h in a_list:
fw.write(h[0] + '\t' + 'A' + '\t' + h[1] + '\t' + '0' +
'\n')
self.produce(update_file_address)
self.update_mysql(db, update_list)
def update_domain_dns_db():
"""
域名的DNS数据存入到数据库中
"""
# 解析关键字段信息
rc_result = []
for rc in domain_dns_rc_db:
domain = rc['domain']
domain_ns = rc['domain_ns']
ns_ns = rc['ns_ns']
unknown_ns = rc['unknown_ns']
invalid_ns = rc['invalid_ns']
verify_strategy = rc['verify_strategy']
tld_ns = rc['tld_domain_ns']
domain_a = rc['domain_a']
domain_unknown_a = rc['domain_unknown_a']
domain_aaaa = rc['domain_aaaa']
domain_unknown_aaaa = rc['domain_unknown_aaaa']
domain_cname = rc['domain_cname']
domain_unknown_cname = rc['domain_unknown_cname']
domain_soa = rc['domain_soa']
domain_mx = rc['domain_mx']
insert_time = rc['insert_time']
rc_result.append((domain,domain_ns,tld_ns,ns_ns,invalid_ns,unknown_ns,verify_strategy,domain_a,domain_cname, \
domain_unknown_a,domain_unknown_cname,domain_soa,domain_aaaa,domain_unknown_aaaa,domain_mx,insert_time))
try:
db = MySQL(SOURCE_CONFIG)
except:
logger.logger.error("数据库连接失败")
return
rc_sql = 'INSERT INTO domain_valid_dns_periodic (domain,domain_ns,tld_ns,ns_ns,invalid_ns,unknown_ns,verify_strategy, \
domain_a,domain_cname, domain_unknown_a,domain_unknown_cname,domain_soa,domain_aaaa,domain_unknown_aaaa,domain_mx,insert_time) \
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) \
ON DUPLICATE KEY UPDATE domain_ns=VALUES(domain_ns),tld_ns=VALUES (tld_ns),ns_ns = VALUES (ns_ns),invalid_ns=VALUES (invalid_ns), \
unknown_ns=VALUES (unknown_ns), verify_strategy=VALUES (verify_strategy),domain_a = VALUES (domain_a), \
domain_cname = VALUES (domain_cname),domain_unknown_a = VALUES (domain_unknown_a),domain_unknown_cname = VALUES (domain_unknown_cname), \
domain_soa = VALUES (domain_soa),domain_aaaa = VALUES (domain_aaaa),domain_mx = VALUES (domain_mx),insert_time = VALUES (insert_time)' # 存在则更新，不存在则插入
try:
db.update_many(rc_sql, rc_result)
except Exception as e:
logger.logger.error("更新域名的NS记录失败:" + str(e))
db.close()