def get_ns_history_record(self): try: db = MySQL(SOURCE_CONFIG) except: pass full_list = [] sql = 'SELECT domain,ns,ns_ip FROM domain_ns WHERE ns!=""' try: db.query(sql) data = db.fetch_all_rows() except Exception as e: return "read tld Error" + str(e) for i in data: tld = str(i['domain']) ns = str(i['ns']) ns_list = ns.split(';') ns = ','.join(ns_list) ns_ip = str(i['ns_ip']) flag = '0' full_list.append((tld,ns,ns_ip,flag)) taskid = str(random.randint(10000000, 99999999)) self.write_tld(taskid) new_file_name = 'full_' + taskid with open("domain_dns_data/sec_compared/"+new_file_name, "w") as f: for t in full_list: f.write(t[0]+'\t'+'NS'+'\t'+t[1]+'\t'+t[3]+'\n') ns_list = t[1].split(',') ip_list = t[2].split(';') for i in range(0,len(ns_list)): f.write(ns_list[i]+'\t'+'A'+'\t'+ip_list[i]+'\t'+'0'+'\n') self.produce(new_file_name) self.produce('tld_'+taskid)
def main(): result = [] tld_result1 = extract_ns_ip_by_root() # 通过询问根域名服务器获得数据, todo 不能有这种命名 tld_result2 = get_tld_ns_by_zone() # 通过下载zone文件获得数据 # 取两个结果的并集 for i in tld_result1: for t in tld_result2: if i[0] == t[0]: ns_list1 = i[1].split(';') # 将字符串变为列表然后取并集 ns_list = list(set(ns_list1) | set(t[1])) ns_str = ';'.join(ns_list) # 列表变为字符串 ipv4_list1 = i[2].split(';') # 将字符串变为列表然后取并集 ipv4_list = list(set(ipv4_list1) | set(t[2])) ipv4_str = ';'.join(ipv4_list) # 列表变为字符串 ipv6_list1 = i[3].split(';') # 将字符串变为列表然后取并集 ipv6_list = list(set(ipv6_list1) | set(t[3])) ipv6_str = ';'.join(ipv6_list) # 列表变为字符串 result.append((i[0], ns_str, ipv4_str, ipv6_str)) # for i in result: # print i try: db = MySQL(SOURCE_CONFIG) except: logger.logger.error("数据库异常:获取域名失败") return insert_tld_ns_db(db, tld_result1) root_ns,root_ipv4,root_ipv6 = get_root() insert_root_ns_db(db, root_ns, root_ipv4, root_ipv6) db.close()
def fetch_tld_ns(): """ 获取顶级域名的权威服务器(ns)IP地址 """ tld_ns = defaultdict(set) try: db = MySQL(SOURCE_CONFIG) sql = 'SELECT tld,server_ipv4 from tld_ns_zone' db.query(sql) tld_ns_query = db.fetch_all_rows() # 获取已存储的顶级域名的权威服务器信息 except Exception, e: logger.logger.error("获取顶级域名异常:", e) return tld_ns
def insert_domains_db(domains): """将域名插入到数据库中""" try: db = MySQL(SOURCE_CONFIG) except Exception,e: logger.logger.error(e) return False
def save_sec_domains(domains): """将sec的域名存入到数据库中""" try: db = MySQL(SOURCE_CONFIG) except Exception, e: logger.logger.error(e) return False
def start(self): try: db = MySQL(SOURCE_CONFIG) except: logger.logger.error("数据库异常:获取域名失败") return tld_list = self.read_tld(db) mysql_data = self.read_domain_ns(db) file_list = self.get_file_name(unres_address, res_address) for i in file_list: update_list = [] res_list = [] file_address = unres_address + '/' + i file_data = self.read_file(file_address) for k, v in file_data.items(): if k not in mysql_data.keys(): update_list.append((k, v[0], v[1])) res_list.append((k, v[0], v[1])) else: file_flag = int(v[1]) mysql = mysql_data[k] ns = mysql[0] flag = mysql[1] mysql_flag = int(mysql[1]) if file_flag > mysql_flag: print k update_list.append((k, v[0], v[1])) res_list.append((k, v[0], v[1])) else: res_list.append((k, ns, flag)) self.update_mysql(db, update_list) update_file_address = res_address + '/' + i + '_update' with open(update_file_address, 'w') as fw: for s in update_list: if s[2] == '0' or s[2] == '-1': pass else: fw.write(s[0] + '\t' + 'NS' + '\t' + s[1] + '\t' + s[2] + '\n') self.produce(update_file_address) res_file_address = res_address + '/' + i + '_res' with open(res_file_address, 'w') as fw: for i in res_list: if i[2] == '0' or i[2] == '-1': pass else: fw.write(i[0] + '\t' + 'NS' + '\t' + i[1] + '\t' + i[2] + '\n') for t in tld_list: if t[2] == '0' or t[2] == '-1': pass else: fw.write(t[0] + '\t' + 'NS' + '\t' + t[1] + '\t' + t[2] + '\n') self.produce(res_file_address)
def write_tld(self, task_id): try: db = MySQL(SOURCE_CONFIG) except: pass tld_list = self.read_tld(db) with open('./domain_dns_data/sec_compared/tld_'+task_id, 'w') as fw: for t in tld_list: fw.write(t[0]+'\t'+'NS'+'\t'+t[1]+'\t'+t[3]+'\n') ns_list = t[1].split(',') ip_list = t[2].split(';') for i in range(0,len(ns_list)): fw.write(ns_list[i]+'\t'+'A'+'\t'+ip_list[i]+'\t'+'0'+'\n')
def read_domains(): """ 从数据库中读取要探测的域名,并解析出其主域名和顶级域名(一级) 注意:若是不符合规范的域名,则丢弃 """ domains = [] main_domains = [] tlds = [] no_fetch_extract = tldextract.TLDExtract(suffix_list_urls=None) try: db = MySQL(SOURCE_CONFIG) except Exception, e: logger.logger.error(e) return False
def start(self): try: db = MySQL(SOURCE_CONFIG) except: logger.logger.error("数据库异常:获取域名失败") return tld_list = self.read_tld(db) mysql_data = self.read_domain_ns(db) file_list = self.get_file_name(unres_address, res_address) for file_name in file_list: update_list = [] a_list = [] res_list = [] file_address = unres_address + '/' + str(file_name) file_data, file_a = self.read_file(file_address) for k, v in file_data.items(): if v[1] == '1': ip = [] ns = v[0] ns_list = ns.split(',') for i in ns_list: ns_ip = list(file_a[i]) ns_a = ','.join(ns_ip) ip.append(ns_a) a_list.append((i, ns_a, '0')) data_ip = ';'.join(ip) update_list.append((k, v[0], data_ip, '0')) if v[0] != '': res_list.append((k, v[0], '0')) print len(update_list) update_file_address = res_address + '/' + 'update_' + str( file_name) with open(update_file_address, 'w') as fw: for s in update_list: if s[1] != '': fw.write(s[0] + '\t' + 'NS' + '\t' + s[1] + '\t' + '0' + '\n') for h in a_list: fw.write(h[0] + '\t' + 'A' + '\t' + h[1] + '\t' + '0' + '\n') self.produce(update_file_address) self.update_mysql(db, update_list)
def update_domain_dns_db(): """ 域名的DNS数据存入到数据库中 """ # 解析关键字段信息 rc_result = [] for rc in domain_dns_rc_db: domain = rc['domain'] domain_ns = rc['domain_ns'] ns_ns = rc['ns_ns'] unknown_ns = rc['unknown_ns'] invalid_ns = rc['invalid_ns'] verify_strategy = rc['verify_strategy'] tld_ns = rc['tld_domain_ns'] domain_a = rc['domain_a'] domain_unknown_a = rc['domain_unknown_a'] domain_aaaa = rc['domain_aaaa'] domain_unknown_aaaa = rc['domain_unknown_aaaa'] domain_cname = rc['domain_cname'] domain_unknown_cname = rc['domain_unknown_cname'] domain_soa = rc['domain_soa'] domain_mx = rc['domain_mx'] insert_time = rc['insert_time'] rc_result.append((domain,domain_ns,tld_ns,ns_ns,invalid_ns,unknown_ns,verify_strategy,domain_a,domain_cname, \ domain_unknown_a,domain_unknown_cname,domain_soa,domain_aaaa,domain_unknown_aaaa,domain_mx,insert_time)) try: db = MySQL(SOURCE_CONFIG) except: logger.logger.error("数据库连接失败") return rc_sql = 'INSERT INTO domain_valid_dns_periodic (domain,domain_ns,tld_ns,ns_ns,invalid_ns,unknown_ns,verify_strategy, \ domain_a,domain_cname, domain_unknown_a,domain_unknown_cname,domain_soa,domain_aaaa,domain_unknown_aaaa,domain_mx,insert_time) \ VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) \ ON DUPLICATE KEY UPDATE domain_ns=VALUES(domain_ns),tld_ns=VALUES (tld_ns),ns_ns = VALUES (ns_ns),invalid_ns=VALUES (invalid_ns), \ unknown_ns=VALUES (unknown_ns), verify_strategy=VALUES (verify_strategy),domain_a = VALUES (domain_a), \ domain_cname = VALUES (domain_cname),domain_unknown_a = VALUES (domain_unknown_a),domain_unknown_cname = VALUES (domain_unknown_cname), \ domain_soa = VALUES (domain_soa),domain_aaaa = VALUES (domain_aaaa),domain_mx = VALUES (domain_mx),insert_time = VALUES (insert_time)' # 存在则更新,不存在则插入 try: db.update_many(rc_sql, rc_result) except Exception as e: logger.logger.error("更新域名的NS记录失败:" + str(e)) db.close()