def cal_time(ip, show_flag=False): during_time_list = [] col = get_col('ip_scan_result_socket1') scan_info = col.find({'ip': ip, 'state': True}) for i in scan_info: during_time_list.append(i['during_time']) if show_flag: print ip, np.mean(during_time_list), np.var(during_time_list, ddof=1) return { 'ip': ip, 'mean': np.mean(during_time_list), 'var': np.var(during_time_list, ddof=1) }
def pure_domain_state(ips): """ 分析全部关闭或者打开的服务器域名 :param ips: ip列表 :return: domain: list,服务器域名 """ col = get_col('com_svr') domains = set() for ip in ips: domain = col.find({'ips': ip}, {'_id': 0, 'domain': 1}) for d in domain: print d['domain'] domains.add(d['domain']) return list(domains)
def state_domain_count(diff_ips): """ 分析ip所对应的WHOIS服务器的情况 :param diff_ips: :return: """ col = get_col('com_svr') svr_domain = Counter() # 服务器计数器 for ip in diff_ips: domain = col.find({'ips': ip}, {'_id': 0, 'domain': 1}) print ip print domain.count() for d in domain: print d svr_domain[d['domain']] += 1 print svr_domain
def ip_state_count(ips): """ 统计不稳定IP的状态 :param ips: :return: """ col = get_col('ip_scan_result1') for ip in ips: c = Counter() scan_info = col.find({'ip': ip}) scan_count = scan_info.count() for i in scan_info: c[i['state']] += 1 print ip, print '%.2f%%' % (c['up'] / float(scan_count) * 100), '%.2f%%' % ( c['down'] / float(scan_count) * 100)
def state_count(ips): """ ip状态计数 :param ips: :return: """ col = get_col('ip_scan_result_socket1') for ip in ips: scan_info = col.find({'ip': ip}) c = Counter() for i in scan_info: print i['ip'], print i['state'], c[i['state']] += 1 print local2utc(i['detected_time']) print ip, c[True], c[False]
def state_count(diff_ips): """ ip状态计数 :param diff_ips: :return: """ col = get_col('ip_scan_result1') for ip in diff_ips: scan_info = col.find({'ip': ip}, { '_id': 0, 'detected_time': 1, 'state': 1 }) c = Counter() for i in scan_info: c[i['state']] += 1 # print ip['state'], local2utc(ip['detected_time']) print ip, c['up'], c['down']
def get_ip_port(): col = get_col('ip_scan_result_80') test = col.find({'state': 'up'}) c = Counter() for i in test: c[i['port_state']] += 1 ip_port_filtered_cur = col.find({ 'state': 'up', 'port_state': 'filtered' }, { '_id': 0, 'ip': 1 }) ip_port_filtered = [] for i in ip_port_filtered_cur: print i['ip'] ip_port_filtered.append(i['ip']) ip_port_open = [] ip_port_open_cur = col.find({ 'state': 'up', 'port_state': 'open' }, { '_id': 0, 'ip': 1 }) for i in ip_port_open_cur: print i['ip'] ip_port_open.append(i['ip']) print len(ip_port_filtered) print len(ip_port_open) ip_port_filtered = list(set(ip_port_filtered)) ip_port_open = list(set(ip_port_open)) print len(ip_port_filtered) print len(ip_port_open) print set(ip_port_filtered) & set(ip_port_open) print set(ip_port_open) - set(ip_port_filtered) print set(ip_port_filtered) - set(ip_port_open)
def extract_field(result): insert_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") col = get_col(target_col_name, target_db_name) # print result for d in result: try: domain = d['domain'] reg_phone = d['reg_phone'] updated_date = d['updated_date'] reg_email = d['reg_email'] expiration_date = d['expiration_date'] reg_name = d['reg_name'] top_whois_server = d['top_whois_server'] name_server = d['name_server'] creation_date = d['creation_date'] sec_whois_server = d['sec_whois_server'] org_name = d['org_name'] sponsoring_registrar = d['sponsoring_registrar'] # 域名记录若不存在则插入,存在则不做任何操作 col.insert({ "domain": domain, "reg_phone": reg_phone, 'updated_date': updated_date, 'reg_email': reg_email, 'expiration_date': expiration_date, 'reg_name': reg_name, 'top_whois_server': top_whois_server, 'name_server': name_server, 'creation_date': creation_date, 'sec_whois_server': sec_whois_server, "record_time": insert_time, # 文档插入时间 "org_name": org_name, "sponsoring_registrar": sponsoring_registrar }) except: print "出错" continue
def update_data(flag, domain_cname): """ 若记录存在,则检查该条记录是否需要进行更新 """ if not flag: # 不存在返回 print "域名新插入" return False col = get_col(target_col) insert_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") domain = domain_cname['domain'] domain_data = col.find({'domain': domain}) cnames = domain_cname['cnames'] ttls = domain_cname['ttls'] cur_record = list(domain_data)[0]['dm_cname'][-1] original_cnames, original_insert_time = cur_record['cnames'], cur_record[ 'insert_time'] # 如果两者相同,则返回,不修改 if is_cname_same(cnames, original_cnames): print "与最近记录一致,更新时间" update_time(col, domain, cnames, original_insert_time) return False print "与最近记录不一致,新添加记录" col.update({'domain': domain}, { "$push": { "dm_cname": { "cnames": cnames, "ttls": ttls, "insert_time": insert_time } }, "$inc": { "visit_times": 1 } }) return True
def update_data(ip_cname): """ 若记录存在,则检查该条记录是否需要进行更新 """ cur_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") domain = ip_cname['domain'] ips = ip_cname['ips'] geos = ip_cname['geos'] cnames = ip_cname['cnames'] col = get_col(target_col) domain_data = col.find({'domain': domain}) # 得到数据库中已存的记录信息 d = list(domain_data)[:] original_ip_record = d[0]['dm_ip'][-1] original_cname_record = d[0]['dm_cname'][-1] original_ips, original_ip_insert_time = original_ip_record[ 'ips'], original_ip_record['insert_time'] original_cnames, original_cname_insert_time = original_cname_record[ 'cnames'], original_cname_record['insert_time'] # 判断IP是否相同 if is_same(ips, original_ips): print "与最近IP记录一致,更新时间" update_time(col, domain, 'A', original_ip_insert_time, cur_time) else: print "与最近IP记录不一致,新添加记录" insert_record(col, "A", domain, ips, cur_time, geos) # 判断cname是否相同 if is_same(cnames, original_cnames): print "与最近CNME记录一致,更新时间" update_time(col, domain, 'CNAME', original_cname_insert_time, cur_time) else: print "与最近CNAME记录不一致,新添加记录" insert_record(col, 'CNAME', domain, cnames, cur_time, geos)
# encoding: utf-8 """ 数据库操作 """ from db_manage import get_col col = get_col('com_svr') domains = col.find({'ips': '216.21.238.34'}, {'domain': 1, '_id': 0}) domain_list = [] for d in domains: domain_list.append(d['domain']) domain_list.sort() save_file = open('svrCanotWork.txt', 'w') for i in domain_list: print i save_file.write(i + '\n') print len(domain_list) save_file.close()
从根域名向下进行查询 作者:程亚楠 时间:2017.8.25 """ import DNS import random import tldextract from datetime import datetime from db_manage import get_col from pandas import Series timeout = 5 # 超时时间 # server = '222.194.15.253' target_col = 'mal_dns_ttl' col = get_col(target_col) ## 全局变量 g_cnames = [] g_cnames_ttl = [] g_ips = [] g_ips_ttl = [] g_ns = [] g_ns_ttl = [] def fetch_domain_ns(domain, server='222.194.15.253'): """ 获取域名的NS记录 """ ns, ns_ttl, qry_result = [], [], []