def cal_time(ip, show_flag=False):
during_time_list = []
col = get_col('ip_scan_result_socket1')
scan_info = col.find({'ip': ip, 'state': True})
for i in scan_info:
during_time_list.append(i['during_time'])
if show_flag:
print ip, np.mean(during_time_list), np.var(during_time_list, ddof=1)
return {
'ip': ip,
'mean': np.mean(during_time_list),
'var': np.var(during_time_list, ddof=1)
}
def pure_domain_state(ips):
"""
分析全部关闭或者打开的服务器域名
:param ips: ip列表
:return:
domain: list，服务器域名
"""
col = get_col('com_svr')
domains = set()
for ip in ips:
domain = col.find({'ips': ip}, {'_id': 0, 'domain': 1})
for d in domain:
print d['domain']
domains.add(d['domain'])
return list(domains)
def state_domain_count(diff_ips):
"""
分析ip所对应的WHOIS服务器的情况
:param diff_ips:
:return:
"""
col = get_col('com_svr')
svr_domain = Counter() # 服务器计数器
for ip in diff_ips:
domain = col.find({'ips': ip}, {'_id': 0, 'domain': 1})
print ip
print domain.count()
for d in domain:
print d
svr_domain[d['domain']] += 1
print svr_domain
def ip_state_count(ips):
"""
统计不稳定IP的状态
:param ips:
:return:
"""
col = get_col('ip_scan_result1')
for ip in ips:
c = Counter()
scan_info = col.find({'ip': ip})
scan_count = scan_info.count()
for i in scan_info:
c[i['state']] += 1
print ip,
print '%.2f%%' % (c['up'] / float(scan_count) * 100), '%.2f%%' % (
c['down'] / float(scan_count) * 100)
def state_count(ips):
"""
ip状态计数
:param ips:
:return:
"""
col = get_col('ip_scan_result_socket1')
for ip in ips:
scan_info = col.find({'ip': ip})
c = Counter()
for i in scan_info:
print i['ip'],
print i['state'],
c[i['state']] += 1
print local2utc(i['detected_time'])
print ip, c[True], c[False]
def state_count(diff_ips):
"""
ip状态计数
:param diff_ips:
:return:
"""
col = get_col('ip_scan_result1')
for ip in diff_ips:
scan_info = col.find({'ip': ip}, {
'_id': 0,
'detected_time': 1,
'state': 1
})
c = Counter()
for i in scan_info:
c[i['state']] += 1
# print ip['state'], local2utc(ip['detected_time'])
print ip, c['up'], c['down']
def get_ip_port():
col = get_col('ip_scan_result_80')
test = col.find({'state': 'up'})
c = Counter()
for i in test:
c[i['port_state']] += 1
ip_port_filtered_cur = col.find({
'state': 'up',
'port_state': 'filtered'
}, {
'_id': 0,
'ip': 1
})
ip_port_filtered = []
for i in ip_port_filtered_cur:
print i['ip']
ip_port_filtered.append(i['ip'])
ip_port_open = []
ip_port_open_cur = col.find({
'state': 'up',
'port_state': 'open'
}, {
'_id': 0,
'ip': 1
})
for i in ip_port_open_cur:
print i['ip']
ip_port_open.append(i['ip'])
print len(ip_port_filtered)
print len(ip_port_open)
ip_port_filtered = list(set(ip_port_filtered))
ip_port_open = list(set(ip_port_open))
print len(ip_port_filtered)
print len(ip_port_open)
print set(ip_port_filtered) & set(ip_port_open)
print set(ip_port_open) - set(ip_port_filtered)
print set(ip_port_filtered) - set(ip_port_open)
def extract_field(result):
insert_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
col = get_col(target_col_name, target_db_name)
# print result
for d in result:
try:
domain = d['domain']
reg_phone = d['reg_phone']
updated_date = d['updated_date']
reg_email = d['reg_email']
expiration_date = d['expiration_date']
reg_name = d['reg_name']
top_whois_server = d['top_whois_server']
name_server = d['name_server']
creation_date = d['creation_date']
sec_whois_server = d['sec_whois_server']
org_name = d['org_name']
sponsoring_registrar = d['sponsoring_registrar']
# 域名记录若不存在则插入，存在则不做任何操作
col.insert({
"domain": domain,
"reg_phone": reg_phone,
'updated_date': updated_date,
'reg_email': reg_email,
'expiration_date': expiration_date,
'reg_name': reg_name,
'top_whois_server': top_whois_server,
'name_server': name_server,
'creation_date': creation_date,
'sec_whois_server': sec_whois_server,
"record_time": insert_time, # 文档插入时间
"org_name": org_name,
"sponsoring_registrar": sponsoring_registrar
})
except:
print "出错"
continue
def update_data(flag, domain_cname):
"""
若记录存在，则检查该条记录是否需要进行更新
"""
if not flag: # 不存在返回
print "域名新插入"
return False
col = get_col(target_col)
insert_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
domain = domain_cname['domain']
domain_data = col.find({'domain': domain})
cnames = domain_cname['cnames']
ttls = domain_cname['ttls']
cur_record = list(domain_data)[0]['dm_cname'][-1]
original_cnames, original_insert_time = cur_record['cnames'], cur_record[
'insert_time']
# 如果两者相同，则返回，不修改
if is_cname_same(cnames, original_cnames):
print "与最近记录一致，更新时间"
update_time(col, domain, cnames, original_insert_time)
return False
print "与最近记录不一致，新添加记录"
col.update({'domain': domain}, {
"$push": {
"dm_cname": {
"cnames": cnames,
"ttls": ttls,
"insert_time": insert_time
}
},
"$inc": {
"visit_times": 1
}
})
return True
def update_data(ip_cname):
"""
若记录存在，则检查该条记录是否需要进行更新
"""
cur_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
domain = ip_cname['domain']
ips = ip_cname['ips']
geos = ip_cname['geos']
cnames = ip_cname['cnames']
col = get_col(target_col)
domain_data = col.find({'domain': domain}) # 得到数据库中已存的记录信息
d = list(domain_data)[:]
original_ip_record = d[0]['dm_ip'][-1]
original_cname_record = d[0]['dm_cname'][-1]
original_ips, original_ip_insert_time = original_ip_record[
'ips'], original_ip_record['insert_time']
original_cnames, original_cname_insert_time = original_cname_record[
'cnames'], original_cname_record['insert_time']
# 判断IP是否相同
if is_same(ips, original_ips):
print "与最近IP记录一致，更新时间"
update_time(col, domain, 'A', original_ip_insert_time, cur_time)
else:
print "与最近IP记录不一致，新添加记录"
insert_record(col, "A", domain, ips, cur_time, geos)
# 判断cname是否相同
if is_same(cnames, original_cnames):
print "与最近CNME记录一致，更新时间"
update_time(col, domain, 'CNAME', original_cname_insert_time, cur_time)
else:
print "与最近CNAME记录不一致，新添加记录"
insert_record(col, 'CNAME', domain, cnames, cur_time, geos)
# encoding: utf-8
"""
数据库操作
"""
from db_manage import get_col
col = get_col('com_svr')
domains = col.find({'ips': '216.21.238.34'}, {'domain': 1, '_id': 0})
domain_list = []
for d in domains:
domain_list.append(d['domain'])
domain_list.sort()
save_file = open('svrCanotWork.txt', 'w')
for i in domain_list:
print i
save_file.write(i + '\n')
print len(domain_list)
save_file.close()
从根域名向下进行查询
作者：程亚楠
时间：2017.8.25
"""
import DNS
import random
import tldextract
from datetime import datetime
from db_manage import get_col
from pandas import Series
timeout = 5 # 超时时间
# server = '222.194.15.253'
target_col = 'mal_dns_ttl'
col = get_col(target_col)
## 全局变量
g_cnames = []
g_cnames_ttl = []
g_ips = []
g_ips_ttl = []
g_ns = []
g_ns_ttl = []
def fetch_domain_ns(domain, server='222.194.15.253'):
"""
获取域名的NS记录
"""
ns, ns_ttl, qry_result = [], [], []
