def main(gte,lte,timestamp):
time=lte.split(" ")
blacklist_dir = source_store_path[1]+source_store_path[0]+'-'+str(time[0])+".json"
# print blacklist_dir
blacklist_Trie_dir = trie_store_path[1]+trie_store_path[0]+'-'+str(time[0])+".json"
# print blacklist_Trie_dir
count = 0
temp_time = datetime.datetime.strptime(lte,'%Y-%m-%d %H:%M:%S')
while (not (os.path.exists(blacklist_dir) and os.path.exists(blacklist_Trie_dir))) and count<30:
temp_time = temp_time + datetime.timedelta(days = -1)
time = temp_time.strftime('%Y-%m-%d %H:%M:%S').split(" ")
blacklist_dir = source_store_path[1]+source_store_path[0]+'-'+str(time[0])+".json"
blacklist_Trie_dir = trie_store_path[1]+trie_store_path[0]+'-'+str(time[0])+".json"
count += 1
if count == 30:
logger_error.error('No blacklist data in last 30 days.')
return 1
es = ESclient()
try:
logger_info.info('Getting ES DNS domain completed.')
search_result = es.get_es_domain(gte,lte)
logger_info.info('Get ES DNS domain completed.')
except Exception as e:
logger_error.error("Get ES DNS domain failed.\n{0}".format(e))
split_DNSList = get_split_DNSList(search_result)
blacklist_Trie = load_dict(blacklist_Trie_dir)
match_DNSList,match_blacklist = find_match_DNS(blacklist_Trie,split_DNSList)
# print match_DNSList
# print match_blacklist
logger_info.info('Match DNS list :\n{0}'.format(match_DNSList))
logger_info.info('Match DNS blacklist :\n{0}'.format(match_blacklist))
# 匹配的DNS回插到es
if match_DNSList:
try:
blacklist = load_dict(blacklist_dir)
for i in range(len(match_blacklist)):
domain = u'{}'.format('.'.join(match_blacklist[i]))
doc = blacklist[domain]
doc['domain'] = '.'.join(match_DNSList[i])
doc['@timestamp'] = timestamp
doc['level'] = judge_level(fp=doc['false_positive'],status=doc['status'])
search_result = es.get_domain_info(gte,lte,domain)
sip_answer_list = get_sip_answer_list(search_result)
for sip_answer in sip_answer_list:
doc['sip'] = sip_answer[0]
doc['answer'] = sip_answer[1]
es.es_index(doc)
# print doc
except Exception as e:
logger_error.error("Insert the alert of theat DNS to ES failed.\n{0}".format(e))
def checkAndInsert(path,filelist,ip_es_list,index,aggs_name,timestamp,serverNum,dport):
# check each file
for fname in filelist:
fpath = path + fname
dataset = load_dict(fpath)
if (dataset):
msg = dataset[dataset.keys()[0]]
# get match result
fullmatch, segmentmatch, subnetlpm, subnetfull = treatip(dataset,ip_es_list)
insert_result(index,aggs_name,timestamp,serverNum,dport,fullmatch,segmentmatch,subnetlpm,subnetfull,dataset)
def main(tday,index, gte, lte, aggs_name, timestamp,serverNum,dport):
path=parser_config.get_store_path()[1]+str(tday)+'\\'
filelist=get_all_file(path)
#get es list
es = ESclient(server =serverNum,port=dport)
ip_es_list = es.get_es_ip(index,gte,lte,aggs_name)
#check each file
for fname in filelist:
fpath=path+fname
dataset=load_dict(fpath)
#get match result
fullmatch,segmentmatch=treatip(dataset,ip_es_list)
insert_result(index,aggs_name,timestamp,serverNum,dport,fullmatch,segmentmatch,dataset)
def checkAndInsert(path,filelist,ip_es_list,index,aggs_name,timestamp,serverNum,dport):
# check each file
mylog=blacklist_tools.getlog()
all_threatIP={}
for fname in filelist:
mylog.info('-*-*-*-*-file:{}-*-*-*-*-'.format(fname))
fpath = path + fname
dataset = load_dict(fpath)
if (dataset):
msg = dataset[dataset.keys()[0]]
# get match result
fullmatch, segmentmatch, subnetlpm, subnetfull = treatip(dataset,ip_es_list)
threatIP=insert_result(index,aggs_name,timestamp,serverNum,dport,fullmatch,segmentmatch,subnetlpm,subnetfull,dataset)
# merge
if(threatIP):
all_threatIP=dict(all_threatIP,**threatIP)
return all_threatIP
def start(stype, values, checkflg=1):
# 程序入口， checkflg表示是否检查和写入本地文件，默认值为1表示检查和写入。
final_dic = {} # return
if (not type(values) == list): # values is not list
print 'values type error!'
return final_dic
# first step: check the local file
if (checkflg == 1):
# load data from local file
if (not os.path.exists(lpath)): # not exists,then create dir
os.mkdir(lpath)
fpath = lpath + filename
if (os.path.exists(fpath)):
ldic = blacklist_tools.load_dict(fpath)
ldata = ldic.keys() #all ips and urls
else:
ldic = {}
ldata = [] # 写文件时再创建
indic = set(values) & set(ldata) # 交集
for ii in indic:
final_dic[ii] = ldic[ii]
outvalues = list(set(values) - set(ldata)) # 差集,不在文件里
# second step: remains data check the X-Force
ftype = search_type[stype]
tmp_dic = {}
for value in outvalues:
try:
resdata = get_info(host_ibm, ftype, value) # resdata is dic
for tmp in resdata.keys():
tmp_dic[tmp] = resdata[tmp]
except Exception, e:
if ("ReadTimeout" in e):
time.sleep(2)
# retry
try:
resdata = get_info(host_ibm, ftype,
value) # resdata is dic
for tmp in resdata.keys():
tmp_dic[tmp] = resdata[tmp]
except Exception, e:
print(e)
continue
else:
print(e)
continue
def main(gte, lte, timestamp, time_zone):
time = datetime.datetime.now().strftime('%Y-%m-%d')
blacklist_dir = os.path.join(data_path,
'source' + '-' + str(time) + ".json")
blacklist_Trie_dir = os.path.join(data_path,
'trie' + '-' + str(time) + ".json")
if not (os.path.exists(blacklist_dir)
and os.path.exists(blacklist_Trie_dir)):
blacklist_dir = os.path.join(data_path, 'source-default.json')
blacklist_Trie_dir = os.path.join(data_path, 'trie-default.json')
es = ESclient()
try:
log.debug('Getting ES DNS domain.')
search_result = es.get_es_domain(gte=gte, lte=lte, time_zone=time_zone)
log.debug('Get ES DNS domain completed.')
except Exception as e:
log.error("Get ES DNS domain failed.\n{0}".format(e))
raise e
split_DNSList = get_split_DNSList(search_result)
blacklist_Trie = load_dict(blacklist_Trie_dir)
match_DNSList, match_blacklist = find_match_DNS(blacklist_Trie,
split_DNSList)
match_DNSList, match_blacklist = check_whitelist(match_DNSList,
match_blacklist)
log.debug('Match DNS list : {0}'.format(match_DNSList))
log.debug('Match DNS blacklist : {0}'.format(match_blacklist))
# 匹配的DNS回插到es
if match_DNSList:
ipv4_pattern = re.compile(
'^(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}$'
)
try:
blacklist = load_dict(blacklist_dir)
for i in range(len(match_blacklist)):
domain = u'{}'.format('.'.join(match_blacklist[i]))
domain_es = '.'.join(match_DNSList[i])
doc = dict(blacklist[domain])
source = doc.pop('source')
doc['domain'] = domain_es
doc['eventid'] = 102002
doc['@timestamp'] = timestamp
doc['level'] = "info"
doc['type'] = "mal_dns"
doc['desc_type'] = "[mal_dns] Request of Malicious Domain Name Detection"
doc['desc_subtype'] = "[{0}] Intelligence comes from:{1}".format(
doc['subtype'], source)
if others["alert_level"] == "info":
es.es_index(doc)
if syslogger:
syslogger.info(doc)
continue
search_result = es.get_domain_info(gte=gte,
lte=lte,
domain=domain_es,
time_zone=time_zone)
sip_answer_dict = get_sip_answer_dict(search_result)
for sip in sip_answer_dict:
answer_list = sip_answer_dict[sip]
doc['level'] = "info"
doc['sip'] = sip
doc['src_dept'] = get_dept_info(sip)
dip_list = []
for answer in answer_list:
doc['answer'] = answer
if ipv4_pattern.findall(answer):
dip_list.append(answer)
doc['dip'] = answer
dipGeo = get_ipip_geo(answer)
doc['dst_country'] = dipGeo[0]
doc['dst_province'] = dipGeo[1]
else:
doc.pop("dip", "")
doc.pop("dst_country", "")
doc.pop("dst_province", "")
es.es_index(doc)
if syslogger:
syslogger.info(doc)
doc.pop("dip", "")
doc.pop("answer", "")
for dip in dip_list:
sip_list = es.second_check(gte=gte,
lte=lte,
time_zone=time_zone,
dip=dip)
# print sip_list
for sip in sip_list:
doc['dip'] = dip
doc["sip"] = sip
doc['src_dept'] = get_dept_info(sip)
dipGeo = get_ipip_geo(dip)
doc['dst_country'] = dipGeo[0]
doc['dst_province'] = dipGeo[1]
doc["level"] = "warn"
es.es_index(doc)
if syslogger:
syslogger.info(doc)
doc.pop("dip", "")
doc.pop("sip", "")
doc.pop("level", "")
doc.pop("src_dept", "")
doc.pop("dst_country", "")
doc.pop("dst_province", "")
except Exception as e:
log.error(
"Insert the alert of threat DNS to ES failed.\n{0}".format(e))
raise e
def main(gte, lte, timestamp, time_zone):
time = datetime.datetime.now().strftime('%Y-%m-%d')
blacklist_dir = os.path.join(data_path,
'source' + '-' + str(time) + ".json")
blacklist_Trie_dir = os.path.join(data_path,
'trie' + '-' + str(time) + ".json")
if not (os.path.exists(blacklist_dir)
and os.path.exists(blacklist_Trie_dir)):
blacklist_dir = os.path.join(data_path, 'source-default.json')
blacklist_Trie_dir = os.path.join(data_path, 'trie-default.json')
es = ESclient()
try:
logger_info.info('Getting ES DNS domain.')
search_result = es.get_es_domain(gte=gte, lte=lte, time_zone=time_zone)
logger_info.info('Get ES DNS domain completed.')
except Exception as e:
logger_error.error("Get ES DNS domain failed.\n{0}".format(e))
raise e
split_DNSList = get_split_DNSList(search_result)
blacklist_Trie = load_dict(blacklist_Trie_dir)
match_DNSList, match_blacklist = find_match_DNS(blacklist_Trie,
split_DNSList)
match_DNSList, match_blacklist = check_whitelist(match_DNSList,
match_blacklist)
logger_info.info('Match DNS list :\n{0}'.format(match_DNSList))
logger_info.info('Match DNS blacklist :\n{0}'.format(match_blacklist))
syslogger = get_syslog_config()
# 匹配的DNS回插到es
if match_DNSList:
dip_list = []
ipv4_pattern = re.compile(
'^(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}$'
)
try:
blacklist = load_dict(blacklist_dir)
for i in range(len(match_blacklist)):
domain = u'{}'.format('.'.join(match_blacklist[i]))
domain_es = '.'.join(match_DNSList[i])
doc = dict(blacklist[domain])
source = doc.pop('source')
doc['domain'] = domain_es
doc['@timestamp'] = timestamp
doc['level'] = "info"
doc['type'] = "mal_dns"
doc['desc_type'] = "[mal_dns] Request of Malicious Domain Name Detection"
doc['desc_subtype'] = "[{0}] Intelligence comes from:{1}".format(
doc['subtype'], source)
if others["only_info"] == "true":
es.es_index(doc)
if syslogger:
syslogger.info(doc)
continue
search_result = es.get_domain_info(gte=gte,
lte=lte,
domain=domain_es,
time_zone=time_zone)
answer_list = get_answer_list(search_result)
dip_list = dip_list + answer_list
for answer in answer_list:
doc['answer'] = answer
if ipv4_pattern.findall(answer):
doc['dip'] = answer
es.es_index(doc)
if syslogger:
syslogger.info(doc)
# print doc
if ipv4_pattern.findall(answer):
sip_list = es.second_check(gte=gte,
lte=lte,
time_zone=time_zone,
dip=answer)
# print sip_list
if sip_list:
for sip in sip_list:
doc["sip"] = sip
doc["level"] = "warn"
es.es_index(doc)
if syslogger:
syslogger.info(doc)
except Exception as e:
logger_error.error(
"Insert the alert of theat DNS to ES failed.\n{0}".format(e))
raise e
def subnet_lpm(subnet, es_ip):
mylog = blacklist_tools.getlog()
lpm.init()
sndict = {}
fpath = parser_config.get_store_path()[1]
sn_lte16 = {}
lpmdict = {}
sn_gte24 = {}
ip_subnet = subnet.keys()
for sn in ip_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): #略过私网
continue
# return 'False'
elif (netMask < 16): #暂时不处理
sn_lte16[sn] = subnet[sn]
# return 'False'
elif (netMask == 16):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask >= 21 and netMask <= 24):
lpmdict[sn] = subnet[sn]
idx = pow(2, 24 - netMask) - 1
# print idx
ip_base = ip_num[2] & (255 - idx)
i = 0
while (i <= idx):
newip1 = []
ipstr1 = ''
ip_num[2] = ip_base + i
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
# print ipstr1
lpm.insert_rule(ipstr1)
i = i + 1
# elif(netMask==24):
# #/25当/24处理
# lpmdict[sn] = subnet[sn]
# newip1 = []
# newip1.append(str(ip_num[0]))
# newip1.append(str(ip_num[1]))
# newip1.append(str(ip_num[2]))
# newip1.append('*')
# ipstr1 = '.'.join(newip1)
# lpm.insert_rule(ipstr1)
elif (netMask > 24): # range match
sn_gte24[sn] = subnet[sn]
else:
#netMask>16 and netMask<21,save them
sndict[sn] = subnet[sn]
mylog.info('lpm data size: %d' % len(lpmdict))
mylog.info('remaining subnet size:%d' % len(sndict))
mylog.info('lte16 size:%d' % len(sn_lte16))
mylog.info('gte24 size:%d' % len(sn_gte24))
#save
snpath = getsavepath(fpath, 'remaining_subnet')
ltepath = getsavepath(fpath, 'lte16_subnet')
lpmpath = getsavepath(fpath, 'lpm_subnet_data')
gtepath = getsavepath(fpath, 'gte24_subnet')
if (sndict):
if (os.path.exists(snpath)):
newsndict = blacklist_tools.load_dict(snpath)
newsndict1 = dict(newsndict, **sndict) #merge
saveToJSON(newsndict1, fpath, "remaining_subnet")
else:
saveToJSON(sndict, fpath, "remaining_subnet")
if (sn_lte16):
if (os.path.exists(ltepath)):
newlte = blacklist_tools.load_dict(ltepath)
newlte16 = dict(newlte, **sn_lte16) #merge
saveToJSON(newlte16, fpath, 'lte16_subnet')
else:
saveToJSON(sn_lte16, fpath, 'lte16_subnet')
if (lpmdict):
if (os.path.exists(lpmpath)):
newlpmdict = blacklist_tools.load_dict(lpmpath)
newlpmdict1 = dict(newlpmdict, **lpmdict) #merge
saveToJSON(newlpmdict1, fpath, 'lpm_subnet_data')
else:
saveToJSON(lpmdict, fpath, 'lpm_subnet_data')
if (sn_gte24):
if (os.path.exists(gtepath)):
newlpmdict = blacklist_tools.load_dict(gtepath)
newlpmdict1 = dict(newlpmdict, **sn_gte24) #merge
saveToJSON(newlpmdict1, fpath, 'gte24_subnet')
else:
saveToJSON(sn_gte24, fpath, 'gte24_subnet')
sn_gte24 = dict(sn_gte24, **sndict) # merge
#match
subnet_result = []
for ips in es_ip:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
subnet_result.append({ips: 'subnet_lpm_match'})
return subnet_result, sndict, sn_lte16, sn_gte24
allThreatIP = dict(allThreatIP, **tmpThreatIP)
except Exception, e:
mylog.error('[mal_ip] Check blacklist error:{}'.format(e))
else:
mylog.warning('[mal_ip] Check blapwcklist warn: no files!')
#blacklist match，本地黑名单检查
blflg, blackpath = parser_config.get_self_filelist('blacklist')
if (blflg == 1):
if (os.path.exists(blackpath)): # 黑名单目录是否存在
filelist = get_all_file(blackpath)
# 黑名单处理与普通文件不一样。
# check each file
for fname in filelist:
fpath = blackpath + fname
# mylog.info(' -*-*-*-*- local file:{} -*-*-*-*-'.format(fname))
dataset = blacklist_tools.load_dict(fpath)
if (dataset):
try:
fullmatch, segmentmatch, subnetlpm, subnetfull = treatip(
dataset, ip_es_list)
tmpIP = insert_result(index, aggs_name, timestamp,
serverNum, dport, fullmatch,
segmentmatch, subnetlpm,
subnetfull, dataset)
if (tmpIP):
allThreatIP = dict(allThreatIP, **tmpIP)
except Exception, e:
mylog.error(
'[mal_ip] Check local blacklist error:{}'.format(
e))
time.sleep(1)
def subnet_lpm(subnet, es_ip):
mylog = blacklist_tools.getlog()
lpm.init()
sndict = {}
fpath = parser_config.get_store_path()[1]
sn_lte16 = {}
lpmdict = {}
ip_subnet = subnet.keys()
for sn in ip_subnet:
subnet_split = sn.split('/')
ip_num = ip_split_num(subnet_split[0])
netMask = int(subnet_split[1])
if (sn == '192.168.0.0/16' or sn == '172.16.0.0/12'
or sn == '10.0.0.0/8'): #略过私网
continue
# return 'False'
elif (netMask < 16): #暂时不处理
sn_lte16[sn] = subnet[sn]
# return 'False'
elif (netMask == 16):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append('*')
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
elif (netMask == 23):
lpmdict[sn] = subnet[sn]
newip1 = []
ip_num[2] = ip_num[2] | 1
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
newip2 = []
ip_num[2] = ip_num[2] & 254
newip2.append(str(ip_num[0]))
newip2.append(str(ip_num[1]))
newip2.append(str(ip_num[2]))
newip2.append('*')
ipstr2 = '.'.join(newip2)
lpm.insert_rule(ipstr2)
elif (netMask == 25 or netMask == 24):
#/25当/24处理
lpmdict[sn] = subnet[sn]
newip1 = []
newip1.append(str(ip_num[0]))
newip1.append(str(ip_num[1]))
newip1.append(str(ip_num[2]))
newip1.append('*')
ipstr1 = '.'.join(newip1)
lpm.insert_rule(ipstr1)
else:
#netMask>16 and not in [16,23,24,25],save them
sndict[sn] = subnet[sn]
mylog.info('lpm data size: %d' % len(lpmdict))
#save
snpath = getsavepath(fpath, 'remain_subnet')
ltepath = getsavepath(fpath, 'lte16_subnet')
lpmpath = getsavepath(fpath, 'lpm_subnet_data')
if (not sndict):
if (os.path.exists(snpath)):
newsndict = blacklist_tools.load_dict(snpath)
newsndict1 = dict(newsndict, **sndict) #merge
saveToJSON(newsndict1, fpath, "remain_subnet")
else:
saveToJSON(sndict, fpath, "remain_subnet")
if (not sn_lte16):
if (os.path.exists(ltepath)):
newlte = blacklist_tools.load_dict(ltepath)
newlte16 = dict(newlte, **sn_lte16) #merge
saveToJSON(newlte16, fpath, 'lte16_subnet')
else:
saveToJSON(sn_lte16, fpath, 'lte16_subnet')
if (not lpmdict):
if (os.path.exists(lpmpath)):
newlpmdict = blacklist_tools.load_dict(lpmpath)
newlpmdict1 = dict(newlpmdict, **lpmdict) #merge
saveToJSON(newlpmdict1, fpath, 'lpm_subnet_data')
else:
saveToJSON(lpmdict, fpath, 'lpm_subnet_data')
#match
subnet_result = []
for ips in es_ip:
ip_es_num = socket.ntohl(
struct.unpack("I", socket.inet_aton(str(ips)))[0])
if (lpm.search_ip(ip_es_num)):
subnet_result.append({ips: 'subnet_lpm_match'})
return subnet_result, sndict, sn_lte16