def main(tday,index, gte, lte, aggs_name, timestamp,serverNum,dport,time_zone):
mylog = blacklist_tools.getlog()
path=parser_config.get_store_path()[1]+str(tday)+os.path.sep
cnt=0
while(cnt<8):
if(os.path.exists(path)):
filelist=get_all_file(path)
break
elif cnt==7:
#default file path
dflg, defaultpath = parser_config.get_self_filelist('defaultlist')
if(dflg==1):
filelist = get_all_file(defaultpath)
else:
filelist=[]
break
else:
# check last 7 days file
lday=tday+datetime.timedelta(-1)
path = parser_config.get_store_path()[1] + str(lday) + os.path.sep
cnt=cnt+1
# mylog.warning('no path!')
# filelist=[]
#get es list
es = ESclient(server =serverNum,port=dport)
# mylog.info('connected with es')
ip_es_list = es.get_es_ip(index,gte,lte,aggs_name,time_zone)
# mylog.info('get es data,data size:%d'%len(ip_es_list))
if(filelist):
try:
#check each file and insert match results
checkAndInsert(path,filelist,ip_es_list,index,aggs_name,timestamp,serverNum,dport)
except Exception, e:
mylog.error(e)
def treatip(dataset, es_ip):
mylog = set_logger()
full, segment, subnet = blacklist_tools.separate_ip(
dataset) #dataset is dict
# match procedure
# full match
full_list = full.keys()
# return fullmatchlist,type is list
fullmatchlist = blacklist_tools.ip_full_match(full_list, es_ip)
# segment match, segmentlist:[{ip:ipsegment},{},...]
segmentlist = blacklist_tools.int_ip_range(segment, es_ip)
subnet_lpm = {}
subnet_full = {}
sndict = {}
sn_lte16 = {}
# read conf file to choose the methods
flg_lpm, flg_full = parser_config.get_method()
if (1 == flg_lpm):
# subnet match by lpm,subnet_lpm is match results;sndict and sn_lte16 is original subnet data
#mylog.info('start lpm match')
#atime=time.time()
subnet_lpm, sndict, sn_lte16, sn_gte24 = blacklist_tools.subnet_lpm(
subnet, es_ip)
#ftime=time.time()-atime
#mylog.info('times:{}'.format(ftime))
#mylog.info('finish lpm match')
if (1 == flg_full):
#subnet match by zhou, parameters are snlist and es_ip
# mylog.info('sndict size: %d'%len(sndict))
# mylog.info('sn_lte16 size: %d' % len(sn_lte16))
#mylog.info('start range subnet match')
subnet_full = blacklist_tools.subnet_range_match(sn_gte24, es_ip)
#mylog.info('finish range subnet match')
#whitelist
wlflg, whitepath = parser_config.get_self_filelist('whitelist')
if (wlflg == 1):
#get whilelist
if (os.path.exists(whitepath)):
filelist = get_all_file(whitepath)
for fname in filelist:
fpath = whitepath + fname
#白名单读取方式不一样, whitedata is dict
whitedata = blacklist_tools.load_whitelist(fpath)
#filter procedure
fullmatchlist, segmentlist, subnet_lpm, subnet_full = blacklist_tools.whitelist_filter(
fullmatchlist, segmentlist, subnet_lpm, subnet_full,
whitedata)
else:
mylog.warn('[mal_ip] Match_insert warn: no self_whitelist_path')
# return match results
return fullmatchlist, segmentlist, subnet_lpm, subnet_full
def main(tday, index, gte, lte, aggs_name, timestamp, serverNum, dport,
time_zone, querys_str):
mylog = set_logger()
path = parser_config.get_store_path() + str(tday) + os.path.sep
cnt = 0
allThreatIP = {} # 有问题的dip
# 不联网情况下，尝试使用过去7天的数据检查
while (cnt < 8):
if (cnt < 7 and os.path.exists(path)):
filelist = get_all_file(path)
if (not filelist): # 目录没有文件则同样检查前一天的数据
lday = tday + datetime.timedelta(-1)
path = parser_config.get_store_path() + str(lday) + os.path.sep
cnt = cnt + 1
continue
else:
break
elif (cnt == 7 or not os.path.exists(path)):
#default file path，达到7天或当前目录不存在
dflg, defaultpath = parser_config.get_self_filelist('defaultlist')
if (dflg == 1):
filelist = get_all_file(defaultpath)
path = defaultpath
else:
filelist = []
break
else:
# check last 7 days file
lday = tday + datetime.timedelta(-1)
path = parser_config.get_store_path() + str(lday) + os.path.sep
cnt = cnt + 1
#get es list
es = ESclient(server=serverNum, port=dport)
# mylog.info('connected with es')
ip_es_list = es.get_es_ip(index, gte, lte, aggs_name, time_zone,
querys_str)
mylog.debug('[mal_ip] ES data size:%d ' % len(ip_es_list))
# 检查下载的网络情报
if (filelist):
try:
#check each file and insert match results
tmpThreatIP = checkAndInsert(path, filelist, ip_es_list, index,
aggs_name, timestamp, serverNum,
dport)
# mylog.info('main_insert Threat_ip size:{}'.format(len(tmpThreatIP)))
if (tmpThreatIP):
allThreatIP = dict(allThreatIP, **tmpThreatIP)
except Exception, e:
mylog.error('[mal_ip] Check blacklist error:{}'.format(e))
mylog.info('ES data size:%d ' % len(ip_es_list))
if (filelist):
try:
#check each file and insert match results
tmpThreatIP = checkAndInsert(path, filelist, ip_es_list, index,
aggs_name, timestamp, serverNum,
dport)
# mylog.info('main_insert Threat_ip size:{}'.format(len(tmpThreatIP)))
if (tmpThreatIP):
allThreatIP = dict(allThreatIP, **tmpThreatIP)
except Exception, e:
mylog.error('check blacklist:{}'.format(e))
else:
mylog.warning('no files!')
#blacklist match
blflg, blackpath = parser_config.get_self_filelist('blacklist')
if (blflg == 1):
if (os.path.exists(blackpath)):
filelist = get_all_file(blackpath)
# 黑名单处理与普通文件不一样。
# check each file
for fname in filelist:
fpath = blackpath + fname
mylog.info(' -*-*-*-*- local file:{} -*-*-*-*-'.format(fname))
dataset = blacklist_tools.load_blacklist(fpath)
if (dataset):
# msg = dataset[dataset.keys()[0]]
# get match result
try:
fullmatch, segmentmatch, subnetlpm, subnetfull = treatip(
dataset, ip_es_list)