def main():
print_detail = False
print_json = True
for iti in range(23):
if iti < 10:
continue
for jti in range(2):
if jti == 0:
hour = iti
minute = 0
shour = str(hour)
ehour = str(hour)
sminute = '00'
eminute = '30'
start = str(hour) + ':00:00'
end = str(hour) + ':30:00'
start_time = datetime.datetime(2017, 11, 16, hour, 0, 0)
stop_time = datetime.datetime(2017, 11, 16, hour, 30, 0)
start_time_server = datetime.datetime(2017, 11, 16, hour - 1,
58, 0)
stop_time_server = datetime.datetime(2017, 11, 16, hour, 28, 0)
else:
hour = iti
minute = 30
shour = str(hour)
ehour = str(hour + 1)
sminute = '30'
eminute = '00'
start = str(hour) + ':30:00'
end = str(hour + 1) + ':00:00'
start_time = datetime.datetime(2017, 11, 16, hour, 30, 0)
stop_time = datetime.datetime(2017, 11, 16, hour + 1, 0, 0)
start_time_server = datetime.datetime(2017, 11, 16, hour, 28,
0)
stop_time_server = datetime.datetime(2017, 11, 16, hour, 58, 0)
fout_string = 'out.' + start
#d = pynfdump.Dumper('/data2/datasource/',profile='16/',sources=['nfcapd.201711161000','nfcapd.201711161005'])
d = pynfdump.Dumper()
# ponce = 0
# d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000')
# dstring = '/data2/datasource/16/'
# for i in range(6):
# nstr = str(i*5)
# if len(nstr) < 2:
# nstr = '0' + nstr
# dstring += 'nfcapd.2017111610' + nstr + ':'
#
# dstring = dstring[:-1]
dfiles = '/data2/datasource/16/nfcapd.20171116' + shour + sminute + ':nfcapd.20171116' + ehour + eminute
d.set_where(start=None, end=None, dirfiles=dfiles)
# d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000')
records = d.search('proto icmp and host 166.111.8.241')
fin = open('/data2/datasource/ICMP/time/' + start + '.txt', 'r')
ip_dict = {}
agg_dict = {}
print dfiles, start
for line in fin.readlines():
items = line.split('#')
ip = items[0].strip()
string = items[1].split(',')
if ip not in ip_dict:
ip_dict[ip] = {
'IN': [],
'OUT': [],
'ICMP': [string[1], string[2], string[3].strip()]
}
agg_dict[ip]= {\
'IN':[],\
'OUT':[],\
'ICMP':[string[1],string[2],string[3].strip()],\
'time_itv':-3,\
'flow_time_error_in':[0,0],\
'flow_time_error_out':[0,0],\
'first_time_error':False,\
'last_time_error':False,\
'itvf':-1,\
'itvl':-1}
# -3: initialized but not assigned; -2 host unreachable -1: first_time_error(first time out is later than in) and last_time_error
fin.close()
for r in records:
first = r['first']
last = r['last']
msec_first = r['msec_first']
msec_last = r['msec_last']
srcip = str(r['srcip'])
dstip = str(r['dstip'])
srcport = r['srcport']
dstport = r['dstport']
packets = r['packets']
tbytes = r['bytes']
srcip_prefix = srcip.split('.')[0] + '.' + srcip.split('.')[1]
dstip_prefix = dstip.split('.')[0] + '.' + dstip.split('.')[1]
# first_time = first + datetime.timedelta(microseconds = msec_first)
# last_time = last + datetime.timedelta(microseconds = msec_last)
if dstip in ip_dict:
key = dstip
flag = 'OUT'
elif srcip in ip_dict:
key = srcip
flag = 'IN'
else:
continue
# prtstr = srcip + ' => ' + dstip + ':' + str(dstport) + ' '+ first_time.strftime("%Y-%m-%d %H:%M:%S.%f") + ' ' + last_time.strftime("%Y-%m-%d %H:%M:%S.%f")+ ' ' + str(packets) + ' ' + str(tbytes)
ip_dict[key][flag].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
for key in ip_dict:
if len(ip_dict[key]['IN']) == 0:
agg_dict[key]['time_itv'] = -2
continue
if len(ip_dict[key]['OUT']) == 0:
agg_dict[key]['time_itv'] = -2
continue
#Here to gathering the flows
ip = key
srcip, srcport = ip_dict[key]['OUT'][0].get_srcip()
dstip, dstport = ip_dict[key]['OUT'][0].get_dstip()
lgin = len(ip_dict[key]['IN'])
lgout = len(ip_dict[key]['OUT'])
real_flow_dict = {'IN': [], 'OUT': []}
real_flow_list = []
real_flow_dict['IN'] = [
Flow.FlowGroup(srcip, srcport, dstip, dstport)
]
real_flow_dict['OUT'] = [
Flow.FlowGroup(srcip, srcport, dstip, dstport)
]
i = 0
for j in range(lgout):
agg_dict[key]['flow_time_error_out'][1] += 1
if ip_dict[key]['OUT'][j].get_first_time(
) > ip_dict[key]['OUT'][j].get_last_time():
agg_dict[key]['flow_time_error_out'][0] += 1
if real_flow_dict['OUT'][i].add_flow(
ip_dict[key]['OUT'][j]):
pass
else:
i += 1
real_flow_dict['OUT'].append(
Flow.FlowGroup(srcip, srcport, dstip, dstport))
real_flow_dict['OUT'][i].add_flow(
ip_dict[key]['OUT'][j])
# except TypeError:
# print key,ip_dict[key]
# print j,ip_dict[key]['OUT'][j]
# exit()
# for item in ip_dict[key]:
# print item.print_string()
# print
# for item in real_flow_dict[key]:
# print item.display_string():
# exit()
i = 0
for j in range(lgin):
agg_dict[key]['flow_time_error_in'][1] += 1
if ip_dict[key]['IN'][j].get_first_time(
) > ip_dict[key]['IN'][j].get_last_time():
agg_dict[key]['flow_time_error_in'][0] += 1
if real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j]):
pass
else:
i += 1
real_flow_dict['IN'].append(
Flow.FlowGroup(srcip, srcport, dstip, dstport))
real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j])
for item in real_flow_dict['OUT']:
agg_dict[key]['OUT'].append(item)
for item in real_flow_dict['IN']:
agg_dict[key]['IN'].append(item)
lenin = len(real_flow_dict['IN'])
for item in real_flow_dict['OUT']:
for i in range(lenin):
if datetime.timedelta(0, -10, 0) < item.get_first_time(
) - real_flow_dict['IN'][i].get_first_time(
) < datetime.timedelta(0, 10, 0):
real_flow_list.append({
'IN': real_flow_dict['IN'][i],
'OUT': item
})
break
if i == range(lenin):
pass
for item in real_flow_list:
if agg_dict[key]['time_itv'] in [-2, -4]:
break
elif agg_dict[key]['time_itv'] != -3:
if item['OUT'].get_first_time() > stop_time_server:
break
else:
agg_dict[key]['time_itv'] = -4
time_itvf = item['IN'].get_first_time(
) - item['OUT'].get_first_time()
time_itvl = item['IN'].get_last_time(
) - item['OUT'].get_last_time()
dtzero = datetime.timedelta(0, 0, 0)
if time_itvf > dtzero:
agg_dict[key][
'itvf'] = time_itvf.seconds * 1000 + time_itvf.microseconds / 1000
else:
agg_dict[key]['itvf'] = -1
if time_itvl > dtzero:
agg_dict[key][
'itvl'] = time_itvl.seconds * 1000 + time_itvl.microseconds / 1000
else:
agg_dict[key]['itvl'] = -1
if time_itvf < dtzero:
agg_dict[key]['first_time_error'] = True
if time_itvl < dtzero:
agg_dict[key]['time_itv'] = -1
agg_dict[key]['last_time_error'] = True
else:
agg_dict[key]['time_itv'] = time_itvl
else:
agg_dict[key]['time_itv'] = time_itvf
if time_itvl < dtzero:
agg_dict[key]['last_time_error'] = True
elif time_itvf > datetime.timedelta(0, 0, 100000):
if time_itvl < datetime.timedelta(0, 0, 100000):
agg_dict[key]['time_itv'] = time_itvl
elif datetime.timedelta(
0, 0, -50000
) < time_itvl - time_itvf < datetime.timedelta(
0, 0, 50000):
agg_dict[key]['time_itv'] = (time_itvf +
time_itvl) / 2
if print_detail:
p_detail(ip_dict, agg_dict, fout_string)
if print_json:
p_json(ip_dict, agg_dict, fout_string)
def search(ip_list):
middle_print = [
False, False, False, False
] #Prior one is middlefile, latter one is middlefile without those without IN or OUT
middle_process = True
if middle_print[0]:
fout0 = open('middlefile_total.txt', 'w')
if middle_print[1]:
fout1 = open('middlefile_only_O', 'w')
if middle_print[2]:
fout2 = open('middlefile_not_equal', 'w')
if middle_print[3]:
fout3 = open('middlefile_equal', 'w')
logger = logging.getLogger()
hdlr = logging.StreamHandler()
logger.addHandler(hdlr)
logger.setLevel(logging.WARNING)
InnerIP_list = [
'166.111', '59.66', '101.5', '101.6', '183.172', '183.173', '118.229',
'202.38', '202.112'
]
ipdict_dict = {}
datain = pynfdump.Dumper()
datain.set_where(
start=None,
end=None,
filename='/lrjapps/netflowdata/datasource/flowdata/nfcapd.201705171800'
)
query = ""
for ip in ip_list:
query = query + ' or host ' + ip
query = 'proto icmp and (' + query[4:] + ')'
logger.debug(query)
records = datain.search(query)
for r in records:
af = r['af']
first = r['first']
last = r['last']
msec_first = r['msec_first']
msec_last = r['msec_last']
proto = r['prot']
srcip = str(r['srcip'])
srcport = r['srcport']
dstip = str(r['dstip'])
dstport = r['dstport']
srcas = r['srcas']
dstas = r['dstas']
packets = r['packets']
tbytes = r['bytes']
srcip_prefix = srcip.split('.')[0] + '.' + srcip.split('.')[1]
dstip_prefix = dstip.split('.')[0] + '.' + dstip.split('.')[1]
if srcip in ip_list and dstip_prefix in InnerIP_list:
flag = 'IN'
elif dstip in ip_list and srcip_prefix in InnerIP_list:
flag = 'OUT'
elif srcip not in ip_list and dstip not in ip_list:
logging.error(srcip + ' ' + dscip +
' UNEXCEPTED source/destination address')
continue
else: #This means srcip or dstip is in ip_list, but dstip/srcip is located out of campus
continue
if flag == 'IN':
key = (dstip, srcip) #Tsinghua IP is [0] and target IP is [1]
else:
key = (srcip, dstip)
if key not in ipdict_dict:
ipdict_dict[key] = {'IN': [], 'OUT': []}
if flag == 'IN':
ipdict_dict[key]['IN'].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
else:
ipdict_dict[key]['OUT'].append(
Flow.IcmpFlow(srcip, srcport, dstip, dstport, first,
msec_first, last, msec_last, packets, tbytes,
flag))
if middle_print[0]:
for key in ipdict_dict:
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout0.write(prtln)
for item in ipdict_dict[key]['IN']:
fout0.write('\t\t' + item.display_string() + '\n')
fout0.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout0.write('\t\t' + item.display_string() + '\n')
fout0.write('\n')
fout0.close()
if middle_print[1]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) == 0:
prtln = key[0] + ' => ' + key[1] + '\n' + '\tOUT:\n'
fg = 0
for item in ipdict_dict[key]['OUT']:
if item.display_dict()['type'] == 'REQUEST_OUT':
if fg == 0:
fout1.write(prtln)
fg = 1
fout1.write('\t\t' + item.display_string() + '\n')
fout1.write('\n')
fout1.close()
if middle_print[2]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) == len(
ipdict_dict[key]['OUT']) or len(
ipdict_dict[key]['IN']) == 0:
continue
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout2.write(prtln)
for item in ipdict_dict[key]['IN']:
fout2.write('\t\t' + item.display_string() + '\n')
fout2.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout2.write('\t\t' + item.display_string() + '\n')
fout2.write('\n')
fout2.close()
if middle_print[3]:
for key in ipdict_dict:
if len(ipdict_dict[key]['IN']) != len(ipdict_dict[key]['OUT']):
continue
prtln = key[0] + ' => ' + key[1] + '\n' + '\tIN:\n'
fout3.write(prtln)
for item in ipdict_dict[key]['IN']:
fout3.write('\t\t' + item.display_string() + '\n')
fout3.write('\tOUT:\n')
for item in ipdict_dict[key]['OUT']:
fout3.write('\t\t' + item.display_string() + '\n')
fout3.write('\n')
fout3.close()
if middle_process:
middle_process_f(ipdict_dict)
second = int(hms[2])
msec_last = int(l.split('.')[1][:-3])
last = datetime.datetime(year,month,day,hour,minite,second)
packets = int(r[4])
tbytes = int(r[5])
if dstip in ip_dict:
flag = 'OUT'
elif srcip in ip_dict:
flag = 'IN'
else:
continue
ip_dict[key][flag].append(Flow.IcmpFlow(srcip,srcport,dstip,dstport,first,msec_first,last,msec_last,packets,tbytes,flag))
for key in ip_dict:
if len(ip_dict[key]['IN']) == 0:
agg_dict[key]['time_itv'] = -2
continue
if len(ip_dict[key]['OUT']) == 0:
agg_dict[key]['time_itv'] = -2
continue
#Here to gathering the flows
ip = key
srcip, srcport = ip_dict[key]['OUT'][0].get_srcip()
dstip, dstport = ip_dict[key]['OUT'][0].get_dstip()
lgin = len(ip_dict[key]['IN'])
lgout = len(ip_dict[key]['OUT'])
real_flow_dict = {'IN':[],'OUT':[]}
