def main(): print_detail = False print_json = True for iti in range(23): if iti < 10: continue for jti in range(2): if jti == 0: hour = iti minute = 0 shour = str(hour) ehour = str(hour) sminute = '00' eminute = '30' start = str(hour) + ':00:00' end = str(hour) + ':30:00' start_time = datetime.datetime(2017, 11, 16, hour, 0, 0) stop_time = datetime.datetime(2017, 11, 16, hour, 30, 0) start_time_server = datetime.datetime(2017, 11, 16, hour - 1, 58, 0) stop_time_server = datetime.datetime(2017, 11, 16, hour, 28, 0) else: hour = iti minute = 30 shour = str(hour) ehour = str(hour + 1) sminute = '30' eminute = '00' start = str(hour) + ':30:00' end = str(hour + 1) + ':00:00' start_time = datetime.datetime(2017, 11, 16, hour, 30, 0) stop_time = datetime.datetime(2017, 11, 16, hour + 1, 0, 0) start_time_server = datetime.datetime(2017, 11, 16, hour, 28, 0) stop_time_server = datetime.datetime(2017, 11, 16, hour, 58, 0) fout_string = 'out.' + start #d = pynfdump.Dumper('/data2/datasource/',profile='16/',sources=['nfcapd.201711161000','nfcapd.201711161005']) d = pynfdump.Dumper() # ponce = 0 # d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000') # dstring = '/data2/datasource/16/' # for i in range(6): # nstr = str(i*5) # if len(nstr) < 2: # nstr = '0' + nstr # dstring += 'nfcapd.2017111610' + nstr + ':' # # dstring = dstring[:-1] dfiles = '/data2/datasource/16/nfcapd.20171116' + shour + sminute + ':nfcapd.20171116' + ehour + eminute d.set_where(start=None, end=None, dirfiles=dfiles) # d.set_where(start=None,end=None,filename='/data2/datasource/16/nfcapd.201711161000') records = d.search('proto icmp and host 166.111.8.241') fin = open('/data2/datasource/ICMP/time/' + start + '.txt', 'r') ip_dict = {} agg_dict = {} print dfiles, start for line in fin.readlines(): items = line.split('#') ip = items[0].strip() string = items[1].split(',') if ip not in ip_dict: ip_dict[ip] = { 'IN': [], 'OUT': [], 'ICMP': [string[1], string[2], string[3].strip()] } agg_dict[ip]= {\ 'IN':[],\ 'OUT':[],\ 'ICMP':[string[1],string[2],string[3].strip()],\ 'time_itv':-3,\ 'flow_time_error_in':[0,0],\ 'flow_time_error_out':[0,0],\ 'first_time_error':False,\ 'last_time_error':False,\ 'itvf':-1,\ 'itvl':-1} # -3: initialized but not assigned; -2 host unreachable -1: first_time_error(first time out is later than in) and last_time_error fin.close() for r in records: first = r['first'] last = r['last'] msec_first = r['msec_first'] msec_last = r['msec_last'] srcip = str(r['srcip']) dstip = str(r['dstip']) srcport = r['srcport'] dstport = r['dstport'] packets = r['packets'] tbytes = r['bytes'] srcip_prefix = srcip.split('.')[0] + '.' + srcip.split('.')[1] dstip_prefix = dstip.split('.')[0] + '.' + dstip.split('.')[1] # first_time = first + datetime.timedelta(microseconds = msec_first) # last_time = last + datetime.timedelta(microseconds = msec_last) if dstip in ip_dict: key = dstip flag = 'OUT' elif srcip in ip_dict: key = srcip flag = 'IN' else: continue # prtstr = srcip + ' => ' + dstip + ':' + str(dstport) + ' '+ first_time.strftime("%Y-%m-%d %H:%M:%S.%f") + ' ' + last_time.strftime("%Y-%m-%d %H:%M:%S.%f")+ ' ' + str(packets) + ' ' + str(tbytes) ip_dict[key][flag].append( Flow.IcmpFlow(srcip, srcport, dstip, dstport, first, msec_first, last, msec_last, packets, tbytes, flag)) for key in ip_dict: if len(ip_dict[key]['IN']) == 0: agg_dict[key]['time_itv'] = -2 continue if len(ip_dict[key]['OUT']) == 0: agg_dict[key]['time_itv'] = -2 continue #Here to gathering the flows ip = key srcip, srcport = ip_dict[key]['OUT'][0].get_srcip() dstip, dstport = ip_dict[key]['OUT'][0].get_dstip() lgin = len(ip_dict[key]['IN']) lgout = len(ip_dict[key]['OUT']) real_flow_dict = {'IN': [], 'OUT': []} real_flow_list = [] real_flow_dict['IN'] = [ Flow.FlowGroup(srcip, srcport, dstip, dstport) ] real_flow_dict['OUT'] = [ Flow.FlowGroup(srcip, srcport, dstip, dstport) ] i = 0 for j in range(lgout): agg_dict[key]['flow_time_error_out'][1] += 1 if ip_dict[key]['OUT'][j].get_first_time( ) > ip_dict[key]['OUT'][j].get_last_time(): agg_dict[key]['flow_time_error_out'][0] += 1 if real_flow_dict['OUT'][i].add_flow( ip_dict[key]['OUT'][j]): pass else: i += 1 real_flow_dict['OUT'].append( Flow.FlowGroup(srcip, srcport, dstip, dstport)) real_flow_dict['OUT'][i].add_flow( ip_dict[key]['OUT'][j]) # except TypeError: # print key,ip_dict[key] # print j,ip_dict[key]['OUT'][j] # exit() # for item in ip_dict[key]: # print item.print_string() # print # for item in real_flow_dict[key]: # print item.display_string(): # exit() i = 0 for j in range(lgin): agg_dict[key]['flow_time_error_in'][1] += 1 if ip_dict[key]['IN'][j].get_first_time( ) > ip_dict[key]['IN'][j].get_last_time(): agg_dict[key]['flow_time_error_in'][0] += 1 if real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j]): pass else: i += 1 real_flow_dict['IN'].append( Flow.FlowGroup(srcip, srcport, dstip, dstport)) real_flow_dict['IN'][i].add_flow(ip_dict[key]['IN'][j]) for item in real_flow_dict['OUT']: agg_dict[key]['OUT'].append(item) for item in real_flow_dict['IN']: agg_dict[key]['IN'].append(item) lenin = len(real_flow_dict['IN']) for item in real_flow_dict['OUT']: for i in range(lenin): if datetime.timedelta(0, -10, 0) < item.get_first_time( ) - real_flow_dict['IN'][i].get_first_time( ) < datetime.timedelta(0, 10, 0): real_flow_list.append({ 'IN': real_flow_dict['IN'][i], 'OUT': item }) break if i == range(lenin): pass for item in real_flow_list: if agg_dict[key]['time_itv'] in [-2, -4]: break elif agg_dict[key]['time_itv'] != -3: if item['OUT'].get_first_time() > stop_time_server: break else: agg_dict[key]['time_itv'] = -4 time_itvf = item['IN'].get_first_time( ) - item['OUT'].get_first_time() time_itvl = item['IN'].get_last_time( ) - item['OUT'].get_last_time() dtzero = datetime.timedelta(0, 0, 0) if time_itvf > dtzero: agg_dict[key][ 'itvf'] = time_itvf.seconds * 1000 + time_itvf.microseconds / 1000 else: agg_dict[key]['itvf'] = -1 if time_itvl > dtzero: agg_dict[key][ 'itvl'] = time_itvl.seconds * 1000 + time_itvl.microseconds / 1000 else: agg_dict[key]['itvl'] = -1 if time_itvf < dtzero: agg_dict[key]['first_time_error'] = True if time_itvl < dtzero: agg_dict[key]['time_itv'] = -1 agg_dict[key]['last_time_error'] = True else: agg_dict[key]['time_itv'] = time_itvl else: agg_dict[key]['time_itv'] = time_itvf if time_itvl < dtzero: agg_dict[key]['last_time_error'] = True elif time_itvf > datetime.timedelta(0, 0, 100000): if time_itvl < datetime.timedelta(0, 0, 100000): agg_dict[key]['time_itv'] = time_itvl elif datetime.timedelta( 0, 0, -50000 ) < time_itvl - time_itvf < datetime.timedelta( 0, 0, 50000): agg_dict[key]['time_itv'] = (time_itvf + time_itvl) / 2 if print_detail: p_detail(ip_dict, agg_dict, fout_string) if print_json: p_json(ip_dict, agg_dict, fout_string)
for key in ip_dict: if len(ip_dict[key]['IN']) == 0: agg_dict[key]['time_itv'] = -2 continue if len(ip_dict[key]['OUT']) == 0: agg_dict[key]['time_itv'] = -2 continue #Here to gathering the flows ip = key srcip, srcport = ip_dict[key]['OUT'][0].get_srcip() dstip, dstport = ip_dict[key]['OUT'][0].get_dstip() lgin = len(ip_dict[key]['IN']) lgout = len(ip_dict[key]['OUT']) real_flow_dict = {'IN':[],'OUT':[]} real_flow_list = [] real_flow_dict['IN'] = [Flow.FlowGroup(dstip,srcport,srcip,0)] real_flow_dict['OUT'] = [Flow.FlowGroup(srcip,srcport,dstip,dstport)] i = 0 # print lgout for j in range(lgout): agg_dict[key]['flow_time_error_out'][1] += 1 if ip_dict[key]['OUT'][j].get_first_time() > ip_dict[key]['OUT'][j].get_last_time(): agg_dict[key]['flow_time_error_out'][0] += 1 if real_flow_dict['OUT'][i].add_flow(ip_dict[key]['OUT'][j]): pass # print i # print j,i,True # print real_flow_dict['OUT'][i].display_string() else: # print j,i,False