def main():
traceDict = at.readTraceJSON(MES_TRACE_FILE)
ipDict = {}
countPrivate = 0
countUnkown = 0
countUniqueASN = 0
as_set = set([])
#cmdARIN = "whois "
#grep = " | grep 'netname\|descr\|origin'"
for pb in traceDict:
for p in traceDict[pb]['ip_path']:
for hop in p:
if hop != '*' and hop not in ipDict:
ipDict[hop]={}
ipadd = ipaddress.ip_address(hop)
if ipadd.is_private:
ipDict[hop]['asn'] = -1
ipDict[hop]['as_name'] = 'private'
ipDict[hop]['descr'] = 'private'
countPrivate += 1
else:
ipDict[hop] = pt.ip2asARIN(hop)
if 'asn' not in ipDict[hop]:
ipDict[hop]['asn'] = -2
countUnkown += 1
elif ipDict[hop]['asn'] not in as_set:
as_set.add(ipDict[hop]['asn'])
countUniqueASN += 1
if 'as_name' not in ipDict[hop]:
ipDict[hop]['as_name'] = 'unknown'
if 'descr' not in ipDict[hop]:
ipDict[hop]['descr'] = 'unknown'
if not ipDict[hop]:
ipDict[hop]['asn'] = -2
ipDict[hop]['as_name'] = 'unknown'
ipDict[hop]['descr'] = 'unknown'
countUnkown += 1
f = open(DIC_IP_2_ASN_ARIN, 'w')
for hop in ipDict:
line = '%s;%d;%s;%s\n' % (hop, ipDict[hop]['asn'], ipDict[hop]['as_name'], ipDict[hop]['descr'])
f.write(line)
f.close()
print("%d entries in all.\n\
%d of them are private address.\n\
%d don't have resolution results.\n\
%d unique ASN appeared." \
% (len(ipDict), countPrivate, countUnkown, countUniqueASN))
def main(argv):
traceflag =False
if len(argv) != 2:
print "Usage: python cleaning.py t/p(trace/ping) filename"
exit()
trace = argv[0]
if trace in TRACE:
traceflag = True
filename = argv[1]
if not os.path.isfile(filename):
print "Measurement file %s doesn't exist." % filename
exit()
pb_to_rm = set([])
if traceflag:
print "Traceroute trace.\n" + \
"An ip-path\n" + \
"- ends with *;\n" + \
"- or contains more than %f *;\n" % INV_PATH + \
"- or contains five or more consecutive *" + \
"is considered invalid."
trace_dict = at.readTraceJSON(filename)
min_len = LEN_P * TRACE_LEN
max_intv = INTV_MX * TRACE_INTV
inv_len = INV_TRACE * TRACE_LEN
#fsave = 'trace_rm.txt'
fsave = PROBE_ID_TRACE_RM_FILE
fval = PROBE_ID_TRACE_VAL_FILE
for pbid in trace_dict:
path_val_flag = []
for p in trace_dict[pbid]['ip_path']:
if path_val(p, INV_PATH):
path_val_flag.append(1)
else:
path_val_flag.append(-1)
trace_dict[pbid]['path_val'] = path_val_flag
val_check = 'path_val'
else:
print "Ping trace\n" + \
"An RTT measurement\n" + \
"- equals -1; \n" + \
"- missing value;\n" + \
"- contains err field; \n" + \
"is considered invalid."
trace_dict = at.readPingJSON(filename)
min_len = LEN_P * PING_LEN
max_intv = INTV_MX * PING_INTV
inv_len = INV_PING * PING_LEN
val_check = 'avg'
#fsave = 'ping_rm.txt'
fsave = PROBE_ID_PING_RM_FILE
fval = PROBE_ID_PING_VAL_FILE
print "\nCleaning criteria:\n\
Minimum length: %f,\n\
Maximum neighbour interval: %f,\n\
Maximum invalid values: %f." % (min_len, max_intv, inv_len)
for pbid in trace_dict:
if not plft_stab(trace_dict[pbid]['time_epc'], max_intv, min_len):
pb_to_rm.add(pbid)
if trace_dict[pbid][val_check].count(-1) > inv_len:
pb_to_rm.add(pbid)
if pb_to_rm:
print "Probes to be removed:"
print "{id:<7}{len_:>10}{intv:>10}{invd:>10}".format(id='ID', len_='Len.',
intv='Max. Intv', invd='# Invd.')
for pb in pb_to_rm:
print "{id:>7d}{len_:>10d}{intv:>10d}{invd:>10d}".format(id=pb,
len_=len(trace_dict[pb]['time_epc']),
intv=max(interv(trace_dict[pb]['time_epc'])),
invd=trace_dict[pb][val_check].count(-1))
clean_trace = removekey(trace_dict, list(pb_to_rm))
else:
clean_trace = trace_dict.copy()
print "%d probes in all, %d probes after cleaning" % (len(trace_dict), len(clean_trace))
f = open(fsave, 'w')
for pb in list(pb_to_rm):
f.write("%d\n" % pb)
f.close()
f = open(fval, 'w')
for pb in clean_trace:
f.write("%d\n" % pb)
f.close()
return pbMeta
def readNoMercy(file):
noMercy = set()
f = open(file, 'r')
for line in f:
cols = line.split(',')
if len(cols) >= 2:
a = int(cols[0].strip())
b = int(cols[1].strip())
noMercy.add((a,b))
noMercy.add((b,a))
return noMercy
alltrace = at.readTraceJSON(MES_TRACE_FILE)
valid_id = read_pdid(PROBE_ID_VALID_FILE)
pbAct = alltrace.keys()
pbVal = list(set(valid_id) & set(pbAct))
clean_trace = {k: alltrace[k] for k in pbVal}
ipDictCY = pt.loadIPDict(DIC_IP_2_ASN)
ipDictMG = pt.loadIPDict(DIC_IP_2_ASN_MERG)
pbASN = read_pbMeta(PROBE_META_FILE)
noMercy = readNoMercy(PAIR_NO_MERCY)
countLoopCY = 0
countLoopMG = 0
pathCountTotal = 0
pathEq = 0
pathNq = 0
