def main(): traceDict = at.readTraceJSON(MES_TRACE_FILE) ipDict = {} countPrivate = 0 countUnkown = 0 countUniqueASN = 0 as_set = set([]) #cmdARIN = "whois " #grep = " | grep 'netname\|descr\|origin'" for pb in traceDict: for p in traceDict[pb]['ip_path']: for hop in p: if hop != '*' and hop not in ipDict: ipDict[hop]={} ipadd = ipaddress.ip_address(hop) if ipadd.is_private: ipDict[hop]['asn'] = -1 ipDict[hop]['as_name'] = 'private' ipDict[hop]['descr'] = 'private' countPrivate += 1 else: ipDict[hop] = pt.ip2asARIN(hop) if 'asn' not in ipDict[hop]: ipDict[hop]['asn'] = -2 countUnkown += 1 elif ipDict[hop]['asn'] not in as_set: as_set.add(ipDict[hop]['asn']) countUniqueASN += 1 if 'as_name' not in ipDict[hop]: ipDict[hop]['as_name'] = 'unknown' if 'descr' not in ipDict[hop]: ipDict[hop]['descr'] = 'unknown' if not ipDict[hop]: ipDict[hop]['asn'] = -2 ipDict[hop]['as_name'] = 'unknown' ipDict[hop]['descr'] = 'unknown' countUnkown += 1 f = open(DIC_IP_2_ASN_ARIN, 'w') for hop in ipDict: line = '%s;%d;%s;%s\n' % (hop, ipDict[hop]['asn'], ipDict[hop]['as_name'], ipDict[hop]['descr']) f.write(line) f.close() print("%d entries in all.\n\ %d of them are private address.\n\ %d don't have resolution results.\n\ %d unique ASN appeared." \ % (len(ipDict), countPrivate, countUnkown, countUniqueASN))
def main(argv): traceflag =False if len(argv) != 2: print "Usage: python cleaning.py t/p(trace/ping) filename" exit() trace = argv[0] if trace in TRACE: traceflag = True filename = argv[1] if not os.path.isfile(filename): print "Measurement file %s doesn't exist." % filename exit() pb_to_rm = set([]) if traceflag: print "Traceroute trace.\n" + \ "An ip-path\n" + \ "- ends with *;\n" + \ "- or contains more than %f *;\n" % INV_PATH + \ "- or contains five or more consecutive *" + \ "is considered invalid." trace_dict = at.readTraceJSON(filename) min_len = LEN_P * TRACE_LEN max_intv = INTV_MX * TRACE_INTV inv_len = INV_TRACE * TRACE_LEN #fsave = 'trace_rm.txt' fsave = PROBE_ID_TRACE_RM_FILE fval = PROBE_ID_TRACE_VAL_FILE for pbid in trace_dict: path_val_flag = [] for p in trace_dict[pbid]['ip_path']: if path_val(p, INV_PATH): path_val_flag.append(1) else: path_val_flag.append(-1) trace_dict[pbid]['path_val'] = path_val_flag val_check = 'path_val' else: print "Ping trace\n" + \ "An RTT measurement\n" + \ "- equals -1; \n" + \ "- missing value;\n" + \ "- contains err field; \n" + \ "is considered invalid." trace_dict = at.readPingJSON(filename) min_len = LEN_P * PING_LEN max_intv = INTV_MX * PING_INTV inv_len = INV_PING * PING_LEN val_check = 'avg' #fsave = 'ping_rm.txt' fsave = PROBE_ID_PING_RM_FILE fval = PROBE_ID_PING_VAL_FILE print "\nCleaning criteria:\n\ Minimum length: %f,\n\ Maximum neighbour interval: %f,\n\ Maximum invalid values: %f." % (min_len, max_intv, inv_len) for pbid in trace_dict: if not plft_stab(trace_dict[pbid]['time_epc'], max_intv, min_len): pb_to_rm.add(pbid) if trace_dict[pbid][val_check].count(-1) > inv_len: pb_to_rm.add(pbid) if pb_to_rm: print "Probes to be removed:" print "{id:<7}{len_:>10}{intv:>10}{invd:>10}".format(id='ID', len_='Len.', intv='Max. Intv', invd='# Invd.') for pb in pb_to_rm: print "{id:>7d}{len_:>10d}{intv:>10d}{invd:>10d}".format(id=pb, len_=len(trace_dict[pb]['time_epc']), intv=max(interv(trace_dict[pb]['time_epc'])), invd=trace_dict[pb][val_check].count(-1)) clean_trace = removekey(trace_dict, list(pb_to_rm)) else: clean_trace = trace_dict.copy() print "%d probes in all, %d probes after cleaning" % (len(trace_dict), len(clean_trace)) f = open(fsave, 'w') for pb in list(pb_to_rm): f.write("%d\n" % pb) f.close() f = open(fval, 'w') for pb in clean_trace: f.write("%d\n" % pb) f.close()
return pbMeta def readNoMercy(file): noMercy = set() f = open(file, 'r') for line in f: cols = line.split(',') if len(cols) >= 2: a = int(cols[0].strip()) b = int(cols[1].strip()) noMercy.add((a,b)) noMercy.add((b,a)) return noMercy alltrace = at.readTraceJSON(MES_TRACE_FILE) valid_id = read_pdid(PROBE_ID_VALID_FILE) pbAct = alltrace.keys() pbVal = list(set(valid_id) & set(pbAct)) clean_trace = {k: alltrace[k] for k in pbVal} ipDictCY = pt.loadIPDict(DIC_IP_2_ASN) ipDictMG = pt.loadIPDict(DIC_IP_2_ASN_MERG) pbASN = read_pbMeta(PROBE_META_FILE) noMercy = readNoMercy(PAIR_NO_MERCY) countLoopCY = 0 countLoopMG = 0 pathCountTotal = 0 pathEq = 0 pathNq = 0