def parse_caida_json_streaming(fname): num_trcrts = 0 with open(fname) as fi: dest_based_aspaths = {} for line in fi: num_trcrts += 1 try: trcrt = json.loads(line) except ValueError: continue if trcrt['stop_reason'] != 'COMPLETED': continue src = trcrt['src'] dst = trcrt['dst'] ixp_match = ixp.ixp_radix.search_best(dst) if ixp_match: continue dst_asn = ip2asn.ip2asn_bgp(dst) src_asn = ip2asn.ip2asn_bgp(src) if not dst_asn: continue if dst_asn not in dest_based_aspaths: dest_based_aspaths[dst_asn] = [] aslinks = [] last_hop_nr = None this_hop_nr = None this_hop_asn = None last_hop_asn = None for hop in trcrt['hops']: addr = hop['addr'] ixp_match = ixp.ixp_radix.search_best(addr) if ixp_match: continue asn = ip2asn.ip2asn_bgp(addr) if asn in ixp.IXPs: continue last_hop_nr = this_hop_nr this_hop_nr = hop['probe_ttl'] last_hop_asn = this_hop_asn this_hop_asn = asn if this_hop_asn and last_hop_asn: if last_hop_asn != this_hop_asn: if (this_hop_nr - last_hop_nr) == 1: link_type = 'd' else: link_type = 'i' link = (int(last_hop_asn), int(this_hop_asn), link_type) aslinks.append(link) if aslinks: dest_based_aspaths[dst_asn].append(aslinks) print len(dest_based_aspaths) print num_trcrts return dest_based_aspaths
def parse_caida_json_streaming(fname): mmt_path_list = [] with open(fname) as fi: dest_based_aspaths = {} for line in fi: trcrt = json.loads(line) if trcrt['stop_reason'] != 'COMPLETED': continue src = trcrt['src'] dst = trcrt['dst'] dst_asn = ip2asn.ip2asn_bgp(dst) src_asn = ip2asn.ip2asn_bgp(src) if src_asn and dst_asn: mmt_path_list.append((int(src_asn), int(dst_asn))) return mmt_path_list
def get_pc_or_bgp_path(src_asn, dst_pref, portnum, measured_path): typ = None dst_asn = ip2asn.ip2asn_bgp(dst_pref.split('_')[0]) path_key = "%s-%s" % (src_asn, dst_pref) if path_key in mc and mc.get(path_key): return paths = get_pc_path(src_asn, dst_pref) if not paths: if dst_asn in ip2asn.asn_to_prefs: prefs = ip2asn.asn_to_prefs[dst_asn] else: prefs = [] sister_paths = [] while prefs: sister_pref = prefs.pop(0) print "trying sister pref", sister_pref sister_pref = sister_pref.replace('/', '_') sister_paths = get_pc_path(src_asn, sister_pref, add_single_homed=False) if sister_paths: break if not sister_paths: bgp_path = get_bgp_path(src_asn, dst_asn, portnum) if not bgp_path: mc.set(path_key, "") return if bgp_path.split()[-1] != dst_asn: print "bgpsim: error, not querying" mc.set(path_key, "") return spliced_paths = get_spliced_bgp_pathcache(dst_pref, src_asn, bgp_path) if sister_paths: paths = sister_paths del sister_paths typ = "sis" elif spliced_paths: paths = spliced_paths del spliced_paths typ = "spliced" else: typ = "pc" if not paths: mc.set(path_key, ([], measured_path, -1, -1, None)) return print "Got these as paths:", paths, typ, "for query:", src_asn, dst_pref pred_path = paths[0] if not pred_path: mc.set(path_key, ([], measured_path, -1, -1, None)) #mc.set(path_key, ()) return rank, prob = find_rank_prob(measured_path, paths, dst_asn) mc.set(path_key, (paths, measured_path, rank, prob, typ))
def compute_dest_based_graphs(msms): measured_paths = [] for msm in msms: info = parse.mmt_info(msm) # Start time should not be < Jan 1st 2016 if info['start_time'] < 1451606400: continue if info['type']['af'] != 4: continue dst_asn = int(info['dst_asn']) data = parse.parse_msm_trcrt(msm, count=1000) for d in data: src_asn = ip2asn.ip2asn_bgp(d['from']) if not src_asn: continue measured_paths.append((int(src_asn), dst_asn)) return list(frozenset(measured_paths))
def wrap_function(qr_set, portnum): pid = None try: current = mp.current_process() identity = current._identity #identity = (1,) print "ID:", identity proc = Popen([ 'mono', 'bgp_sim/TestingApplication/bin/Release/TestingApplication.exe', '-server%d' % (portnum), '20160101-cyclops', 'bgp_sim/precomp/US-precomp367.txt', 'bgp_sim/cache/exit_asns.txt' ], preexec_fn=os.setsid, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) time.sleep(10) pid = proc.pid mc_bgpsim_pid.set(str(portnum), str(pid)) for query in qr_set: src_asn = int(query.split('-')[0]) dst_pref = query.split('-')[1] dst_asn = ip2asn.ip2asn_bgp(dst_pref.split('_')[0]) meas_path = query.split('-')[-1] path_key = "%s-%s" % (src_asn, dst_pref) if path_key in mc and mc.get(path_key): print "Found it in the cache", path_key continue get_pc_or_bgp_path(src_asn, dst_pref, portnum, meas_path) except: print "".join(traceback.format_exception(*sys.exc_info())) try: proc.kill() os.killpg(pid, signal.SIGTERM) os.system("kill -9 %s" % pid) mc_bgpsim_pid.set(str(portnum), None) except: pass return pid
def get_asn(ip): try: return ip2asn_bgp(ip) except Exception as e: agenda.subfailure(f"ip {ip}: error {e}") return None
def parse_caida_json_streaming(fname): global num_mmts print num_mmts num_trcrts = 0 with open(fname) as fi: dest_based_aspaths = {} for line in fi: num_trcrts += 1 try: trcrt = json.loads(line) except: continue if trcrt['stop_reason'] != 'COMPLETED': continue num_mmts += 1 src = trcrt['src'] dst = trcrt['dst'] rnode = ip2asn.ip_to_pref(dst) if rnode: dst_prefix = rnode.prefix.replace('/', '_') else: continue ixp_match = ixp.ixp_radix.search_best(dst) if ixp_match: continue dst_asn = int(ip2asn.ip2asn_bgp(dst)) src_asn = ip2asn.ip2asn_bgp(src) if not dst_asn: continue if (dst_asn, dst_prefix) not in dest_based_aspaths: dest_based_aspaths[(dst_asn, dst_prefix)] = [] aslinks = [] last_hop_nr = None this_hop_nr = None this_hop_asn = None last_hop_asn = None for hop in trcrt['hops']: addr = hop['addr'] ixp_match = ixp.ixp_radix.search_best(addr) if ixp_match: continue asn = ip2asn.ip2asn_bgp(addr) if not asn: continue if asn in ixp.IXPs: continue last_hop_nr = this_hop_nr this_hop_nr = hop['probe_ttl'] last_hop_asn = this_hop_asn this_hop_asn = asn if this_hop_asn and last_hop_asn: if last_hop_asn != this_hop_asn: if (this_hop_nr - last_hop_nr) == 1: link_type = 'd' else: link_type = 'i' link = (int(last_hop_asn), int(this_hop_asn), link_type) aslinks.append(link) if aslinks: dest_based_aspaths[(int(dst_asn), dst_prefix)].append(aslinks) #print len(dest_based_aspaths) #print num_trcrts return dest_based_aspaths
return True else: if dst in all_graphs_bgp: gr = all_graphs_bgp[asn] if src in gr.nodes: return True return False NUM_CONTENT = 500 websites = alexa.top_list(NUM_CONTENT) content_asns = [] content_ips = {} for w in websites: try: asn = int(ip2asn.ip2asn_bgp(socket.gethostbyname(w[1]))) content_asns.append(asn) content_ips[asn] = socket.gethostbyname(w[1]) except: print w continue content_asns = list(set(content_asns)) f = open("data/aspop") entries = list() for table in f: records = table.split("[") for record in records: record = record.split("]")[0] entry = dict()
msms = [(59, 8075, 3793363), (73, 8075, 3793364), (262438, 8075, 3793366), (553, 8075, 3793367), (681, 8075, 3793368), (59, 23820, 3793369), (73, 23820, 3793370), (553, 23820, 3793371), (681, 23820, 3793372), (271, 23820, 3793375), (59, 22414, 3793376), (73, 22414, 3793377), (553, 22414, 3793378), (681, 22414, 3793380), (271, 22414, 3793381), (59, 62787, 3793382), (73, 62787, 3793383), (553, 62787, 3793386), (681, 62787, 3793387), (271, 62787, 3793388), (59, 47764, 3793389), (327750, 47764, 3793390), (73, 47764, 3793391), (196705, 47764, 3793392), (239, 47764, 3793393), (59, 13335, 3793394), (73, 13335, 3793395), (553, 13335, 3793409), (681, 13335, 3793410), (33480, 13335, 3793411), (59, 23576, 3793412), (73, 23576, 3793413), (553, 23576, 3793414), (681, 23576, 3793415), (271, 23576, 3793417), (59, 30361, 3793418), (73, 30361, 3793419), (553, 30361, 3793420), (681, 30361, 3793421), (197422, 30361, 3793422), (251, 14618, 3793423), (553, 14618, 3793425), (197422, 14618, 3793426), (1267, 14618, 3793427), (34177, 14618, 3793429), (59, 16798, 3793430), (73, 16798, 3793431), (553, 16798, 3793432), (681, 16798, 3793433), (1267, 16798, 3793434), (1267, 46489, 3793441), (1764, 46489, 3793442), (35244, 46489, 3793443), (35540, 46489, 3793444), (3242, 46489, 3793446), (59, 32934, 3793447), (73, 32934, 3793448), (251, 32934, 3793449), (262438, 32934, 3793450), (196965, 32934, 3793451), (59, 23724, 3793452), (327750, 23724, 3793453), (73, 23724, 3793454), (196705, 23724, 3793455), (251, 23724, 3793456), (59, 45102, 3793457), (73, 45102, 3793458), (553, 45102, 3793459), (681, 45102, 3793460), (1267, 45102, 3793463), (59, 10929, 3793465), (73, 10929, 3793466), (553, 10929, 3793467), (681, 10929, 3793468), (197422, 10929, 3793469), (59, 47541, 3793470), (73, 47541, 3793471), (553, 47541, 3793472), (681, 47541, 3793473), (197422, 47541, 3793474), (59, 13238, 3793475), (73, 13238, 3793476), (553, 13238, 3793477), (681, 13238, 3793478), (271, 13238, 3793480), (197422, 5662, 3793481), (1267, 5662, 3793482), (1764, 5662, 3793483), (35244, 5662, 3793484), (35366, 5662, 3793485), (59, 14907, 3793486), (327750, 14907, 3793487), (73, 14907, 3793488), (196705, 14907, 3793489), (239, 14907, 3793490), (59, 46652, 3793491), (327750, 46652, 3793492), (73, 46652, 3793493), (196705, 46652, 3793494), (251, 46652, 3793495), (73, 9802, 3793496), (553, 9802, 3793497), (681, 9802, 3793498), (271, 9802, 3793501), (1653, 9802, 3793502), (59, 15169, 3793503), (73, 15169, 3793504), (262438, 15169, 3793505), (196965, 15169, 3793506), (553, 15169, 3793507), (59, 37963, 3793508), (73, 37963, 3793509), (251, 37963, 3793510), (553, 37963, 3793511), (681, 37963, 3793512), (59, 9924, 3793513), (73, 9924, 3793514), (553, 9924, 3793517), (681, 9924, 3793518), (719, 9924, 3793519), (59, 4808, 3793520), (327750, 4808, 3793521), (73, 4808, 3793522), (196705, 4808, 3793523), (251, 4808, 3793524), (59, 33612, 3793525), (73, 33612, 3793526), (553, 33612, 3793527), (681, 33612, 3793528), (1267, 33612, 3793529), (59, 714, 3793530), (73, 714, 3793531), (553, 714, 3793532), (681, 714, 3793533), (197422, 714, 3793534), (59, 2635, 3793535), (73, 2635, 3793536), (553, 2635, 3793537), (681, 2635, 3793538), (1267, 2635, 3793539), (59, 4812, 3793540), (73, 4812, 3793541), (553, 4812, 3793542), (681, 4812, 3793543), (271, 4812, 3793545), (59, 14413, 3793546), (553, 14413, 3793549), (681, 14413, 3793550), (1267, 14413, 3793551), (271, 14413, 3793552), (73, 2510, 3793553), (553, 2510, 3793554), (681, 2510, 3793555), (1653, 2510, 3793560), (1887, 2510, 3793561), (59, 19024, 3793562), (327750, 19024, 3793563), (73, 19024, 3793564), (196705, 19024, 3793566), (251, 19024, 3793567), (59, 53334, 3793568), (73, 53334, 3793569), (553, 53334, 3793578), (681, 53334, 3793579), (197422, 53334, 3793580), (59, 35415, 3793581), (73, 35415, 3793582), (553, 35415, 3793583), (681, 35415, 3793584), (1267, 35415, 3793586), (59, 7643, 3793587), (73, 7643, 3793588), (553, 7643, 3793589), (681, 7643, 3793591), (271, 7643, 3793593), (59, 29789, 3793594), (73, 29789, 3793595), (553, 29789, 3793597), (681, 29789, 3793598), (197422, 29789, 3793600), (59, 19679, 3793601), (73, 19679, 3793602), (553, 19679, 3793604), (681, 19679, 3793605), (197422, 19679, 3793606), (59, 54113, 3793607), (73, 54113, 3793608), (196965, 54113, 3793611), (553, 54113, 3793613), (681, 54113, 3793624), (59, 4134, 3793639), (327750, 4134, 3793644), (73, 4134, 3793646), (196705, 4134, 3793654), (251, 4134, 3793676), (59, 13414, 3793682), (73, 13414, 3793685), (262438, 13414, 3793693), (553, 13414, 3793695), (681, 13414, 3793696), (59, 5719, 3793697), (73, 5719, 3793698), (553, 5719, 3793699), (681, 5719, 3793701), (271, 5719, 3793745), (197422, 36459, 3793757), (1267, 36459, 3793758), (35244, 36459, 3793763), (35366, 36459, 3793764), (35592, 36459, 3793765), (59, 4847, 3793766), (73, 4847, 3793767), (553, 4847, 3793770), (681, 4847, 3793771), (197422, 4847, 3793777), (59, 17012, 3793778), (73, 17012, 3793779), (251, 17012, 3793803), (553, 17012, 3793804), (681, 17012, 3793806), (59, 26101, 3793807), (73, 26101, 3793808), (251, 26101, 3793809), (262438, 26101, 3793810), (553, 26101, 3793811), (59, 39572, 3793812), (73, 39572, 3793813), (553, 39572, 3793815), (681, 39572, 3793820), (197422, 39572, 3793821), (197422, 11643, 3793823), (1267, 11643, 3793825), (35244, 11643, 3793831), (35366, 11643, 3793832), (35592, 11643, 3793833), (59, 24572, 3793834), (251, 24572, 3793835), (553, 24572, 3793836), (681, 24572, 3793837), (197422, 24572, 3793838), (59, 16509, 3793839), (327750, 16509, 3793840), (73, 16509, 3793841), (251, 16509, 3793842), (553, 16509, 3793843), (59, 36351, 3793845), (73, 36351, 3793846), (553, 36351, 3793849), (681, 36351, 3793851), (197422, 36351, 3793852)] path_comp = [] for tup in msms: msm = tup[-1] src_asn = tup[0] dst_asn = tup[1] try: data = parse.parse_msm_trcrt(msm) except: pdb.set_trace() continue for d in data: src_asn = int(ripeprobes.get_probe_asn(d['prb_id'])) dst_asn = int(ip2asn.ip2asn_bgp(d['dst_addr'])) if not path_in_cache(src_asn, dst_asn): continue aslinks = asp.traceroute_to_aspath(d) if not aslinks['_links']: continue aslinks = ixp.remove_ixps(aslinks) hops = [] for link in aslinks: hops.append(int(link['src'])) hops.append(int(link['dst'])) pc_path = get_most_probable_path(src_asn, dst_asn, threshold=5) path_comp.append([pc_path, hops]) pdb.set_trace() accuracy = 0 for p in path_comp: if p[-1] in p[0]:
if len(sys.argv) > 2: if sys.argv[2] == 'random': greedy_max_coverage = random_coverage strategy = "random" elif sys.argv[2] == 'geod': greedy_max_coverage = geo_distributed_coverage strategy = "geod" else: strategy = "greedy" overall_mmt_gain = {} overall_single_homed_gain_content = set() for top_content_pref in top_content_prefs: if top_content_pref == '0.0.0.0': continue print "Top content prefix", top_content_pref asn = ip2asn.ip2asn_bgp(top_content_pref) if asn not in asn_graphs: print "DO NOT HAVE ASN GRAPH OF PREF", top_content_pref continue print "Prefix belongs to asn", asn print "Getting the ASN graph for", top_content_pref print "Graph is a tree?", nx.is_tree(gr) gr = asn_graphs[asn] print "Greedily define the utilities of all paths from RIPE probes toward", top_content_pref init_utility_per_mmt = {} superset = set( ) # contains all the nodes that can be covered if we were to run measurements from # all ripe nodes. for ripe_asn in probes_per_asn: single_homed_gain = get_single_homed_customers(ripe_asn)
from mkit.inference import ip_to_asn as ip2asn import json import csv import pdb with open("dest_pref.json") as fi: dest_pref = json.load(fi) per_pref_count = {} per_asn_count = {} for dest, pref_list in dest_pref.iteritems(): for pref in pref_list: if pref in per_pref_count: per_pref_count[pref] += 1 else: per_pref_count[pref] = 1 asn = ip2asn.ip2asn_bgp(pref) if asn in per_asn_count: per_asn_count[asn] += 1 else: per_asn_count[asn] = 1 per_pref_count_sorted = sorted(per_pref_count.items(), key=lambda x: x[1], reverse=True) csv_list = [['pref', 'count']] for pref, count in per_pref_count_sorted: csv_list.append([pref, count]) pdb.set_trace() with open("per_prefix_count.csv", "w") as fi: writer = csv.writer(fi) writer.writerows(csv_list)