def parse_caida_json_streaming(fname):
num_trcrts = 0
with open(fname) as fi:
dest_based_aspaths = {}
for line in fi:
num_trcrts += 1
try:
trcrt = json.loads(line)
except ValueError:
continue
if trcrt['stop_reason'] != 'COMPLETED':
continue
src = trcrt['src']
dst = trcrt['dst']
ixp_match = ixp.ixp_radix.search_best(dst)
if ixp_match:
continue
dst_asn = ip2asn.ip2asn_bgp(dst)
src_asn = ip2asn.ip2asn_bgp(src)
if not dst_asn:
continue
if dst_asn not in dest_based_aspaths:
dest_based_aspaths[dst_asn] = []
aslinks = []
last_hop_nr = None
this_hop_nr = None
this_hop_asn = None
last_hop_asn = None
for hop in trcrt['hops']:
addr = hop['addr']
ixp_match = ixp.ixp_radix.search_best(addr)
if ixp_match:
continue
asn = ip2asn.ip2asn_bgp(addr)
if asn in ixp.IXPs:
continue
last_hop_nr = this_hop_nr
this_hop_nr = hop['probe_ttl']
last_hop_asn = this_hop_asn
this_hop_asn = asn
if this_hop_asn and last_hop_asn:
if last_hop_asn != this_hop_asn:
if (this_hop_nr - last_hop_nr) == 1:
link_type = 'd'
else:
link_type = 'i'
link = (int(last_hop_asn), int(this_hop_asn),
link_type)
aslinks.append(link)
if aslinks:
dest_based_aspaths[dst_asn].append(aslinks)
print len(dest_based_aspaths)
print num_trcrts
return dest_based_aspaths
def parse_caida_json_streaming(fname):
mmt_path_list = []
with open(fname) as fi:
dest_based_aspaths = {}
for line in fi:
trcrt = json.loads(line)
if trcrt['stop_reason'] != 'COMPLETED':
continue
src = trcrt['src']
dst = trcrt['dst']
dst_asn = ip2asn.ip2asn_bgp(dst)
src_asn = ip2asn.ip2asn_bgp(src)
if src_asn and dst_asn:
mmt_path_list.append((int(src_asn), int(dst_asn)))
return mmt_path_list
def get_pc_or_bgp_path(src_asn, dst_pref, portnum, measured_path):
typ = None
dst_asn = ip2asn.ip2asn_bgp(dst_pref.split('_')[0])
path_key = "%s-%s" % (src_asn, dst_pref)
if path_key in mc and mc.get(path_key):
return
paths = get_pc_path(src_asn, dst_pref)
if not paths:
if dst_asn in ip2asn.asn_to_prefs:
prefs = ip2asn.asn_to_prefs[dst_asn]
else:
prefs = []
sister_paths = []
while prefs:
sister_pref = prefs.pop(0)
print "trying sister pref", sister_pref
sister_pref = sister_pref.replace('/', '_')
sister_paths = get_pc_path(src_asn,
sister_pref,
add_single_homed=False)
if sister_paths:
break
if not sister_paths:
bgp_path = get_bgp_path(src_asn, dst_asn, portnum)
if not bgp_path:
mc.set(path_key, "")
return
if bgp_path.split()[-1] != dst_asn:
print "bgpsim: error, not querying"
mc.set(path_key, "")
return
spliced_paths = get_spliced_bgp_pathcache(dst_pref, src_asn,
bgp_path)
if sister_paths:
paths = sister_paths
del sister_paths
typ = "sis"
elif spliced_paths:
paths = spliced_paths
del spliced_paths
typ = "spliced"
else:
typ = "pc"
if not paths:
mc.set(path_key, ([], measured_path, -1, -1, None))
return
print "Got these as paths:", paths, typ, "for query:", src_asn, dst_pref
pred_path = paths[0]
if not pred_path:
mc.set(path_key, ([], measured_path, -1, -1, None))
#mc.set(path_key, ())
return
rank, prob = find_rank_prob(measured_path, paths, dst_asn)
mc.set(path_key, (paths, measured_path, rank, prob, typ))
def compute_dest_based_graphs(msms):
measured_paths = []
for msm in msms:
info = parse.mmt_info(msm)
# Start time should not be < Jan 1st 2016
if info['start_time'] < 1451606400: continue
if info['type']['af'] != 4: continue
dst_asn = int(info['dst_asn'])
data = parse.parse_msm_trcrt(msm, count=1000)
for d in data:
src_asn = ip2asn.ip2asn_bgp(d['from'])
if not src_asn: continue
measured_paths.append((int(src_asn), dst_asn))
return list(frozenset(measured_paths))
def wrap_function(qr_set, portnum):
pid = None
try:
current = mp.current_process()
identity = current._identity
#identity = (1,)
print "ID:", identity
proc = Popen([
'mono',
'bgp_sim/TestingApplication/bin/Release/TestingApplication.exe',
'-server%d' % (portnum), '20160101-cyclops',
'bgp_sim/precomp/US-precomp367.txt', 'bgp_sim/cache/exit_asns.txt'
],
preexec_fn=os.setsid,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
time.sleep(10)
pid = proc.pid
mc_bgpsim_pid.set(str(portnum), str(pid))
for query in qr_set:
src_asn = int(query.split('-')[0])
dst_pref = query.split('-')[1]
dst_asn = ip2asn.ip2asn_bgp(dst_pref.split('_')[0])
meas_path = query.split('-')[-1]
path_key = "%s-%s" % (src_asn, dst_pref)
if path_key in mc and mc.get(path_key):
print "Found it in the cache", path_key
continue
get_pc_or_bgp_path(src_asn, dst_pref, portnum, meas_path)
except:
print "".join(traceback.format_exception(*sys.exc_info()))
try:
proc.kill()
os.killpg(pid, signal.SIGTERM)
os.system("kill -9 %s" % pid)
mc_bgpsim_pid.set(str(portnum), None)
except:
pass
return pid
def get_asn(ip):
try:
return ip2asn_bgp(ip)
except Exception as e:
agenda.subfailure(f"ip {ip}: error {e}")
return None
def parse_caida_json_streaming(fname):
global num_mmts
print num_mmts
num_trcrts = 0
with open(fname) as fi:
dest_based_aspaths = {}
for line in fi:
num_trcrts += 1
try:
trcrt = json.loads(line)
except:
continue
if trcrt['stop_reason'] != 'COMPLETED':
continue
num_mmts += 1
src = trcrt['src']
dst = trcrt['dst']
rnode = ip2asn.ip_to_pref(dst)
if rnode:
dst_prefix = rnode.prefix.replace('/', '_')
else:
continue
ixp_match = ixp.ixp_radix.search_best(dst)
if ixp_match:
continue
dst_asn = int(ip2asn.ip2asn_bgp(dst))
src_asn = ip2asn.ip2asn_bgp(src)
if not dst_asn:
continue
if (dst_asn, dst_prefix) not in dest_based_aspaths:
dest_based_aspaths[(dst_asn, dst_prefix)] = []
aslinks = []
last_hop_nr = None
this_hop_nr = None
this_hop_asn = None
last_hop_asn = None
for hop in trcrt['hops']:
addr = hop['addr']
ixp_match = ixp.ixp_radix.search_best(addr)
if ixp_match:
continue
asn = ip2asn.ip2asn_bgp(addr)
if not asn: continue
if asn in ixp.IXPs:
continue
last_hop_nr = this_hop_nr
this_hop_nr = hop['probe_ttl']
last_hop_asn = this_hop_asn
this_hop_asn = asn
if this_hop_asn and last_hop_asn:
if last_hop_asn != this_hop_asn:
if (this_hop_nr - last_hop_nr) == 1:
link_type = 'd'
else:
link_type = 'i'
link = (int(last_hop_asn), int(this_hop_asn), link_type)
aslinks.append(link)
if aslinks:
dest_based_aspaths[(int(dst_asn), dst_prefix)].append(aslinks)
#print len(dest_based_aspaths)
#print num_trcrts
return dest_based_aspaths
return True
else:
if dst in all_graphs_bgp:
gr = all_graphs_bgp[asn]
if src in gr.nodes:
return True
return False
NUM_CONTENT = 500
websites = alexa.top_list(NUM_CONTENT)
content_asns = []
content_ips = {}
for w in websites:
try:
asn = int(ip2asn.ip2asn_bgp(socket.gethostbyname(w[1])))
content_asns.append(asn)
content_ips[asn] = socket.gethostbyname(w[1])
except:
print w
continue
content_asns = list(set(content_asns))
f = open("data/aspop")
entries = list()
for table in f:
records = table.split("[")
for record in records:
record = record.split("]")[0]
entry = dict()
msms = [(59, 8075, 3793363), (73, 8075, 3793364), (262438, 8075, 3793366), (553, 8075, 3793367), (681, 8075, 3793368), (59, 23820, 3793369), (73, 23820, 3793370), (553, 23820, 3793371), (681, 23820, 3793372), (271, 23820, 3793375), (59, 22414, 3793376), (73, 22414, 3793377), (553, 22414, 3793378), (681, 22414, 3793380), (271, 22414, 3793381), (59, 62787, 3793382), (73, 62787, 3793383), (553, 62787, 3793386), (681, 62787, 3793387), (271, 62787, 3793388), (59, 47764, 3793389), (327750, 47764, 3793390), (73, 47764, 3793391), (196705, 47764, 3793392), (239, 47764, 3793393), (59, 13335, 3793394), (73, 13335, 3793395), (553, 13335, 3793409), (681, 13335, 3793410), (33480, 13335, 3793411), (59, 23576, 3793412), (73, 23576, 3793413), (553, 23576, 3793414), (681, 23576, 3793415), (271, 23576, 3793417), (59, 30361, 3793418), (73, 30361, 3793419), (553, 30361, 3793420), (681, 30361, 3793421), (197422, 30361, 3793422), (251, 14618, 3793423), (553, 14618, 3793425), (197422, 14618, 3793426), (1267, 14618, 3793427), (34177, 14618, 3793429), (59, 16798, 3793430), (73, 16798, 3793431), (553, 16798, 3793432), (681, 16798, 3793433), (1267, 16798, 3793434), (1267, 46489, 3793441), (1764, 46489, 3793442), (35244, 46489, 3793443), (35540, 46489, 3793444), (3242, 46489, 3793446), (59, 32934, 3793447), (73, 32934, 3793448), (251, 32934, 3793449), (262438, 32934, 3793450), (196965, 32934, 3793451), (59, 23724, 3793452), (327750, 23724, 3793453), (73, 23724, 3793454), (196705, 23724, 3793455), (251, 23724, 3793456), (59, 45102, 3793457), (73, 45102, 3793458), (553, 45102, 3793459), (681, 45102, 3793460), (1267, 45102, 3793463), (59, 10929, 3793465), (73, 10929, 3793466), (553, 10929, 3793467), (681, 10929, 3793468), (197422, 10929, 3793469), (59, 47541, 3793470), (73, 47541, 3793471), (553, 47541, 3793472), (681, 47541, 3793473), (197422, 47541, 3793474), (59, 13238, 3793475), (73, 13238, 3793476), (553, 13238, 3793477), (681, 13238, 3793478), (271, 13238, 3793480), (197422, 5662, 3793481), (1267, 5662, 3793482), (1764, 5662, 3793483), (35244, 5662, 3793484), (35366, 5662, 3793485), (59, 14907, 3793486), (327750, 14907, 3793487), (73, 14907, 3793488), (196705, 14907, 3793489), (239, 14907, 3793490), (59, 46652, 3793491), (327750, 46652, 3793492), (73, 46652, 3793493), (196705, 46652, 3793494), (251, 46652, 3793495), (73, 9802, 3793496), (553, 9802, 3793497), (681, 9802, 3793498), (271, 9802, 3793501), (1653, 9802, 3793502), (59, 15169, 3793503), (73, 15169, 3793504), (262438, 15169, 3793505), (196965, 15169, 3793506), (553, 15169, 3793507), (59, 37963, 3793508), (73, 37963, 3793509), (251, 37963, 3793510), (553, 37963, 3793511), (681, 37963, 3793512), (59, 9924, 3793513), (73, 9924, 3793514), (553, 9924, 3793517), (681, 9924, 3793518), (719, 9924, 3793519), (59, 4808, 3793520), (327750, 4808, 3793521), (73, 4808, 3793522), (196705, 4808, 3793523), (251, 4808, 3793524), (59, 33612, 3793525), (73, 33612, 3793526), (553, 33612, 3793527), (681, 33612, 3793528), (1267, 33612, 3793529), (59, 714, 3793530), (73, 714, 3793531), (553, 714, 3793532), (681, 714, 3793533), (197422, 714, 3793534), (59, 2635, 3793535), (73, 2635, 3793536), (553, 2635, 3793537), (681, 2635, 3793538), (1267, 2635, 3793539), (59, 4812, 3793540), (73, 4812, 3793541), (553, 4812, 3793542), (681, 4812, 3793543), (271, 4812, 3793545), (59, 14413, 3793546), (553, 14413, 3793549), (681, 14413, 3793550), (1267, 14413, 3793551), (271, 14413, 3793552), (73, 2510, 3793553), (553, 2510, 3793554), (681, 2510, 3793555), (1653, 2510, 3793560), (1887, 2510, 3793561), (59, 19024, 3793562), (327750, 19024, 3793563), (73, 19024, 3793564), (196705, 19024, 3793566), (251, 19024, 3793567), (59, 53334, 3793568), (73, 53334, 3793569), (553, 53334, 3793578), (681, 53334, 3793579), (197422, 53334, 3793580), (59, 35415, 3793581), (73, 35415, 3793582), (553, 35415, 3793583), (681, 35415, 3793584), (1267, 35415, 3793586), (59, 7643, 3793587), (73, 7643, 3793588), (553, 7643, 3793589), (681, 7643, 3793591), (271, 7643, 3793593), (59, 29789, 3793594), (73, 29789, 3793595), (553, 29789, 3793597), (681, 29789, 3793598), (197422, 29789, 3793600), (59, 19679, 3793601), (73, 19679, 3793602), (553, 19679, 3793604), (681, 19679, 3793605), (197422, 19679, 3793606), (59, 54113, 3793607), (73, 54113, 3793608), (196965, 54113, 3793611), (553, 54113, 3793613), (681, 54113, 3793624), (59, 4134, 3793639), (327750, 4134, 3793644), (73, 4134, 3793646), (196705, 4134, 3793654), (251, 4134, 3793676), (59, 13414, 3793682), (73, 13414, 3793685), (262438, 13414, 3793693), (553, 13414, 3793695), (681, 13414, 3793696), (59, 5719, 3793697), (73, 5719, 3793698), (553, 5719, 3793699), (681, 5719, 3793701), (271, 5719, 3793745), (197422, 36459, 3793757), (1267, 36459, 3793758), (35244, 36459, 3793763), (35366, 36459, 3793764), (35592, 36459, 3793765), (59, 4847, 3793766), (73, 4847, 3793767), (553, 4847, 3793770), (681, 4847, 3793771), (197422, 4847, 3793777), (59, 17012, 3793778), (73, 17012, 3793779), (251, 17012, 3793803), (553, 17012, 3793804), (681, 17012, 3793806), (59, 26101, 3793807), (73, 26101, 3793808), (251, 26101, 3793809), (262438, 26101, 3793810), (553, 26101, 3793811), (59, 39572, 3793812), (73, 39572, 3793813), (553, 39572, 3793815), (681, 39572, 3793820), (197422, 39572, 3793821), (197422, 11643, 3793823), (1267, 11643, 3793825), (35244, 11643, 3793831), (35366, 11643, 3793832), (35592, 11643, 3793833), (59, 24572, 3793834), (251, 24572, 3793835), (553, 24572, 3793836), (681, 24572, 3793837), (197422, 24572, 3793838), (59, 16509, 3793839), (327750, 16509, 3793840), (73, 16509, 3793841), (251, 16509, 3793842), (553, 16509, 3793843), (59, 36351, 3793845), (73, 36351, 3793846), (553, 36351, 3793849), (681, 36351, 3793851), (197422, 36351, 3793852)]
path_comp = []
for tup in msms:
msm = tup[-1]
src_asn = tup[0]
dst_asn = tup[1]
try:
data = parse.parse_msm_trcrt(msm)
except:
pdb.set_trace()
continue
for d in data:
src_asn = int(ripeprobes.get_probe_asn(d['prb_id']))
dst_asn = int(ip2asn.ip2asn_bgp(d['dst_addr']))
if not path_in_cache(src_asn, dst_asn): continue
aslinks = asp.traceroute_to_aspath(d)
if not aslinks['_links']: continue
aslinks = ixp.remove_ixps(aslinks)
hops = []
for link in aslinks:
hops.append(int(link['src']))
hops.append(int(link['dst']))
pc_path = get_most_probable_path(src_asn, dst_asn, threshold=5)
path_comp.append([pc_path, hops])
pdb.set_trace()
accuracy = 0
for p in path_comp:
if p[-1] in p[0]:
if len(sys.argv) > 2:
if sys.argv[2] == 'random':
greedy_max_coverage = random_coverage
strategy = "random"
elif sys.argv[2] == 'geod':
greedy_max_coverage = geo_distributed_coverage
strategy = "geod"
else:
strategy = "greedy"
overall_mmt_gain = {}
overall_single_homed_gain_content = set()
for top_content_pref in top_content_prefs:
if top_content_pref == '0.0.0.0': continue
print "Top content prefix", top_content_pref
asn = ip2asn.ip2asn_bgp(top_content_pref)
if asn not in asn_graphs:
print "DO NOT HAVE ASN GRAPH OF PREF", top_content_pref
continue
print "Prefix belongs to asn", asn
print "Getting the ASN graph for", top_content_pref
print "Graph is a tree?", nx.is_tree(gr)
gr = asn_graphs[asn]
print "Greedily define the utilities of all paths from RIPE probes toward", top_content_pref
init_utility_per_mmt = {}
superset = set(
) # contains all the nodes that can be covered if we were to run measurements from
# all ripe nodes.
for ripe_asn in probes_per_asn:
single_homed_gain = get_single_homed_customers(ripe_asn)
from mkit.inference import ip_to_asn as ip2asn
import json
import csv
import pdb
with open("dest_pref.json") as fi:
dest_pref = json.load(fi)
per_pref_count = {}
per_asn_count = {}
for dest, pref_list in dest_pref.iteritems():
for pref in pref_list:
if pref in per_pref_count:
per_pref_count[pref] += 1
else:
per_pref_count[pref] = 1
asn = ip2asn.ip2asn_bgp(pref)
if asn in per_asn_count:
per_asn_count[asn] += 1
else:
per_asn_count[asn] = 1
per_pref_count_sorted = sorted(per_pref_count.items(),
key=lambda x: x[1],
reverse=True)
csv_list = [['pref', 'count']]
for pref, count in per_pref_count_sorted:
csv_list.append([pref, count])
pdb.set_trace()
with open("per_prefix_count.csv", "w") as fi:
writer = csv.writer(fi)
writer.writerows(csv_list)
