def main():
ips=set()
with open('measurementset.json','r') as infile:
msms = json.load( infile )
msm_list = msms['v4'] + msms['v6']
# random.shuffle( msm_list )
count=0
for m in msm_list:
print >>sys.stderr, "(%d/%d) msm gathering, now fetching %s" % ( count, len(msm_list), m )
for data in MeasurementFetch.fetch( m['msm_id'] ):
tr = ripe.atlas.sagan.TracerouteResult( data )
for hop in tr.hops:
for pkt in hop.packets:
ip = pkt.origin
if pkt.arrived_late_by: ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
continue
if ip != None:
ips.add( ip )
count+=1
no_ips = len(ips)
print >>sys.stderr, "ip gathering finished, now analysing. ip count: %s" % ( no_ips )
ipcache = IPInfoCache.IPInfoCache()
counter=1
ips = list(ips)
ips.sort()
asns = set()
for ip in ips:
res= ipcache.findIPInfo( ip )
print "(%d/%d) %s / %s" % ( counter, no_ips, ip, res )
counter += 1
if 'asn' in res and res['asn'] != None and res['asn'] != '':
asns.add( res['asn'] )
# writes this file
ipcache.toJsonFragments('ips.json-fragments')
def process_msm(msm_spec, protocol):
# msm_spec has msm_id
msm_id = msm_spec['msm_id']
print >> sys.stderr, "starting processing of %s" % (msm_id)
## exit if .msm.%s file already exists
outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id)
if os.path.exists(outfilename):
print >> sys.stderr, "file already exists %s" % (outfilename)
return
outdata = []
for data in MeasurementFetch.fetch(msm_id):
tr = ripe.atlas.sagan.TracerouteResult(data)
tracetxt = MeasurementPrint.trace2txt(data)
src_prb_id = data['prb_id']
dst_prb_id = probes_by_ip[data['dst_addr']]
ixps = check_if_via_ixp(tr, ixp_radix)
via_ixp = False
if len(ixps) > 0: via_ixp = True
#print "IXPS: %s" % ( ixps )
#print tracetxt
locs = MeasurementPrint.trace2locs(data)
as_links = MeasurementEnhance.aslinksplus(data, ixp_radix)
geojson = MeasurementEnhance.togeojson(data,
probes_by_id[src_prb_id],
probes_by_id[dst_prb_id])
#print as_links
countries = conf['country']
if type(countries) != list:
countries = [countries]
countries = map(lambda x: x.upper(), countries)
is_in_country = check_if_is_in_country(countries, locs)
#print "INCOUNTRY: %s" % (is_in_country)
dst_rtts = get_destination_rtts(tr)
outdata.append({
'ts': data['timestamp'],
'result': data['result'],
'protocol': protocol,
'msm_id': msm_id,
'as_links': as_links,
'src_prb_id': src_prb_id,
'dst_prb_id': dst_prb_id,
#'src_asn': src_asn,
#'dst_asn': dst_asn,
#'last_rtt': tr.last_rtt,
'dst_rtts': dst_rtts,
#'target_responded': tr.target_responded,
#'src_is_member': srcmb,
#'dst_is_member': dstmb,
### more correctly: geojson linestring array
'geojson': geojson,
'in_country': is_in_country,
'via_ixp': via_ixp,
'ixps': ixps,
'tracetxt': tracetxt,
'locations': list(locs)
})
with open(outfilename, 'w') as outfile:
json.dump(outdata, outfile, indent=2)
def process_msm( msm_spec, protocol ):
# msm_spec has msm_id
msm_id = msm_spec['msm_id']
print >>sys.stderr, "starting processing of %s" % ( msm_id )
## exit if .msm.%s file already exists
outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
if os.path.exists( outfilename ):
print >>sys.stderr, "file already exists %s" % ( outfilename )
return
outdata = []
for data in MeasurementFetch.fetch( msm_id ):
tr = ripe.atlas.sagan.TracerouteResult( data )
tracetxt = MeasurementPrint.trace2txt( data )
src_prb_id = data['prb_id']
dst_prb_id = probes_by_ip[ data['dst_addr'] ]
ixps = check_if_via_ixp( tr, ixp_radix )
via_ixp = False
if len(ixps) > 0: via_ixp = True
#print "IXPS: %s" % ( ixps )
#print tracetxt
locs = MeasurementPrint.trace2locs( data )
as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
geojson = MeasurementEnhance.togeojson( data, probes_by_id[ src_prb_id ] , probes_by_id[ dst_prb_id ] )
#print as_links
countries = conf['country']
if type(countries) != list:
countries = [ countries ]
countries = map(lambda x:x.upper(), countries)
is_in_country = check_if_is_in_country( countries, locs )
#print "INCOUNTRY: %s" % (is_in_country)
dst_rtts = get_destination_rtts( tr )
outdata.append( {
'ts': data['timestamp'],
'result': data['result'],
'protocol': protocol,
'msm_id': msm_id,
'as_links': as_links,
'src_prb_id': src_prb_id,
'dst_prb_id': dst_prb_id,
#'src_asn': src_asn,
#'dst_asn': dst_asn,
#'last_rtt': tr.last_rtt,
'dst_rtts': dst_rtts,
#'target_responded': tr.target_responded,
#'src_is_member': srcmb,
#'dst_is_member': dstmb,
### more correctly: geojson linestring array
'geojson': geojson,
'in_country': is_in_country,
'via_ixp': via_ixp,
'ixps': ixps,
'tracetxt': tracetxt,
'locations': list(locs)
} )
with open(outfilename,'w') as outfile:
json.dump( outdata, outfile, indent=2 )
def main():
ips = set()
with open('measurementset.json', 'r') as infile:
msms = json.load(infile)
msm_list = msms['v4'] + msms['v6']
# random.shuffle( msm_list )
count = 0
for m in msm_list:
print >> sys.stderr, "(%d/%d) msm gathering, now fetching %s" % (
count, len(msm_list), m)
for data in MeasurementFetch.fetch(m['msm_id']):
tr = ripe.atlas.sagan.TracerouteResult(data)
for hop in tr.hops:
for pkt in hop.packets:
ip = pkt.origin
if pkt.arrived_late_by: ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
continue
if ip != None:
ips.add(ip)
count += 1
no_ips = len(ips)
print >> sys.stderr, "ip gathering finished, now analysing. ip count: %s" % (
no_ips)
ipcache = IPInfoCache.IPInfoCache()
ips = list(ips)
ips.sort()
asns = set()
lock = threading.Lock()
def findInfo(ip):
global counter
res = ipcache.findIPInfo(ip)
with lock:
print '(%d/%d) %s / %s' % (counter, no_ips, ip, res)
counter += 1
sys.stdout.flush()
if 'asn' in res and res['asn'] != None and res['asn'] != '':
asns.add(res['asn'])
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpu_count()) as executor:
executor.map(findInfo, ips)
# writes this file
ipcache.toJsonFragments('ips-openipmap.json-fragments')
if p.origin == trace.destination_address:
dest_hop = hop.index
break
return ips
# for latest data
stop_t = time.time()
start_t = stop_t - interval
## ips seen as penultimate hop for responding dst
penult_ips = set()
prb_id2tr = {}
## properties per probe
for data in MeasurementFetch.fetch( 1767679, start=start_t, stop=stop_t ):
tr = TracerouteResult( data )
prb_id2tr[ tr.probe_id ] = data
if tr.target_responded:
pre_dest_ips = get_pre_dest_ips( tr )
for ip in pre_dest_ips:
penult_ips.add( ip )
count = 0
printit=[]
for data in MeasurementFetch.fetch( 1767680, start=start_t, stop=stop_t ):
tr = TracerouteResult( data )
if not tr.target_responded:
last_ips = last_responding_ips( tr )
for ip in last_ips:
def process_msm( msm_spec, protocol ):
# msm_spec has msm_id
msm_id = msm_spec['msm_id']
print >>sys.stderr, "starting processing of %s" % ( msm_id )
## exit if .msm.%s file already exists
outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
if os.path.exists( outfilename ):
print >>sys.stderr, "file already exists %s" % ( outfilename )
return
outdata = []
for data in MeasurementFetch.fetch( msm_id ):
data = filter_cruft( data )
assert 'edst' not in repr( data ), data
tr = ripe.atlas.sagan.TracerouteResult( data )
ip_list = tr.ip_path
hops = []
for i in ip_list:
if len(i) > 0:
hops.append(i[0])
tracetxt = MeasurementPrint.trace2txt( data )
#hops_ips = []
#print ("tracetxt: " + tracetxt)
src_prb_id = data['prb_id']
src_prb = probes_by_id[ src_prb_id ]
dst_prb_id = None
dst_prb = None
try:
dst_prb_id = probes_by_ip[ data['dst_addr'] ]
dst_prb = probes_by_id[ dst_prb_id ]
except: pass
if src_prb_id == dst_prb_id:
### probe to itself is not interesting/useful
## TODO filter this out in the measurement creation
continue
ixps = check_if_via_ixp( tr, ixp_radix )
via_ixp = False
if len(ixps) > 0: via_ixp = True
#print "IXPS: %s" % ( ixps )
#print tracetxt
locs = MeasurementPrint.trace2locs( data )
as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
geojson = MeasurementEnhance.togeojson( data, src_prb , dst_prb )
#print as_links
countries = basedata['countries']
is_in_country = check_if_is_in_country( countries, locs )
#print "INCOUNTRY: %s" % (is_in_country)
dst_rtts = get_destination_rtts( tr )
outdata.append( {
'ts': data['timestamp'],
'result': data['result'],
'protocol': protocol,
'msm_id': msm_id,
'as_links': as_links,
'src_prb_id': src_prb_id,
'dst_prb_id': dst_prb_id,
#'src_asn': src_asn,
#'dst_asn': dst_asn,
#'last_rtt': tr.last_rtt,
'dst_rtts': dst_rtts,
#'target_responded': tr.target_responded,
#'src_is_member': srcmb,
#'dst_is_member': dstmb,
### more correctly: geojson linestring array
'geojson': geojson,
'in_country': is_in_country,
'via_ixp': via_ixp,
'ixps': ixps,
'tracetxt': tracetxt,
'locations': list(locs)
} )
outdatatraixroute.append( {
'msm_id': msm_id,
'ip_path' : hops,
'src_prb_id' : src_prb_id,
'dst_prb_id': dst_prb_id
} )
with open(outfilename,'w') as outfile:
json.dump( outdata, outfile, indent=2 )
def process_msm(msm_spec, protocol):
# msm_spec has msm_id
msm_id = msm_spec['msm_id']
print >> sys.stderr, "starting processing of %s" % (msm_id)
## exit if .msm.%s file already exists
outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id)
if os.path.exists(outfilename):
print >> sys.stderr, "file already exists %s" % (outfilename)
return
outdata = []
for data in MeasurementFetch.fetch(msm_id):
data = filter_cruft(data)
assert 'edst' not in repr(data), data
tr = ripe.atlas.sagan.TracerouteResult(data)
if 'dst_addr' in data:
tracetxt = MeasurementPrint.trace2txt(data)
else:
continue
src_prb_id = data['prb_id']
src_prb = probes_by_id[src_prb_id]
src_asn = None
if data['af'] == 4:
src_asn = src_prb['asn_v4']
elif data['af'] == 6:
src_asn = src_prb['asn_v6']
dst_prb_id = None
dst_prb = None
try:
dst_prb_id = probes_by_ip[data[
'dst_name']] # dst name always has the IP that is in msmset/probeset
dst_prb = probes_by_id[dst_prb_id]
except:
### 2a01:7700:0:1033:220:4aff:fee0:2694 vs. 2a01:7700::1033:220:4aff:fee0:2694
##AAAAAAAAAAA
print >> sys.stderr, "can't find dst_prb_id for this dst_name. SHOULD NOT HAPPEN"
if src_prb_id == dst_prb_id:
### probe to itself is not interesting/useful
## TODO filter this out in the measurement creation
continue
ixps = check_if_via_ixp(tr, ixp_radix)
via_ixp = False
if len(ixps) > 0: via_ixp = True
#print "IXPS: %s" % ( ixps )
#print tracetxt
locs = MeasurementPrint.trace2locs(data)
as_links = MeasurementEnhance.aslinksplus(data,
ixp_radix,
src_asn=src_asn)
geojson = MeasurementEnhance.togeojson(data, src_prb, dst_prb)
#print as_links
countries = basedata['countries']
is_in_country = check_if_is_in_country(countries, locs)
#print "INCOUNTRY: %s" % (is_in_country)
dst_rtts = get_destination_rtts(tr)
outdata.append({
'ts': data['timestamp'],
'result': data['result'],
'protocol': protocol,
'msm_id': msm_id,
'as_links': as_links,
'src_prb_id': src_prb_id,
'dst_prb_id': dst_prb_id,
#'src_asn': src_asn,
#'dst_asn': dst_asn,
#'last_rtt': tr.last_rtt,
'dst_rtts': dst_rtts,
#'target_responded': tr.target_responded,
#'src_is_member': srcmb,
#'dst_is_member': dstmb,
### more correctly: geojson linestring array
'geojson': geojson,
'in_country': is_in_country,
'via_ixp': via_ixp,
'ixps': ixps,
'tracetxt': tracetxt,
'locations': list(locs)
})
with open(outfilename, 'w') as outfile:
json.dump(outdata, outfile, indent=2)
def main():
iprtt = {}
with open('measurementset.json', 'r') as infile:
msms = json.load(infile)
msm_list = msms['v4'] + msms['v6']
# random.shuffle( msm_list )
count = 0
for m in msm_list:
print >> sys.stderr, "(%d/%d) msm gathering, now fetching %s" % (
count, len(msm_list), m)
for data in MeasurementFetch.fetch(m['msm_id']):
tr = ripe.atlas.sagan.TracerouteResult(data)
for hop in tr.hops:
for pkt in hop.packets:
ip = pkt.origin
rtt = pkt.rtt
if pkt.arrived_late_by: ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
continue
if ip != None:
if not ip in iprtt or iprtt[ip]['rtt'] > rtt:
iprtt[ip] = {
'rtt': rtt,
'prb_id': tr.probe_id,
'msm_id': tr.measurement_id,
}
count += 1
ips = list(iprtt.keys())
ips.sort()
ip_count = len(ips)
print >> sys.stderr, "ip gathering finished, now analysing. ip count: %s" % (
ip_count)
outf = open("ips.json-fragments", "w")
outf_geoerr = open('geo-err.json-fragments', "w")
no_result_cnt = 0
for ip in ips:
print >> sys.stderr, "attempting %s" % ip
j = None
#if True:
try:
out = {
'ip': ip,
'lon': None,
'location': '',
'lat': None,
'hostname': "",
'asn': "",
'geo_error': None
}
req = requests.get(
"https://ipmap.ripe.net/api/v1/locate/%s/partials?engines=probeslocation,crowdsourced,ixp"
% ip,
timeout=25)
j = req.json()
j['ip'] = ip
#del( j['meta'] )
#print "%s" % json.dumps( j )
have_result = False
try:
out['hostname'] = ip2hostname(ip)
except:
print >> sys.stderr, "hostname lookup failed for %s" % ip
pass
try:
out['asn'] = ip2asn(ip)
except:
print >> sys.stderr, "asn lookup failed for %s" % ip
pass
if 'partials' in j and len(j['partials']) > 0:
for p in j['partials']:
if not have_result and p['engine'] in (
'probelocations', 'crowdsourced',
'ixp') and len(p['locations']) > 0:
#print '# %s' % ( p['locations'][0], )
for loc in p['locations']:
if 'cityNameAscii' in loc:
out['lat'] = loc['latitude']
out['lon'] = loc['longitude']
out['location'] = u"%s,%s" % (
loc['cityNameAscii'],
loc['countryCodeAlpha2'])
# {"ip": "213.19.197.126", "hostname": "infopact-ne.ear3.amsterdam1.level3.net", "lon": null, "location": "", "lat": null, "asn": 3356}
#print u'%s %s "%s"' % ( ip, p['engine'], loc['cityName'], loc['countryCodeAlpha2'] )
have_result = True
# now get the probe lat/lon and compare to ipmap + RTT
if out['lon'] and out['lat']:
prb = run.probes_by_id[iprtt[ip]['prb_id']]
geodist = haversine(
out['lon'], out['lat'], prb['lon'],
prb['lat'])
rttdist = iprtt[ip]['rtt'] * 100
# don't make geo_err judgements when prb-ip distance is under 100km
if geodist < 100 or geodist < rttdist:
out['geo_error'] = False
else:
out['geo_error'] = True
g = out
g['prb_id'] = iprtt[ip]['prb_id']
g['msm_id'] = iprtt[ip]['msm_id']
g['rtt'] = iprtt[ip]['rtt']
g['geo_km'] = geodist
g['rtt_km'] = rttdist
print >> outf_geoerr, json.dumps(g)
## now reset the location, as we don't want it to be used in the real jedi
out['lat'] = None
out['lon'] = None
out['location'] = ''
print >> sys.stderr, "#RTT %s geo_km:%.2f rtt_km:%.2f prb_id:%s rtt:%.2f (%s)" % (
ip, geodist, rttdist,
iprtt[ip]['prb_id'], iprtt[ip]['rtt'],
out['geo_error'])
else:
print >> sys.stderr, "#error: no lat/lon in new ipmap?! %s" % (
out)
break
except:
print >> sys.stderr, "#error on json for %s: %s" % (ip, j)
print >> outf, u'%s' % json.dumps(out)
if not have_result:
no_result_cnt += 1
while start_t < stop_stop:
stop_t = start_t + interval
## properties per probe
probe_prop = {}
for msm_id in sorted( measurements ):
pfx_size = measurements[ msm_id ]['size']
has_irr = measurements[ msm_id ]['irr']
# index for 'fingerprinting' behaviours
msm_idx = measurements[ msm_id ]['idx']
responded=0
count=0
# last_ips will contain the last-responding IP, if the intended dest didn't respond
last_ips = Counter()
asns = {}
for data in MeasurementFetch.fetch( msm_id, start=start_t , stop=stop_t ):
tr = TracerouteResult( data )
## init probe_prop
if not tr.probe_id in probe_prop:
probe_prop[ tr.probe_id ] = {
'fingerprint': [None]*len( measurements ),
}
prb_asn = probes[ tr.probe_id ]['asn_v4']
if not prb_asn in asns:
asns[ prb_asn ] = {'count':0, 'responses':0}
asns[prb_asn]['count'] += 1
#if tr.probe_id == 14353:
# print MeasurementPrint.trace2txt( data, hostnames=False )
#print "%s %s %s" % ( tr.measurement_id, tr.probe_id, tr.target_responded )
last_resp_ips = last_responding_ip( tr )
pre_dest_ips = get_pre_dest_ips( tr )