示例#1
0
def main():
   ips=set()
   with open('measurementset.json','r') as infile:
      msms = json.load( infile )
      msm_list = msms['v4'] + msms['v6']
#      random.shuffle( msm_list ) 
      count=0
      for m in msm_list:
         print >>sys.stderr, "(%d/%d) msm gathering, now fetching %s" % ( count, len(msm_list), m )
         for data in MeasurementFetch.fetch( m['msm_id'] ):
            tr = ripe.atlas.sagan.TracerouteResult( data )
            for hop in tr.hops:
               for pkt in hop.packets:
                  ip = pkt.origin
                  if pkt.arrived_late_by: ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
                     continue
                  if ip != None:
                     ips.add( ip )
         count+=1
   no_ips = len(ips)
   print >>sys.stderr, "ip gathering finished, now analysing. ip count: %s" % ( no_ips )
   ipcache = IPInfoCache.IPInfoCache()
   counter=1
   ips = list(ips)
   ips.sort()
   asns = set()
   for ip in ips:
      res= ipcache.findIPInfo( ip )
      print "(%d/%d) %s / %s" % ( counter, no_ips, ip, res )
      counter += 1
      if 'asn' in res and res['asn'] != None and res['asn'] != '':
         asns.add( res['asn'] )
   # writes this file
   ipcache.toJsonFragments('ips.json-fragments')
示例#2
0
 def process_msm(msm_spec, protocol):
     # msm_spec has msm_id
     msm_id = msm_spec['msm_id']
     print >> sys.stderr, "starting processing of %s" % (msm_id)
     ## exit if .msm.%s file already exists
     outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id)
     if os.path.exists(outfilename):
         print >> sys.stderr, "file already exists %s" % (outfilename)
         return
     outdata = []
     for data in MeasurementFetch.fetch(msm_id):
         tr = ripe.atlas.sagan.TracerouteResult(data)
         tracetxt = MeasurementPrint.trace2txt(data)
         src_prb_id = data['prb_id']
         dst_prb_id = probes_by_ip[data['dst_addr']]
         ixps = check_if_via_ixp(tr, ixp_radix)
         via_ixp = False
         if len(ixps) > 0: via_ixp = True
         #print "IXPS: %s" % ( ixps )
         #print tracetxt
         locs = MeasurementPrint.trace2locs(data)
         as_links = MeasurementEnhance.aslinksplus(data, ixp_radix)
         geojson = MeasurementEnhance.togeojson(data,
                                                probes_by_id[src_prb_id],
                                                probes_by_id[dst_prb_id])
         #print as_links
         countries = conf['country']
         if type(countries) != list:
             countries = [countries]
         countries = map(lambda x: x.upper(), countries)
         is_in_country = check_if_is_in_country(countries, locs)
         #print "INCOUNTRY: %s" % (is_in_country)
         dst_rtts = get_destination_rtts(tr)
         outdata.append({
             'ts': data['timestamp'],
             'result': data['result'],
             'protocol': protocol,
             'msm_id': msm_id,
             'as_links': as_links,
             'src_prb_id': src_prb_id,
             'dst_prb_id': dst_prb_id,
             #'src_asn': src_asn,
             #'dst_asn': dst_asn,
             #'last_rtt': tr.last_rtt,
             'dst_rtts': dst_rtts,
             #'target_responded': tr.target_responded,
             #'src_is_member': srcmb,
             #'dst_is_member': dstmb,
             ### more correctly: geojson linestring array
             'geojson': geojson,
             'in_country': is_in_country,
             'via_ixp': via_ixp,
             'ixps': ixps,
             'tracetxt': tracetxt,
             'locations': list(locs)
         })
     with open(outfilename, 'w') as outfile:
         json.dump(outdata, outfile, indent=2)
 def process_msm( msm_spec, protocol ):
    # msm_spec has msm_id
    msm_id = msm_spec['msm_id']
    print >>sys.stderr, "starting processing of %s" % ( msm_id )
    ## exit if .msm.%s file already exists
    outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
    if os.path.exists( outfilename ):
       print >>sys.stderr, "file already exists %s" % ( outfilename )
       return
    outdata = []      
    for data in MeasurementFetch.fetch( msm_id ):
       tr = ripe.atlas.sagan.TracerouteResult( data )
       tracetxt = MeasurementPrint.trace2txt( data )
       src_prb_id = data['prb_id']
       dst_prb_id = probes_by_ip[ data['dst_addr'] ]
       ixps = check_if_via_ixp( tr, ixp_radix ) 
       via_ixp = False
       if len(ixps) > 0: via_ixp = True
       #print "IXPS: %s" % ( ixps )
       #print tracetxt
       locs = MeasurementPrint.trace2locs( data )
       as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
       geojson = MeasurementEnhance.togeojson( data, probes_by_id[ src_prb_id ] , probes_by_id[ dst_prb_id ] )
       #print as_links
       countries = conf['country']
       if type(countries) != list:
          countries = [ countries ]
       countries = map(lambda x:x.upper(), countries)
       is_in_country = check_if_is_in_country( countries, locs )
       #print "INCOUNTRY: %s" % (is_in_country)
       dst_rtts = get_destination_rtts( tr )
       outdata.append( {
          'ts': data['timestamp'],
          'result': data['result'],
          'protocol': protocol,
          'msm_id': msm_id,
          'as_links': as_links,
          'src_prb_id': src_prb_id,
          'dst_prb_id': dst_prb_id,
          #'src_asn': src_asn,
          #'dst_asn': dst_asn,
          #'last_rtt': tr.last_rtt,
          'dst_rtts': dst_rtts,
          #'target_responded': tr.target_responded,
          #'src_is_member': srcmb,
          #'dst_is_member': dstmb,
          ### more correctly: geojson linestring array
          'geojson': geojson,
          'in_country': is_in_country,
          'via_ixp': via_ixp,
          'ixps': ixps,
          'tracetxt': tracetxt,
          'locations': list(locs)
       } )
    with open(outfilename,'w') as outfile:
       json.dump( outdata, outfile, indent=2 )
示例#4
0
def main():
    ips = set()
    with open('measurementset.json', 'r') as infile:
        msms = json.load(infile)
        msm_list = msms['v4'] + msms['v6']
        #      random.shuffle( msm_list )
        count = 0
        for m in msm_list:
            print >> sys.stderr, "(%d/%d) msm gathering, now fetching %s" % (
                count, len(msm_list), m)
            for data in MeasurementFetch.fetch(m['msm_id']):
                tr = ripe.atlas.sagan.TracerouteResult(data)
                for hop in tr.hops:
                    for pkt in hop.packets:
                        ip = pkt.origin
                        if pkt.arrived_late_by:  ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
                            continue
                        if ip != None:
                            ips.add(ip)
            count += 1
    no_ips = len(ips)
    print >> sys.stderr, "ip gathering finished, now analysing. ip count: %s" % (
        no_ips)
    ipcache = IPInfoCache.IPInfoCache()

    ips = list(ips)
    ips.sort()
    asns = set()

    lock = threading.Lock()

    def findInfo(ip):
        global counter
        res = ipcache.findIPInfo(ip)
        with lock:
            print '(%d/%d) %s / %s' % (counter, no_ips, ip, res)
            counter += 1
            sys.stdout.flush()
        if 'asn' in res and res['asn'] != None and res['asn'] != '':
            asns.add(res['asn'])

    with concurrent.futures.ThreadPoolExecutor(
            max_workers=cpu_count()) as executor:
        executor.map(findInfo, ips)

    # writes this file
    ipcache.toJsonFragments('ips-openipmap.json-fragments')
         if p.origin == trace.destination_address:
            dest_hop = hop.index
            break
   return ips

# for latest data
stop_t = time.time()
start_t = stop_t - interval

## ips seen as penultimate hop for responding dst
penult_ips = set()

prb_id2tr = {}

## properties per probe
for data in MeasurementFetch.fetch( 1767679, start=start_t, stop=stop_t ):
   tr = TracerouteResult( data )
   prb_id2tr[ tr.probe_id ] = data
   if tr.target_responded:
      pre_dest_ips = get_pre_dest_ips( tr )
      for ip in pre_dest_ips:
         penult_ips.add( ip )
      
count = 0
printit=[]

for data in MeasurementFetch.fetch( 1767680, start=start_t, stop=stop_t ):
   tr = TracerouteResult( data )
   if not tr.target_responded:
      last_ips = last_responding_ips( tr )
      for ip in last_ips:
   def process_msm( msm_spec, protocol ):
      # msm_spec has msm_id
      msm_id = msm_spec['msm_id']
      print >>sys.stderr, "starting processing of %s" % ( msm_id )
      ## exit if .msm.%s file already exists
      outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
      if os.path.exists( outfilename ):
         print >>sys.stderr, "file already exists %s" % ( outfilename )
         return
      outdata = []
      for data in MeasurementFetch.fetch( msm_id ):
         data = filter_cruft( data )
         assert 'edst' not in repr( data ), data
         tr = ripe.atlas.sagan.TracerouteResult( data )
         ip_list = tr.ip_path
         hops = []
         for i in ip_list:
            if len(i) > 0:
               hops.append(i[0])
         tracetxt = MeasurementPrint.trace2txt( data )
         #hops_ips = []

         #print ("tracetxt: " + tracetxt)
         
         src_prb_id = data['prb_id']
         src_prb = probes_by_id[ src_prb_id ]
         dst_prb_id = None
         dst_prb = None
         try:
            dst_prb_id = probes_by_ip[ data['dst_addr'] ]
            dst_prb = probes_by_id[  dst_prb_id ]
         except: pass
         if src_prb_id == dst_prb_id:
            ### probe to itself is not interesting/useful
            ## TODO filter this out in the measurement creation
            continue
         ixps = check_if_via_ixp( tr, ixp_radix ) 
         via_ixp = False
         if len(ixps) > 0: via_ixp = True
         #print "IXPS: %s" % ( ixps )
         #print tracetxt
         locs = MeasurementPrint.trace2locs( data )
         as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
         geojson = MeasurementEnhance.togeojson( data, src_prb , dst_prb )
         #print as_links
         countries = basedata['countries']
         is_in_country = check_if_is_in_country( countries, locs )
         #print "INCOUNTRY: %s" % (is_in_country)
         dst_rtts = get_destination_rtts( tr )
         outdata.append( {
            'ts': data['timestamp'],
            'result': data['result'],
            'protocol': protocol,
            'msm_id': msm_id,
            'as_links': as_links,
            'src_prb_id': src_prb_id,
            'dst_prb_id': dst_prb_id,
            #'src_asn': src_asn,
            #'dst_asn': dst_asn,
            #'last_rtt': tr.last_rtt,
            'dst_rtts': dst_rtts,
            #'target_responded': tr.target_responded,
            #'src_is_member': srcmb,
            #'dst_is_member': dstmb,
            ### more correctly: geojson linestring array
            'geojson': geojson,
            'in_country': is_in_country,
            'via_ixp': via_ixp,
            'ixps': ixps,
            'tracetxt': tracetxt,
            'locations': list(locs)
         } )

         outdatatraixroute.append( {
            'msm_id': msm_id,
            'ip_path' : hops, 
            'src_prb_id' : src_prb_id, 
            'dst_prb_id': dst_prb_id 
         } )
   

      with open(outfilename,'w') as outfile:
         json.dump( outdata, outfile, indent=2 )
 def process_msm(msm_spec, protocol):
     # msm_spec has msm_id
     msm_id = msm_spec['msm_id']
     print >> sys.stderr, "starting processing of %s" % (msm_id)
     ## exit if .msm.%s file already exists
     outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id)
     if os.path.exists(outfilename):
         print >> sys.stderr, "file already exists %s" % (outfilename)
         return
     outdata = []
     for data in MeasurementFetch.fetch(msm_id):
         data = filter_cruft(data)
         assert 'edst' not in repr(data), data
         tr = ripe.atlas.sagan.TracerouteResult(data)
         if 'dst_addr' in data:
             tracetxt = MeasurementPrint.trace2txt(data)
         else:
             continue
         src_prb_id = data['prb_id']
         src_prb = probes_by_id[src_prb_id]
         src_asn = None
         if data['af'] == 4:
             src_asn = src_prb['asn_v4']
         elif data['af'] == 6:
             src_asn = src_prb['asn_v6']
         dst_prb_id = None
         dst_prb = None
         try:
             dst_prb_id = probes_by_ip[data[
                 'dst_name']]  # dst name always has the IP that is in msmset/probeset
             dst_prb = probes_by_id[dst_prb_id]
         except:
             ### 2a01:7700:0:1033:220:4aff:fee0:2694 vs. 2a01:7700::1033:220:4aff:fee0:2694
             ##AAAAAAAAAAA
             print >> sys.stderr, "can't find dst_prb_id for this dst_name. SHOULD NOT HAPPEN"
         if src_prb_id == dst_prb_id:
             ### probe to itself is not interesting/useful
             ## TODO filter this out in the measurement creation
             continue
         ixps = check_if_via_ixp(tr, ixp_radix)
         via_ixp = False
         if len(ixps) > 0: via_ixp = True
         #print "IXPS: %s" % ( ixps )
         #print tracetxt
         locs = MeasurementPrint.trace2locs(data)
         as_links = MeasurementEnhance.aslinksplus(data,
                                                   ixp_radix,
                                                   src_asn=src_asn)
         geojson = MeasurementEnhance.togeojson(data, src_prb, dst_prb)
         #print as_links
         countries = basedata['countries']
         is_in_country = check_if_is_in_country(countries, locs)
         #print "INCOUNTRY: %s" % (is_in_country)
         dst_rtts = get_destination_rtts(tr)
         outdata.append({
             'ts': data['timestamp'],
             'result': data['result'],
             'protocol': protocol,
             'msm_id': msm_id,
             'as_links': as_links,
             'src_prb_id': src_prb_id,
             'dst_prb_id': dst_prb_id,
             #'src_asn': src_asn,
             #'dst_asn': dst_asn,
             #'last_rtt': tr.last_rtt,
             'dst_rtts': dst_rtts,
             #'target_responded': tr.target_responded,
             #'src_is_member': srcmb,
             #'dst_is_member': dstmb,
             ### more correctly: geojson linestring array
             'geojson': geojson,
             'in_country': is_in_country,
             'via_ixp': via_ixp,
             'ixps': ixps,
             'tracetxt': tracetxt,
             'locations': list(locs)
         })
     with open(outfilename, 'w') as outfile:
         json.dump(outdata, outfile, indent=2)
示例#8
0
def main():
    iprtt = {}
    with open('measurementset.json', 'r') as infile:
        msms = json.load(infile)
        msm_list = msms['v4'] + msms['v6']
        #      random.shuffle( msm_list )
        count = 0
        for m in msm_list:
            print >> sys.stderr, "(%d/%d) msm gathering, now fetching %s" % (
                count, len(msm_list), m)
            for data in MeasurementFetch.fetch(m['msm_id']):
                tr = ripe.atlas.sagan.TracerouteResult(data)
                for hop in tr.hops:
                    for pkt in hop.packets:
                        ip = pkt.origin
                        rtt = pkt.rtt
                        if pkt.arrived_late_by:  ## these are 'weird' packets ignore ehm (better would be to filter out pkts with 'edst')
                            continue
                        if ip != None:
                            if not ip in iprtt or iprtt[ip]['rtt'] > rtt:
                                iprtt[ip] = {
                                    'rtt': rtt,
                                    'prb_id': tr.probe_id,
                                    'msm_id': tr.measurement_id,
                                }

            count += 1
    ips = list(iprtt.keys())
    ips.sort()
    ip_count = len(ips)
    print >> sys.stderr, "ip gathering finished, now analysing. ip count: %s" % (
        ip_count)
    outf = open("ips.json-fragments", "w")
    outf_geoerr = open('geo-err.json-fragments', "w")

    no_result_cnt = 0
    for ip in ips:
        print >> sys.stderr, "attempting %s" % ip
        j = None
        #if True:
        try:
            out = {
                'ip': ip,
                'lon': None,
                'location': '',
                'lat': None,
                'hostname': "",
                'asn': "",
                'geo_error': None
            }
            req = requests.get(
                "https://ipmap.ripe.net/api/v1/locate/%s/partials?engines=probeslocation,crowdsourced,ixp"
                % ip,
                timeout=25)
            j = req.json()
            j['ip'] = ip
            #del( j['meta'] )
            #print "%s" % json.dumps( j )
            have_result = False
            try:
                out['hostname'] = ip2hostname(ip)
            except:
                print >> sys.stderr, "hostname lookup failed for %s" % ip
                pass
            try:
                out['asn'] = ip2asn(ip)
            except:
                print >> sys.stderr, "asn lookup failed for %s" % ip
                pass
            if 'partials' in j and len(j['partials']) > 0:
                for p in j['partials']:
                    if not have_result and p['engine'] in (
                            'probelocations', 'crowdsourced',
                            'ixp') and len(p['locations']) > 0:
                        #print '# %s' % ( p['locations'][0], )
                        for loc in p['locations']:
                            if 'cityNameAscii' in loc:
                                out['lat'] = loc['latitude']
                                out['lon'] = loc['longitude']
                                out['location'] = u"%s,%s" % (
                                    loc['cityNameAscii'],
                                    loc['countryCodeAlpha2'])
                                # {"ip": "213.19.197.126", "hostname": "infopact-ne.ear3.amsterdam1.level3.net", "lon": null, "location": "", "lat": null, "asn": 3356}
                                #print u'%s %s "%s"' % ( ip, p['engine'], loc['cityName'], loc['countryCodeAlpha2'] )
                                have_result = True
                                # now get the probe lat/lon and compare to ipmap + RTT
                                if out['lon'] and out['lat']:
                                    prb = run.probes_by_id[iprtt[ip]['prb_id']]
                                    geodist = haversine(
                                        out['lon'], out['lat'], prb['lon'],
                                        prb['lat'])
                                    rttdist = iprtt[ip]['rtt'] * 100
                                    # don't make geo_err judgements when prb-ip distance is under 100km
                                    if geodist < 100 or geodist < rttdist:
                                        out['geo_error'] = False
                                    else:
                                        out['geo_error'] = True
                                        g = out
                                        g['prb_id'] = iprtt[ip]['prb_id']
                                        g['msm_id'] = iprtt[ip]['msm_id']
                                        g['rtt'] = iprtt[ip]['rtt']
                                        g['geo_km'] = geodist
                                        g['rtt_km'] = rttdist
                                        print >> outf_geoerr, json.dumps(g)
                                        ## now reset the location, as we don't want it to be used in the real jedi
                                        out['lat'] = None
                                        out['lon'] = None
                                        out['location'] = ''
                                    print >> sys.stderr, "#RTT %s geo_km:%.2f rtt_km:%.2f prb_id:%s rtt:%.2f (%s)" % (
                                        ip, geodist, rttdist,
                                        iprtt[ip]['prb_id'], iprtt[ip]['rtt'],
                                        out['geo_error'])
                                else:
                                    print >> sys.stderr, "#error: no lat/lon in new ipmap?! %s" % (
                                        out)
                                break
        except:
            print >> sys.stderr, "#error on json for %s: %s" % (ip, j)
        print >> outf, u'%s' % json.dumps(out)
        if not have_result:
            no_result_cnt += 1
while start_t < stop_stop:
   stop_t = start_t + interval
   ## properties per probe
   probe_prop = {}
   for msm_id in sorted( measurements ):
      pfx_size = measurements[ msm_id ]['size']
      has_irr = measurements[ msm_id ]['irr']
      # index for 'fingerprinting' behaviours
      msm_idx = measurements[ msm_id ]['idx']
      responded=0
      count=0
      # last_ips will contain the last-responding IP, if the intended dest didn't respond
      last_ips = Counter()
      asns = {}
      for data in MeasurementFetch.fetch( msm_id, start=start_t , stop=stop_t ):
         tr = TracerouteResult( data )
         ## init probe_prop 
         if not tr.probe_id in probe_prop:
            probe_prop[ tr.probe_id ] = {
               'fingerprint': [None]*len( measurements ),
            }
         prb_asn = probes[ tr.probe_id ]['asn_v4']
         if not prb_asn in asns:
            asns[ prb_asn ] = {'count':0, 'responses':0}
         asns[prb_asn]['count'] += 1
         #if tr.probe_id == 14353:
         #   print MeasurementPrint.trace2txt( data, hostnames=False )
         #print "%s %s %s" % ( tr.measurement_id, tr.probe_id, tr.target_responded )
         last_resp_ips =  last_responding_ip( tr )
         pre_dest_ips = get_pre_dest_ips( tr )