Пример #1
0
def main():
    msms = {}
    with open('measurementset.json', 'r') as infile:
        msms = json.load(infile)
    probes = {}
    with open('probeset.json', 'r') as infile:
        probes = json.load(infile)
    probes_by_ip = {}
    probes_by_id = {}
    for p in probes:
        probes_by_id[p['probe_id']] = p
        if 'address_v4' in p and p['address_v4'] != None:
            probes_by_ip[p['address_v4']] = p['probe_id']
        if 'address_v6' in p and p['address_v6'] != None:
            probes_by_ip[p['address_v6']] = p['probe_id']
    #NOTE: there are IPs with multiple probes behind them, this just picks one.
    conf = {}
    with open('config.json', 'r') as infile:
        conf = json.load(infile)
    basedata = {}
    with open('basedata.json', 'r') as infile:
        basedata = json.load(infile)
    ixp_radix = create_ixp_radix(conf)
    MeasurementPrint.IPInfoCacheFromFile('ips.json-fragments')
    MeasurementPrint.setCacheOnly(True)
    MeasurementEnhance.IPInfoCacheFromFile('ips.json-fragments')
    MeasurementEnhance.setCacheOnly(True)
    if not os.path.exists(RESULTDIR):
        os.makedirs(RESULTDIR)

    def process_msm(msm_spec, protocol):
        # msm_spec has msm_id
        msm_id = msm_spec['msm_id']
        print >> sys.stderr, "starting processing of %s" % (msm_id)
        ## exit if .msm.%s file already exists
        outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id)
        if os.path.exists(outfilename):
            print >> sys.stderr, "file already exists %s" % (outfilename)
            return
        outdata = []
        for data in MeasurementFetch.fetch(msm_id):
            tr = ripe.atlas.sagan.TracerouteResult(data)
            tracetxt = MeasurementPrint.trace2txt(data)
            src_prb_id = data['prb_id']
            dst_prb_id = probes_by_ip[data['dst_addr']]
            ixps = check_if_via_ixp(tr, ixp_radix)
            via_ixp = False
            if len(ixps) > 0: via_ixp = True
            #print "IXPS: %s" % ( ixps )
            #print tracetxt
            locs = MeasurementPrint.trace2locs(data)
            as_links = MeasurementEnhance.aslinksplus(data, ixp_radix)
            geojson = MeasurementEnhance.togeojson(data,
                                                   probes_by_id[src_prb_id],
                                                   probes_by_id[dst_prb_id])
            #print as_links
            countries = conf['country']
            if type(countries) != list:
                countries = [countries]
            countries = map(lambda x: x.upper(), countries)
            is_in_country = check_if_is_in_country(countries, locs)
            #print "INCOUNTRY: %s" % (is_in_country)
            dst_rtts = get_destination_rtts(tr)
            outdata.append({
                'ts': data['timestamp'],
                'result': data['result'],
                'protocol': protocol,
                'msm_id': msm_id,
                'as_links': as_links,
                'src_prb_id': src_prb_id,
                'dst_prb_id': dst_prb_id,
                #'src_asn': src_asn,
                #'dst_asn': dst_asn,
                #'last_rtt': tr.last_rtt,
                'dst_rtts': dst_rtts,
                #'target_responded': tr.target_responded,
                #'src_is_member': srcmb,
                #'dst_is_member': dstmb,
                ### more correctly: geojson linestring array
                'geojson': geojson,
                'in_country': is_in_country,
                'via_ixp': via_ixp,
                'ixps': ixps,
                'tracetxt': tracetxt,
                'locations': list(locs)
            })
        with open(outfilename, 'w') as outfile:
            json.dump(outdata, outfile, indent=2)

    ## loop over measurements
    parallel_proc = 0
    children = set()
    msm_list = msms['v4'] + msms['v6']
    for m in msm_list:
        child_pid = os.fork()
        if child_pid == 0:
            process_msm(m, 4)  # only v4 atm
            sys.exit(0)
        else:
            children.add(child_pid)
            parallel_proc += 1
            if parallel_proc >= MAX_PARALLEL_PROCESSES:
                cpid, cstatus = os.wait()
                children.remove(cpid)
                parallel_proc -= 1
    for cpid in children:
        print "was still waiting for cpid: %s" % (cpid)
        os.waitpid(cpid, 0)
    print "FINISHED!"
def main():
   msms = {}
   with open('measurementset.json','r') as infile:
      msms = json.load( infile )
   probes = {}
   with open('probeset.json','r') as infile:
      probes = json.load( infile )
   probes_by_ip = {}
   probes_by_id = {}
   for p in probes:
      probes_by_id[ p['probe_id'] ] = p
      if 'address_v4' in p and p['address_v4'] != None:
         probes_by_ip[ p['address_v4'] ] = p['probe_id']
      if 'address_v6' in p and p['address_v6'] != None:
         probes_by_ip[ p['address_v6'] ] = p['probe_id']
   #NOTE: there are IPs with multiple probes behind them, this just picks one.

   # all auxilliary data should come from 'basedata' prepare-step should put it there
   # this is so we can fill out the blanks in prepare-stage
   #conf = {}
   #with open('config.json','r') as infile:
   #   conf = json.load ( infile )
   basedata = {}
   with open('basedata.json','r') as infile:
      basedata = json.load ( infile )
   ixp_radix = create_ixp_radix( basedata )
   MeasurementPrint.IPInfoCacheFromFile('ips.json-fragments')
   MeasurementPrint.setCacheOnly( True )
   MeasurementEnhance.IPInfoCacheFromFile('ips.json-fragments')
   MeasurementEnhance.setCacheOnly( True )
   if not os.path.exists(RESULTDIR):
      os.makedirs(RESULTDIR)
   
   outdatatraixroute = []

   def process_msm( msm_spec, protocol ):
      # msm_spec has msm_id
      msm_id = msm_spec['msm_id']
      print >>sys.stderr, "starting processing of %s" % ( msm_id )
      ## exit if .msm.%s file already exists
      outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
      if os.path.exists( outfilename ):
         print >>sys.stderr, "file already exists %s" % ( outfilename )
         return
      outdata = []
      for data in MeasurementFetch.fetch( msm_id ):
         data = filter_cruft( data )
         assert 'edst' not in repr( data ), data
         tr = ripe.atlas.sagan.TracerouteResult( data )
         ip_list = tr.ip_path
         hops = []
         for i in ip_list:
            if len(i) > 0:
               hops.append(i[0])
         tracetxt = MeasurementPrint.trace2txt( data )
         #hops_ips = []

         #print ("tracetxt: " + tracetxt)
         
         src_prb_id = data['prb_id']
         src_prb = probes_by_id[ src_prb_id ]
         dst_prb_id = None
         dst_prb = None
         try:
            dst_prb_id = probes_by_ip[ data['dst_addr'] ]
            dst_prb = probes_by_id[  dst_prb_id ]
         except: pass
         if src_prb_id == dst_prb_id:
            ### probe to itself is not interesting/useful
            ## TODO filter this out in the measurement creation
            continue
         ixps = check_if_via_ixp( tr, ixp_radix ) 
         via_ixp = False
         if len(ixps) > 0: via_ixp = True
         #print "IXPS: %s" % ( ixps )
         #print tracetxt
         locs = MeasurementPrint.trace2locs( data )
         as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
         geojson = MeasurementEnhance.togeojson( data, src_prb , dst_prb )
         #print as_links
         countries = basedata['countries']
         is_in_country = check_if_is_in_country( countries, locs )
         #print "INCOUNTRY: %s" % (is_in_country)
         dst_rtts = get_destination_rtts( tr )
         outdata.append( {
            'ts': data['timestamp'],
            'result': data['result'],
            'protocol': protocol,
            'msm_id': msm_id,
            'as_links': as_links,
            'src_prb_id': src_prb_id,
            'dst_prb_id': dst_prb_id,
            #'src_asn': src_asn,
            #'dst_asn': dst_asn,
            #'last_rtt': tr.last_rtt,
            'dst_rtts': dst_rtts,
            #'target_responded': tr.target_responded,
            #'src_is_member': srcmb,
            #'dst_is_member': dstmb,
            ### more correctly: geojson linestring array
            'geojson': geojson,
            'in_country': is_in_country,
            'via_ixp': via_ixp,
            'ixps': ixps,
            'tracetxt': tracetxt,
            'locations': list(locs)
         } )

         outdatatraixroute.append( {
            'msm_id': msm_id,
            'ip_path' : hops, 
            'src_prb_id' : src_prb_id, 
            'dst_prb_id': dst_prb_id 
         } )
   

      with open(outfilename,'w') as outfile:
         json.dump( outdata, outfile, indent=2 )
   ## loop over measurements
   #parallel_proc = 0
   #children = set()
   for m in msms['v4']:
        process_msm(m,4)
   for m in msms['v6']:
        process_msm(m,6)

   with open("traixroute.json",'w') as outfile:
       json.dump(outdatatraixroute, outfile, indent=2 )
def main():
   msms = {}
   with open('measurementset.json','r') as infile:
      msms = json.load( infile )
   probes = {}
   with open('probeset.json','r') as infile:
      probes = json.load( infile )
   probes_by_ip = {}
   probes_by_id = {}
   for p in probes:
      probes_by_id[ p['probe_id'] ] = p
      if 'address_v4' in p and p['address_v4'] != None:
         probes_by_ip[ p['address_v4'] ] = p['probe_id']
      if 'address_v6' in p and p['address_v6'] != None:
         probes_by_ip[ p['address_v6'] ] = p['probe_id']
   #NOTE: there are IPs with multiple probes behind them, this just picks one.
   conf = {}
   with open('config.json','r') as infile:
      conf = json.load ( infile )
   basedata = {}
   with open('basedata.json','r') as infile:
      basedata = json.load ( infile )
   ixp_radix = create_ixp_radix( conf )
   MeasurementPrint.IPInfoCacheFromFile('ips.json-fragments')
   MeasurementPrint.setCacheOnly( True )
   MeasurementEnhance.IPInfoCacheFromFile('ips.json-fragments')
   MeasurementEnhance.setCacheOnly( True )
   if not os.path.exists(RESULTDIR):
      os.makedirs(RESULTDIR)
   def process_msm( msm_spec, protocol ):
      # msm_spec has msm_id
      msm_id = msm_spec['msm_id']
      print >>sys.stderr, "starting processing of %s" % ( msm_id )
      ## exit if .msm.%s file already exists
      outfilename = "%s/msm.%s.json" % (RESULTDIR, msm_id )
      if os.path.exists( outfilename ):
         print >>sys.stderr, "file already exists %s" % ( outfilename )
         return
      outdata = []      
      for data in MeasurementFetch.fetch( msm_id ):
         tr = ripe.atlas.sagan.TracerouteResult( data )
         tracetxt = MeasurementPrint.trace2txt( data )
         src_prb_id = data['prb_id']
         dst_prb_id = probes_by_ip[ data['dst_addr'] ]
         ixps = check_if_via_ixp( tr, ixp_radix ) 
         via_ixp = False
         if len(ixps) > 0: via_ixp = True
         #print "IXPS: %s" % ( ixps )
         #print tracetxt
         locs = MeasurementPrint.trace2locs( data )
         as_links = MeasurementEnhance.aslinksplus( data, ixp_radix )
         geojson = MeasurementEnhance.togeojson( data, probes_by_id[ src_prb_id ] , probes_by_id[ dst_prb_id ] )
         #print as_links
         countries = conf['country']
         if type(countries) != list:
            countries = [ countries ]
         countries = map(lambda x:x.upper(), countries)
         is_in_country = check_if_is_in_country( countries, locs )
         #print "INCOUNTRY: %s" % (is_in_country)
         dst_rtts = get_destination_rtts( tr )
         outdata.append( {
            'ts': data['timestamp'],
            'result': data['result'],
            'protocol': protocol,
            'msm_id': msm_id,
            'as_links': as_links,
            'src_prb_id': src_prb_id,
            'dst_prb_id': dst_prb_id,
            #'src_asn': src_asn,
            #'dst_asn': dst_asn,
            #'last_rtt': tr.last_rtt,
            'dst_rtts': dst_rtts,
            #'target_responded': tr.target_responded,
            #'src_is_member': srcmb,
            #'dst_is_member': dstmb,
            ### more correctly: geojson linestring array
            'geojson': geojson,
            'in_country': is_in_country,
            'via_ixp': via_ixp,
            'ixps': ixps,
            'tracetxt': tracetxt,
            'locations': list(locs)
         } )
      with open(outfilename,'w') as outfile:
         json.dump( outdata, outfile, indent=2 )
   ## loop over measurements
   parallel_proc = 0
   children = set()
   msm_list = msms['v4'] + msms['v6']
   for m in msm_list:
      child_pid = os.fork()
      if child_pid == 0:
         process_msm( m, 4) # only v4 atm
         sys.exit(0)
      else:
         children.add( child_pid )
         parallel_proc += 1
         if parallel_proc >= MAX_PARALLEL_PROCESSES:
            cpid,cstatus = os.wait()
            children.remove( cpid )
            parallel_proc -= 1
   for cpid in children:
      print "was still waiting for cpid: %s" % ( cpid )
      os.waitpid(cpid,0)
   print "FINISHED!"