def calculate_effective_prefixes(self, start_interval=1451692800): ''' This method calculates the number of effective /24 prefixes that lie underneath a specified route prefix :param start_interval: the start interval of the data stream :return: number of effective prefixes under the route's root prefix ''' stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', self.collector_name) stream.add_filter('record-type', 'ribs') if isinstance(start_interval, datetime.datetime): interval_start_utc = self.convert_interval_to_utc(start_interval) stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300) else: stream.add_interval_filter(start_interval - 300, start_interval + 300) stream.start() print "Starting routing table parsing" while (stream.get_next_record(rec)): elem = rec.get_next_elem() while elem: # Get the peer ASN and IP. We then construct a peer_id, since a collector # can establish multiple connections with the same ASN. peer_asn = elem.peer_asn peer_asn_ip = elem.peer_address # make this an unmodifiable tuple peer_id = (peer_asn, peer_asn_ip) peer_route_trie = self.peer_trie_handles[peer_id] # Do a single level search of the route prefix to # find the left and right prefix advertisements single_level_prefixes = \ peer_route_trie.single_level_search(elem.fields['prefix']) print single_level_prefixes prefix_count = len(single_level_prefixes) root_24_prefix_count = 0 lr_24_prefix_count1 = lr_24_prefix_count2 = 0 # The /24 prefixes below the advertised prefix are calculated # as all the /24 prefixes served by the root - sum of the /24 # prefixes served by root's children if prefix_count == 1: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) elif prefix_count == 2: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count1 = \ 2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<') .rstrip('>').split('/')[1])) else: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count1 = \ 2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count2 = \ 2 ** (24 - int(str(single_level_prefixes[2]).lstrip('<') .rstrip('>').split('/')[1])) effective_24_prefix_count = \ root_24_prefix_count - (lr_24_prefix_count1 + lr_24_prefix_count2) print "Effective Prefix Count : ", \ effective_24_prefix_count for prefix in single_level_prefixes: trie_node = peer_route_trie.search_exact(str(prefix)) as_path = trie_node.data['as-path'].split(" ") as_path_headless = as_path[1:-1] print "AS-Path : ", as_path as_headless_len = len(as_path_headless) if as_headless_len > 1: for i in range(0, as_headless_len - 1): print "Headless nodes : ", as_path_headless[i], \ as_path_headless[i + 1] if as_path_headless[i] in self.aslink_datastore: self.aslink_datastore[as_path_headless[i]] += 1 else: self.aslink_datastore[as_path_headless[i]] = 1 elem = rec.get_next_elem()
def run(self): stream = BGPStream() rec = BGPRecord() if self.upd_file is None: stream.add_filter('collector', self.collector) stream.add_filter('record-type', self.record_type) stream.add_interval_filter(self.from_date, self.to_date) stream.set_live_mode() else: stream.set_data_interface('singlefile') if self.upd_file: stream.set_data_interface_option('singlefile', 'upd-file', self.upd_file) if self.rib_file: stream.set_data_interface_option('singlefile', 'rib-file', self.rib_file) if self.prefix_filter is not None: for prefix in self.prefix_filter: stream.add_filter('prefix', prefix) if self.peer_as_filter: for asn in self.peer_as_filter: stream.add_filter('peer-asn', str(asn)) if self.communities_filter: for community in self.communities_filter: stream.add_filter('community', community) stream.start() stream.get_next_record(rec) prev = rec.time while (stream.get_next_record(rec)): now = rec.time if rec.status == 'valid': elem = rec.get_next_elem() while (elem): statement = None peer_address = elem.peer_address peer_asn = elem.peer_asn if peer_asn in self.asn_to_nexthop: if elem.type == 'A' or elem.type == 'R': prefix = elem.fields['prefix'] as_path = elem.fields['as-path'] nexthop = elem.fields['next-hop'] if peer_asn in self.asn_to_nexthop: nexthop = self.asn_to_nexthop[peer_asn] statement = 'announce route %s next-hop %s as-path' \ ' [ %s ]' % (prefix, nexthop, as_path) elif elem.type == 'W': prefix = elem.fields['prefix'] statement = 'withdraw route %s' % prefix if statement: sys.stdout.write("%s\n" % statement) sys.stdout.flush() elem = rec.get_next_elem() time.sleep(self.delay + now - prev) prev = now
def sendMessageToKafka(self, col_name, col_data): print "lalalala" stream = BGPStream() record = BGPRecord() time_start = int(col_data.get('ribs').get('latestDumpTime')) time_end = time_start + int(col_data.get('ribs').get('dumpPeriod')) stream.add_filter('collector', col_name) stream.add_filter('record-type', 'ribs') stream.add_interval_filter(time_start, time_end) print "Before Start" stream.start() print "After Start" while stream.get_next_record(record): if record.status == "valid": elem = record.get_next_elem() while elem: # print "Record:{}".format(elem) producer = Kafka_producer() producer.send_data(col_name, json.dumps(elem)) elem = record.get_next_elem() else: print "## Current record not valid!" print "One Collector Finished"
def downloader(start_date, duration): """Download BGP paths from Routeviews and RIPE NCC from a start date for a certain duration.""" # Start of UNIX time base = int(datetime.datetime.strptime(start_date, '%m/%d/%Y').strftime('%s')) # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() stream.add_interval_filter(base, base + int(duration)) stream.add_filter('record-type', 'ribs') stream.start() path_set = set() f = open('rib.txt', 'w') while True: rec = stream.get_next_record() if rec is None: return if rec.status != "valid": continue else: elem = rec.get_next_elem() while(elem): path = elem.fields['as-path'] if '{' in path or '(' in path: elem = rec.get_next_elem() continue prefix = elem.fields['prefix'] # Focus on IPv4 prefixes if ":" not in prefix and path not in path_set: f.write(path.replace(' ', '|') + '\n') path_set.add(path) elem = rec.get_next_elem() f.close()
def _get_data(self, prefix, datetime): """ output example: [['15547', '8220', '1853', '1205'],[..another AS path..]] :param prefix: :param datetime: end interval :return: list of AS paths """ print('[*] ris.py: _get_data() called') print('[*] ris.py: _get_data() prefix: {}'.format(prefix)) start = int(datetime) - 20000 # 20000 second seems to be the shortest interval to get data from BGPstream stop = int(datetime) result = [] stream = BGPStream() rec = BGPRecord() stream.add_filter('prefix', prefix) stream.add_filter('record-type', 'ribs') stream.add_filter('project', 'ris') stream.add_interval_filter(start, stop) stream.start() while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: as_path = elem.fields['as-path'].split() as_path.append(' ') # for tree creation result.append(as_path) elem = rec.get_next_elem() print('[*] ris.py: _get_data() finished.') return result
def create_trie_from_bgpstream_info(self, interval_start=1451692800): stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', self.collector_name) stream.add_filter('record-type', 'ribs') if isinstance(interval_start, datetime.datetime): interval_start_utc = self.convert_interval_to_utc(interval_start) stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300) else: stream.add_interval_filter(interval_start - 300, interval_start + 300) stream.start() while (stream.get_next_record(rec)): elem = rec.get_next_elem() while elem: # Get the peer ASN and IP. We then construct a peer_id, since a collector # can establish multiple connections with the same ASN. peer_asn = elem.peer_asn peer_asn_ip = elem.peer_address # make this an unmodifiable tuple peer_id = (peer_asn, peer_asn_ip) peer_route_trie = self.peer_trie_handles[peer_id] trie_node = peer_route_trie.add(elem.fields['prefix']) trie_node.data['as-path'] = elem.fields['as-path'] elem = rec.get_next_elem()
def recv_bgpstream_rib(begin, until, collector): """ Receive and parse BGP RIB records from a given bgpstream collector. """ logging.info("CALL recv_bgpstream_rib") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type', 'updates') stream.add_interval_filter(begin, until) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping.") continue bgp_message = None while (elem): if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'): bgp_message = BGPmessage(elem.time, 'update') aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) bgp_message.add_announce(elem.fields['prefix']) output(bgp_message) elem = rec.get_next_elem() # end while (elem) # end while (stream...) output('FLUSH')
def get_data_graph(self): stream = BGPStream() rec = BGPRecord() stream.add_filter('record-type', 'ribs') stream.add_interval_filter(self.start, self.end) stream.start() as_graph = nx.Graph() while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: # the list is a list, which stores an as-path as_path = [ k for k, g in groupby(elem.fields['as-path'].split(" ")) ] peer = str(elem.peer_asn) # judge whether the as-path is legal if len(as_path) > 1 and as_path[0] == peer: if re.match(self.pattern, elem.fields['prefix']): self.as_prefix[as_path[-1]].add( elem.fields['prefix']) # add edges to the graph for i in range(0, len(as_path) - 1): as_graph.add_edge(as_path[i], as_path[i + 1]) elem = rec.get_next_elem() as_graph = self.add_geo_loc(as_graph) return as_graph
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # to track the peers that have elems in this record peer_signatures = set() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) peer_signatures.add(sig) # if this is the first time we have ever seen this peer, create # an empty result: (Pfxs_v4_set, Pfxs_v6_set, ASNs_v4_set, ASNs_v6_set) if sig not in peers_data: peers_data[sig] =[set(),set(), set(),set()] if('prefix' in elem.fields): pfx=elem.fields['prefix']; if(":" in pfx): peers_data[sig][1].add(pfx) else: peers_data[sig][0].add(pfx) if('as-path' in elem.fields): path_split=elem.fields['as-path'].split() if(":" in pfx): for ASN in path_split: peers_data[sig][3].add(ASN) else: for ASN in path_split: peers_data[sig][2].add(ASN) elem = rec.get_next_elem() # one peer that was present in this record (allows a true, per-collector # count of records since each record can contain elems for many peers) # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def main(): (options, args) = getopts() start = options.start_time end = options.end_time target_prefs = Set() with open('./../../atlas/anchor_prefix.txt', 'rb') as br: for l in br: target_prefs.add(l.strip()) # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() with open('./data/stream_{0}'.format(start), 'wb') as bw: #for pref in target_prefs: for ptmp in target_prefs: stream.add_filter('prefix', ptmp) # stream.add_filter('prefix','0.0.0.0/0') # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record cnt = 0 while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: cnt += 1 elem = rec.get_next_elem() while elem: if elem.type == 'S': continue # Print record and elem information # print rec.project, rec.collector, rec.type, rec.time, rec.status, # print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref bw.write( '{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'. format(rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.fields['prefix'], elem.peer_address, elem.peer_asn, elem.fields)) bw.flush() elem = rec.get_next_elem() print 'Successful termination; Start time: {0}'.format(start)
def dump2file(type, name, latestDumpTime, dumpDuration, dumpPeriod): print type + ' of ' + name + ': ' csv_header = ['type', 'addr', 'as', 'prefix', 'next_hop', 'as_path'] _file = None stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', c_name) if type == 'ribs': stream.add_filter('record-type', 'ribs') _file = open(c_name + '_ribs.csv', 'w+') elif type == 'updates': stream.add_filter('record-type', 'updates') _file = open(c_name + '_updates.csv', 'w+') stream.add_interval_filter(latestDumpTime, latestDumpTime + dumpPeriod) stream.start() count = 0 useless_c = 0 writer = csv.writer(_file) writer.writerow(csv_header) # Get next record while (stream.get_next_record(rec)): # Print the record information only if it is not a valid record if rec.status != "valid": # print rec.project, rec.collector, rec.type, rec.time, rec.status print 'current rec not valid.' else: elem = rec.get_next_elem() while (elem): useless_c += 1 if useless_c % 1000 == 0: print 'Got ' + str(useless_c) + ' elem totally.' # Print record and elem information if isIPV6(elem): # ipv6 packet ignored elem = rec.get_next_elem() continue count += 1 # print rec.project, rec.collector, rec.type, rec.time, rec.status, # print elem.type, elem.peer_address, elem.peer_asn # elem.fields contains four column: communities, next-hop, prefix, as-path field = elem.fields prefix = field['prefix'] if 'prefix' in field.keys() else '' next_hop = field['next-hop'] if 'next-hop' in field.keys( ) else '' as_path = field['as-path'] if 'as-path' in field.keys() else '' as_path = as_path.replace(' ', '|') writer.writerow([ elem.type, elem.peer_address, elem.peer_asn, prefix, next_hop, as_path ]) elem = rec.get_next_elem() _file.close() print 'count: ' + str(count) return count
def build_sql_db(): conn = sqlite3.connect('bgp_stage.db') c = conn.cursor() # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() stream.add_filter('collector','rrc11') stream.add_interval_filter(1438417216,1438417316) stream.start() while(stream.get_next_record(rec)): if rec.status != "valid": continue else: elem = rec.get_next_elem() while(elem): if elem.type == "AB": prefix = elem.fields["prefix"] as_path = elem.fields["as-path"].split(" ") origin = as_path[-1] time = elem.time #IP Prefix database ip_min, ip_max = calculate_min_max(prefix) c.execute("SELECT ip_min FROM prefix_as WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin)) row = c.fetchone() if len(row) != 0: c.execute("UPDATE prefix_as SET count = count + 1 WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin)) else: c.execute("INSERT INTO prefix_as VALUES(?,?,?,?,?)", (ip_min, ip_max, origin, 1, time)) #AS link database for as1,as2 in zip(as_path, as_path[1:]) : c.execute("SELECT as_o FROM as_link WHERE as_o = (?) AND as_n = (?)",(as1,as2)) row = c.fetchone() if len(row) != 0: c.execute("UPDATE as_link SET count = count + 1 WHERE as_o = (?) AND as_n = (?)", (as1, as2)) else: c.execute("INSERT INTO as_link VALUES(?,?,?,?)", (as1, as2, 1, 0)) elif elem.type == "WA": prefix = elem.fields["prefix"] time = elem.time #Needs research print(rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.peer_address, elem.peer_asn, elem.fields) print(prefix,elem.time, "W") print(rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.peer_address, elem.peer_asn, elem.fields) elem = rec.get_next_elem() conn.commit() conn.close()
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) # if this is the first time we have ever seen this peer, create # an empty result: (dict(Pfx_v4)=Pfx_origins, dict(Pfx_v6)=Pfx_origins)) if sig not in peers_data: peers_data[sig] =[{},{}] if('prefix' in elem.fields): pfx=elem.fields['prefix']; origin="" if('as-path' in elem.fields): path_split=elem.fields['as-path'].split() if(len(path_split)!=0): origin=path_split[len(path_split)-1] if(":" in pfx): if(pfx not in peers_data[sig][1]): peers_data[sig][1][pfx]=set() #discard as origin: AS sets, and ASN=23456 [AS_TRANS] if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][1][pfx].add(origin) else: if(pfx not in peers_data[sig][0]): peers_data[sig][0][pfx]=set() #discard as origin: AS sets, and ASN=23456 [AS_TRANS] if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][0][pfx].add(origin) elem = rec.get_next_elem() # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def main(): (options, args) = getopts() start = options.start_time end = options.end_time target_prefs = Set() with open('./../../atlas/anchor_prefix.txt', 'rb') as br: for l in br: target_prefs.add(l.strip()) # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() with open('./data/stream_{0}'.format(start), 'wb') as bw: #for pref in target_prefs: for ptmp in target_prefs: stream.add_filter('prefix', ptmp) # stream.add_filter('prefix','0.0.0.0/0') # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record cnt = 0 while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: cnt += 1 elem = rec.get_next_elem() while elem: if elem.type == 'S': continue # Print record and elem information # print rec.project, rec.collector, rec.type, rec.time, rec.status, # print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref bw.write('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'.format( rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.fields['prefix'], elem.peer_address, elem.peer_asn, elem.fields)) bw.flush() elem = rec.get_next_elem() print 'Successful termination; Start time: {0}'.format(start)
def main(): (options, args) = getopts() limit = options.limit start = options.start_time end = options.end_time # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') stream.add_filter('prefix', '0.0.0.0/0') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264) #stream.add_interval_filter(1454630400, 1454716800) # 1 hour #1454284800 - 1454288400 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record prefixes_update = defaultdict(int) prefixes_withdraw = defaultdict(int) while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while elem: if elem.type == 'A': #print elem.fields['as-path'] prefixes_update[elem.fields['prefix']] += 1 if elem.type == 'W': prefixes_withdraw[elem.fields['prefix']] += 1 #print rec.project, rec.collector, rec.type, rec.time, rec.status, #print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem() for k in prefixes_update: if prefixes_update[k] >= limit: print k + "\t" + str(prefixes_update[k]) + "\t" + str( prefixes_withdraw[k])
def main(): (options, args) = getopts() limit = options.limit start = options.start_time end = options.end_time # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') stream.add_filter('prefix', '0.0.0.0/0') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264) #stream.add_interval_filter(1454630400, 1454716800) # 1 hour #1454284800 - 1454288400 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record prefixes_update = defaultdict(int) prefixes_withdraw = defaultdict(int) while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while elem: if elem.type == 'A': #print elem.fields['as-path'] prefixes_update[elem.fields['prefix']] += 1 if elem.type == 'W': prefixes_withdraw[elem.fields['prefix']] += 1 #print rec.project, rec.collector, rec.type, rec.time, rec.status, #print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem() for k in prefixes_update: if prefixes_update[k] >= limit: print k + "\t" + str(prefixes_update[k]) + "\t" + str(prefixes_withdraw[k])
def download_data(): peer_state = defaultdict(dict) results = defaultdict(defaultdict_list) current_bin = 0 # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (startts, endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(startts, endts) stream.start() while (stream.get_next_record(rec)): if rec.status != "valid": print(rec.project, rec.collector, rec.type, rec.time, rec.status) # from IPython import embed # embed() if current_bin == 0: current_bin = rec.time # slide the time window: if current_bin + bin_size < rec.time: timebins = range(current_bin, rec.time, bin_size) for i in timebins[:-1]: results["other"]["timebin"].append(i) for pfx, p_s in peer_state.items(): for peeras, state in p_s.items(): results[pfx][peeras].append(state) current_bin = timebins[-1] elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn prefix = elem.fields["prefix"] peer_state[prefix][peeras] = elem.type elem = rec.get_next_elem() return results
def main(): parser = argparse.ArgumentParser() parser.formatter_class = argparse.RawDescriptionHelpFormatter parser.description = textwrap.dedent('''\ a proof-of-concept utility for watching updates from BGPstream and then printing out if an unexpected update is heard ''') parser.epilog = textwrap.dedent('''\ Example: watch these route announcements %(prog)s -f routes.yaml ''') required = parser.add_argument_group('required arguments') required.add_argument("-f", "--file", required=True, help="yaml file of prefixes to origin asn") parser.add_argument("-d", "--debug", action='store_true', help="print out all updates containing these prefixes") args = parser.parse_args() routes = pytricia.PyTricia(48) # longest reasonable pfx in dfz with open(args.file, 'r') as f: routesfile = yaml.safe_load(f) for pfx in routesfile: routes[pfx] = routesfile[pfx] stream = BGPStream() rec = BGPRecord() stream.add_filter('record-type', 'updates') stream.add_interval_filter(int(time.time()), 0) stream.set_live_mode() stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() while (elem): if 'as-path' in elem.fields: path = elem.fields['as-path'].split() prefix = elem.fields['prefix'] if prefix in routes and (routes[prefix] != path[-1] or args.debug): print('Heard prefix:', elem.fields['prefix'], 'AS-PATH:', elem.fields['as-path'], ' Found by project:', rec.project, 'collector:', rec.collector, 'type:', rec.type, 'at time:', rec.time, 'Type:', elem.type, 'Peer:', elem.peer_address, 'AS', elem.peer_asn) elem = rec.get_next_elem()
def MOAS_prefixes(): # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider Route Views Singapore only stream.add_filter('collector', 'route-views.sg') # Consider RIBs dumps only stream.add_filter('record-type', 'ribs') # Consider this time interval: # Sat, 01 Aug 2015 7:50:00 GMT - 08:10:00 GMT stream.add_interval_filter(1438415400, 1438416600) # Start the stream stream.start() # <prefix, origin-ASns-set > dictionary prefix_origin = defaultdict(set) prefix_origin_dict = defaultdict(list) # Get next record while (stream.get_next_record(rec)): elem = rec.get_next_elem() while (elem): # Get the prefix pfx = elem.fields['prefix'] # Get the list of ASes in the AS path ases = elem.fields['as-path'].split(" ") if len(ases) > 0: # Get the origin ASn (rightmost) origin = ases[-1] if 262857 == origin: #AS262857 - UFRN: AS262857 177.20.128.0/19 UNIVERSIDADE FEDERAL DO RIO GRANDE DO NORTE ''' BI 177.20.128.0/19 198.32.125.84 280 100 0 1916, 262857 IGP E 177.20.128.0/19 213.248.67.117 0 70 0 1299, 2914, 1916, 262857 IGP E 177.20.128.0/19 213.248.98.93 0 70 0 1299, 2914, 1916, 262857 IGP ''' print "Achou UFRN" # Insert the origin ASn in the set of # origins for the prefix prefix_origin[pfx].add(origin) prefix_origin_dict[pfx].append(ases) elem = rec.get_next_elem() # Print the list of MOAS prefix and their origin ASns for pfx in prefix_origin: if len(prefix_origin[pfx]) > 1: pass #print pfx, ",".join(prefix_origin[pfx])
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time - 1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # to track the peers that have elems in this record peer_signatures = set() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) peer_signatures.add(sig) # if this is the first time we have ever seen this peer, create # an empty result: (elem_cnt, peer_record_cnt, coll_record_cnt) if sig not in peers_data: peers_data[sig] = [0, 0, 0] peers_data[sig][0] += 1 # increment elem cnt for this peer elem = rec.get_next_elem() # done with elems, increment the 'coll_record_cnt' field for just # one peer that was present in this record (allows a true, per-collector # count of records since each record can contain elems for many peers) if len(peer_signatures): first = True for sig in peer_signatures: # increment peer_record_cnt for all if first: peers_data[sig][2] += 1 # increment the coll_record_cnt first = False peers_data[sig][1] += 1 # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def getting_BGP_update(): stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', 'rrc11') stream.add_interval_filter(1438417216, 1438417216) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status != "valid": print rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while (elem): print rec.project, rec.collector, rec.type, rec.time, rec.status, print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem()
def generate_stream(): bs = BGPStream() rec = BGPRecord() #initialize MySql a = MySqlDAL() a.start() #initialize the publisher in port number 12345 publisher = ZmqPublisher(12345) bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0) # bs.add_filter('collector', 'route-views.sg') bs.add_filter('record-type', 'updates') bs.start() print('Beginning to read from stream') input_id = 0 while bs.get_next_record(rec): elem = rec.get_next_elem() while elem is not None: # sleep until it is time to send this record ''' now = calendar.timegm(time.gmtime()) sim_time = now - delay if elem.time > sim_time: time.sleep(elem.time - sim_time) ''' if elem.type not in ['A', 'W']: continue input_id += 1 msg = elem2bgplay(rec, elem) msg['type'] = 'A' msg['id'] = input_id print(msg) # Publish the message publisher.publish(msg) # Write it to DB if elem.type == 'A': a.add(msg) elif elem.type == 'W': a.remove(msg) else: print "Error: Unknown type: " + elem.type elem = rec.get_next_elem()
def generate_stream(): bs = BGPStream() rec = BGPRecord() #initialize MySql db_writer = MysqlWriter() db_writer.start() #initialize the publisher in port number 12345 publisher = ZmqPublisher(12345) bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0) # bs.add_filter('collector', 'route-views.sg') bs.add_filter('record-type', 'updates') bs.start() print('Beginning to read from stream') input_id = 0 while bs.get_next_record(rec): elem = rec.get_next_elem() while elem is not None: # sleep until it is time to send this record ''' now = calendar.timegm(time.gmtime()) sim_time = now - delay if elem.time > sim_time: time.sleep(elem.time - sim_time) ''' if elem.type not in ['A', 'W']: continue input_id += 1 msg = elem2bgplay(rec, elem) msg['type'] = 'A' msg['id'] = input_id print(msg) # Publish the message publisher.publish(msg) # Write it to DB if elem.type == 'A': db_writer.add(msg) elif elem.type == 'W': db_writer.remove(msg) else: print "Error: Unknown type: " + elem.type elem = rec.get_next_elem()
def get_ribs(collector, start, period): stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) stream.add_interval_filter( int(start.timestamp()) - period, int(start.timestamp())) stream.add_filter('record-type', 'ribs') stream.start() while (stream.get_next_record(rec)): if rec.status == "valid": elem = rec.get_next_elem() while (elem): yield (rec.project, rec.collector, datetime.utcfromtimestamp(rec.time).isoformat(), elem.peer_address, elem.peer_asn, elem.fields['prefix'], elem.fields['as-path']) elem = rec.get_next_elem()
def recv_bgpstream_updates(begin, until, collector, output_queue): logging.info ("CALL recv_bgpstream_updates") # wait for first RIB table dump to complete while (rib_ts < 0): time.sleep(RIB_TS_WAIT/10) time.sleep(RIB_TS_WAIT) # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type','updates') stream.add_interval_filter(rib_ts,until) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue logging.info("Record TS: "+str(rec.time)) while (elem): logging.info(" -- Record Element Type: " + elem.type + ", TS: " + str(elem.time)) bgp_message = BGPmessage(elem.time, 'update') src_peer = dict() src_addr = elem.peer_address src_asn = elem.peer_asn src_peer['addr'] = src_addr src_peer['port'] = 0 src_peer['asn'] = src_asn bgp_message.set_source(src_peer) if elem.type.upper() == 'A': bgp_message.add_announce(elem.fields['prefix']) bgp_message.set_nexthop(elem.fields['next-hop']) aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) output_queue.put(bgp_message) elif elem.type.upper() == 'W': bgp_message.add_withdraw(elem.fields['prefix']) output_queue.put(bgp_message) elem = rec.get_next_elem()
def main(rib, target_prefs): # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() with open('./data/stream_{0}'.format(start), 'wb') as bw: for ptmp in target_prefs: stream.add_filter('prefix', ptmp) # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('record-type', 'ribs') #stream.add_filter('collector', 'rrc04') stream.add_filter('project', 'ris') stream.add_filter('project', 'routeviews') stream.add_interval_filter(start-60*60*8, start) stream.add_rib_period_filter(10000000000000) # Start the stream stream.start() while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": continue #if rec.time < start: elem = rec.get_next_elem() while elem: if elem.type == 'A' or elem.type == 'R': rib.add_to_rib(rec.collector, elem.peer_address, elem.fields['prefix'], elem.time, elem.fields['as-path']) elem = rec.get_next_elem() #else: rib.flush() print 'Successful termination; Start time: {0}'.format(start)
def recv_bgpstream_rib(begin, until, collector, output_queue): logging.info ("CALL recv_bgpstream_rib") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type','updates') stream.add_interval_filter(begin,until) # Start the stream stream.start() while (stream.get_next_record(rec)): global rib_ts if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping.") continue if (rib_ts > 0) and (rec.time > (rib_ts + RIB_TS_INTERVAL/2)): logging.info("received full RIB table dump.") break bgp_message = None while (elem): if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'): rib_ts = elem.time bgp_message = BGPmessage(elem.time, 'update') bgp_message.set_nexthop(elem.fields['next-hop']) src_peer = dict() src_addr = elem.peer_address src_asn = elem.peer_asn src_peer['addr'] = src_addr src_peer['port'] = 0 src_peer['asn'] = src_asn bgp_message.set_source(src_peer) aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) bgp_message.add_announce(elem.fields['prefix']) output_queue.put(bgp_message) elem = rec.get_next_elem()
def count_valid_withdrawals_per_second(t0,num_seconds): # div0 in progress tracking if <100 seconds assert num_seconds>100; # Create a new bgpstream instance stream = BGPStream(); # use Routeviews Saopaulo records stream.add_filter('project','routeviews'); stream.add_filter('collector','route-views.saopaulo'); stream.add_filter('record-type','updates'); # time interval stream.add_interval_filter(t0,t0+tt-1); # progress tracking prog = t0; # Create a reusable bgprecord instance rec = BGPRecord(); # list of zeroes ctr = [0]*tt; # Start the stream stream.start(); # iterate over records while(stream.get_next_record(rec)): if rec.status == "valid": elem = rec.get_next_elem(); while(elem): if elem.type == 'W': ctr[rec.time-t0] += 1; elem = rec.get_next_elem(); # el-cheapo progress indication with dots. comment out if you don't want the dots. # """ if rec.time > prog and (rec.time - t0) % floor(num_seconds/100) == 0: sys.stdout.write('.'); sys.stdout.flush(); prog = rec.time; # """ # print newline after the dots print ""; return ctr;
def readUpdates(self): # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in self.prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) stream.start() while (stream.get_next_record(rec)): if (rec.status != "valid" and rec.status != "filtered-source" and rec.status != "empty-source"): raise Exception(rec.project, rec.collector, rec.type, rec.time, rec.status) zDt = rec.time elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn sPrefix = elem.fields["prefix"] if elem.type == "W": self.withdraws[sPrefix][peeras] = True else: sPath = elem.fields["as-path"] self.paths[sPrefix][peeras] = sPath self.withdraws[sPrefix][peeras] = False elem = rec.get_next_elem()
def get_updates(collector, start, end): print("getting updates for {} from {} to {}".format(collector, start, end)) stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) stream.add_interval_filter(int(start.timestamp()), int(end.timestamp())) stream.add_filter('record-type', 'updates') stream.start() while (stream.get_next_record(rec)): if rec.status == "valid": elem = rec.get_next_elem() while (elem): if elem.type == 'A': yield ('A', rec.project, rec.collector, datetime.utcfromtimestamp(rec.time).isoformat(), elem.peer_address, elem.peer_asn, elem.fields['prefix'], elem.fields['as-path']) elif elem.type == 'W': yield ('W', rec.project, rec.collector, datetime.utcfromtimestamp(rec.time).isoformat(), elem.peer_address, elem.peer_asn, elem.fields['prefix'], None) elem = rec.get_next_elem()
def recv_bgpstream_updates(begin, until, collector): """ Receive and parse BGP update records from a given bgpstream collector """ logging.info("CALL recv_bgpstream_updates") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type', 'updates') stream.add_interval_filter(begin, until) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue logging.info("Record TS: " + str(rec.time)) while (elem): logging.info(" -- Record Element Type: " + elem.type + ", TS: " + str(elem.time)) bgp_message = BGPmessage(elem.time, 'update') if elem.type.upper() == 'A': bgp_message.add_announce(elem.fields['prefix']) aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) output(bgp_message) elif elem.type.upper() == 'W': bgp_message.add_withdraw(elem.fields['prefix']) output(bgp_message) elem = rec.get_next_elem()
if (not prefix in prefix_set): prefix_set.add(prefix) prefix_file.write("{pfx}|IP_Block\n".format(pfx=prefix)) if (not (as_path[-1], prefix) in connections_set): connections_set.add((as_path[-1], prefix)) connect_rels_file.write("{asn}|{pfx}|ORIGINATES\n".format( asn=as_path[-1], pfx=prefix)) elem = rec.get_next_elem() # Main loop route_id = None started = False prev_rib_dump = 0 while (stream.get_next_record(rec)): if rec.dump_time != prev_rib_dump: dt = datetime.datetime.fromtimestamp(rec.dump_time) time_file.write("{time}|{Y}|{M}|{D}|{h}|{m}|{s}|Time\n".format( time=rec.dump_time, Y=dt.year, M=dt.month, D=dt.day, h=dt.hour, m=dt.minute, s=dt.second)) if (prev_rib_dump != 0): time_rels_file.write("{t1}|{t2}|NEXT\n".format(t1=prev_rib_dump, t2=rec.dump_time))
def main(): parser = argparse.ArgumentParser(description='', epilog='') parser.add_argument('-b', '--begin', help='Begin date (inclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-u', '--until', help='Until date (exclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-c', '--collector', help='Route collector from RIPE RIS or Route-Views project.', type=str, required=True) parser.add_argument('-m', '--mongodb', help='MongoDB connection parameters.', type=str, default=None) parser.add_argument('-l', '--loglevel', help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].', type=str, default='WARNING') args = vars(parser.parse_args()) numeric_level = getattr(logging, args['loglevel'].upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, format='%(asctime)s : %(levelname)s : %(message)s') ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds()) ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds()) mongodbstr = None if args['mongodb']: mongodbstr = args['mongodb'].strip() # BEGIN logging.info("START") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector',args['collector']) stream.add_filter('record-type','ribs') stream.add_interval_filter(ts_begin,ts_until) # Start the stream stream.start() rib_ts = 0 rib_origins = dict() while(stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue if rec.time > (rib_ts + RIB_TS_INTERVAL): rib_ts = rec.time if mongodbstr: store_rib_origins(rib_ts, rib_origins, mongodbstr) else: print_rib_origins(rib_ts, rib_origins) rib_origins = dict() while(elem): prefix = elem.fields['prefix'] aspath = elem.fields['as-path'].split() for a in aspath: # remove AS-SETs if '{' in a: aspath.remove(a) origin = aspath[-1] if prefix not in rib_origins: rib_origins[prefix] = list() if origin not in rib_origins[prefix]: rib_origins[prefix].append(origin) elem = rec.get_next_elem()
stream.add_filter('prefix','140.78.0.0/16') stream.add_filter('record-type','ribs') stream.add_filter('collector','rrc11') # stream.add_filter('project','ris') stream.add_interval_filter(1464681000, 1464682200) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC) stream.start() # # while(stream.get_next_record(rec)): # # Print the record information only if it is not a valid record # if rec.status == "valid": # elem = rec.get_next_elem() # while elem: # # Print record and elem information # print rec.collector, elem.peer_address, elem.peer_asn, elem.fields['as-path'] # elem = rec.get_next_elem() result = [] while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields as_path = elem.fields['as-path'].split() as_path.append(' ') # for tree creation result.append(as_path) elem = rec.get_next_elem() print('-- elapsed time: {} seconds'.format(time.time() - start_time)) print(result)
def main(): parser = argparse.ArgumentParser(description='', epilog='') parser.add_argument('-b', '--begin', help='Begin date (inclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-u', '--until', help='Until date (exclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-c', '--collector', help='Route collector from RIPE RIS or Route-Views project.', type=str, required=True) parser.add_argument('-m', '--mongodb', help='MongoDB connection parameters.', type=str, default=None) parser.add_argument('-k', '--keepsnapshots', help='Keep all snapshots, works only with -s.', action='store_true') parser.add_argument('-s', '--snapshot', help='Enable snapshoting.', action='store_true') parser.add_argument('-l', '--loglevel', help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].', type=str, default='WARNING') args = vars(parser.parse_args()) numeric_level = getattr(logging, args['loglevel'].upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, format='#> %(asctime)s : %(levelname)s : %(message)s') ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds()) ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds()) mongodbstr = None if args['mongodb']: mongodbstr = args['mongodb'].strip() rib_ts = 0 rib_origins = dict() origins_lt = list() if args['snapshot']: rib_ts, rib_origins = load_snapshot(mongodbstr) if rib_ts > ts_begin: logging.info ("SKIP, found snapshot with newer ts") ts_begin = rib_ts - RIB_TS_THRESHOLD # BEGIN logging.info("START") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector',args['collector']) stream.add_filter('record-type','ribs') stream.add_interval_filter(ts_begin,ts_until) # Start the stream stream.start() while(stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue #end if if rec.time > (rib_ts + RIB_TS_THRESHOLD): for p in rib_origins: for o in rib_origins[p]: if rib_origins[p][o][1] < (rib_ts - RIB_TS_THRESHOLD): origins_lt.append( (p,o,rib_origins[p][o][0],rib_origins[p][o][1]) ) #end if #end for #end for if args['snapshot'] and (len(rib_origins.keys()) > 0): store_snapshot(rec.time, rib_origins, mongodbstr) if not args['keepsnapshots']: remove_snapshot(rib_ts, mongodbstr) # end if keepsnapshots # end if snapshot rib_ts = rec.time logging.info("ts: "+str(rib_ts)) if len(origins_lt) > 0: if mongodbstr: store_origins_lt(rib_ts,origins_lt, mongodbstr) else: print_origins_lt(rib_ts,origins_lt) #end if for l in origins_lt: del rib_origins[l[0]][l[1]] #end for origins_lt = list() # end if #end if while(elem): prefix = elem.fields['prefix'] aspath = elem.fields['as-path'].split() for a in aspath: # remove AS-SETs if '{' in a: aspath.remove(a) #end if #end for origin = aspath[-1] if prefix not in rib_origins: rib_origins[prefix] = dict() #end if if origin not in rib_origins[prefix]: rib_origins[prefix][origin] = (rib_ts,rib_ts) else: rib_origins[prefix][origin] = (rib_origins[prefix][origin][0],rib_ts) #end if elem = rec.get_next_elem() #end while #end while if args['snapshot']: print "NOTE: remaining origin lifetimes are stored in latest snapshot (%d)!\n" % rib_ts if (len(rib_origins.keys()) > 0): store_snapshot(rib_ts, rib_origins, mongodbstr) # end if else: print "NOTE: output remaining origin lifetimes with current ts (%d)\n" % rib_ts origins_lt = list() for p in rib_origins: for o in rib_origins[p]: origins_lt.append( (p,o,rib_origins[p][o][0],rib_ts) ) if mongodbstr: store_origins_lt(rib_ts,origins_lt, mongodbstr) else: print_origins_lt(rib_ts,origins_lt)
#stream_start = 1454284800 #stream_end = 1454288400 buckets = create_time_buckets(stream_start, stream_end) prefixList = [] raw_bgp_stream_data = {} stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', 'rrc06') stream.add_filter('record-type', 'updates') stream.add_interval_filter(stream_start, stream_end) stream.start() while(stream.get_next_record(rec)): elem = rec.get_next_elem() while(elem): prefix = elem.fields.get("prefix", "") time_stamp = rec.time # unix epoc timestamp 1427846670 if prefix != "": deal_with_time_bucket_junk(prefix, time_stamp) elem = rec.get_next_elem() for prefix in list(raw_bgp_stream_data):
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) # if this is the first time we have ever seen this peer, create # an empty result: (Pfxs_v4_set , Pfxs_v6_set, # dict(Transit_ASN)=Pfxs_v4_set , dict(Transit_ASN)=Pfxs_v6_set) if sig not in peers_data: peers_data[sig] =[set(),set(),{},{}] pfx="" if('prefix' in elem.fields): pfx=elem.fields['prefix']; if(":" in pfx): peers_data[sig][1].add(pfx) else: peers_data[sig][0].add(pfx) if('as-path' in elem.fields): #Squash the AS Path to keep only distinct ASNs, i.e., remove prepending path_split = [k for k, g in groupby(elem.fields['as-path'].split(" "))] if(len(path_split)!=0): for i in range(1,len(path_split)-1): transit=path_split[i] if(":" in pfx): if(transit not in peers_data[sig][3]): peers_data[sig][3][transit]=set() peers_data[sig][3][transit].add(pfx) elif(pfx!=""): if(transit not in peers_data[sig][2]): peers_data[sig][2][transit]=set() peers_data[sig][2][transit].add(pfx) elem = rec.get_next_elem() # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
rec = BGPRecord() stream.add_filter('project', 'ris') timestamp = int(time.time()) # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 stream.add_recent_interval_filter('12 h', False) # Start the stream stream.start() l = 0 out = list() # Get next record while (stream.get_next_record(rec) and l < 100): # Print the record information only if it is not a valid record if rec.status == "valid": elem = rec.get_next_elem() while (elem and l < 100): el = dict() el['project'] = rec.project el['collector'] = rec.collector el['record_type'] = rec.type el['time'] = rec.time el['status'] = rec.status el['elem_type'] = elem.type el['peer_address'] = elem.peer_address el['peer_asn'] = elem.peer_asn el['fields'] = elem.fields out.append(el)
class BGPDataAggregator(object): """docstring for BGPDataAggregator""" def __init__( self, filters={"collector": ["rrc00"]}, rpki_validator="rpki-validator.realmv6.org:8282", settings_file="../settings.json", ): self.stream = BGPStream() self.filters = filters self.route_table = dict() self.i = 0 self.metadata_vp = dict() self.metadata_rc = dict() self.peers = Counter() self.prefix4 = Counter() self.prefix6 = Counter() start_timestamp = get_push_timestamp(datetime.now(timezone.utc)) for filter_type, filter_array in filters.items(): for filter_value in filter_array: self.stream.add_filter(filter_type, filter_value) for collector in filters["collector"]: self.route_table[collector] = defaultdict(dict) self.metadata_vp[collector] = defaultdict(list) self.metadata_rc[collector] = RouteCollectorMeta(None, 0, 0, 0, 0) self.peers[collector] = defaultdict(int) self.prefix4[collector] = defaultdict(int) self.prefix6[collector] = defaultdict(int) settings = get_settings(settings_file) settings["db"]["password"] = os.environ["PGPASS"] self.db = DBConnector(settings["db"]) rpki = rpki_validator.split(":") self.mgr = RTRManager(rpki[0], rpki[1]) self.mgr.start() self.start_collecting(start_timestamp) def __del__(self): if self.mgr.is_synced(): self.mgr.stop() def push_data(self, timestamp): print("UPDATE:", timestamp) self.db.update_vp_meta(self.metadata_vp) for rc in self.metadata_rc.keys(): self.metadata_rc[rc] = RouteCollectorMeta( rc, timestamp, len(self.peers[rc].keys()), len(self.prefix4[rc]), len(self.prefix6[rc]), ) self.db.update_rc_meta(self.metadata_rc) def start_collecting(self, start_timestamp, end_timestamp=0): self.stream.add_interval_filter(start_timestamp, end_timestamp) print("Start BGPStream:", start_timestamp, end_timestamp) next_timestamp = init_next_timestamp(start_timestamp, 5) print("Next Push to DB at:", next_timestamp) self.stream.start() rec = BGPRecord() while (self.stream.get_next_record(rec)): if rec.status == "valid": if rec.time >= next_timestamp: self.push_data(next_timestamp) next_timestamp += 300 elem = rec.get_next_elem() while elem: origin_asn = "" if elem.type is "R" or elem.type is "A": origin_asn = elem.fields["as-path"].split(" ")[-1] try: origin_asn = int(origin_asn) except ValueError: elem = rec.get_next_elem() continue prefix = elem.fields["prefix"] ip, mask_len = split_prefix(prefix) # Check if v4 or v6 is_v4 = check_ipv4(ip) validated = self.mgr.validate(origin_asn, ip, mask_len) old_elem = self.route_table[rec.collector][( elem.peer_asn, elem.peer_address)].get(prefix) if elem.type is "R" or elem.type is "A": self.route_table[rec.collector][( elem.peer_asn, elem.peer_address)][prefix] = Route( origin_asn, rec.collector, prefix, is_v4, validated.state.value, ) if old_elem: if old_elem.type != validated.state.value: """Make use of the fact that: 0: valid in enum 1: unknown in enum 2: invalid in enum We designed the namedtuple the way to represent that. So valid is a pos 3 and so on. """ self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)][4 + old_elem.type] -= 1 self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address )][4 + validated.state.value] += 1 else: if not self.metadata_vp[rec.collector].get( (elem.peer_asn, elem.peer_address)): """Init the metadata-entry if it not exists already""" self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)] = [ elem.peer_asn, elem.peer_address, rec.collector, next_timestamp, 0, 0, 0, ] # Update the VantagePoint Metadate the same way like above. self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)][4 + validated.state.value] += 1 self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)][3] = next_timestamp self.peers[rec.collector][elem.peer_asn] += 1 if is_v4: self.prefix4[rec.collector][prefix] += 1 else: self.prefix6[rec.collector][prefix] += 1 elif elem.type is "W": if old_elem: # Reduce the number of IPv4/v6 Addresses for this prefix if is_v4: self.prefix4[rec.collector][prefix] -= 1 if self.prefix4[rec.collector][prefix] == 0: del (self.prefix4[rec.collector][prefix]) else: self.prefix6[rec.collector][prefix] -= 1 if self.prefix6[rec.collector][prefix] == 0: del (self.prefix6[rec.collector][prefix]) # Reduce number of prefixes belonging to this ASN self.peers[rec.collector][elem.peer_asn] -= 1 if self.peers[rec.collector][elem.peer_asn] == 0: del (self.peers[rec.collector][elem.peer_asn]) # Update the metadata valid/unknown/invalid count self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)][4 + old_elem.type] -= 1 # Update the metadata timestamp self.metadata_vp[rec.collector][( elem.peer_asn, elem.peer_address)][3] = next_timestamp # Remove the entry from the route_table self.route_table[rec.collector][( elem.peer_asn, elem.peer_address)].pop(prefix, None) else: # !!TODO: write log about that! pass elem = rec.get_next_elem()
def load_data(start, stop, collectors, window, threshold): peers = {} # collectors is a list of the collectors we want to include # Start and stop define the interval we are looking in the data # Create a new BGPStream instance and a reusable BGPRecord instance stream = BGPStream() rec = BGPRecord() # Add filter for each collector. # If no collector is mentioned, it will consider 16 of them if collectors: for collector in collectors: print collector stream.add_filter('collector', collector) else: for i in range(0, 10): stream.add_filter('collector', 'rrc0' + str(i)) for i in range(10, 16): stream.add_filter('collector', 'rrc' + str(i)) stream.add_filter('record-type', 'updates') # Consider the interval from "start" to "stop" in seconds since epoch stream.add_interval_filter(start, stop) # Start the stream stream.start() # For each record (one record = one second, can have multiple elements for the same second) we handle its updates while stream.get_next_record(rec): timestamp = rec.time if rec.status != "valid": print rec.project, rec.collector, rec.type, timestamp, rec.status else: # Go through all elements of the record elem = rec.get_next_elem() while elem: # Consider only the A and W updates if elem.type not in ['A', 'W']: elem = rec.get_next_elem() continue peer = elem.peer_address updatetype = elem.type prefix = elem.fields['prefix'] if peer not in peers: peers[peer] = { 'A': [], 'W': [] } update = {'tst': timestamp, 'prefix': prefix} if updatetype == 'A': handleUpdate(peers[peer]['A'], burst2writeA, update, peer, updatetype, timestamp, window, threshold) saveGraphPoint(peers[peer]['A'], updatetype, peer, timestamp, collectors, threshold) else: handleUpdate(peers[peer]['W'], burst2writeW, update, peer, updatetype, timestamp, window, threshold) saveGraphPoint(peers[peer]['W'], updatetype, peer, timestamp, collectors, threshold) elem = rec.get_next_elem() # After processing all records, we write the graph json files with the graph points recorded for each peer for peer in graph_points: peer_file_name = peer.replace(':', '_') if not os.path.exists(peer_file_name): os.makedirs(peer_file_name) with open(peer_file_name+'/'+peer_file_name + '-graph.json', 'w') as outfile: json.dump(graph_points[peer], outfile, indent=2) # Write the last burst of A updates if there is one left if burst2writeA: for peer in burst2writeA: if burst2writeA[peer]: for timestamp in burst2writeA[peer]: writeBurst(peer, burst2writeA, 'A', timestamp) # Write the last burst of W updates if there is one left if burst2writeW: for peer in burst2writeW: if burst2writeW[peer]: for timestamp in burst2writeW[peer]: writeBurst(peer, burst2writeW, 'W', timestamp) # transform csv names in json file to use getJSON in plotGrap # step to CSV is used to avoid appending to the end of a json file directly as appending # to a json file overwrite the whole file jsonlist = [] with open('csv_peernames-'+'-'.join(collectors)+'.csv', 'rb') as f: reader = csv.reader(f) for row in reader: jsonlist.append(row[0]) jsondata = json.dumps(jsonlist, indent=2) fd = open('json_file_names-' + '-'.join(collectors) + '.json', 'w') fd.write(jsondata) fd.close()
def build_bgp_graph(project=None, collector=None): print project, collector global bgp_graphs stream = BGPStream() rec = BGPRecord() if collector: stream.add_filter('collector', collector) else: stream.add_filter('project', project) # Consider RIBs dumps only stream.add_filter('record-type', 'ribs') # Consider this time interval: #cur_time = int( time.time() ) cur_time = 1471219200 if collector and 'views' in collector: interval = 3 else: interval = 10 #prev_time = int(cur_time - (60 * 60 * interval)) prev_time = 1471132800 print "Starting stream with %d interval" % interval stream.add_interval_filter(prev_time, cur_time) stream.start() ribEntryCount = 0 loopCount = 0 while (stream.get_next_record(rec)): elem = rec.get_next_elem() while (elem): pdb.set_trace() ribEntryCount += 1 peer = str(elem.peer_asn) hops = [k for k, g in groupby(elem.fields['as-path'].split(" "))] loops = [i for i, x in enumerate(hops) if hops.count(x) > 1] if loops: # print( "Routing loop! AS path %s" % elem.fields['as-path'] ) loopCount += 1 elem = rec.get_next_elem() continue if '{' in elem.fields['as-path']: elem = rec.get_next_elem() continue if len(hops) > 1 and hops[0] == peer: # Get the origin ASN, is the destination for traffic # toward this prefix origin = hops[-1] if origin in ixp.IXPs: print "Origin", origin, "is an IXP, it announced prefix", elem.fields[ 'prefix'] elem = rec.get_next_elem() continue if origin in bgp_graphs: as_graph = bgp_graphs[origin] else: as_graph = nx.DiGraph() bgp_graphs[origin] = as_graph # Add new edges to the NetworkX graph new_hops = [] for hop in hops: if hop in ixp.IXPs: continue new_hops.append(hop) if len(new_hops) <= 1: elem = rec.get_next_elem() continue for i in range(0, len(new_hops) - 1): as_graph.add_edge(new_hops[i], new_hops[i + 1]) # Making sure, root has 0 out degree #if as_graph.out_degree( origin ) != 0: # pdb.set_trace() elem = rec.get_next_elem() print "Total RIB entries parsed", ribEntryCount, "Number of AS paths with loops", loopCount
def get_data(timestamp): graph = Graph(password="******") stream = BGPStream() rec = BGPRecord() rec_time = None # stream.add_filter('prefix', '198.41.0.0/24') # A-root # stream.add_filter('prefix', '192.228.79.0/24') # B-root, only 1 site # stream.add_filter('prefix', '192.33.4.0/24') # C-root # stream.add_filter('prefix', '199.7.91.0/24') # D-root # stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only # stream.add_filter('prefix', '192.5.5.0/24') # F-root # stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only # stream.add_filter('prefix', '198.97.190.0/24') # H-root # stream.add_filter('prefix', '192.36.148.0/24') # I-root # stream.add_filter('prefix', '192.58.128.0/24') # J-root stream.add_filter('prefix', '193.0.14.0/24') # K-root # stream.add_filter('prefix', '199.7.83.0/24') # L-root # stream.add_filter('prefix', '202.12.27.0/24') # M-root # IPv6 # stream.add_filter('prefix', '2001:503:ba3e::/48') # A ## stream.add_filter('prefix', '2001:500:84::/48') # B, only 1 site # stream.add_filter('prefix', '2001:500:2::/48') # C # stream.add_filter('prefix', '2001:500:2d::/48') # D # stream.add_filter('prefix', '2001:500:2f::/48') # F # stream.add_filter('prefix', '2001:500:1::/48') # H # stream.add_filter('prefix', '2001:7fe::/33') # I # stream.add_filter('prefix', '2001:503:c27::/48') # J # stream.add_filter('prefix', '2001:7fd::/48') # K # stream.add_filter('prefix', '2001:500:9f::/48') # L # stream.add_filter('prefix', '2001:dc3::/32') # M stream.add_filter('record-type', 'ribs') # stream.add_filter('collector', 'rrc01') stream.add_filter('project', 'routeviews') stream.add_interval_filter(timestamp, timestamp) stream.start() result = {} while stream.get_next_record(rec): rec_time = rec.time if rec.status == "valid": elem = rec.get_next_elem() while elem: print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields as_path = elem.fields['as-path'].split() as_path.reverse() prefix = elem.fields['prefix'] if prefix not in result: result[prefix] = [] result[prefix].append(as_path) elem = rec.get_next_elem() # get only unique lists in result for prefix in result: result[prefix] = [list(x) for x in set(tuple(x) for x in result[prefix])] print('timestamp {} ==> result: {}'.format(rec_time, result)) for prefix in result: for path in result[prefix]: print('path: {}'.format(path)) cur_node = None prev_node = None counter_as_prepend = 0 for index, asn in enumerate(path): searched_node = graph.find('asn', property_key='label', property_value=asn) try: cur_node = searched_node.next() # see if the AS node is already in the db or not. If yes, cur_node == prev_node except StopIteration: cur_node = Node('asn', label=str(asn)) # if not exists, then create a new one if index > 0: if index == len(path) - 1: cur_node['path'] = path # attach AS path to the last ASN if cur_node != prev_node: if counter_as_prepend > 0: cur_node['prepended'] = counter_as_prepend counter_as_prepend = 0 # reset peering = Relationship(cur_node, 'TO', prev_node, time=rec_time, prefix=prefix) # peering['time'] = rec_time # peering['prefix'] = prefix graph.create(peering) else: # AS prepending counter_as_prepend += 1 prev_node = cur_node
def readrib(self): stream = None rec = None if self.txtFile is None: # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type ribs" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" for c in self.collectors: bgprFilter += " and collector %s " % c # if not self.asnFilter is None: # bgprFilter += ' and path %s$' % self.asnFilter for p in self.includedPeers: bgprFilter += " and peer %s " % p for p in self.includedPrefix: bgprFilter += " and prefix more %s " % p logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts - 3600, self.startts + 3600)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts - 3600, self.startts + 3600) if self.livemode: stream.set_live_mode() stream.start() else: rec = txtReader.txtReader(self.txtFile) # for line in p1.stdout: while (self.txtFile and not rec.running) or (stream and stream.get_next_record(rec)): if rec.status != "valid": print rec.project, rec.collector, rec.type, rec.time, rec.status zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] sPath = elem.fields["as-path"] # print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields)) if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes (and using IP addresses as spatial # resolution) self.peersASN[zOrig].add(zAS) if len(path) < 2: # Ignore paths with only one AS elem = rec.get_next_elem() continue node = self.rtree.add(zPfx) if zOrig in node.data: # Already read this entry, we should read only one RIB per peer elem = rec.get_next_elem() continue if self.ribQueue is not None: self.ribQueue.put((zDt, zOrig, zAS, zPfx, path)) node.data[zOrig] = { "path": set(path), "count": 0, "origAS": origAS } # print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields) if self.spatialResolution: # compute weight for this path count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # assert count >= 0 node.data[zOrig]["count"] = count # Update above nodes parent = self.findParent(node, zOrig) if not parent is None: # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs(parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount pOrigAS = parent.data[zOrig]["origAS"] asns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, pOrigAS, zAS, asns) self.incTotalCount(pdiff, zOrig, pOrigAS, zAS) else: count = 1 node.data[zOrig]["count"] = count asns = node.data[zOrig]["path"] self.incTotalCount(count, zOrig, origAS, zAS) self.incCount(count, zOrig, origAS, zAS, asns) elem = rec.get_next_elem()
def sendMessageToKafka(col_name, col_data): # print "Collector-{} Sending Message...".format(col_name) size_total = 0 num_total = 0 num_ipv4 = 0 stream = BGPStream() record = BGPRecord() time_start = int(col_data.get('updates').get('latestDumpTime')) time_end = time_start + int(col_data.get('updates').get('dumpPeriod')) # print "Start Time:{}, End Time:{}".format(time_start, time_end) stream.add_filter('collector', col_name) stream.add_filter('record-type', 'ribs') # stream.add_interval_filter(time_start, time_end) # stream.add_interval_filter(time_start, time_start+300) stream.add_interval_filter(1503475200, 1503475200 + 7200) # print "Before Start>>>>>" stream.start() print col_name # print "After Start>>>>>>" producer = Kafka_producer() while stream.get_next_record(record): if record.status == "valid": elem = record.get_next_elem() while elem: if filter(lambda x: ':' in x.peer_address, [elem]): num_total += 1 elem = record.get_next_elem() continue #print "Element:{},{},{}".format(elem.type, elem.peer_address, elem.peer_asn) field = elem.fields #print type(field) prefix = field['prefix'] if field.has_key('prefix') else '' next_hop = field['next-hop'] if field.has_key( 'next-hop') else '' as_path = field['as-path'] if field.has_key('as-path') else '' as_path = as_path.replace(' ', '|') text = [ elem.type, elem.peer_address, str(elem.peer_asn), prefix, next_hop, as_path, str(record.time) ] text = ','.join(text) # producer = Kafka_producer() producer.send_data(col_name, text) num_total += 1 num_ipv4 += 1 # print "[{}]-{}".format(col_name, num_total) # size_total += len(text) #NUM_TOTAL += 1 #print "[{}]-{}-{}-{}-{}".format(col_name, num_total, num_ipv4, size_total, time.ctime(record.time)) #print "No.{} Message Send Success-[{}]".format(num_total, text) elem = record.get_next_elem() else: pass # print "## Current record not valid!" # break # print "One Collector Finished" else: # print "-----------------------------" # print "Collector[{}] And Records Send Finished\nTotal Num:{}, IPv4 Num:{}, Total Size:{}".format(col_name, num_total, num_ipv4, size_total) # print "-----------------------------" print "Collector:[{}]".format(col_name) print "Total Num:{}, IPv4 Num:{}, Total Size:{}".format( num_total, num_ipv4, size_total)
def readupdates(self): #TODO implement txt file for update messages? if self.txtFile: return # create a new bgpstream instance stream = BGPStream() bgprFilter = "type updates" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" # bgprFilter += " and collector rrc10 " for c in self.collectors: bgprFilter += " and collector %s " % c # if self.asnFilter is not None: # # TOFIX filter is now deprecated, we need to have both # # announcements and withdrawals # bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) if self.livemode: stream.set_live_mode() stream.start() # for line in p1.stdout: # create a reusable bgprecord instance rec = BGPRecord() while (stream.get_next_record(rec)): if rec.status != "valid": logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" % (rec.project, rec.collector, rec.type, rec.time, rec.status)) zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address if zOrig not in self.peers: # no need to update the counts for non-full feed peers elem = rec.get_next_elem() continue zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue msgTs = zDt # set first time bin! if self.ts is None: self.slideTimeWindow(msgTs) elif self.ts + self.timeWindow <= msgTs: self.slideTimeWindow(msgTs) elif self.ts > msgTs: #Old update, ignore this to update the graph logging.warn( "Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s" % (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields))) elem = rec.get_next_elem() continue node = self.rtree.search_exact(zPfx) if elem.type == "W": # Withdraw: remove the corresponding node if not node is None and zOrig in node.data: origAS = node.data[zOrig]["origAS"] if self.spatialResolution: count = node.data[zOrig]["count"] # Update count for above node parent = self.findParent(node, zOrig) if parent is None: self.incTotalCount(-count, zOrig, origAS, zAS) asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) else: node.data[zOrig]["count"] = 0 # Add ips to above node and corresponding ASes # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) # oldpCount = parent.data[zOrig]["count"] # pCount = self.nbIPs(parent.prefixlen) - pcountBelow # parent.data[zOrig]["count"] = pCount # pdiff = pCount - oldpCount # assert pdiff==count # Update count for origAS and path from the # parent node porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(count, zOrig, porigAS, zAS, pasns) self.incTotalCount(count, zOrig, porigAS, zAS) # Update count for withdrawn origAS and path asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) self.incTotalCount(-count, zOrig, origAS, zAS) else: asns = node.data[zOrig]["path"] self.incCount(-1, zOrig, origAS, zAS, asns) self.incTotalCount(-1, zOrig, origAS, zAS) del node.data[zOrig] else: # Announce: update counters sPath = elem.fields["as-path"] path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes or implicit withdraws if len(path) < 2: # Ignoring paths with only one AS elem = rec.get_next_elem() continue if self.announceQueue is not None: self.announceQueue.put((zDt, zOrig, zAS, zPfx, path)) # Announce: if node is None or not zOrig in node.data: # Add a new node node = self.rtree.add(zPfx) if self.spatialResolution: # Compute the exact number of IPs count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) parent = self.findParent(node, zOrig) if not parent is None: # Update above nodes # print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count)) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs( parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount # print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount)) # print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ] porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, porigAS, zAS, pasns) self.incTotalCount(pdiff, zOrig, porigAS, zAS) else: self.incTotalCount(1, zOrig, origAS, zAS) count = 1 # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asn = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) else: #Update node path and counts if self.spatialResolution: count = node.data[zOrig]["count"] else: count = 1 porigAS = node.data[zOrig]["origAS"] asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, porigAS, zAS, asns) self.incTotalCount(-count, zOrig, porigAS, zAS) node.data[zOrig]["path"] = set(path) node.data[zOrig]["origAS"] = origAS asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) elem = rec.get_next_elem()
def get_data(timestamp): graph = Graph(password="******") stream = BGPStream() rec = BGPRecord() rec_time = None # IPv4 stream.add_filter('prefix', '198.41.0.0/24') # A-root stream.add_filter('prefix', '192.33.4.0/24') # C-root stream.add_filter('prefix', '199.7.91.0/24') # D-root stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only stream.add_filter('prefix', '192.5.5.0/24') # F-root stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only stream.add_filter('prefix', '198.97.190.0/24') # H-root stream.add_filter('prefix', '192.36.148.0/24') # I-root stream.add_filter('prefix', '192.58.128.0/24') # J-root stream.add_filter('prefix', '193.0.14.0/24') # K-root stream.add_filter('prefix', '199.7.83.0/24') # L-root stream.add_filter('prefix', '202.12.27.0/24') # M-root # IPv6 stream.add_filter('prefix', '2001:503:ba3e::/48') # A stream.add_filter('prefix', '2001:500:2::/48') # C stream.add_filter('prefix', '2001:500:2d::/48') # D stream.add_filter('prefix', '2001:500:2f::/48') # F stream.add_filter('prefix', '2001:500:1::/48') # H stream.add_filter('prefix', '2001:7fe::/33') # I stream.add_filter('prefix', '2001:503:c27::/48') # J stream.add_filter('prefix', '2001:7fd::/48') # K stream.add_filter('prefix', '2001:500:9f::/48') # L stream.add_filter('prefix', '2001:dc3::/32') # M stream.add_filter('record-type', 'ribs') # stream.add_filter('collector', 'route-views.soxrs') stream.add_filter('project', 'routeviews') stream.add_interval_filter(timestamp, timestamp) stream.start() result = {} while stream.get_next_record(rec): rec_time = rec.time if rec.status == "valid": elem = rec.get_next_elem() while elem: print('{} {} {} {} {}'.format(rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields)) as_path = elem.fields['as-path'].split() as_path.reverse() prefix = elem.fields['prefix'] if prefix not in result: result[prefix] = [] result[prefix].append(as_path) elem = rec.get_next_elem() # get only unique lists in result for prefix in result: result[prefix] = [list(x) for x in set(tuple(x) for x in result[prefix])] print('timestamp {} ==> result: {}'.format(rec_time, result)) for prefix in result: for path in result[prefix]: print('path: {}'.format(path)) cur_node = None prev_node = None counter_as_prepend = 0 for index, asn in enumerate(path): cur_node = asn graph.run('MERGE(s:asn{{name:"{0}", label:"{0}"}})'.format(asn)) # create new node if not exist. if index > 0: if cur_node != prev_node: query = 'MATCH (s:asn),(d:asn) ' \ 'WHERE s.name="{0}" AND d.name="{1}" ' \ 'MERGE (s)-[r:TO {{prefix: "{3}", time: {2}, prepended: {4}}}]->(d)'\ .format(cur_node, prev_node, rec_time, prefix, counter_as_prepend) graph.run(query) if counter_as_prepend > 0: counter_as_prepend = 0 # reset else: # AS prepending counter_as_prepend += 1 # else: # origin AS # graph.run('MATCH (s:asn)' # 'WHERE s.name="{0}"' # 'SET s.origin="{1}_{2}"' # 'RETURN s'.format(asn, prefix, rec_time)) prev_node = cur_node
class BGPTest(object): def __init__(self, route_collector="rrc00", rpki_validator="rpki-validator.realmv6.org:8282"): self.rc = route_collector rpki = rpki_validator.split(":") self.mgr = RTRManager(rpki[0], rpki[1]) # self._start_rtr_manager() self.stream = BGPStream() self.rec = BGPRecord() def _start_rtr_manager(self): self.mgr.start() while not self.mgr.is_synced(): sleep(0.2) if status.error: print("Connection error") exit() def start_stream(self, start_time=None, end_time=0, route_collector=""): """ Starts the """ if route_collector == "": route_collector = self.rc self.stream.add_filter('collector', route_collector) self.stream.add_filter('record-type', 'ribs') if (start_time is None) or not isinstance(start_time, datetime): start_time = datetime.utcnow() if isinstance(end_time, datetime): end = int(end_time.strftime("%s")) else: end = 0 start = int(datetime.utcnow().strftime("%s")) print(start) self.stream.add_interval_filter(start, 0) # print('Start stream with', start_time, end_time) self.stream.start() def get_records(self): while(self.stream.get_next_record(self.rec)): # Print the self.record information only if it is not a valid self.record if self.rec.status != "valid": pass else: elem = self.rec.get_next_elem() while(elem): # Print self.record and elem information print(self.rec.project, self.rec.collector, self.rec.type, self.rec.time, self.rec.status) print(elem.type, elem.peer_address, elem.peer_asn, elem.fields) # prefix = elem.fields["prefix"].split('/') # result = mgr.validate((int) elem.fields["as-path"].split(" ")[-1], prefix[0], prefix[1]) elem = self.rec.get_next_elem() print("done.")