def calculate_effective_prefixes(self, start_interval=1451692800): ''' This method calculates the number of effective /24 prefixes that lie underneath a specified route prefix :param start_interval: the start interval of the data stream :return: number of effective prefixes under the route's root prefix ''' stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', self.collector_name) stream.add_filter('record-type', 'ribs') if isinstance(start_interval, datetime.datetime): interval_start_utc = self.convert_interval_to_utc(start_interval) stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300) else: stream.add_interval_filter(start_interval - 300, start_interval + 300) stream.start() print "Starting routing table parsing" while (stream.get_next_record(rec)): elem = rec.get_next_elem() while elem: # Get the peer ASN and IP. We then construct a peer_id, since a collector # can establish multiple connections with the same ASN. peer_asn = elem.peer_asn peer_asn_ip = elem.peer_address # make this an unmodifiable tuple peer_id = (peer_asn, peer_asn_ip) peer_route_trie = self.peer_trie_handles[peer_id] # Do a single level search of the route prefix to # find the left and right prefix advertisements single_level_prefixes = \ peer_route_trie.single_level_search(elem.fields['prefix']) print single_level_prefixes prefix_count = len(single_level_prefixes) root_24_prefix_count = 0 lr_24_prefix_count1 = lr_24_prefix_count2 = 0 # The /24 prefixes below the advertised prefix are calculated # as all the /24 prefixes served by the root - sum of the /24 # prefixes served by root's children if prefix_count == 1: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) elif prefix_count == 2: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count1 = \ 2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<') .rstrip('>').split('/')[1])) else: root_24_prefix_count = \ 2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count1 = \ 2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<') .rstrip('>').split('/')[1])) lr_24_prefix_count2 = \ 2 ** (24 - int(str(single_level_prefixes[2]).lstrip('<') .rstrip('>').split('/')[1])) effective_24_prefix_count = \ root_24_prefix_count - (lr_24_prefix_count1 + lr_24_prefix_count2) print "Effective Prefix Count : ", \ effective_24_prefix_count for prefix in single_level_prefixes: trie_node = peer_route_trie.search_exact(str(prefix)) as_path = trie_node.data['as-path'].split(" ") as_path_headless = as_path[1:-1] print "AS-Path : ", as_path as_headless_len = len(as_path_headless) if as_headless_len > 1: for i in range(0, as_headless_len - 1): print "Headless nodes : ", as_path_headless[i], \ as_path_headless[i + 1] if as_path_headless[i] in self.aslink_datastore: self.aslink_datastore[as_path_headless[i]] += 1 else: self.aslink_datastore[as_path_headless[i]] = 1 elem = rec.get_next_elem()
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # to track the peers that have elems in this record peer_signatures = set() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) peer_signatures.add(sig) # if this is the first time we have ever seen this peer, create # an empty result: (Pfxs_v4_set, Pfxs_v6_set, ASNs_v4_set, ASNs_v6_set) if sig not in peers_data: peers_data[sig] =[set(),set(), set(),set()] if('prefix' in elem.fields): pfx=elem.fields['prefix']; if(":" in pfx): peers_data[sig][1].add(pfx) else: peers_data[sig][0].add(pfx) if('as-path' in elem.fields): path_split=elem.fields['as-path'].split() if(":" in pfx): for ASN in path_split: peers_data[sig][3].add(ASN) else: for ASN in path_split: peers_data[sig][2].add(ASN) elem = rec.get_next_elem() # one peer that was present in this record (allows a true, per-collector # count of records since each record can contain elems for many peers) # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def __init__(self): # Create a new bgpstream instance # and a reusable bgprecord instances self.stream = BGPStream() self.rec = BGPRecord() self.origin_ases = set() self.bgp_lens = defaultdict(lambda: defaultdict(lambda: None))
def __init__(self, filters={'collector': ['rrc00']}, rpki_validator="rpki-validator.realmv6.org:8282", db="metasnap.db"): self.stream = BGPStream() self.filters = filters self.route_table = dict() self.i = 0 for filter_type, filter_array in filters.items(): for filter_value in filter_array: self.stream.add_filter(filter_type, filter_value) for collector in filters['collector']: self.route_table[collector] = defaultdict(dict) # self.db = DBConnector(db, read_only=False) rpki = rpki_validator.split(":") self.mgr = RTRManager(rpki[0], rpki[1]) self.mgr.start() self.counter = Counter() start_timestamp = self.get_push_timestamp(datetime.now(timezone.utc)) # self.start_collecting(start_timestamp, int(datetime.now(timezone.utc).strftime("%s"))) self.start_collecting(start_timestamp, start_timestamp)
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) # if this is the first time we have ever seen this peer, create # an empty result: (dict(Pfx_v4)=Pfx_origins, dict(Pfx_v6)=Pfx_origins)) if sig not in peers_data: peers_data[sig] =[{},{}] if('prefix' in elem.fields): pfx=elem.fields['prefix']; origin="" if('as-path' in elem.fields): path_split=elem.fields['as-path'].split() if(len(path_split)!=0): origin=path_split[len(path_split)-1] if(":" in pfx): if(pfx not in peers_data[sig][1]): peers_data[sig][1][pfx]=set() #discard as origin: AS sets, and ASN=23456 [AS_TRANS] if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][1][pfx].add(origin) else: if(pfx not in peers_data[sig][0]): peers_data[sig][0][pfx]=set() #discard as origin: AS sets, and ASN=23456 [AS_TRANS] if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][0][pfx].add(origin) elem = rec.get_next_elem() # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def __init__(self, route_collector="rrc00", rpki_validator="rpki-validator.realmv6.org:8282"): self.rc = route_collector rpki = rpki_validator.split(":") self.mgr = RTRManager(rpki[0], rpki[1]) # self._start_rtr_manager() self.stream = BGPStream() self.rec = BGPRecord()
def get_data_graph(self): stream = BGPStream() rec = BGPRecord() stream.add_filter('record-type', 'ribs') stream.add_interval_filter(self.start, self.end) stream.start() as_graph = nx.Graph() while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: # the list is a list, which stores an as-path as_path = [ k for k, g in groupby(elem.fields['as-path'].split(" ")) ] peer = str(elem.peer_asn) # judge whether the as-path is legal if len(as_path) > 1 and as_path[0] == peer: if re.match(self.pattern, elem.fields['prefix']): self.as_prefix[as_path[-1]].add( elem.fields['prefix']) # add edges to the graph for i in range(0, len(as_path) - 1): as_graph.add_edge(as_path[i], as_path[i + 1]) elem = rec.get_next_elem() as_graph = self.add_geo_loc(as_graph) return as_graph
def sendMessageToKafka(self, col_name, col_data): print "lalalala" stream = BGPStream() record = BGPRecord() time_start = int(col_data.get('ribs').get('latestDumpTime')) time_end = time_start + int(col_data.get('ribs').get('dumpPeriod')) stream.add_filter('collector', col_name) stream.add_filter('record-type', 'ribs') stream.add_interval_filter(time_start, time_end) print "Before Start" stream.start() print "After Start" while stream.get_next_record(record): if record.status == "valid": elem = record.get_next_elem() while elem: # print "Record:{}".format(elem) producer = Kafka_producer() producer.send_data(col_name, json.dumps(elem)) elem = record.get_next_elem() else: print "## Current record not valid!" print "One Collector Finished"
def downloader(start_date, duration): """Download BGP paths from Routeviews and RIPE NCC from a start date for a certain duration.""" # Start of UNIX time base = int(datetime.datetime.strptime(start_date, '%m/%d/%Y').strftime('%s')) # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() stream.add_interval_filter(base, base + int(duration)) stream.add_filter('record-type', 'ribs') stream.start() path_set = set() f = open('rib.txt', 'w') while True: rec = stream.get_next_record() if rec is None: return if rec.status != "valid": continue else: elem = rec.get_next_elem() while(elem): path = elem.fields['as-path'] if '{' in path or '(' in path: elem = rec.get_next_elem() continue prefix = elem.fields['prefix'] # Focus on IPv4 prefixes if ":" not in prefix and path not in path_set: f.write(path.replace(' ', '|') + '\n') path_set.add(path) elem = rec.get_next_elem() f.close()
def build_sql_db(): conn = sqlite3.connect('bgp_stage.db') c = conn.cursor() # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() stream.add_filter('collector','rrc11') stream.add_interval_filter(1438417216,1438417316) stream.start() while(stream.get_next_record(rec)): if rec.status != "valid": continue else: elem = rec.get_next_elem() while(elem): if elem.type == "AB": prefix = elem.fields["prefix"] as_path = elem.fields["as-path"].split(" ") origin = as_path[-1] time = elem.time #IP Prefix database ip_min, ip_max = calculate_min_max(prefix) c.execute("SELECT ip_min FROM prefix_as WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin)) row = c.fetchone() if len(row) != 0: c.execute("UPDATE prefix_as SET count = count + 1 WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin)) else: c.execute("INSERT INTO prefix_as VALUES(?,?,?,?,?)", (ip_min, ip_max, origin, 1, time)) #AS link database for as1,as2 in zip(as_path, as_path[1:]) : c.execute("SELECT as_o FROM as_link WHERE as_o = (?) AND as_n = (?)",(as1,as2)) row = c.fetchone() if len(row) != 0: c.execute("UPDATE as_link SET count = count + 1 WHERE as_o = (?) AND as_n = (?)", (as1, as2)) else: c.execute("INSERT INTO as_link VALUES(?,?,?,?)", (as1, as2, 1, 0)) elif elem.type == "WA": prefix = elem.fields["prefix"] time = elem.time #Needs research print(rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.peer_address, elem.peer_asn, elem.fields) print(prefix,elem.time, "W") print(rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.peer_address, elem.peer_asn, elem.fields) elem = rec.get_next_elem() conn.commit() conn.close()
def getBGPStream(recordType, AF, collectors, startts, endts): stream = BGPStream() # recordType is supposed to be ribs or updates bgprFilter = "type " + recordType if AF == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" for c in collectors: bgprFilter += " and collector %s " % c if isinstance(startts, str): startts = datetime.strptime(startts + "UTC", "%Y-%m-%dT%H:%M:%S%Z") startts = dt2ts(startts) if isinstance(endts, str): endts = datetime.strptime(endts + "UTC", "%Y-%m-%dT%H:%M:%S%Z") endts = dt2ts(endts) currentts = dt2ts(datetime.now()) if endts > currentts: stream.set_live_mode() stream.parse_filter_string(bgprFilter) stream.add_interval_filter(startts, endts) return stream
def getBGPStream(self): logging.info( f"creating BGPstream {self.collector} {self.record_type} {self.start.year}-{self.start.month} " ) stream = BGPStream() stream.add_filter('collector', self.collector) stream.add_filter('record-type', self.record_type) if self.record_type == "updates": stream.add_interval_filter(dt2ts(self.start), dt2ts(self.end)) elif self.record_type == "ribs": _start = dt2ts(self.start - timedelta(hours=2)) _end = dt2ts(self.start + timedelta(hours=2)) stream.add_interval_filter(_start, _end) return stream
def download_data(): peer_state = defaultdict(dict) results = defaultdict(defaultdict_list) current_bin = 0 # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (startts, endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(startts, endts) stream.start() while (stream.get_next_record(rec)): if rec.status != "valid": print(rec.project, rec.collector, rec.type, rec.time, rec.status) # from IPython import embed # embed() if current_bin == 0: current_bin = rec.time # slide the time window: if current_bin + bin_size < rec.time: timebins = range(current_bin, rec.time, bin_size) for i in timebins[:-1]: results["other"]["timebin"].append(i) for pfx, p_s in peer_state.items(): for peeras, state in p_s.items(): results[pfx][peeras].append(state) current_bin = timebins[-1] elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn prefix = elem.fields["prefix"] peer_state[prefix][peeras] = elem.type elem = rec.get_next_elem() return results
def recv_bgpstream_updates(begin, until, collector, output_queue): logging.info ("CALL recv_bgpstream_updates") # wait for first RIB table dump to complete while (rib_ts < 0): time.sleep(RIB_TS_WAIT/10) time.sleep(RIB_TS_WAIT) # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type','updates') stream.add_interval_filter(rib_ts,until) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue logging.info("Record TS: "+str(rec.time)) while (elem): logging.info(" -- Record Element Type: " + elem.type + ", TS: " + str(elem.time)) bgp_message = BGPmessage(elem.time, 'update') src_peer = dict() src_addr = elem.peer_address src_asn = elem.peer_asn src_peer['addr'] = src_addr src_peer['port'] = 0 src_peer['asn'] = src_asn bgp_message.set_source(src_peer) if elem.type.upper() == 'A': bgp_message.add_announce(elem.fields['prefix']) bgp_message.set_nexthop(elem.fields['next-hop']) aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) output_queue.put(bgp_message) elif elem.type.upper() == 'W': bgp_message.add_withdraw(elem.fields['prefix']) output_queue.put(bgp_message) elem = rec.get_next_elem()
def getting_BGP_update(): stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', 'rrc11') stream.add_interval_filter(1438417216, 1438417216) # Start the stream stream.start() while (stream.get_next_record(rec)): if rec.status != "valid": print rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while (elem): print rec.project, rec.collector, rec.type, rec.time, rec.status, print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem()
def generate_stream(): bs = BGPStream() rec = BGPRecord() #initialize MySql db_writer = MysqlWriter() db_writer.start() #initialize the publisher in port number 12345 publisher = ZmqPublisher(12345) bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0) # bs.add_filter('collector', 'route-views.sg') bs.add_filter('record-type', 'updates') bs.start() print('Beginning to read from stream') input_id = 0 while bs.get_next_record(rec): elem = rec.get_next_elem() while elem is not None: # sleep until it is time to send this record ''' now = calendar.timegm(time.gmtime()) sim_time = now - delay if elem.time > sim_time: time.sleep(elem.time - sim_time) ''' if elem.type not in ['A', 'W']: continue input_id += 1 msg = elem2bgplay(rec, elem) msg['type'] = 'A' msg['id'] = input_id print(msg) # Publish the message publisher.publish(msg) # Write it to DB if elem.type == 'A': db_writer.add(msg) elif elem.type == 'W': db_writer.remove(msg) else: print "Error: Unknown type: " + elem.type elem = rec.get_next_elem()
def recv_bgpstream_rib(begin, until, collector, output_queue): logging.info ("CALL recv_bgpstream_rib") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector', collector) stream.add_filter('record-type','updates') stream.add_interval_filter(begin,until) # Start the stream stream.start() while (stream.get_next_record(rec)): global rib_ts if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping.") continue if (rib_ts > 0) and (rec.time > (rib_ts + RIB_TS_INTERVAL/2)): logging.info("received full RIB table dump.") break bgp_message = None while (elem): if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'): rib_ts = elem.time bgp_message = BGPmessage(elem.time, 'update') bgp_message.set_nexthop(elem.fields['next-hop']) src_peer = dict() src_addr = elem.peer_address src_asn = elem.peer_asn src_peer['addr'] = src_addr src_peer['port'] = 0 src_peer['asn'] = src_asn bgp_message.set_source(src_peer) aspath = elem.fields['as-path'].split() for a in aspath: if not '{' in a: # ignore AS-SETs bgp_message.add_as_to_path(a) bgp_message.add_announce(elem.fields['prefix']) output_queue.put(bgp_message) elem = rec.get_next_elem()
def get_stream(self): logging.debug( f"[ZombieRecordFinder-{self.collector}] try to create BGPstream") _start = datetime.datetime(self.year, self.month, 10) _end = datetime.datetime(self.year, self.month, 20) stream = BGPStream() stream.add_interval_filter(dt2ts(_start), dt2ts(_end)) stream.add_filter('collector', self.collector) for _, p in self.zombies: stream.add_filter('prefix-exact', p) return stream
def __init__( self, filters={"collector": ["rrc00"]}, rpki_validator="rpki-validator.realmv6.org:8282", settings_file="../settings.json", ): self.stream = BGPStream() self.filters = filters self.route_table = dict() self.i = 0 self.metadata_vp = dict() self.metadata_rc = dict() self.peers = Counter() self.prefix4 = Counter() self.prefix6 = Counter() start_timestamp = get_push_timestamp(datetime.now(timezone.utc)) for filter_type, filter_array in filters.items(): for filter_value in filter_array: self.stream.add_filter(filter_type, filter_value) for collector in filters["collector"]: self.route_table[collector] = defaultdict(dict) self.metadata_vp[collector] = defaultdict(list) self.metadata_rc[collector] = RouteCollectorMeta(None, 0, 0, 0, 0) self.peers[collector] = defaultdict(int) self.prefix4[collector] = defaultdict(int) self.prefix6[collector] = defaultdict(int) settings = get_settings(settings_file) settings["db"]["password"] = os.environ["PGPASS"] self.db = DBConnector(settings["db"]) rpki = rpki_validator.split(":") self.mgr = RTRManager(rpki[0], rpki[1]) self.mgr.start() self.start_collecting(start_timestamp)
def generate_stream(): bs = BGPStream() rec = BGPRecord() #initialize MySql a = MySqlDAL() a.start() #initialize the publisher in port number 12345 publisher = ZmqPublisher(12345) bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0) # bs.add_filter('collector', 'route-views.sg') bs.add_filter('record-type', 'updates') bs.start() print('Beginning to read from stream') input_id = 0 while bs.get_next_record(rec): elem = rec.get_next_elem() while elem is not None: # sleep until it is time to send this record ''' now = calendar.timegm(time.gmtime()) sim_time = now - delay if elem.time > sim_time: time.sleep(elem.time - sim_time) ''' if elem.type not in ['A', 'W']: continue input_id += 1 msg = elem2bgplay(rec, elem) msg['type'] = 'A' msg['id'] = input_id print(msg) # Publish the message publisher.publish(msg) # Write it to DB if elem.type == 'A': a.add(msg) elif elem.type == 'W': a.remove(msg) else: print "Error: Unknown type: " + elem.type elem = rec.get_next_elem()
def readUpdates(self): # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in self.prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) stream.start() while (stream.get_next_record(rec)): if (rec.status != "valid" and rec.status != "filtered-source" and rec.status != "empty-source"): raise Exception(rec.project, rec.collector, rec.type, rec.time, rec.status) zDt = rec.time elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn sPrefix = elem.fields["prefix"] if elem.type == "W": self.withdraws[sPrefix][peeras] = True else: sPath = elem.fields["as-path"] self.paths[sPrefix][peeras] = sPath self.withdraws[sPrefix][peeras] = False elem = rec.get_next_elem()
def _start_stream(self, **filters): """Initializes, starts, and returns bgp stream with filters""" # Create a new bgpstream instance stream = BGPStream() # Add filters if params exist for key, value in filters.items(): if (key in ['peer-asn', 'prefix', 'collector'] and value is not None): stream.add_filter(key, value) # Time params must be in epoch start_epoch = calendar.timegm(filters.get('start').timetuple()) end_epoch = calendar.timegm(filters.get('end').timetuple()) stream.add_interval_filter(start_epoch, end_epoch) # Start the stream stream.start() self.logger.debug("Started stream for caida announcements") return stream
def main(): parser = argparse.ArgumentParser(description='', epilog='') parser.add_argument('-b', '--begin', help='Begin date (inclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-u', '--until', help='Until date (exclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-c', '--collector', help='Route collector from RIPE RIS or Route-Views project.', type=str, required=True) parser.add_argument('-m', '--mongodb', help='MongoDB connection parameters.', type=str, default=None) parser.add_argument('-l', '--loglevel', help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].', type=str, default='WARNING') args = vars(parser.parse_args()) numeric_level = getattr(logging, args['loglevel'].upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, format='%(asctime)s : %(levelname)s : %(message)s') ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds()) ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds()) mongodbstr = None if args['mongodb']: mongodbstr = args['mongodb'].strip() # BEGIN logging.info("START") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector',args['collector']) stream.add_filter('record-type','ribs') stream.add_interval_filter(ts_begin,ts_until) # Start the stream stream.start() rib_ts = 0 rib_origins = dict() while(stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue if rec.time > (rib_ts + RIB_TS_INTERVAL): rib_ts = rec.time if mongodbstr: store_rib_origins(rib_ts, rib_origins, mongodbstr) else: print_rib_origins(rib_ts, rib_origins) rib_origins = dict() while(elem): prefix = elem.fields['prefix'] aspath = elem.fields['as-path'].split() for a in aspath: # remove AS-SETs if '{' in a: aspath.remove(a) origin = aspath[-1] if prefix not in rib_origins: rib_origins[prefix] = list() if origin not in rib_origins[prefix]: rib_origins[prefix].append(origin) elem = rec.get_next_elem()
from _pybgpstream import BGPStream, BGPRecord, BGPElem # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() start = 1454800000 end = 1454802000 target_pref = '200.7.6.0/24' print start, end, target_pref print target_pref stream.add_filter('prefix', target_pref) # Consider RIPE RRC 10 only # stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') stream.add_interval_filter(start, end) # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # """ Very short period for test """ # stream.add_interval_filter(start, end) # """ Jan till now """ # stream.add_interval_filter(1451606400,1454785264) # """ yesterday """ # stream.add_interval_filter(1454630400, 1454716800) # Start the stream stream.start()
from _pybgpstream import BGPStream, BGPElem, BGPRecord from py2neo import Graph, Node, Relationship graph = Graph(password="******") stream = BGPStream() rec = BGPRecord() # stream.add_filter('prefix', '198.41.0.0/24') # A-root # stream.add_filter('prefix', '192.33.4.0/24') # C-root # stream.add_filter('prefix', '199.7.91.0/24') # D-root # stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only # stream.add_filter('prefix', '192.5.5.0/24') # F-root # stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only # stream.add_filter('prefix', '198.97.190.0/24') # H-root # stream.add_filter('prefix', '192.36.148.0/24') # I-root # stream.add_filter('prefix', '192.58.128.0/24') # J-root stream.add_filter('prefix', '193.0.14.0/24') # K-root stream.add_filter('prefix', '199.7.83.0/24') # L-root stream.add_filter('prefix', '202.12.27.0/24') # M-root stream.add_filter('record-type','ribs') # stream.add_filter('collector','route-views.sg') stream.add_filter('project','routeviews') timestamp = 1464739200 # 2016/6/1 00:00 stream.add_interval_filter(timestamp, timestamp) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC); 1464682200 stream.start() result = {} while stream.get_next_record(rec):
def main(): (options, args) = getopts() limit = options.limit start = options.start_time end = options.end_time # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') stream.add_filter('prefix', '0.0.0.0/0') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264) #stream.add_interval_filter(1454630400, 1454716800) # 1 hour #1454284800 - 1454288400 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record prefixes_update = defaultdict(int) prefixes_withdraw = defaultdict(int) while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while elem: if elem.type == 'A': #print elem.fields['as-path'] prefixes_update[elem.fields['prefix']] += 1 if elem.type == 'W': prefixes_withdraw[elem.fields['prefix']] += 1 #print rec.project, rec.collector, rec.type, rec.time, rec.status, #print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem() for k in prefixes_update: if prefixes_update[k] >= limit: print k + "\t" + str(prefixes_update[k]) + "\t" + str( prefixes_withdraw[k])
stream_start = int(sys.argv[1]) stream_end = int(sys.argv[2]) out_file_name = sys.argv[3] except: print "Usage: %s [start time] [end time] [output file name]" %(sys.argv[0]) exit() #stream_start = 1454284800 #stream_end = 1454288400 buckets = create_time_buckets(stream_start, stream_end) prefixList = [] raw_bgp_stream_data = {} stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', 'rrc06') stream.add_filter('record-type', 'updates') stream.add_interval_filter(stream_start, stream_end) stream.start() while(stream.get_next_record(rec)): elem = rec.get_next_elem() while(elem): prefix = elem.fields.get("prefix", "") time_stamp = rec.time # unix epoc timestamp 1427846670
def readupdates(self): #TODO implement txt file for update messages? if self.txtFile: return # create a new bgpstream instance stream = BGPStream() bgprFilter = "type updates" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" # bgprFilter += " and collector rrc10 " for c in self.collectors: bgprFilter += " and collector %s " % c # if self.asnFilter is not None: # # TOFIX filter is now deprecated, we need to have both # # announcements and withdrawals # bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) if self.livemode: stream.set_live_mode() stream.start() # for line in p1.stdout: # create a reusable bgprecord instance rec = BGPRecord() while (stream.get_next_record(rec)): if rec.status != "valid": logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" % (rec.project, rec.collector, rec.type, rec.time, rec.status)) zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address if zOrig not in self.peers: # no need to update the counts for non-full feed peers elem = rec.get_next_elem() continue zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue msgTs = zDt # set first time bin! if self.ts is None: self.slideTimeWindow(msgTs) elif self.ts + self.timeWindow <= msgTs: self.slideTimeWindow(msgTs) elif self.ts > msgTs: #Old update, ignore this to update the graph logging.warn( "Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s" % (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields))) elem = rec.get_next_elem() continue node = self.rtree.search_exact(zPfx) if elem.type == "W": # Withdraw: remove the corresponding node if not node is None and zOrig in node.data: origAS = node.data[zOrig]["origAS"] if self.spatialResolution: count = node.data[zOrig]["count"] # Update count for above node parent = self.findParent(node, zOrig) if parent is None: self.incTotalCount(-count, zOrig, origAS, zAS) asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) else: node.data[zOrig]["count"] = 0 # Add ips to above node and corresponding ASes # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) # oldpCount = parent.data[zOrig]["count"] # pCount = self.nbIPs(parent.prefixlen) - pcountBelow # parent.data[zOrig]["count"] = pCount # pdiff = pCount - oldpCount # assert pdiff==count # Update count for origAS and path from the # parent node porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(count, zOrig, porigAS, zAS, pasns) self.incTotalCount(count, zOrig, porigAS, zAS) # Update count for withdrawn origAS and path asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) self.incTotalCount(-count, zOrig, origAS, zAS) else: asns = node.data[zOrig]["path"] self.incCount(-1, zOrig, origAS, zAS, asns) self.incTotalCount(-1, zOrig, origAS, zAS) del node.data[zOrig] else: # Announce: update counters sPath = elem.fields["as-path"] path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes or implicit withdraws if len(path) < 2: # Ignoring paths with only one AS elem = rec.get_next_elem() continue if self.announceQueue is not None: self.announceQueue.put((zDt, zOrig, zAS, zPfx, path)) # Announce: if node is None or not zOrig in node.data: # Add a new node node = self.rtree.add(zPfx) if self.spatialResolution: # Compute the exact number of IPs count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) parent = self.findParent(node, zOrig) if not parent is None: # Update above nodes # print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count)) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs( parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount # print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount)) # print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ] porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, porigAS, zAS, pasns) self.incTotalCount(pdiff, zOrig, porigAS, zAS) else: self.incTotalCount(1, zOrig, origAS, zAS) count = 1 # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asn = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) else: #Update node path and counts if self.spatialResolution: count = node.data[zOrig]["count"] else: count = 1 porigAS = node.data[zOrig]["origAS"] asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, porigAS, zAS, asns) self.incTotalCount(-count, zOrig, porigAS, zAS) node.data[zOrig]["path"] = set(path) node.data[zOrig]["origAS"] = origAS asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) elem = rec.get_next_elem()
def run_bgpstream(args): (collector, start_time, end_time, data_type) = args # initialize and configure BGPStream stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) # NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the # end time since we are using inclusive/exclusive intervals stream.add_interval_filter(start_time, end_time-1) stream.add_filter('record-type', data_type) stream.start() # per-peer data peers_data = {} # loop over all records in the stream while stream.get_next_record(rec): elem = rec.get_next_elem() # loop over all elems in the record while elem: # create a peer signature for this elem sig = peer_signature(rec, elem) # if this is the first time we have ever seen this peer, create # an empty result: (Pfxs_v4_set , Pfxs_v6_set, # dict(Transit_ASN)=Pfxs_v4_set , dict(Transit_ASN)=Pfxs_v6_set) if sig not in peers_data: peers_data[sig] =[set(),set(),{},{}] pfx="" if('prefix' in elem.fields): pfx=elem.fields['prefix']; if(":" in pfx): peers_data[sig][1].add(pfx) else: peers_data[sig][0].add(pfx) if('as-path' in elem.fields): #Squash the AS Path to keep only distinct ASNs, i.e., remove prepending path_split = [k for k, g in groupby(elem.fields['as-path'].split(" "))] if(len(path_split)!=0): for i in range(1,len(path_split)-1): transit=path_split[i] if(":" in pfx): if(transit not in peers_data[sig][3]): peers_data[sig][3][transit]=set() peers_data[sig][3][transit].add(pfx) elif(pfx!=""): if(transit not in peers_data[sig][2]): peers_data[sig][2][transit]=set() peers_data[sig][2][transit].add(pfx) elem = rec.get_next_elem() # the time in the output row is truncated down to a multiple of # RESULT_GRANULARITY so that slices can be merged correctly start_time = \ int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY) # for each peer that we processed data for, create an output row return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def dump2file(type, name, latestDumpTime, dumpDuration, dumpPeriod): print type + ' of ' + name + ': ' csv_header = ['type', 'addr', 'as', 'prefix', 'next_hop', 'as_path'] _file = None stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', c_name) if type == 'ribs': stream.add_filter('record-type', 'ribs') _file = open(c_name + '_ribs.csv', 'w+') elif type == 'updates': stream.add_filter('record-type', 'updates') _file = open(c_name + '_updates.csv', 'w+') stream.add_interval_filter(latestDumpTime, latestDumpTime + dumpPeriod) stream.start() count = 0 useless_c = 0 writer = csv.writer(_file) writer.writerow(csv_header) # Get next record while (stream.get_next_record(rec)): # Print the record information only if it is not a valid record if rec.status != "valid": # print rec.project, rec.collector, rec.type, rec.time, rec.status print 'current rec not valid.' else: elem = rec.get_next_elem() while (elem): useless_c += 1 if useless_c % 1000 == 0: print 'Got ' + str(useless_c) + ' elem totally.' # Print record and elem information if isIPV6(elem): # ipv6 packet ignored elem = rec.get_next_elem() continue count += 1 # print rec.project, rec.collector, rec.type, rec.time, rec.status, # print elem.type, elem.peer_address, elem.peer_asn # elem.fields contains four column: communities, next-hop, prefix, as-path field = elem.fields prefix = field['prefix'] if 'prefix' in field.keys() else '' next_hop = field['next-hop'] if 'next-hop' in field.keys( ) else '' as_path = field['as-path'] if 'as-path' in field.keys() else '' as_path = as_path.replace(' ', '|') writer.writerow([ elem.type, elem.peer_address, elem.peer_asn, prefix, next_hop, as_path ]) elem = rec.get_next_elem() _file.close() print 'count: ' + str(count) return count
def create_trie_from_bgpstream_info(self, interval_start=1451692800): stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', self.collector_name) stream.add_filter('record-type', 'ribs') if isinstance(interval_start, datetime.datetime): interval_start_utc = self.convert_interval_to_utc(interval_start) stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300) else: stream.add_interval_filter(interval_start - 300, interval_start + 300) stream.start() while (stream.get_next_record(rec)): elem = rec.get_next_elem() while elem: # Get the peer ASN and IP. We then construct a peer_id, since a collector # can establish multiple connections with the same ASN. peer_asn = elem.peer_asn peer_asn_ip = elem.peer_address # make this an unmodifiable tuple peer_id = (peer_asn, peer_asn_ip) peer_route_trie = self.peer_trie_handles[peer_id] trie_node = peer_route_trie.add(elem.fields['prefix']) trie_node.data['as-path'] = elem.fields['as-path'] elem = rec.get_next_elem()
#!/usr/bin/env python import os, sys, time from datetime import datetime import yaml from _pybgpstream import BGPStream, BGPRecord, BGPElem stream = BGPStream() class BGPStreamReader(object): defaults = { 'mrt_file': None, 'collector': 'rrc00', 'record_type': 'update', 'from_date': int(time.time()) - 3600 * 24 * 7, # back up a week 'until_date': 0, 'prefix_filter': None, 'peer_as_filter': None, 'communities_filter': None, } def __init__(self, config={}): self.config = config for k, v in self.defaults.items(): self.config.setdefault(k, v) if self.config['mrt_file'] is None: stream.add_filter('collector', self.config['collector']) stream.add_filter('record-type', self.config['record_type']) stream.add_interval_filter(self.config['from_date'], self.config['until_date']) stream.set_live_mode()
def main(): (options, args) = getopts() limit = options.limit start = options.start_time end = options.end_time # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') stream.add_filter('prefix', '0.0.0.0/0') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264) #stream.add_interval_filter(1454630400, 1454716800) # 1 hour #1454284800 - 1454288400 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record prefixes_update = defaultdict(int) prefixes_withdraw = defaultdict(int) while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: elem = rec.get_next_elem() while elem: if elem.type == 'A': #print elem.fields['as-path'] prefixes_update[elem.fields['prefix']] += 1 if elem.type == 'W': prefixes_withdraw[elem.fields['prefix']] += 1 #print rec.project, rec.collector, rec.type, rec.time, rec.status, #print elem.type, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem() for k in prefixes_update: if prefixes_update[k] >= limit: print k + "\t" + str(prefixes_update[k]) + "\t" + str(prefixes_withdraw[k])
import sys from BlackHoleDB import BlackHoleDb from _pybgpstream import BGPStream, BGPRecord, BGPElem data_source = "DE-CIX" sys.stderr.write("Inserting data from DE-CIX\n") # get connection to database bh_db = BlackHoleDb() # export LD_LIBRARY_PATH="/usr/local/lib" # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RRC12 stream.add_filter('collector','rrc12') # Consider RIBs dumps only stream.add_filter('record-type','ribs') # One rib per day stream.add_rib_period_filter(3600*24) # Consider this time interval: May 2016 stream.add_interval_filter(1462060800,1464739200) # Start the stream
from _pybgpstream import BGPRecord, BGPStream stop = 1422778200 start = stop - 1200 # 15000 second seems to be the shortest interval to get data from BGPstream result = [] stream = BGPStream() rec = BGPRecord() stream.add_filter('prefix', '192.33.4.0/24') stream.add_filter('record-type', 'ribs') stream.add_filter('project', 'ris') # stream.add_filter('collector', 'router-route-views.routeviews.org.peer-IPV4_route-spews.cbbtier3.att.net') stream.add_interval_filter(start, stop) stream.start() print('start') # test = stream.get_data_interfaces() # print('test: {}'.format(test)) while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: as_path = elem.fields['as-path'].split() as_path.append(' ') # for tree creation result.append(as_path) elem = rec.get_next_elem()
def _get_data(self, prefix, datetime): """ output example: [['15547', '8220', '1853', '1205'],[..another AS path..]] :param prefix: :param datetime: end interval :return: list of AS paths """ print('[*] ris.py: _get_data() called') print('[*] ris.py: _get_data() prefix: {}'.format(prefix)) start = int(datetime) - 20000 # 20000 second seems to be the shortest interval to get data from BGPstream stop = int(datetime) result = [] stream = BGPStream() rec = BGPRecord() stream.add_filter('prefix', prefix) stream.add_filter('record-type', 'ribs') stream.add_filter('project', 'ris') stream.add_interval_filter(start, stop) stream.start() while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: as_path = elem.fields['as-path'].split() as_path.append(' ') # for tree creation result.append(as_path) elem = rec.get_next_elem() print('[*] ris.py: _get_data() finished.') return result
global peer_id_dict global next_peer_id if collector_name not in peer_id_dict: peer_id_dict[collector_name] = dict() if peer_asn not in peer_id_dict[collector_name]: peer_id_dict[collector_name][peer_asn] = dict() if peer_address not in peer_id_dict[collector_name][peer_asn]: next_peer_id = next_peer_id peer_id_dict[collector_name][peer_asn][peer_address] = next_peer_id next_peer_id += 1 return peer_id_dict[collector_name][peer_asn][peer_address] # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() stream.add_filter('project','ris') stream.add_filter('project','routeviews') # Consider RIBs dumps only stream.add_filter('record-type','ribs') # Consider this time interval: jan_02_2016 = 1451692800 stream.add_interval_filter(jan_02_2016 - 300, jan_02_2016 + 300) stream.start() edge_peer_asn = dict()
#!/usr/bin/env python # Collects snapshot data from a ribs file and adds those nodes and # connections to a database file. Contains routing data as well as AS connections from _pybgpstream import BGPStream, BGPRecord, BGPElem from collections import defaultdict import string import sys # Initialize BGPStream with relevant filters stream = BGPStream() rec = BGPRecord() mode = "" if len(sys.argv) == 1: collectors = ['rrc00', 'rrc01', 'rrc03', 'rrc04', 'rrc05'] for collector in collectors: stream.add_filter('collector', collector) stream.add_filter('record-type', 'ribs') stream.add_interval_filter(1475310000, 1475350000) stream.add_filter('prefix', '8.0.0.0/8') mode = 'ripe' elif len(sys.argv) == 2: filename = sys.argv[1] stream.set_data_interface('singlefile') stream.set_data_interface_option('singlefile', 'rib-file', filename) collectors = ['singlefile_ds'] mode = 'file' stream.start()
#!/usr/bin/env python # Collects data from ribs files and adds those nodes and # connections to a database file along with a node representing # the time from _pybgpstream import BGPStream, BGPRecord, BGPElem from collections import defaultdict import string import datetime # Initialize BGPStream with relevant filters stream = BGPStream() rec = BGPRecord() collectors = ['rrc01'] for collector in collectors: stream.add_filter('collector', collector) stream.add_filter('record-type', 'ribs') stream.add_interval_filter(1438000000, 1439000000) stream.add_filter('prefix', '8.8.0.0/16') stream.start() # open files for neo4j-import collector_file = open("csv/collector.csv", 'w') AS_file = open("csv/AS.csv", 'w') prefix_file = open("csv/prefix.csv", 'w') route_file = open("csv/route.csv", 'w') time_file = open("csv/time.csv", 'w') connect_rels_file = open("csv/connect_rels.csv", 'w')
def main(): parser = argparse.ArgumentParser() parser.formatter_class = argparse.RawDescriptionHelpFormatter parser.description = textwrap.dedent('''\ a proof-of-concept utility for watching updates from BGPstream and then printing out if an unexpected update is heard ''') parser.epilog = textwrap.dedent('''\ Example: watch these route announcements %(prog)s -f routes.yaml ''') required = parser.add_argument_group('required arguments') required.add_argument("-f", "--file", required=True, help="yaml file of prefixes to origin asn") parser.add_argument("-d", "--debug", action='store_true', help="print out all updates containing these prefixes") args = parser.parse_args() routes = pytricia.PyTricia(48) # longest reasonable pfx in dfz with open(args.file, 'r') as f: routesfile = yaml.safe_load(f) for pfx in routesfile: routes[pfx] = routesfile[pfx] stream = BGPStream() rec = BGPRecord() stream.add_filter('record-type', 'updates') stream.add_interval_filter(int(time.time()), 0) stream.set_live_mode() stream.start() while (stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() while (elem): if 'as-path' in elem.fields: path = elem.fields['as-path'].split() prefix = elem.fields['prefix'] if prefix in routes and (routes[prefix] != path[-1] or args.debug): print('Heard prefix:', elem.fields['prefix'], 'AS-PATH:', elem.fields['as-path'], ' Found by project:', rec.project, 'collector:', rec.collector, 'type:', rec.type, 'at time:', rec.time, 'Type:', elem.type, 'Peer:', elem.peer_address, 'AS', elem.peer_asn) elem = rec.get_next_elem()
import time from _pybgpstream import BGPStream, BGPRecord, BGPElem # read here: http://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html start_time = time.time() stream = BGPStream() rec = BGPRecord() stream.add_filter('prefix','140.78.0.0/16') stream.add_filter('record-type','updates') # stream.add_filter('collector','rrc11') stream.add_filter('project','ris') stream.add_interval_filter(1462942850, 1462962850) stream.start() while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status == "valid": elem = rec.get_next_elem() while elem: # Print record and elem information print rec.collector, elem.peer_address, elem.peer_asn, elem.fields elem = rec.get_next_elem() # result = [] # while stream.get_next_record(rec): # if rec.status == "valid": # elem = rec.get_next_elem() # while elem:
def readrib(self): stream = None rec = None if self.txtFile is None: # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type ribs" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" for c in self.collectors: bgprFilter += " and collector %s " % c # if not self.asnFilter is None: # bgprFilter += ' and path %s$' % self.asnFilter for p in self.includedPeers: bgprFilter += " and peer %s " % p for p in self.includedPrefix: bgprFilter += " and prefix more %s " % p logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts - 3600, self.startts + 3600)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts - 3600, self.startts + 3600) if self.livemode: stream.set_live_mode() stream.start() else: rec = txtReader.txtReader(self.txtFile) # for line in p1.stdout: while (self.txtFile and not rec.running) or (stream and stream.get_next_record(rec)): if rec.status != "valid": print rec.project, rec.collector, rec.type, rec.time, rec.status zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] sPath = elem.fields["as-path"] # print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields)) if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes (and using IP addresses as spatial # resolution) self.peersASN[zOrig].add(zAS) if len(path) < 2: # Ignore paths with only one AS elem = rec.get_next_elem() continue node = self.rtree.add(zPfx) if zOrig in node.data: # Already read this entry, we should read only one RIB per peer elem = rec.get_next_elem() continue if self.ribQueue is not None: self.ribQueue.put((zDt, zOrig, zAS, zPfx, path)) node.data[zOrig] = { "path": set(path), "count": 0, "origAS": origAS } # print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields) if self.spatialResolution: # compute weight for this path count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # assert count >= 0 node.data[zOrig]["count"] = count # Update above nodes parent = self.findParent(node, zOrig) if not parent is None: # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs(parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount pOrigAS = parent.data[zOrig]["origAS"] asns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, pOrigAS, zAS, asns) self.incTotalCount(pdiff, zOrig, pOrigAS, zAS) else: count = 1 node.data[zOrig]["count"] = count asns = node.data[zOrig]["path"] self.incTotalCount(count, zOrig, origAS, zAS) self.incCount(count, zOrig, origAS, zAS, asns) elem = rec.get_next_elem()
def __init__(self): self.collector_list = [] self.collector_types = ('routeviews', 'ris') self.stream = BGPStream() self.record = BGPRecord()
if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("exp_name") parser.add_argument("collector") parser.add_argument("event_number", type=int) parser.add_argument( "--anchors", help="if set, downloads the anchor, otherwise downloads beacons", action='store_true') args = parser.parse_args() exp_name = args.exp_name collector = args.collector event_number = args.event_number stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector) if args.anchors: prefixes = anchor_list(exp_name) else: prefixes = beacon_list(exp_name) for prefix in prefixes: # print('Adding filter for prefix {}'.format(prefix)) stream.add_filter('prefix', prefix) fn = download_filename(exp_name, collector, args.anchors, event_number) if os.path.isfile(fn) and os.stat(fn).st_size > 0: # To debug
from _pybgpstream import BGPStream, BGPRecord, BGPElem from collections import defaultdict from itertools import groupby # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RIS RRC 00 only stream.add_filter('collector','route-views.sfmix') # Consider RIBs dumps only stream.add_filter('record-type','ribs') # Consider this time interval: jan_02_2016 = 1451692800 stream.add_interval_filter(jan_02_2016 - 300, jan_02_2016 + 300) stream.start() tier1_str = "174 209 286 701 1239 1299 2828 2914 3257 3320 3356 5511 6453 6461 6762 7018 12956" tier1s = tier1_str.split() edge_pfx = dict() transit = set() while(stream.get_next_record(rec)): elem = rec.get_next_elem() while(elem):
print buf print >> f, buf list_linkset = [] dictup = {} dictdown = {} nevents = {} history = {} current = start current_time_short = strftime("%Y/%m/%d-%H:%M:%S", gmtime(current)) current_time_long = strftime("%a, %d %b %Y %H:%M:%S", gmtime(current)) while current <= end: stream = BGPStream() rec = BGPRecord() stream.add_filter('collector', collector_filter) stream.add_filter('record-type','ribs') stream.add_interval_filter(current - 10 * 60, current + 10 * 60) stream.add_filter('peer-asn','3356') stream.add_filter('peer-asn','174') stream.add_filter('peer-asn','3257') stream.add_filter('peer-asn','1299') stream.add_filter('peer-asn','2914') stream.add_filter('peer-asn','6453') stream.add_filter('peer-asn','6762') stream.add_filter('peer-asn','6939') stream.add_filter('peer-asn','2828') stream.add_filter('peer-asn','3549')
# Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program. If not, see <http://www.gnu.org/licenses/>. # from _pybgpstream import BGPStream, BGPRecord, BGPElem # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() # configure the stream to retrieve RIB records from the RRC06 collector at # 2015/05/01 00:00 UTC stream.add_filter('collector', 'rrc06') stream.add_filter('record-type', 'ribs') stream.add_interval_filter(1427846400, 1427846700) # start the stream stream.start() as_topology = set() rib_entries = 0
import time from _pybgpstream import BGPStream, BGPRecord, BGPElem start_time = time.time() stream = BGPStream() rec = BGPRecord() stream.add_filter('prefix','140.78.0.0/16') stream.add_filter('record-type','ribs') stream.add_filter('collector','rrc11') # stream.add_filter('project','ris') stream.add_interval_filter(1464681000, 1464682200) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC) stream.start() # # while(stream.get_next_record(rec)): # # Print the record information only if it is not a valid record # if rec.status == "valid": # elem = rec.get_next_elem() # while elem: # # Print record and elem information # print rec.collector, elem.peer_address, elem.peer_asn, elem.fields['as-path'] # elem = rec.get_next_elem() result = [] while stream.get_next_record(rec): if rec.status == "valid": elem = rec.get_next_elem() while elem: print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields
def main(): parser = argparse.ArgumentParser(description='', epilog='') parser.add_argument('-b', '--begin', help='Begin date (inclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-u', '--until', help='Until date (exclusive), format: yyyy-mm-dd HH:MM', type=valid_date, required=True) parser.add_argument('-c', '--collector', help='Route collector from RIPE RIS or Route-Views project.', type=str, required=True) parser.add_argument('-m', '--mongodb', help='MongoDB connection parameters.', type=str, default=None) parser.add_argument('-k', '--keepsnapshots', help='Keep all snapshots, works only with -s.', action='store_true') parser.add_argument('-s', '--snapshot', help='Enable snapshoting.', action='store_true') parser.add_argument('-l', '--loglevel', help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].', type=str, default='WARNING') args = vars(parser.parse_args()) numeric_level = getattr(logging, args['loglevel'].upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level, format='#> %(asctime)s : %(levelname)s : %(message)s') ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds()) ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds()) mongodbstr = None if args['mongodb']: mongodbstr = args['mongodb'].strip() rib_ts = 0 rib_origins = dict() origins_lt = list() if args['snapshot']: rib_ts, rib_origins = load_snapshot(mongodbstr) if rib_ts > ts_begin: logging.info ("SKIP, found snapshot with newer ts") ts_begin = rib_ts - RIB_TS_THRESHOLD # BEGIN logging.info("START") # Create bgpstream stream = BGPStream() rec = BGPRecord() # set filtering stream.add_filter('collector',args['collector']) stream.add_filter('record-type','ribs') stream.add_interval_filter(ts_begin,ts_until) # Start the stream stream.start() while(stream.get_next_record(rec)): if rec.status == 'valid': elem = rec.get_next_elem() else: logging.warn("stream record invalid, skipping ...") continue #end if if rec.time > (rib_ts + RIB_TS_THRESHOLD): for p in rib_origins: for o in rib_origins[p]: if rib_origins[p][o][1] < (rib_ts - RIB_TS_THRESHOLD): origins_lt.append( (p,o,rib_origins[p][o][0],rib_origins[p][o][1]) ) #end if #end for #end for if args['snapshot'] and (len(rib_origins.keys()) > 0): store_snapshot(rec.time, rib_origins, mongodbstr) if not args['keepsnapshots']: remove_snapshot(rib_ts, mongodbstr) # end if keepsnapshots # end if snapshot rib_ts = rec.time logging.info("ts: "+str(rib_ts)) if len(origins_lt) > 0: if mongodbstr: store_origins_lt(rib_ts,origins_lt, mongodbstr) else: print_origins_lt(rib_ts,origins_lt) #end if for l in origins_lt: del rib_origins[l[0]][l[1]] #end for origins_lt = list() # end if #end if while(elem): prefix = elem.fields['prefix'] aspath = elem.fields['as-path'].split() for a in aspath: # remove AS-SETs if '{' in a: aspath.remove(a) #end if #end for origin = aspath[-1] if prefix not in rib_origins: rib_origins[prefix] = dict() #end if if origin not in rib_origins[prefix]: rib_origins[prefix][origin] = (rib_ts,rib_ts) else: rib_origins[prefix][origin] = (rib_origins[prefix][origin][0],rib_ts) #end if elem = rec.get_next_elem() #end while #end while if args['snapshot']: print "NOTE: remaining origin lifetimes are stored in latest snapshot (%d)!\n" % rib_ts if (len(rib_origins.keys()) > 0): store_snapshot(rib_ts, rib_origins, mongodbstr) # end if else: print "NOTE: output remaining origin lifetimes with current ts (%d)\n" % rib_ts origins_lt = list() for p in rib_origins: for o in rib_origins[p]: origins_lt.append( (p,o,rib_origins[p][o][0],rib_ts) ) if mongodbstr: store_origins_lt(rib_ts,origins_lt, mongodbstr) else: print_origins_lt(rib_ts,origins_lt)
# Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program. If not, see <http://www.gnu.org/licenses/>. # from _pybgpstream import BGPStream, BGPRecord, BGPElem # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() # configure the stream to retrieve Updates records from the RRC06 collector # The commented out add_filter lines are the old way, the parse_filter_string # way is the new method for applying filters #stream.add_filter('collector', 'rrc06') #stream.add_filter('record-type', 'updates') stream.parse_filter_string('collector rrc06 and type updates') # select the time interval to process: # Wed Apr 1 00:02:50 UTC 2015 -> Wed Apr 1 00:04:30 stream.add_interval_filter(1427846570, 1427846670)
# profile) before using pybgpstream # export LD_LIBRARY_PATH="/usr/local/pkg/ioda-tools/lib:$LD_LIBRARY_PATH" # this tutorial is not complete, please # refer to this page for a complete documentation # # https://staff.caida.org/~alistair/pybgpstream/index.html from _pybgpstream import BGPStream, BGPRecord, BGPElem import radix import sys import calendar import time stream = BGPStream() rec = BGPRecord() start_interval= 1401623715 ##stream.add_filter('project', 'ris') stream.add_filter('collector', 'route-views2') #stream.add_filter('collector', 'rrc04') stream.add_filter('record-type', 'ribs') stream.add_filter('record-type', 'updates') stream.add_interval_filter(start_interval,1406894115 ) stream.add_rib_period_filter(172800) stream.set_data_interface('broker') #stream.set_data_interface_option('broker', 'db-host', 'loki-ge') #stream.set_data_interface_option('broker', 'db-port', '3306') #stream.set_data_interface_option('broker', 'db-user', 'bgpstream') print "start bgpstream" stream.start()
if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("collector") parser.add_argument("--our-servers", default="localhost:9092") args = parser.parse_args() logging.basicConfig(level=logging.INFO) save_file = "ts-{}".format(args.collector) stream = BGPStream() stream.add_filter('project', args.collector) if os.path.exists(save_file): with open(save_file, "r") as f: last_ts = int(float(f.read().strip())) logger.info("loading timestamp from file: %s", datetime.utcfromtimestamp(last_ts)) else: # Consider RIBs dumps only now = time.time() last_ts = int(now - now % 3600) logger.info("loading from: %s", datetime.utcfromtimestamp(last_ts)) stream.add_filter('record-type', 'ribs')
def main(): (options, args) = getopts() start = options.start_time end = options.end_time target_prefs = Set() with open('./../../atlas/anchor_prefix.txt', 'rb') as br: for l in br: target_prefs.add(l.strip()) # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() with open('./data/stream_{0}'.format(start), 'wb') as bw: #for pref in target_prefs: for ptmp in target_prefs: stream.add_filter('prefix', ptmp) # stream.add_filter('prefix','0.0.0.0/0') # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('collector', 'rrc00') # Consider this time interval: # Sat Aug 1 08:20:11 UTC 2015 # stream.add_interval_filter(1438417216,1438417216) # stream.add_interval_filter(1451606400,1454785264 stream.add_interval_filter(start, end) # Start the stream stream.start() # Get next record cnt = 0 while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": pass # print '*', rec.project, rec.collector, rec.type, rec.time, rec.status else: cnt += 1 elem = rec.get_next_elem() while elem: if elem.type == 'S': continue # Print record and elem information # print rec.project, rec.collector, rec.type, rec.time, rec.status, # print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref bw.write('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'.format( rec.project, rec.collector, rec.type, rec.time, rec.status, elem.type, elem.fields['prefix'], elem.peer_address, elem.peer_asn, elem.fields)) bw.flush() elem = rec.get_next_elem() print 'Successful termination; Start time: {0}'.format(start)
def sendMessageToKafka(col_name, col_data): # print "Collector-{} Sending Message...".format(col_name) size_total = 0 num_total = 0 num_ipv4 = 0 stream = BGPStream() record = BGPRecord() time_start = int(col_data.get('updates').get('latestDumpTime')) time_end = time_start + int(col_data.get('updates').get('dumpPeriod')) # print "Start Time:{}, End Time:{}".format(time_start, time_end) stream.add_filter('collector', col_name) stream.add_filter('record-type', 'ribs') # stream.add_interval_filter(time_start, time_end) # stream.add_interval_filter(time_start, time_start+300) stream.add_interval_filter(1503475200, 1503475200 + 7200) # print "Before Start>>>>>" stream.start() print col_name # print "After Start>>>>>>" producer = Kafka_producer() while stream.get_next_record(record): if record.status == "valid": elem = record.get_next_elem() while elem: if filter(lambda x: ':' in x.peer_address, [elem]): num_total += 1 elem = record.get_next_elem() continue #print "Element:{},{},{}".format(elem.type, elem.peer_address, elem.peer_asn) field = elem.fields #print type(field) prefix = field['prefix'] if field.has_key('prefix') else '' next_hop = field['next-hop'] if field.has_key( 'next-hop') else '' as_path = field['as-path'] if field.has_key('as-path') else '' as_path = as_path.replace(' ', '|') text = [ elem.type, elem.peer_address, str(elem.peer_asn), prefix, next_hop, as_path, str(record.time) ] text = ','.join(text) # producer = Kafka_producer() producer.send_data(col_name, text) num_total += 1 num_ipv4 += 1 # print "[{}]-{}".format(col_name, num_total) # size_total += len(text) #NUM_TOTAL += 1 #print "[{}]-{}-{}-{}-{}".format(col_name, num_total, num_ipv4, size_total, time.ctime(record.time)) #print "No.{} Message Send Success-[{}]".format(num_total, text) elem = record.get_next_elem() else: pass # print "## Current record not valid!" # break # print "One Collector Finished" else: # print "-----------------------------" # print "Collector[{}] And Records Send Finished\nTotal Num:{}, IPv4 Num:{}, Total Size:{}".format(col_name, num_total, num_ipv4, size_total) # print "-----------------------------" print "Collector:[{}]".format(col_name) print "Total Num:{}, IPv4 Num:{}, Total Size:{}".format( num_total, num_ipv4, size_total)
def main(rib, target_prefs): # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() with open('./data/stream_{0}'.format(start), 'wb') as bw: for ptmp in target_prefs: stream.add_filter('prefix', ptmp) # Consider RIPE RRC 10 only stream.add_filter('record-type', 'updates') stream.add_filter('record-type', 'ribs') #stream.add_filter('collector', 'rrc04') stream.add_filter('project', 'ris') stream.add_filter('project', 'routeviews') stream.add_interval_filter(start-60*60*8, start) stream.add_rib_period_filter(10000000000000) # Start the stream stream.start() while stream.get_next_record(rec): # Print the record information only if it is not a valid record if rec.status != "valid": continue #if rec.time < start: elem = rec.get_next_elem() while elem: if elem.type == 'A' or elem.type == 'R': rib.add_to_rib(rec.collector, elem.peer_address, elem.fields['prefix'], elem.time, elem.fields['as-path']) elem = rec.get_next_elem() #else: rib.flush() print 'Successful termination; Start time: {0}'.format(start)
def load_data(start, stop, collectors, window, threshold): peers = {} # collectors is a list of the collectors we want to include # Start and stop define the interval we are looking in the data # Create a new BGPStream instance and a reusable BGPRecord instance stream = BGPStream() rec = BGPRecord() # Add filter for each collector. # If no collector is mentioned, it will consider 16 of them if collectors: for collector in collectors: print collector stream.add_filter('collector', collector) else: for i in range(0, 10): stream.add_filter('collector', 'rrc0' + str(i)) for i in range(10, 16): stream.add_filter('collector', 'rrc' + str(i)) stream.add_filter('record-type', 'updates') # Consider the interval from "start" to "stop" in seconds since epoch stream.add_interval_filter(start, stop) # Start the stream stream.start() # For each record (one record = one second, can have multiple elements for the same second) we handle its updates while stream.get_next_record(rec): timestamp = rec.time if rec.status != "valid": print rec.project, rec.collector, rec.type, timestamp, rec.status else: # Go through all elements of the record elem = rec.get_next_elem() while elem: # Consider only the A and W updates if elem.type not in ['A', 'W']: elem = rec.get_next_elem() continue peer = elem.peer_address updatetype = elem.type prefix = elem.fields['prefix'] if peer not in peers: peers[peer] = { 'A': [], 'W': [] } update = {'tst': timestamp, 'prefix': prefix} if updatetype == 'A': handleUpdate(peers[peer]['A'], burst2writeA, update, peer, updatetype, timestamp, window, threshold) saveGraphPoint(peers[peer]['A'], updatetype, peer, timestamp, collectors, threshold) else: handleUpdate(peers[peer]['W'], burst2writeW, update, peer, updatetype, timestamp, window, threshold) saveGraphPoint(peers[peer]['W'], updatetype, peer, timestamp, collectors, threshold) elem = rec.get_next_elem() # After processing all records, we write the graph json files with the graph points recorded for each peer for peer in graph_points: peer_file_name = peer.replace(':', '_') if not os.path.exists(peer_file_name): os.makedirs(peer_file_name) with open(peer_file_name+'/'+peer_file_name + '-graph.json', 'w') as outfile: json.dump(graph_points[peer], outfile, indent=2) # Write the last burst of A updates if there is one left if burst2writeA: for peer in burst2writeA: if burst2writeA[peer]: for timestamp in burst2writeA[peer]: writeBurst(peer, burst2writeA, 'A', timestamp) # Write the last burst of W updates if there is one left if burst2writeW: for peer in burst2writeW: if burst2writeW[peer]: for timestamp in burst2writeW[peer]: writeBurst(peer, burst2writeW, 'W', timestamp) # transform csv names in json file to use getJSON in plotGrap # step to CSV is used to avoid appending to the end of a json file directly as appending # to a json file overwrite the whole file jsonlist = [] with open('csv_peernames-'+'-'.join(collectors)+'.csv', 'rb') as f: reader = csv.reader(f) for row in reader: jsonlist.append(row[0]) jsondata = json.dumps(jsonlist, indent=2) fd = open('json_file_names-' + '-'.join(collectors) + '.json', 'w') fd.write(jsondata) fd.close()
# profile) before using pybgpstream # export LD_LIBRARY_PATH="/usr/local/pkg/ioda-tools/lib:$LD_LIBRARY_PATH" # this tutorial is not complete, please # refer to this page for a complete documentation # # https://staff.caida.org/~alistair/pybgpstream/index.html from _pybgpstream import BGPStream, BGPRecord, BGPElem import radix import sys import calendar import time stream = BGPStream() rec = BGPRecord() start_interval= 1403042654 end_interval= 1403055754 ##stream.add_filter('project', 'ris') stream.add_filter('collector', 'route-views2') #stream.add_filter('collector', 'rrc04') stream.add_filter('record-type', 'ribs') stream.add_filter('record-type', 'updates') stream.add_interval_filter(start_interval,end_interval ) stream.add_rib_period_filter(172800) stream.set_data_interface('mysql') stream.set_data_interface_option('mysql', 'db-host', 'loki-ge') stream.set_data_interface_option('mysql', 'db-port', '3306') stream.set_data_interface_option('mysql', 'db-user', 'bgpstream')
def run(self): stream = BGPStream() rec = BGPRecord() if self.upd_file is None: stream.add_filter('collector', self.collector) stream.add_filter('record-type', self.record_type) stream.add_interval_filter(self.from_date, self.to_date) stream.set_live_mode() else: stream.set_data_interface('singlefile') if self.upd_file: stream.set_data_interface_option('singlefile', 'upd-file', self.upd_file) if self.rib_file: stream.set_data_interface_option('singlefile', 'rib-file', self.rib_file) if self.prefix_filter is not None: for prefix in self.prefix_filter: stream.add_filter('prefix', prefix) if self.peer_as_filter: for asn in self.peer_as_filter: stream.add_filter('peer-asn', str(asn)) if self.communities_filter: for community in self.communities_filter: stream.add_filter('community', community) stream.start() stream.get_next_record(rec) prev = rec.time while (stream.get_next_record(rec)): now = rec.time if rec.status == 'valid': elem = rec.get_next_elem() while (elem): statement = None peer_address = elem.peer_address peer_asn = elem.peer_asn if peer_asn in self.asn_to_nexthop: if elem.type == 'A' or elem.type == 'R': prefix = elem.fields['prefix'] as_path = elem.fields['as-path'] nexthop = elem.fields['next-hop'] if peer_asn in self.asn_to_nexthop: nexthop = self.asn_to_nexthop[peer_asn] statement = 'announce route %s next-hop %s as-path' \ ' [ %s ]' % (prefix, nexthop, as_path) elif elem.type == 'W': prefix = elem.fields['prefix'] statement = 'withdraw route %s' % prefix if statement: sys.stdout.write("%s\n" % statement) sys.stdout.flush() elem = rec.get_next_elem() time.sleep(self.delay + now - prev) prev = now
#!/usr/bin/env python # docs at bgpstream.caida.org from _pybgpstream import BGPStream, BGPRecord, BGPElem from collections import defaultdict # export LD_LIBRARY_PATH="/usr/local/lib" # Create a new bgpstream instance and a reusable bgprecord instance stream = BGPStream() rec = BGPRecord() # Consider RRC12 stream.add_filter('collector','rrc12') # Consider RIBs dumps only # stream.add_filter('record-type','ribs') # Consider this time interval: # Sat, 01 Aug 2015 7:50:00 GMT - 08:10:00 GMT stream.add_interval_filter(1438415400,1438416600) # Start the stream stream.start() # Get next record while(stream.get_next_record(rec)): elem = rec.get_next_elem() while(elem): # only consider RIBs entries and Announcement messages if elem.type in ["R", "A"]: