def getBGPStream(recordType, AF, collectors, startts, endts): stream = BGPStream() # recordType is supposed to be ribs or updates bgprFilter = "type " + recordType if AF == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" for c in collectors: bgprFilter += " and collector %s " % c if isinstance(startts, str): startts = datetime.strptime(startts + "UTC", "%Y-%m-%dT%H:%M:%S%Z") startts = dt2ts(startts) if isinstance(endts, str): endts = datetime.strptime(endts + "UTC", "%Y-%m-%dT%H:%M:%S%Z") endts = dt2ts(endts) currentts = dt2ts(datetime.now()) if endts > currentts: stream.set_live_mode() stream.parse_filter_string(bgprFilter) stream.add_interval_filter(startts, endts) return stream
def download_data(): peer_state = defaultdict(dict) results = defaultdict(defaultdict_list) current_bin = 0 # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (startts, endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(startts, endts) stream.start() while (stream.get_next_record(rec)): if rec.status != "valid": print(rec.project, rec.collector, rec.type, rec.time, rec.status) # from IPython import embed # embed() if current_bin == 0: current_bin = rec.time # slide the time window: if current_bin + bin_size < rec.time: timebins = range(current_bin, rec.time, bin_size) for i in timebins[:-1]: results["other"]["timebin"].append(i) for pfx, p_s in peer_state.items(): for peeras, state in p_s.items(): results[pfx][peeras].append(state) current_bin = timebins[-1] elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn prefix = elem.fields["prefix"] peer_state[prefix][peeras] = elem.type elem = rec.get_next_elem() return results
def readUpdates(self): # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type updates" bgprFilter += " and project ris " for prefix in self.prefixes: bgprFilter += " and prefix more %s " % prefix logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) stream.start() while (stream.get_next_record(rec)): if (rec.status != "valid" and rec.status != "filtered-source" and rec.status != "empty-source"): raise Exception(rec.project, rec.collector, rec.type, rec.time, rec.status) zDt = rec.time elem = rec.get_next_elem() while (elem): # peerip g= elem.peer_address peeras = elem.peer_asn sPrefix = elem.fields["prefix"] if elem.type == "W": self.withdraws[sPrefix][peeras] = True else: sPath = elem.fields["as-path"] self.paths[sPrefix][peeras] = sPath self.withdraws[sPrefix][peeras] = False elem = rec.get_next_elem()
def readupdates(self): #TODO implement txt file for update messages? if self.txtFile: return # create a new bgpstream instance stream = BGPStream() bgprFilter = "type updates" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" # bgprFilter += " and collector rrc10 " for c in self.collectors: bgprFilter += " and collector %s " % c # if self.asnFilter is not None: # # TOFIX filter is now deprecated, we need to have both # # announcements and withdrawals # bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts, self.endts)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts, self.endts) if self.livemode: stream.set_live_mode() stream.start() # for line in p1.stdout: # create a reusable bgprecord instance rec = BGPRecord() while (stream.get_next_record(rec)): if rec.status != "valid": logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" % (rec.project, rec.collector, rec.type, rec.time, rec.status)) zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address if zOrig not in self.peers: # no need to update the counts for non-full feed peers elem = rec.get_next_elem() continue zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue msgTs = zDt # set first time bin! if self.ts is None: self.slideTimeWindow(msgTs) elif self.ts + self.timeWindow <= msgTs: self.slideTimeWindow(msgTs) elif self.ts > msgTs: #Old update, ignore this to update the graph logging.warn( "Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s" % (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields))) elem = rec.get_next_elem() continue node = self.rtree.search_exact(zPfx) if elem.type == "W": # Withdraw: remove the corresponding node if not node is None and zOrig in node.data: origAS = node.data[zOrig]["origAS"] if self.spatialResolution: count = node.data[zOrig]["count"] # Update count for above node parent = self.findParent(node, zOrig) if parent is None: self.incTotalCount(-count, zOrig, origAS, zAS) asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) else: node.data[zOrig]["count"] = 0 # Add ips to above node and corresponding ASes # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) # oldpCount = parent.data[zOrig]["count"] # pCount = self.nbIPs(parent.prefixlen) - pcountBelow # parent.data[zOrig]["count"] = pCount # pdiff = pCount - oldpCount # assert pdiff==count # Update count for origAS and path from the # parent node porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(count, zOrig, porigAS, zAS, pasns) self.incTotalCount(count, zOrig, porigAS, zAS) # Update count for withdrawn origAS and path asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, origAS, zAS, asns) self.incTotalCount(-count, zOrig, origAS, zAS) else: asns = node.data[zOrig]["path"] self.incCount(-1, zOrig, origAS, zAS, asns) self.incTotalCount(-1, zOrig, origAS, zAS) del node.data[zOrig] else: # Announce: update counters sPath = elem.fields["as-path"] path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes or implicit withdraws if len(path) < 2: # Ignoring paths with only one AS elem = rec.get_next_elem() continue if self.announceQueue is not None: self.announceQueue.put((zDt, zOrig, zAS, zPfx, path)) # Announce: if node is None or not zOrig in node.data: # Add a new node node = self.rtree.add(zPfx) if self.spatialResolution: # Compute the exact number of IPs count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) parent = self.findParent(node, zOrig) if not parent is None: # Update above nodes # print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count)) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs( parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount # print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount)) # print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ] porigAS = parent.data[zOrig]["origAS"] pasns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, porigAS, zAS, pasns) self.incTotalCount(pdiff, zOrig, porigAS, zAS) else: self.incTotalCount(1, zOrig, origAS, zAS) count = 1 # Update the ASes counts node.data[zOrig] = { "path": set(path), "count": count, "origAS": origAS } asn = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) else: #Update node path and counts if self.spatialResolution: count = node.data[zOrig]["count"] else: count = 1 porigAS = node.data[zOrig]["origAS"] asns = node.data[zOrig]["path"] self.incCount(-count, zOrig, porigAS, zAS, asns) self.incTotalCount(-count, zOrig, porigAS, zAS) node.data[zOrig]["path"] = set(path) node.data[zOrig]["origAS"] = origAS asns = node.data[zOrig]["path"] self.incCount(count, zOrig, origAS, zAS, asns) self.incTotalCount(count, zOrig, origAS, zAS) elem = rec.get_next_elem()
def readrib(self): stream = None rec = None if self.txtFile is None: # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() bgprFilter = "type ribs" if self.af == 6: bgprFilter += " and ipversion 6" else: bgprFilter += " and ipversion 4" for c in self.collectors: bgprFilter += " and collector %s " % c # if not self.asnFilter is None: # bgprFilter += ' and path %s$' % self.asnFilter for p in self.includedPeers: bgprFilter += " and peer %s " % p for p in self.includedPrefix: bgprFilter += " and prefix more %s " % p logging.info("Connecting to BGPstream... (%s)" % bgprFilter) logging.info("Timestamps: %s, %s" % (self.startts - 3600, self.startts + 3600)) stream.parse_filter_string(bgprFilter) stream.add_interval_filter(self.startts - 3600, self.startts + 3600) if self.livemode: stream.set_live_mode() stream.start() else: rec = txtReader.txtReader(self.txtFile) # for line in p1.stdout: while (self.txtFile and not rec.running) or (stream and stream.get_next_record(rec)): if rec.status != "valid": print rec.project, rec.collector, rec.type, rec.time, rec.status zDt = rec.time elem = rec.get_next_elem() while (elem): zOrig = elem.peer_address zAS = elem.peer_asn if zAS in self.excludedPeers or (len( self.includedPeers) and zAS not in self.includedPeers): elem = rec.get_next_elem() continue zPfx = elem.fields["prefix"] sPath = elem.fields["as-path"] # print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields)) if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or ( len(self.includedPrefix) and zPfx not in self.includedPrefix): elem = rec.get_next_elem() continue path = sPath.split(" ") origAS = path[-1] if origAS in self.excludedOriginASN or ( len(self.includedOriginASN) and origAS not in self.includedOriginASN): elem = rec.get_next_elem() continue # FIXME: this is not going to work in the case of # delegated prefixes (and using IP addresses as spatial # resolution) self.peersASN[zOrig].add(zAS) if len(path) < 2: # Ignore paths with only one AS elem = rec.get_next_elem() continue node = self.rtree.add(zPfx) if zOrig in node.data: # Already read this entry, we should read only one RIB per peer elem = rec.get_next_elem() continue if self.ribQueue is not None: self.ribQueue.put((zDt, zOrig, zAS, zPfx, path)) node.data[zOrig] = { "path": set(path), "count": 0, "origAS": origAS } # print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields) if self.spatialResolution: # compute weight for this path count = self.nbIPs(node.prefixlen) countBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(zPfx) if zOrig in n.data and n != node ]) count -= countBelow # assert count >= 0 node.data[zOrig]["count"] = count # Update above nodes parent = self.findParent(node, zOrig) if not parent is None: # pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data]) pcountBelow = sum([ n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n != parent ]) oldpCount = parent.data[zOrig]["count"] pCount = self.nbIPs(parent.prefixlen) - pcountBelow pdiff = pCount - oldpCount parent.data[zOrig]["count"] = pCount pOrigAS = parent.data[zOrig]["origAS"] asns = parent.data[zOrig]["path"] self.incCount(pdiff, zOrig, pOrigAS, zAS, asns) self.incTotalCount(pdiff, zOrig, pOrigAS, zAS) else: count = 1 node.data[zOrig]["count"] = count asns = node.data[zOrig]["path"] self.incTotalCount(count, zOrig, origAS, zAS) self.incCount(count, zOrig, origAS, zAS, asns) elem = rec.get_next_elem()
from _pybgpstream import BGPStream, BGPRecord, BGPElem # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() # configure the stream to retrieve Updates records from the RRC06 collector # The commented out add_filter lines are the old way, the parse_filter_string # way is the new method for applying filters #stream.add_filter('collector', 'rrc06') #stream.add_filter('record-type', 'updates') stream.parse_filter_string('collector rrc06 and type updates') # select the time interval to process: # Wed Apr 1 00:02:50 UTC 2015 -> Wed Apr 1 00:04:30 stream.add_interval_filter(1427846570, 1427846670) # start the stream stream.start() # print the stream while (stream.get_next_record(rec)): print rec.status, rec.project + "." + rec.collector, rec.time elem = rec.get_next_elem() while (elem): print "\t", elem.type, elem.peer_address, elem.peer_asn, \ elem.type, elem.fields
from _pybgpstream import BGPStream, BGPRecord, BGPElem # create a new bgpstream instance stream = BGPStream() # create a reusable bgprecord instance rec = BGPRecord() # configure the stream to retrieve Updates records from the RRC06 collector # The commented out add_filter lines are the old way, the parse_filter_string # way is the new method for applying filters #stream.add_filter('collector', 'rrc06') #stream.add_filter('record-type', 'updates') stream.parse_filter_string('collector rrc06 and type updates') # select the time interval to process: # Wed Apr 1 00:02:50 UTC 2015 -> Wed Apr 1 00:04:30 stream.add_interval_filter(1427846570, 1427846670) # start the stream stream.start() # print the stream while(stream.get_next_record(rec)): print rec.status, rec.project +"."+ rec.collector, rec.time elem = rec.get_next_elem() while(elem): print "\t", elem.type, elem.peer_address, elem.peer_asn, \ elem.type, elem.fields
class HistoBGPStream(): def __init__(self): # Create a new bgpstream instance # and a reusable bgprecord instances self.stream = BGPStream() self.rec = BGPRecord() self.origin_ases = set() self.bgp_lens = defaultdict(lambda: defaultdict(lambda: None)) def set_filter(self, string): """ This is optional. For the new version, we use stream.parse_filter_string() instead of stream.add_filter(). (https://github.com/CAIDA/bgpstream/blob/master/FILTERING) :param string: example 'type ribs and prefix more 72.20.0.0/24 ' :return: """ logging.info("Set filter: %s" % string) self.stream.parse_filter_string(string) def convert_dt_to_timestamp(self, dt): """ Return utc timestamp for a given time string, or time """ if isinstance(dt, str): dt = parse(dt) dt = dt.replace(tzinfo=None) timestamp = (dt - datetime(1970, 1, 1)).total_seconds() assert datetime.utcfromtimestamp(timestamp) == dt return int(timestamp) def get_paths(self, start_time, end_time=None): """ Return all paths for each peer for each collector for the given time interval. :param start_time: datimetime/date_string :param end_time: datimetime/date_string/None :return: paths(dict)[collector][peer-address][record-type] = (AS-path,time,prefix) """ # If end_time is None, set as one hour duration. if isinstance(start_time, str): start_time = parse(start_time) if isinstance(end_time, str): end_time = parse(end_time) if end_time == None: end_time = start_time + timedelta(hours=2) # start collection before 8 hours start_time = start_time - timedelta(hours=8) # If start_time or end_time is not timestamp, convert them. if not isinstance(start_time, int): start_time = self.convert_dt_to_timestamp(start_time) if not isinstance(end_time, int): end_time = self.convert_dt_to_timestamp(end_time) paths = self.get_bgpstream(start_time, end_time) return paths def get_bgpstream(self, start_time, end_time=None): """ Read all rib + updates according to filters. You have to set filters with self.set_filter function before run this if you need to set some filters. If the time for collecting paths is too long, we will optimize it later. Let's record the time taken. :param start_time:(timestamp) :param end_time:(timestamp) :return: paths(dict)[collector][peer-address][record-type] = (AS-path,time,prefix) """ logging.info("Collecting.. histo BGPStream [%s, %s]" % (start_time, end_time)) time_taken = time.time() # Time intervals: mandatory self.stream.add_interval_filter(start_time, end_time) # Start the stream self.stream.start() # Get next record # (record is rib/updates from a single peer of a collector) # (each record might have many elems. an elem per a prefix) paths = defaultdict( lambda: defaultdict(lambda: defaultdict(lambda: list()))) while (self.stream.get_next_record(self.rec)): elem = self.rec.get_next_elem() while (elem): if ('prefix' not in elem.fields) or (elem.type == 'S'): logging.debug("No prefix: %s, %s" % (elem.fields, elem.type)) continue # 0.0.0.0/0 is default, means "everything else", thus ignore! if elem.fields['prefix'] == '0.0.0.0/0': elem = self.rec.get_next_elem() continue # [Part 1] If you want to draw a graph, self.draw_graph(elem) # [Part 2] Retrieve record and elem information record_type = 'rib' if (self.rec.type == 'rib') else 'updates' # check as-path as_path = '' if 'as-path' in elem.fields: # Leave it behind if it includes only origin AS if len(elem.fields['as-path']) < 2: elem = self.rec.get_next_elem() continue as_path = elem.fields['as-path'] # get all origin ASes self.origin_ases.add(as_path.split(" ")[-1]) # if withdrawal? if elem.type == 'W': record_type = 'withdrawal' if len(paths[self.rec.collector][elem.peer_address][ elem.fields['prefix']]) != 0: # Traverse paths in reverse order to find the right time line. length = len(paths[self.rec.collector][elem.peer_address][ elem.fields['prefix']]) for index in reversed(range(length)): p_time, p_type, p_as_path =\ paths[self.rec.collector][elem.peer_address][elem.fields['prefix']][index] # If the current time is the last timestamp from announcements until now, # just append it at the end and break. Don't need to finish the travel. if self.rec.time >= p_time: if p_as_path != as_path: # if this is in the middle of the list, we have to check index+1 paths # whether there are identical, if they are identical, # delete index+1 and add this one. if (index + 1 != length): f_time, f_type, f_as_path = \ paths[self.rec.collector][elem.peer_address][elem.fields['prefix']][index+1] if as_path == f_as_path: logging.info( "Delete the same routing info.") logging.debug( "*** prefix: %s\n" "prev: %s, %s, %s\n" "new: %s, %s, %s\n" "next: %s, %s, %s" % (elem.fields['prefix'], p_time, p_type, p_as_path, self.rec.time, record_type, as_path, f_time, f_type, f_as_path)) del paths[self.rec.collector][ elem.peer_address][ elem.fields['prefix']][index + 1] paths[self.rec.collector][elem.peer_address][elem.fields['prefix']]\ .insert(index+1, (self.rec.time, record_type, as_path)) if (index + 1 != length): if as_path == f_as_path: logging.debug("updated: %s" % paths[self.rec.collector] [elem.peer_address] [elem.fields['prefix']]) break # Otherwise, just pass! else: logging.info( "The routing info arrives later than the latest one." ) if (index == 0) and (record_type != 'withdrawal'): logging.debug( "*** prefix: %s\n" "new_path: %s, %s, %s\n" "cur_path: %s, %s, %s\n" % (elem.fields['prefix'], self.rec.time, record_type, as_path, p_time, p_type, p_as_path)) if as_path == p_as_path: del paths[self.rec.collector][ elem.peer_address][ elem.fields['prefix']][index] paths[self.rec.collector][elem.peer_address][elem.fields['prefix']] \ .insert(index, (self.rec.time, record_type, as_path)) logging.debug( "updated: %s" % paths[self.rec.collector] [elem.peer_address][elem.fields['prefix']]) else: paths[self.rec.collector][elem.peer_address][elem.fields['prefix']].append\ ((self.rec.time, record_type, as_path)) elem = self.rec.get_next_elem() time_taken = time.time() - time_taken logging.info("Time taken for gathering histo bgpstream is %s" % time_taken) return paths def get_all_prefixes_given_as(self, ases, start_time): """ Collect all prefixes announced by suspected hijacker 8 hours before the hijack events :param asn: :param start_time: :return: """ logging.info("Start getting all prefixes for a given as: %s" % ases) all_prefixes = dict() # set a filter to get all paths ending with the given ASes filter_string = '' hijackers = [ases] if ',' in ases: hijackers = ases.split(', ') for hijacker in hijackers: if filter_string != '': filter_string += ' and ' filter_string += "path %s$" % hijacker logging.debug("Set a filter to get all prefixes: %s" % filter_string) self.set_filter(filter_string) # Get all prefixes 8 hours before the start_time if isinstance(start_time, str): start_time = parse(start_time) start_collect_time = start_time - timedelta(hours=8) end_collect_time = start_time - timedelta(hours=1) # If start_time or end_time is not timestamp, convert them. if not isinstance(start_collect_time, int): start_collect_time = self.convert_dt_to_timestamp( start_collect_time) if not isinstance(end_collect_time, int): end_collect_time = self.convert_dt_to_timestamp(end_collect_time) # Get all historical paths between start_collect_time and end_collect_time paths = self.get_bgpstream(start_collect_time, end_collect_time) for collector in paths.keys(): for peer in paths[collector].keys(): for pfx in paths[collector][peer].keys(): for t, r_type, path in paths[collector][peer][pfx]: origin_asn = path.split(' ')[-1] if origin_asn not in all_prefixes: all_prefixes[origin_asn] = list() all_prefixes[origin_asn].append(pfx) for asn in all_prefixes.keys(): all_prefixes[asn] = list(set(all_prefixes[asn])) return all_prefixes def store_real_events_to_mongodb(self): """ Read from excel files and store to mongodb """ return def draw_graph(self, elem): """ Draw a graph from updates + ribs :return: """ # [Part 1] # Get the peer ASn import networkx as nx # Create an instance of a simple undirected graph self.as_graph = nx.Graph() peer = str(elem.peer_asn) # Get the array of ASns in the AS path and remove repeatedly prepended ASns hops = [k for k, g in groupby(elem.fields['as-path'].split(" "))] if len(hops) > 1 and hops[0] == peer: # Get the origin ASn origin = hops[-1] # Add new edges to the NetworkX graph for i in range(0, len(hops) - 1): self.as_graph.add_edge(hops[i], hops[i + 1]) # Update the AS path length between 'peer' and 'origin' self.bgp_lens[peer][origin] = \ min(filter(bool, [self.bgp_lens[peer][origin], len(hops)])) return