def getBGPStream(recordType, AF, collectors, startts, endts):
stream = BGPStream()
# recordType is supposed to be ribs or updates
bgprFilter = "type " + recordType
if AF == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
for c in collectors:
bgprFilter += " and collector %s " % c
if isinstance(startts, str):
startts = datetime.strptime(startts + "UTC", "%Y-%m-%dT%H:%M:%S%Z")
startts = dt2ts(startts)
if isinstance(endts, str):
endts = datetime.strptime(endts + "UTC", "%Y-%m-%dT%H:%M:%S%Z")
endts = dt2ts(endts)
currentts = dt2ts(datetime.now())
if endts > currentts:
stream.set_live_mode()
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(startts, endts)
return stream
def run(self):
stream = BGPStream()
rec = BGPRecord()
if self.upd_file is None:
stream.add_filter('collector', self.collector)
stream.add_filter('record-type', self.record_type)
stream.add_interval_filter(self.from_date, self.to_date)
stream.set_live_mode()
else:
stream.set_data_interface('singlefile')
if self.upd_file:
stream.set_data_interface_option('singlefile', 'upd-file',
self.upd_file)
if self.rib_file:
stream.set_data_interface_option('singlefile', 'rib-file',
self.rib_file)
if self.prefix_filter is not None:
for prefix in self.prefix_filter:
stream.add_filter('prefix', prefix)
if self.peer_as_filter:
for asn in self.peer_as_filter:
stream.add_filter('peer-asn', str(asn))
if self.communities_filter:
for community in self.communities_filter:
stream.add_filter('community', community)
stream.start()
stream.get_next_record(rec)
prev = rec.time
while (stream.get_next_record(rec)):
now = rec.time
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
statement = None
peer_address = elem.peer_address
peer_asn = elem.peer_asn
if peer_asn in self.asn_to_nexthop:
if elem.type == 'A' or elem.type == 'R':
prefix = elem.fields['prefix']
as_path = elem.fields['as-path']
nexthop = elem.fields['next-hop']
if peer_asn in self.asn_to_nexthop:
nexthop = self.asn_to_nexthop[peer_asn]
statement = 'announce route %s next-hop %s as-path' \
' [ %s ]' % (prefix, nexthop, as_path)
elif elem.type == 'W':
prefix = elem.fields['prefix']
statement = 'withdraw route %s' % prefix
if statement:
sys.stdout.write("%s\n" % statement)
sys.stdout.flush()
elem = rec.get_next_elem()
time.sleep(self.delay + now - prev)
prev = now
def main():
parser = argparse.ArgumentParser()
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.description = textwrap.dedent('''\
a proof-of-concept utility for watching updates from BGPstream
and then printing out if an unexpected update is heard
''')
parser.epilog = textwrap.dedent('''\
Example: watch these route announcements
%(prog)s -f routes.yaml ''')
required = parser.add_argument_group('required arguments')
required.add_argument("-f",
"--file",
required=True,
help="yaml file of prefixes to origin asn")
parser.add_argument("-d",
"--debug",
action='store_true',
help="print out all updates containing these prefixes")
args = parser.parse_args()
routes = pytricia.PyTricia(48) # longest reasonable pfx in dfz
with open(args.file, 'r') as f:
routesfile = yaml.safe_load(f)
for pfx in routesfile:
routes[pfx] = routesfile[pfx]
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(int(time.time()), 0)
stream.set_live_mode()
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
if 'as-path' in elem.fields:
path = elem.fields['as-path'].split()
prefix = elem.fields['prefix']
if prefix in routes and (routes[prefix] != path[-1]
or args.debug):
print('Heard prefix:', elem.fields['prefix'],
'AS-PATH:', elem.fields['as-path'],
' Found by project:', rec.project, 'collector:',
rec.collector, 'type:', rec.type, 'at time:',
rec.time, 'Type:', elem.type, 'Peer:',
elem.peer_address, 'AS', elem.peer_asn)
elem = rec.get_next_elem()
def readupdates(self):
#TODO implement txt file for update messages?
if self.txtFile:
return
# create a new bgpstream instance
stream = BGPStream()
bgprFilter = "type updates"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
# bgprFilter += " and collector rrc10 "
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if self.asnFilter is not None:
# # TOFIX filter is now deprecated, we need to have both
# # announcements and withdrawals
# bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (self.startts, self.endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts, self.endts)
if self.livemode:
stream.set_live_mode()
stream.start()
# for line in p1.stdout:
# create a reusable bgprecord instance
rec = BGPRecord()
while (stream.get_next_record(rec)):
if rec.status != "valid":
logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" %
(rec.project, rec.collector, rec.type, rec.time,
rec.status))
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
if zOrig not in self.peers:
# no need to update the counts for non-full feed peers
elem = rec.get_next_elem()
continue
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
msgTs = zDt
# set first time bin!
if self.ts is None:
self.slideTimeWindow(msgTs)
elif self.ts + self.timeWindow <= msgTs:
self.slideTimeWindow(msgTs)
elif self.ts > msgTs:
#Old update, ignore this to update the graph
logging.warn(
"Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s"
% (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields)))
elem = rec.get_next_elem()
continue
node = self.rtree.search_exact(zPfx)
if elem.type == "W":
# Withdraw: remove the corresponding node
if not node is None and zOrig in node.data:
origAS = node.data[zOrig]["origAS"]
if self.spatialResolution:
count = node.data[zOrig]["count"]
# Update count for above node
parent = self.findParent(node, zOrig)
if parent is None:
self.incTotalCount(-count, zOrig, origAS, zAS)
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
else:
node.data[zOrig]["count"] = 0
# Add ips to above node and corresponding ASes
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
# oldpCount = parent.data[zOrig]["count"]
# pCount = self.nbIPs(parent.prefixlen) - pcountBelow
# parent.data[zOrig]["count"] = pCount
# pdiff = pCount - oldpCount
# assert pdiff==count
# Update count for origAS and path from the
# parent node
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(count, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(count, zOrig, porigAS, zAS)
# Update count for withdrawn origAS and path
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
self.incTotalCount(-count, zOrig, origAS, zAS)
else:
asns = node.data[zOrig]["path"]
self.incCount(-1, zOrig, origAS, zAS, asns)
self.incTotalCount(-1, zOrig, origAS, zAS)
del node.data[zOrig]
else:
# Announce: update counters
sPath = elem.fields["as-path"]
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes or implicit withdraws
if len(path) < 2:
# Ignoring paths with only one AS
elem = rec.get_next_elem()
continue
if self.announceQueue is not None:
self.announceQueue.put((zDt, zOrig, zAS, zPfx, path))
# Announce:
if node is None or not zOrig in node.data:
# Add a new node
node = self.rtree.add(zPfx)
if self.spatialResolution:
# Compute the exact number of IPs
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
parent = self.findParent(node, zOrig)
if not parent is None:
# Update above nodes
# print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count))
pcountBelow = sum([
n.data[zOrig]["count"] for n in
self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(
parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
# print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount))
# print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ]
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(pdiff, zOrig, porigAS, zAS)
else:
self.incTotalCount(1, zOrig, origAS, zAS)
count = 1
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asn = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
else:
#Update node path and counts
if self.spatialResolution:
count = node.data[zOrig]["count"]
else:
count = 1
porigAS = node.data[zOrig]["origAS"]
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, porigAS, zAS, asns)
self.incTotalCount(-count, zOrig, porigAS, zAS)
node.data[zOrig]["path"] = set(path)
node.data[zOrig]["origAS"] = origAS
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
elem = rec.get_next_elem()
def readrib(self):
stream = None
rec = None
if self.txtFile is None:
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type ribs"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if not self.asnFilter is None:
# bgprFilter += ' and path %s$' % self.asnFilter
for p in self.includedPeers:
bgprFilter += " and peer %s " % p
for p in self.includedPrefix:
bgprFilter += " and prefix more %s " % p
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" %
(self.startts - 3600, self.startts + 3600))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts - 3600,
self.startts + 3600)
if self.livemode:
stream.set_live_mode()
stream.start()
else:
rec = txtReader.txtReader(self.txtFile)
# for line in p1.stdout:
while (self.txtFile
and not rec.running) or (stream
and stream.get_next_record(rec)):
if rec.status != "valid":
print rec.project, rec.collector, rec.type, rec.time, rec.status
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
sPath = elem.fields["as-path"]
# print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields))
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes (and using IP addresses as spatial
# resolution)
self.peersASN[zOrig].add(zAS)
if len(path) < 2:
# Ignore paths with only one AS
elem = rec.get_next_elem()
continue
node = self.rtree.add(zPfx)
if zOrig in node.data:
# Already read this entry, we should read only one RIB per peer
elem = rec.get_next_elem()
continue
if self.ribQueue is not None:
self.ribQueue.put((zDt, zOrig, zAS, zPfx, path))
node.data[zOrig] = {
"path": set(path),
"count": 0,
"origAS": origAS
}
# print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields)
if self.spatialResolution:
# compute weight for this path
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# assert count >= 0
node.data[zOrig]["count"] = count
# Update above nodes
parent = self.findParent(node, zOrig)
if not parent is None:
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
pcountBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
pOrigAS = parent.data[zOrig]["origAS"]
asns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, pOrigAS, zAS, asns)
self.incTotalCount(pdiff, zOrig, pOrigAS, zAS)
else:
count = 1
node.data[zOrig]["count"] = count
asns = node.data[zOrig]["path"]
self.incTotalCount(count, zOrig, origAS, zAS)
self.incCount(count, zOrig, origAS, zAS, asns)
elem = rec.get_next_elem()
