def calculate_effective_prefixes(self, start_interval=1451692800):
'''
This method calculates the number of effective /24
prefixes that lie underneath a specified route prefix
:param start_interval: the start interval of the data
stream
:return: number of effective prefixes under the route's
root prefix
'''
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', self.collector_name)
stream.add_filter('record-type', 'ribs')
if isinstance(start_interval, datetime.datetime):
interval_start_utc = self.convert_interval_to_utc(start_interval)
stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300)
else:
stream.add_interval_filter(start_interval - 300, start_interval + 300)
stream.start()
print "Starting routing table parsing"
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while elem:
# Get the peer ASN and IP. We then construct a peer_id, since a collector
# can establish multiple connections with the same ASN.
peer_asn = elem.peer_asn
peer_asn_ip = elem.peer_address
# make this an unmodifiable tuple
peer_id = (peer_asn, peer_asn_ip)
peer_route_trie = self.peer_trie_handles[peer_id]
# Do a single level search of the route prefix to
# find the left and right prefix advertisements
single_level_prefixes = \
peer_route_trie.single_level_search(elem.fields['prefix'])
print single_level_prefixes
prefix_count = len(single_level_prefixes)
root_24_prefix_count = 0
lr_24_prefix_count1 = lr_24_prefix_count2 = 0
# The /24 prefixes below the advertised prefix are calculated
# as all the /24 prefixes served by the root - sum of the /24
# prefixes served by root's children
if prefix_count == 1:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
elif prefix_count == 2:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count1 = \
2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<')
.rstrip('>').split('/')[1]))
else:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count1 = \
2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count2 = \
2 ** (24 - int(str(single_level_prefixes[2]).lstrip('<')
.rstrip('>').split('/')[1]))
effective_24_prefix_count = \
root_24_prefix_count - (lr_24_prefix_count1 +
lr_24_prefix_count2)
print "Effective Prefix Count : ", \
effective_24_prefix_count
for prefix in single_level_prefixes:
trie_node = peer_route_trie.search_exact(str(prefix))
as_path = trie_node.data['as-path'].split(" ")
as_path_headless = as_path[1:-1]
print "AS-Path : ", as_path
as_headless_len = len(as_path_headless)
if as_headless_len > 1:
for i in range(0, as_headless_len - 1):
print "Headless nodes : ", as_path_headless[i], \
as_path_headless[i + 1]
if as_path_headless[i] in self.aslink_datastore:
self.aslink_datastore[as_path_headless[i]] += 1
else:
self.aslink_datastore[as_path_headless[i]] = 1
elem = rec.get_next_elem()
def run(self):
stream = BGPStream()
rec = BGPRecord()
if self.upd_file is None:
stream.add_filter('collector', self.collector)
stream.add_filter('record-type', self.record_type)
stream.add_interval_filter(self.from_date, self.to_date)
stream.set_live_mode()
else:
stream.set_data_interface('singlefile')
if self.upd_file:
stream.set_data_interface_option('singlefile', 'upd-file',
self.upd_file)
if self.rib_file:
stream.set_data_interface_option('singlefile', 'rib-file',
self.rib_file)
if self.prefix_filter is not None:
for prefix in self.prefix_filter:
stream.add_filter('prefix', prefix)
if self.peer_as_filter:
for asn in self.peer_as_filter:
stream.add_filter('peer-asn', str(asn))
if self.communities_filter:
for community in self.communities_filter:
stream.add_filter('community', community)
stream.start()
stream.get_next_record(rec)
prev = rec.time
while (stream.get_next_record(rec)):
now = rec.time
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
statement = None
peer_address = elem.peer_address
peer_asn = elem.peer_asn
if peer_asn in self.asn_to_nexthop:
if elem.type == 'A' or elem.type == 'R':
prefix = elem.fields['prefix']
as_path = elem.fields['as-path']
nexthop = elem.fields['next-hop']
if peer_asn in self.asn_to_nexthop:
nexthop = self.asn_to_nexthop[peer_asn]
statement = 'announce route %s next-hop %s as-path' \
' [ %s ]' % (prefix, nexthop, as_path)
elif elem.type == 'W':
prefix = elem.fields['prefix']
statement = 'withdraw route %s' % prefix
if statement:
sys.stdout.write("%s\n" % statement)
sys.stdout.flush()
elem = rec.get_next_elem()
time.sleep(self.delay + now - prev)
prev = now
def sendMessageToKafka(self, col_name, col_data):
print "lalalala"
stream = BGPStream()
record = BGPRecord()
time_start = int(col_data.get('ribs').get('latestDumpTime'))
time_end = time_start + int(col_data.get('ribs').get('dumpPeriod'))
stream.add_filter('collector', col_name)
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(time_start, time_end)
print "Before Start"
stream.start()
print "After Start"
while stream.get_next_record(record):
if record.status == "valid":
elem = record.get_next_elem()
while elem:
# print "Record:{}".format(elem)
producer = Kafka_producer()
producer.send_data(col_name, json.dumps(elem))
elem = record.get_next_elem()
else:
print "## Current record not valid!"
print "One Collector Finished"
def downloader(start_date, duration):
"""Download BGP paths from Routeviews and RIPE NCC from a start date for a certain duration."""
# Start of UNIX time
base = int(datetime.datetime.strptime(start_date, '%m/%d/%Y').strftime('%s'))
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
stream.add_interval_filter(base, base + int(duration))
stream.add_filter('record-type', 'ribs')
stream.start()
path_set = set()
f = open('rib.txt', 'w')
while True:
rec = stream.get_next_record()
if rec is None:
return
if rec.status != "valid":
continue
else:
elem = rec.get_next_elem()
while(elem):
path = elem.fields['as-path']
if '{' in path or '(' in path:
elem = rec.get_next_elem()
continue
prefix = elem.fields['prefix']
# Focus on IPv4 prefixes
if ":" not in prefix and path not in path_set:
f.write(path.replace(' ', '|') + '\n')
path_set.add(path)
elem = rec.get_next_elem()
f.close()
def _get_data(self, prefix, datetime):
"""
output example: [['15547', '8220', '1853', '1205'],[..another AS path..]]
:param prefix:
:param datetime: end interval
:return: list of AS paths
"""
print('[*] ris.py: _get_data() called')
print('[*] ris.py: _get_data() prefix: {}'.format(prefix))
start = int(datetime) - 20000 # 20000 second seems to be the shortest interval to get data from BGPstream
stop = int(datetime)
result = []
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('prefix', prefix)
stream.add_filter('record-type', 'ribs')
stream.add_filter('project', 'ris')
stream.add_interval_filter(start, stop)
stream.start()
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
as_path = elem.fields['as-path'].split()
as_path.append(' ') # for tree creation
result.append(as_path)
elem = rec.get_next_elem()
print('[*] ris.py: _get_data() finished.')
return result
def create_trie_from_bgpstream_info(self, interval_start=1451692800):
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', self.collector_name)
stream.add_filter('record-type', 'ribs')
if isinstance(interval_start, datetime.datetime):
interval_start_utc = self.convert_interval_to_utc(interval_start)
stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300)
else:
stream.add_interval_filter(interval_start - 300, interval_start + 300)
stream.start()
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while elem:
# Get the peer ASN and IP. We then construct a peer_id, since a collector
# can establish multiple connections with the same ASN.
peer_asn = elem.peer_asn
peer_asn_ip = elem.peer_address
# make this an unmodifiable tuple
peer_id = (peer_asn, peer_asn_ip)
peer_route_trie = self.peer_trie_handles[peer_id]
trie_node = peer_route_trie.add(elem.fields['prefix'])
trie_node.data['as-path'] = elem.fields['as-path']
elem = rec.get_next_elem()
def recv_bgpstream_rib(begin, until, collector):
"""
Receive and parse BGP RIB records from a given bgpstream collector.
"""
logging.info("CALL recv_bgpstream_rib")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(begin, until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping.")
continue
bgp_message = None
while (elem):
if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'):
bgp_message = BGPmessage(elem.time, 'update')
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
bgp_message.add_announce(elem.fields['prefix'])
output(bgp_message)
elem = rec.get_next_elem()
# end while (elem)
# end while (stream...)
output('FLUSH')
def get_data_graph(self):
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(self.start, self.end)
stream.start()
as_graph = nx.Graph()
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
# the list is a list, which stores an as-path
as_path = [
k
for k, g in groupby(elem.fields['as-path'].split(" "))
]
peer = str(elem.peer_asn)
# judge whether the as-path is legal
if len(as_path) > 1 and as_path[0] == peer:
if re.match(self.pattern, elem.fields['prefix']):
self.as_prefix[as_path[-1]].add(
elem.fields['prefix'])
# add edges to the graph
for i in range(0, len(as_path) - 1):
as_graph.add_edge(as_path[i], as_path[i + 1])
elem = rec.get_next_elem()
as_graph = self.add_geo_loc(as_graph)
return as_graph
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# to track the peers that have elems in this record
peer_signatures = set()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
peer_signatures.add(sig)
# if this is the first time we have ever seen this peer, create
# an empty result: (Pfxs_v4_set, Pfxs_v6_set, ASNs_v4_set, ASNs_v6_set)
if sig not in peers_data:
peers_data[sig] =[set(),set(),
set(),set()]
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
if(":" in pfx):
peers_data[sig][1].add(pfx)
else:
peers_data[sig][0].add(pfx)
if('as-path' in elem.fields):
path_split=elem.fields['as-path'].split()
if(":" in pfx):
for ASN in path_split:
peers_data[sig][3].add(ASN)
else:
for ASN in path_split:
peers_data[sig][2].add(ASN)
elem = rec.get_next_elem()
# one peer that was present in this record (allows a true, per-collector
# count of records since each record can contain elems for many peers)
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def main():
(options, args) = getopts()
start = options.start_time
end = options.end_time
target_prefs = Set()
with open('./../../atlas/anchor_prefix.txt', 'rb') as br:
for l in br:
target_prefs.add(l.strip())
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
with open('./data/stream_{0}'.format(start), 'wb') as bw:
#for pref in target_prefs:
for ptmp in target_prefs:
stream.add_filter('prefix', ptmp)
# stream.add_filter('prefix','0.0.0.0/0')
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
cnt = 0
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
cnt += 1
elem = rec.get_next_elem()
while elem:
if elem.type == 'S':
continue
# Print record and elem information
# print rec.project, rec.collector, rec.type, rec.time, rec.status,
# print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref
bw.write(
'{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'.
format(rec.project, rec.collector, rec.type, rec.time,
rec.status, elem.type, elem.fields['prefix'],
elem.peer_address, elem.peer_asn, elem.fields))
bw.flush()
elem = rec.get_next_elem()
print 'Successful termination; Start time: {0}'.format(start)
def dump2file(type, name, latestDumpTime, dumpDuration, dumpPeriod):
print type + ' of ' + name + ': '
csv_header = ['type', 'addr', 'as', 'prefix', 'next_hop', 'as_path']
_file = None
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', c_name)
if type == 'ribs':
stream.add_filter('record-type', 'ribs')
_file = open(c_name + '_ribs.csv', 'w+')
elif type == 'updates':
stream.add_filter('record-type', 'updates')
_file = open(c_name + '_updates.csv', 'w+')
stream.add_interval_filter(latestDumpTime, latestDumpTime + dumpPeriod)
stream.start()
count = 0
useless_c = 0
writer = csv.writer(_file)
writer.writerow(csv_header)
# Get next record
while (stream.get_next_record(rec)):
# Print the record information only if it is not a valid record
if rec.status != "valid":
# print rec.project, rec.collector, rec.type, rec.time, rec.status
print 'current rec not valid.'
else:
elem = rec.get_next_elem()
while (elem):
useless_c += 1
if useless_c % 1000 == 0:
print 'Got ' + str(useless_c) + ' elem totally.'
# Print record and elem information
if isIPV6(elem): # ipv6 packet ignored
elem = rec.get_next_elem()
continue
count += 1
# print rec.project, rec.collector, rec.type, rec.time, rec.status,
# print elem.type, elem.peer_address, elem.peer_asn
# elem.fields contains four column: communities, next-hop, prefix, as-path
field = elem.fields
prefix = field['prefix'] if 'prefix' in field.keys() else ''
next_hop = field['next-hop'] if 'next-hop' in field.keys(
) else ''
as_path = field['as-path'] if 'as-path' in field.keys() else ''
as_path = as_path.replace(' ', '|')
writer.writerow([
elem.type, elem.peer_address, elem.peer_asn, prefix,
next_hop, as_path
])
elem = rec.get_next_elem()
_file.close()
print 'count: ' + str(count)
return count
def build_sql_db():
conn = sqlite3.connect('bgp_stage.db')
c = conn.cursor()
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector','rrc11')
stream.add_interval_filter(1438417216,1438417316)
stream.start()
while(stream.get_next_record(rec)):
if rec.status != "valid":
continue
else:
elem = rec.get_next_elem()
while(elem):
if elem.type == "AB":
prefix = elem.fields["prefix"]
as_path = elem.fields["as-path"].split(" ")
origin = as_path[-1]
time = elem.time
#IP Prefix database
ip_min, ip_max = calculate_min_max(prefix)
c.execute("SELECT ip_min FROM prefix_as WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin))
row = c.fetchone()
if len(row) != 0:
c.execute("UPDATE prefix_as SET count = count + 1 WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin))
else:
c.execute("INSERT INTO prefix_as VALUES(?,?,?,?,?)", (ip_min, ip_max, origin, 1, time))
#AS link database
for as1,as2 in zip(as_path, as_path[1:]) :
c.execute("SELECT as_o FROM as_link WHERE as_o = (?) AND as_n = (?)",(as1,as2))
row = c.fetchone()
if len(row) != 0:
c.execute("UPDATE as_link SET count = count + 1 WHERE as_o = (?) AND as_n = (?)",
(as1, as2))
else:
c.execute("INSERT INTO as_link VALUES(?,?,?,?)", (as1, as2, 1, 0))
elif elem.type == "WA":
prefix = elem.fields["prefix"]
time = elem.time
#Needs research
print(rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.peer_address, elem.peer_asn, elem.fields)
print(prefix,elem.time, "W")
print(rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.peer_address, elem.peer_asn, elem.fields)
elem = rec.get_next_elem()
conn.commit()
conn.close()
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
# if this is the first time we have ever seen this peer, create
# an empty result: (dict(Pfx_v4)=Pfx_origins, dict(Pfx_v6)=Pfx_origins))
if sig not in peers_data:
peers_data[sig] =[{},{}]
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
origin=""
if('as-path' in elem.fields):
path_split=elem.fields['as-path'].split()
if(len(path_split)!=0):
origin=path_split[len(path_split)-1]
if(":" in pfx):
if(pfx not in peers_data[sig][1]):
peers_data[sig][1][pfx]=set()
#discard as origin: AS sets, and ASN=23456 [AS_TRANS]
if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][1][pfx].add(origin)
else:
if(pfx not in peers_data[sig][0]):
peers_data[sig][0][pfx]=set()
#discard as origin: AS sets, and ASN=23456 [AS_TRANS]
if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][0][pfx].add(origin)
elem = rec.get_next_elem()
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def main():
(options, args) = getopts()
start = options.start_time
end = options.end_time
target_prefs = Set()
with open('./../../atlas/anchor_prefix.txt', 'rb') as br:
for l in br:
target_prefs.add(l.strip())
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
with open('./data/stream_{0}'.format(start), 'wb') as bw:
#for pref in target_prefs:
for ptmp in target_prefs:
stream.add_filter('prefix', ptmp)
# stream.add_filter('prefix','0.0.0.0/0')
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
cnt = 0
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
cnt += 1
elem = rec.get_next_elem()
while elem:
if elem.type == 'S':
continue
# Print record and elem information
# print rec.project, rec.collector, rec.type, rec.time, rec.status,
# print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref
bw.write('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'.format(
rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.fields['prefix'], elem.peer_address, elem.peer_asn, elem.fields))
bw.flush()
elem = rec.get_next_elem()
print 'Successful termination; Start time: {0}'.format(start)
def main():
(options, args) = getopts()
limit = options.limit
start = options.start_time
end = options.end_time
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
stream.add_filter('prefix', '0.0.0.0/0')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264)
#stream.add_interval_filter(1454630400, 1454716800)
# 1 hour
#1454284800 - 1454288400
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
prefixes_update = defaultdict(int)
prefixes_withdraw = defaultdict(int)
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A':
#print elem.fields['as-path']
prefixes_update[elem.fields['prefix']] += 1
if elem.type == 'W':
prefixes_withdraw[elem.fields['prefix']] += 1
#print rec.project, rec.collector, rec.type, rec.time, rec.status,
#print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
for k in prefixes_update:
if prefixes_update[k] >= limit:
print k + "\t" + str(prefixes_update[k]) + "\t" + str(
prefixes_withdraw[k])
def main():
(options, args) = getopts()
limit = options.limit
start = options.start_time
end = options.end_time
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
stream.add_filter('prefix', '0.0.0.0/0')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264)
#stream.add_interval_filter(1454630400, 1454716800)
# 1 hour
#1454284800 - 1454288400
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
prefixes_update = defaultdict(int)
prefixes_withdraw = defaultdict(int)
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A':
#print elem.fields['as-path']
prefixes_update[elem.fields['prefix']] += 1
if elem.type == 'W':
prefixes_withdraw[elem.fields['prefix']] += 1
#print rec.project, rec.collector, rec.type, rec.time, rec.status,
#print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
for k in prefixes_update:
if prefixes_update[k] >= limit:
print k + "\t" + str(prefixes_update[k]) + "\t" + str(prefixes_withdraw[k])
def download_data():
peer_state = defaultdict(dict)
results = defaultdict(defaultdict_list)
current_bin = 0
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type updates"
bgprFilter += " and project ris "
for prefix in prefixes:
bgprFilter += " and prefix more %s " % prefix
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (startts, endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(startts, endts)
stream.start()
while (stream.get_next_record(rec)):
if rec.status != "valid":
print(rec.project, rec.collector, rec.type, rec.time, rec.status)
# from IPython import embed
# embed()
if current_bin == 0:
current_bin = rec.time
# slide the time window:
if current_bin + bin_size < rec.time:
timebins = range(current_bin, rec.time, bin_size)
for i in timebins[:-1]:
results["other"]["timebin"].append(i)
for pfx, p_s in peer_state.items():
for peeras, state in p_s.items():
results[pfx][peeras].append(state)
current_bin = timebins[-1]
elem = rec.get_next_elem()
while (elem):
# peerip g= elem.peer_address
peeras = elem.peer_asn
prefix = elem.fields["prefix"]
peer_state[prefix][peeras] = elem.type
elem = rec.get_next_elem()
return results
def main():
parser = argparse.ArgumentParser()
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.description = textwrap.dedent('''\
a proof-of-concept utility for watching updates from BGPstream
and then printing out if an unexpected update is heard
''')
parser.epilog = textwrap.dedent('''\
Example: watch these route announcements
%(prog)s -f routes.yaml ''')
required = parser.add_argument_group('required arguments')
required.add_argument("-f",
"--file",
required=True,
help="yaml file of prefixes to origin asn")
parser.add_argument("-d",
"--debug",
action='store_true',
help="print out all updates containing these prefixes")
args = parser.parse_args()
routes = pytricia.PyTricia(48) # longest reasonable pfx in dfz
with open(args.file, 'r') as f:
routesfile = yaml.safe_load(f)
for pfx in routesfile:
routes[pfx] = routesfile[pfx]
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(int(time.time()), 0)
stream.set_live_mode()
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
if 'as-path' in elem.fields:
path = elem.fields['as-path'].split()
prefix = elem.fields['prefix']
if prefix in routes and (routes[prefix] != path[-1]
or args.debug):
print('Heard prefix:', elem.fields['prefix'],
'AS-PATH:', elem.fields['as-path'],
' Found by project:', rec.project, 'collector:',
rec.collector, 'type:', rec.type, 'at time:',
rec.time, 'Type:', elem.type, 'Peer:',
elem.peer_address, 'AS', elem.peer_asn)
elem = rec.get_next_elem()
def MOAS_prefixes():
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider Route Views Singapore only
stream.add_filter('collector', 'route-views.sg')
# Consider RIBs dumps only
stream.add_filter('record-type', 'ribs')
# Consider this time interval:
# Sat, 01 Aug 2015 7:50:00 GMT - 08:10:00 GMT
stream.add_interval_filter(1438415400, 1438416600)
# Start the stream
stream.start()
# <prefix, origin-ASns-set > dictionary
prefix_origin = defaultdict(set)
prefix_origin_dict = defaultdict(list)
# Get next record
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while (elem):
# Get the prefix
pfx = elem.fields['prefix']
# Get the list of ASes in the AS path
ases = elem.fields['as-path'].split(" ")
if len(ases) > 0:
# Get the origin ASn (rightmost)
origin = ases[-1]
if 262857 == origin: #AS262857 - UFRN: AS262857 177.20.128.0/19 UNIVERSIDADE FEDERAL DO RIO GRANDE DO NORTE
'''
BI 177.20.128.0/19 198.32.125.84 280 100 0 1916, 262857 IGP
E 177.20.128.0/19 213.248.67.117 0 70 0 1299, 2914, 1916, 262857 IGP
E 177.20.128.0/19 213.248.98.93 0 70 0 1299, 2914, 1916, 262857 IGP
'''
print "Achou UFRN"
# Insert the origin ASn in the set of
# origins for the prefix
prefix_origin[pfx].add(origin)
prefix_origin_dict[pfx].append(ases)
elem = rec.get_next_elem()
# Print the list of MOAS prefix and their origin ASns
for pfx in prefix_origin:
if len(prefix_origin[pfx]) > 1:
pass #print pfx, ",".join(prefix_origin[pfx])
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time - 1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# to track the peers that have elems in this record
peer_signatures = set()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
peer_signatures.add(sig)
# if this is the first time we have ever seen this peer, create
# an empty result: (elem_cnt, peer_record_cnt, coll_record_cnt)
if sig not in peers_data:
peers_data[sig] = [0, 0, 0]
peers_data[sig][0] += 1 # increment elem cnt for this peer
elem = rec.get_next_elem()
# done with elems, increment the 'coll_record_cnt' field for just
# one peer that was present in this record (allows a true, per-collector
# count of records since each record can contain elems for many peers)
if len(peer_signatures):
first = True
for sig in peer_signatures: # increment peer_record_cnt for all
if first:
peers_data[sig][2] += 1 # increment the coll_record_cnt
first = False
peers_data[sig][1] += 1
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def getting_BGP_update():
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', 'rrc11')
stream.add_interval_filter(1438417216, 1438417216)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status != "valid":
print rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while (elem):
print rec.project, rec.collector, rec.type, rec.time, rec.status,
print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
def generate_stream():
bs = BGPStream()
rec = BGPRecord()
#initialize MySql
a = MySqlDAL()
a.start()
#initialize the publisher in port number 12345
publisher = ZmqPublisher(12345)
bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0)
# bs.add_filter('collector', 'route-views.sg')
bs.add_filter('record-type', 'updates')
bs.start()
print('Beginning to read from stream')
input_id = 0
while bs.get_next_record(rec):
elem = rec.get_next_elem()
while elem is not None:
# sleep until it is time to send this record
'''
now = calendar.timegm(time.gmtime())
sim_time = now - delay
if elem.time > sim_time:
time.sleep(elem.time - sim_time)
'''
if elem.type not in ['A', 'W']:
continue
input_id += 1
msg = elem2bgplay(rec, elem)
msg['type'] = 'A'
msg['id'] = input_id
print(msg)
# Publish the message
publisher.publish(msg)
# Write it to DB
if elem.type == 'A':
a.add(msg)
elif elem.type == 'W':
a.remove(msg)
else:
print "Error: Unknown type: " + elem.type
elem = rec.get_next_elem()
def generate_stream():
bs = BGPStream()
rec = BGPRecord()
#initialize MySql
db_writer = MysqlWriter()
db_writer.start()
#initialize the publisher in port number 12345
publisher = ZmqPublisher(12345)
bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0)
# bs.add_filter('collector', 'route-views.sg')
bs.add_filter('record-type', 'updates')
bs.start()
print('Beginning to read from stream')
input_id = 0
while bs.get_next_record(rec):
elem = rec.get_next_elem()
while elem is not None:
# sleep until it is time to send this record
'''
now = calendar.timegm(time.gmtime())
sim_time = now - delay
if elem.time > sim_time:
time.sleep(elem.time - sim_time)
'''
if elem.type not in ['A', 'W']:
continue
input_id += 1
msg = elem2bgplay(rec, elem)
msg['type'] = 'A'
msg['id'] = input_id
print(msg)
# Publish the message
publisher.publish(msg)
# Write it to DB
if elem.type == 'A':
db_writer.add(msg)
elif elem.type == 'W':
db_writer.remove(msg)
else:
print "Error: Unknown type: " + elem.type
elem = rec.get_next_elem()
def get_ribs(collector, start, period):
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
stream.add_interval_filter(
int(start.timestamp()) - period, int(start.timestamp()))
stream.add_filter('record-type', 'ribs')
stream.start()
while (stream.get_next_record(rec)):
if rec.status == "valid":
elem = rec.get_next_elem()
while (elem):
yield (rec.project, rec.collector,
datetime.utcfromtimestamp(rec.time).isoformat(),
elem.peer_address, elem.peer_asn, elem.fields['prefix'],
elem.fields['as-path'])
elem = rec.get_next_elem()
def recv_bgpstream_updates(begin, until, collector, output_queue):
logging.info ("CALL recv_bgpstream_updates")
# wait for first RIB table dump to complete
while (rib_ts < 0):
time.sleep(RIB_TS_WAIT/10)
time.sleep(RIB_TS_WAIT)
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type','updates')
stream.add_interval_filter(rib_ts,until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
logging.info("Record TS: "+str(rec.time))
while (elem):
logging.info(" -- Record Element Type: " + elem.type + ", TS: " + str(elem.time))
bgp_message = BGPmessage(elem.time, 'update')
src_peer = dict()
src_addr = elem.peer_address
src_asn = elem.peer_asn
src_peer['addr'] = src_addr
src_peer['port'] = 0
src_peer['asn'] = src_asn
bgp_message.set_source(src_peer)
if elem.type.upper() == 'A':
bgp_message.add_announce(elem.fields['prefix'])
bgp_message.set_nexthop(elem.fields['next-hop'])
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
output_queue.put(bgp_message)
elif elem.type.upper() == 'W':
bgp_message.add_withdraw(elem.fields['prefix'])
output_queue.put(bgp_message)
elem = rec.get_next_elem()
def main(rib, target_prefs):
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
with open('./data/stream_{0}'.format(start), 'wb') as bw:
for ptmp in target_prefs:
stream.add_filter('prefix', ptmp)
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('record-type', 'ribs')
#stream.add_filter('collector', 'rrc04')
stream.add_filter('project', 'ris')
stream.add_filter('project', 'routeviews')
stream.add_interval_filter(start-60*60*8, start)
stream.add_rib_period_filter(10000000000000)
# Start the stream
stream.start()
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
continue
#if rec.time < start:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A' or elem.type == 'R':
rib.add_to_rib(rec.collector, elem.peer_address, elem.fields['prefix'], elem.time, elem.fields['as-path'])
elem = rec.get_next_elem()
#else:
rib.flush()
print 'Successful termination; Start time: {0}'.format(start)
def recv_bgpstream_rib(begin, until, collector, output_queue):
logging.info ("CALL recv_bgpstream_rib")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type','updates')
stream.add_interval_filter(begin,until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
global rib_ts
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping.")
continue
if (rib_ts > 0) and (rec.time > (rib_ts + RIB_TS_INTERVAL/2)):
logging.info("received full RIB table dump.")
break
bgp_message = None
while (elem):
if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'):
rib_ts = elem.time
bgp_message = BGPmessage(elem.time, 'update')
bgp_message.set_nexthop(elem.fields['next-hop'])
src_peer = dict()
src_addr = elem.peer_address
src_asn = elem.peer_asn
src_peer['addr'] = src_addr
src_peer['port'] = 0
src_peer['asn'] = src_asn
bgp_message.set_source(src_peer)
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
bgp_message.add_announce(elem.fields['prefix'])
output_queue.put(bgp_message)
elem = rec.get_next_elem()
def count_valid_withdrawals_per_second(t0,num_seconds):
# div0 in progress tracking if <100 seconds
assert num_seconds>100;
# Create a new bgpstream instance
stream = BGPStream();
# use Routeviews Saopaulo records
stream.add_filter('project','routeviews');
stream.add_filter('collector','route-views.saopaulo');
stream.add_filter('record-type','updates');
# time interval
stream.add_interval_filter(t0,t0+tt-1);
# progress tracking
prog = t0;
# Create a reusable bgprecord instance
rec = BGPRecord();
# list of zeroes
ctr = [0]*tt;
# Start the stream
stream.start();
# iterate over records
while(stream.get_next_record(rec)):
if rec.status == "valid":
elem = rec.get_next_elem();
while(elem):
if elem.type == 'W':
ctr[rec.time-t0] += 1;
elem = rec.get_next_elem();
# el-cheapo progress indication with dots. comment out if you don't want the dots.
# """
if rec.time > prog and (rec.time - t0) % floor(num_seconds/100) == 0:
sys.stdout.write('.');
sys.stdout.flush();
prog = rec.time;
# """
# print newline after the dots
print "";
return ctr;
def readUpdates(self):
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type updates"
bgprFilter += " and project ris "
for prefix in self.prefixes:
bgprFilter += " and prefix more %s " % prefix
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (self.startts, self.endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts, self.endts)
stream.start()
while (stream.get_next_record(rec)):
if (rec.status != "valid" and rec.status != "filtered-source"
and rec.status != "empty-source"):
raise Exception(rec.project, rec.collector, rec.type, rec.time,
rec.status)
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
# peerip g= elem.peer_address
peeras = elem.peer_asn
sPrefix = elem.fields["prefix"]
if elem.type == "W":
self.withdraws[sPrefix][peeras] = True
else:
sPath = elem.fields["as-path"]
self.paths[sPrefix][peeras] = sPath
self.withdraws[sPrefix][peeras] = False
elem = rec.get_next_elem()
def get_updates(collector, start, end):
print("getting updates for {} from {} to {}".format(collector, start, end))
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
stream.add_interval_filter(int(start.timestamp()), int(end.timestamp()))
stream.add_filter('record-type', 'updates')
stream.start()
while (stream.get_next_record(rec)):
if rec.status == "valid":
elem = rec.get_next_elem()
while (elem):
if elem.type == 'A':
yield ('A', rec.project, rec.collector,
datetime.utcfromtimestamp(rec.time).isoformat(),
elem.peer_address, elem.peer_asn,
elem.fields['prefix'], elem.fields['as-path'])
elif elem.type == 'W':
yield ('W', rec.project, rec.collector,
datetime.utcfromtimestamp(rec.time).isoformat(),
elem.peer_address, elem.peer_asn,
elem.fields['prefix'], None)
elem = rec.get_next_elem()
def recv_bgpstream_updates(begin, until, collector):
"""
Receive and parse BGP update records from a given bgpstream collector
"""
logging.info("CALL recv_bgpstream_updates")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(begin, until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
logging.info("Record TS: " + str(rec.time))
while (elem):
logging.info(" -- Record Element Type: " + elem.type + ", TS: " +
str(elem.time))
bgp_message = BGPmessage(elem.time, 'update')
if elem.type.upper() == 'A':
bgp_message.add_announce(elem.fields['prefix'])
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
output(bgp_message)
elif elem.type.upper() == 'W':
bgp_message.add_withdraw(elem.fields['prefix'])
output(bgp_message)
elem = rec.get_next_elem()
if (not prefix in prefix_set):
prefix_set.add(prefix)
prefix_file.write("{pfx}|IP_Block\n".format(pfx=prefix))
if (not (as_path[-1], prefix) in connections_set):
connections_set.add((as_path[-1], prefix))
connect_rels_file.write("{asn}|{pfx}|ORIGINATES\n".format(
asn=as_path[-1], pfx=prefix))
elem = rec.get_next_elem()
# Main loop
route_id = None
started = False
prev_rib_dump = 0
while (stream.get_next_record(rec)):
if rec.dump_time != prev_rib_dump:
dt = datetime.datetime.fromtimestamp(rec.dump_time)
time_file.write("{time}|{Y}|{M}|{D}|{h}|{m}|{s}|Time\n".format(
time=rec.dump_time,
Y=dt.year,
M=dt.month,
D=dt.day,
h=dt.hour,
m=dt.minute,
s=dt.second))
if (prev_rib_dump != 0):
time_rels_file.write("{t1}|{t2}|NEXT\n".format(t1=prev_rib_dump,
t2=rec.dump_time))
def main():
parser = argparse.ArgumentParser(description='', epilog='')
parser.add_argument('-b', '--begin',
help='Begin date (inclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-u', '--until',
help='Until date (exclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-c', '--collector',
help='Route collector from RIPE RIS or Route-Views project.',
type=str, required=True)
parser.add_argument('-m', '--mongodb',
help='MongoDB connection parameters.',
type=str, default=None)
parser.add_argument('-l', '--loglevel',
help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].',
type=str, default='WARNING')
args = vars(parser.parse_args())
numeric_level = getattr(logging, args['loglevel'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level,
format='%(asctime)s : %(levelname)s : %(message)s')
ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds())
ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds())
mongodbstr = None
if args['mongodb']:
mongodbstr = args['mongodb'].strip()
# BEGIN
logging.info("START")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector',args['collector'])
stream.add_filter('record-type','ribs')
stream.add_interval_filter(ts_begin,ts_until)
# Start the stream
stream.start()
rib_ts = 0
rib_origins = dict()
while(stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
if rec.time > (rib_ts + RIB_TS_INTERVAL):
rib_ts = rec.time
if mongodbstr:
store_rib_origins(rib_ts, rib_origins, mongodbstr)
else:
print_rib_origins(rib_ts, rib_origins)
rib_origins = dict()
while(elem):
prefix = elem.fields['prefix']
aspath = elem.fields['as-path'].split()
for a in aspath: # remove AS-SETs
if '{' in a:
aspath.remove(a)
origin = aspath[-1]
if prefix not in rib_origins:
rib_origins[prefix] = list()
if origin not in rib_origins[prefix]:
rib_origins[prefix].append(origin)
elem = rec.get_next_elem()
stream.add_filter('prefix','140.78.0.0/16')
stream.add_filter('record-type','ribs')
stream.add_filter('collector','rrc11')
# stream.add_filter('project','ris')
stream.add_interval_filter(1464681000, 1464682200) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC)
stream.start()
#
# while(stream.get_next_record(rec)):
# # Print the record information only if it is not a valid record
# if rec.status == "valid":
# elem = rec.get_next_elem()
# while elem:
# # Print record and elem information
# print rec.collector, elem.peer_address, elem.peer_asn, elem.fields['as-path']
# elem = rec.get_next_elem()
result = []
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields
as_path = elem.fields['as-path'].split()
as_path.append(' ') # for tree creation
result.append(as_path)
elem = rec.get_next_elem()
print('-- elapsed time: {} seconds'.format(time.time() - start_time))
print(result)
def main():
parser = argparse.ArgumentParser(description='', epilog='')
parser.add_argument('-b', '--begin',
help='Begin date (inclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-u', '--until',
help='Until date (exclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-c', '--collector',
help='Route collector from RIPE RIS or Route-Views project.',
type=str, required=True)
parser.add_argument('-m', '--mongodb',
help='MongoDB connection parameters.',
type=str, default=None)
parser.add_argument('-k', '--keepsnapshots',
help='Keep all snapshots, works only with -s.',
action='store_true')
parser.add_argument('-s', '--snapshot',
help='Enable snapshoting.',
action='store_true')
parser.add_argument('-l', '--loglevel',
help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].',
type=str, default='WARNING')
args = vars(parser.parse_args())
numeric_level = getattr(logging, args['loglevel'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level,
format='#> %(asctime)s : %(levelname)s : %(message)s')
ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds())
ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds())
mongodbstr = None
if args['mongodb']:
mongodbstr = args['mongodb'].strip()
rib_ts = 0
rib_origins = dict()
origins_lt = list()
if args['snapshot']:
rib_ts, rib_origins = load_snapshot(mongodbstr)
if rib_ts > ts_begin:
logging.info ("SKIP, found snapshot with newer ts")
ts_begin = rib_ts - RIB_TS_THRESHOLD
# BEGIN
logging.info("START")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector',args['collector'])
stream.add_filter('record-type','ribs')
stream.add_interval_filter(ts_begin,ts_until)
# Start the stream
stream.start()
while(stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
#end if
if rec.time > (rib_ts + RIB_TS_THRESHOLD):
for p in rib_origins:
for o in rib_origins[p]:
if rib_origins[p][o][1] < (rib_ts - RIB_TS_THRESHOLD):
origins_lt.append( (p,o,rib_origins[p][o][0],rib_origins[p][o][1]) )
#end if
#end for
#end for
if args['snapshot'] and (len(rib_origins.keys()) > 0):
store_snapshot(rec.time, rib_origins, mongodbstr)
if not args['keepsnapshots']:
remove_snapshot(rib_ts, mongodbstr)
# end if keepsnapshots
# end if snapshot
rib_ts = rec.time
logging.info("ts: "+str(rib_ts))
if len(origins_lt) > 0:
if mongodbstr:
store_origins_lt(rib_ts,origins_lt, mongodbstr)
else:
print_origins_lt(rib_ts,origins_lt)
#end if
for l in origins_lt:
del rib_origins[l[0]][l[1]]
#end for
origins_lt = list()
# end if
#end if
while(elem):
prefix = elem.fields['prefix']
aspath = elem.fields['as-path'].split()
for a in aspath: # remove AS-SETs
if '{' in a:
aspath.remove(a)
#end if
#end for
origin = aspath[-1]
if prefix not in rib_origins:
rib_origins[prefix] = dict()
#end if
if origin not in rib_origins[prefix]:
rib_origins[prefix][origin] = (rib_ts,rib_ts)
else:
rib_origins[prefix][origin] = (rib_origins[prefix][origin][0],rib_ts)
#end if
elem = rec.get_next_elem()
#end while
#end while
if args['snapshot']:
print "NOTE: remaining origin lifetimes are stored in latest snapshot (%d)!\n" % rib_ts
if (len(rib_origins.keys()) > 0):
store_snapshot(rib_ts, rib_origins, mongodbstr)
# end if
else:
print "NOTE: output remaining origin lifetimes with current ts (%d)\n" % rib_ts
origins_lt = list()
for p in rib_origins:
for o in rib_origins[p]:
origins_lt.append( (p,o,rib_origins[p][o][0],rib_ts) )
if mongodbstr:
store_origins_lt(rib_ts,origins_lt, mongodbstr)
else:
print_origins_lt(rib_ts,origins_lt)
#stream_start = 1454284800
#stream_end = 1454288400
buckets = create_time_buckets(stream_start, stream_end)
prefixList = []
raw_bgp_stream_data = {}
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', 'rrc06')
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(stream_start, stream_end)
stream.start()
while(stream.get_next_record(rec)):
elem = rec.get_next_elem()
while(elem):
prefix = elem.fields.get("prefix", "")
time_stamp = rec.time # unix epoc timestamp 1427846670
if prefix != "":
deal_with_time_bucket_junk(prefix, time_stamp)
elem = rec.get_next_elem()
for prefix in list(raw_bgp_stream_data):
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
# if this is the first time we have ever seen this peer, create
# an empty result: (Pfxs_v4_set , Pfxs_v6_set,
# dict(Transit_ASN)=Pfxs_v4_set , dict(Transit_ASN)=Pfxs_v6_set)
if sig not in peers_data:
peers_data[sig] =[set(),set(),{},{}]
pfx=""
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
if(":" in pfx):
peers_data[sig][1].add(pfx)
else:
peers_data[sig][0].add(pfx)
if('as-path' in elem.fields):
#Squash the AS Path to keep only distinct ASNs, i.e., remove prepending
path_split = [k for k, g in groupby(elem.fields['as-path'].split(" "))]
if(len(path_split)!=0):
for i in range(1,len(path_split)-1):
transit=path_split[i]
if(":" in pfx):
if(transit not in peers_data[sig][3]):
peers_data[sig][3][transit]=set()
peers_data[sig][3][transit].add(pfx)
elif(pfx!=""):
if(transit not in peers_data[sig][2]):
peers_data[sig][2][transit]=set()
peers_data[sig][2][transit].add(pfx)
elem = rec.get_next_elem()
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
rec = BGPRecord()
stream.add_filter('project', 'ris')
timestamp = int(time.time())
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
stream.add_recent_interval_filter('12 h', False)
# Start the stream
stream.start()
l = 0
out = list()
# Get next record
while (stream.get_next_record(rec) and l < 100):
# Print the record information only if it is not a valid record
if rec.status == "valid":
elem = rec.get_next_elem()
while (elem and l < 100):
el = dict()
el['project'] = rec.project
el['collector'] = rec.collector
el['record_type'] = rec.type
el['time'] = rec.time
el['status'] = rec.status
el['elem_type'] = elem.type
el['peer_address'] = elem.peer_address
el['peer_asn'] = elem.peer_asn
el['fields'] = elem.fields
out.append(el)
class BGPDataAggregator(object):
"""docstring for BGPDataAggregator"""
def __init__(
self,
filters={"collector": ["rrc00"]},
rpki_validator="rpki-validator.realmv6.org:8282",
settings_file="../settings.json",
):
self.stream = BGPStream()
self.filters = filters
self.route_table = dict()
self.i = 0
self.metadata_vp = dict()
self.metadata_rc = dict()
self.peers = Counter()
self.prefix4 = Counter()
self.prefix6 = Counter()
start_timestamp = get_push_timestamp(datetime.now(timezone.utc))
for filter_type, filter_array in filters.items():
for filter_value in filter_array:
self.stream.add_filter(filter_type, filter_value)
for collector in filters["collector"]:
self.route_table[collector] = defaultdict(dict)
self.metadata_vp[collector] = defaultdict(list)
self.metadata_rc[collector] = RouteCollectorMeta(None, 0, 0, 0, 0)
self.peers[collector] = defaultdict(int)
self.prefix4[collector] = defaultdict(int)
self.prefix6[collector] = defaultdict(int)
settings = get_settings(settings_file)
settings["db"]["password"] = os.environ["PGPASS"]
self.db = DBConnector(settings["db"])
rpki = rpki_validator.split(":")
self.mgr = RTRManager(rpki[0], rpki[1])
self.mgr.start()
self.start_collecting(start_timestamp)
def __del__(self):
if self.mgr.is_synced():
self.mgr.stop()
def push_data(self, timestamp):
print("UPDATE:", timestamp)
self.db.update_vp_meta(self.metadata_vp)
for rc in self.metadata_rc.keys():
self.metadata_rc[rc] = RouteCollectorMeta(
rc,
timestamp,
len(self.peers[rc].keys()),
len(self.prefix4[rc]),
len(self.prefix6[rc]),
)
self.db.update_rc_meta(self.metadata_rc)
def start_collecting(self, start_timestamp, end_timestamp=0):
self.stream.add_interval_filter(start_timestamp, end_timestamp)
print("Start BGPStream:", start_timestamp, end_timestamp)
next_timestamp = init_next_timestamp(start_timestamp, 5)
print("Next Push to DB at:", next_timestamp)
self.stream.start()
rec = BGPRecord()
while (self.stream.get_next_record(rec)):
if rec.status == "valid":
if rec.time >= next_timestamp:
self.push_data(next_timestamp)
next_timestamp += 300
elem = rec.get_next_elem()
while elem:
origin_asn = ""
if elem.type is "R" or elem.type is "A":
origin_asn = elem.fields["as-path"].split(" ")[-1]
try:
origin_asn = int(origin_asn)
except ValueError:
elem = rec.get_next_elem()
continue
prefix = elem.fields["prefix"]
ip, mask_len = split_prefix(prefix)
# Check if v4 or v6
is_v4 = check_ipv4(ip)
validated = self.mgr.validate(origin_asn, ip, mask_len)
old_elem = self.route_table[rec.collector][(
elem.peer_asn, elem.peer_address)].get(prefix)
if elem.type is "R" or elem.type is "A":
self.route_table[rec.collector][(
elem.peer_asn, elem.peer_address)][prefix] = Route(
origin_asn,
rec.collector,
prefix,
is_v4,
validated.state.value,
)
if old_elem:
if old_elem.type != validated.state.value:
"""Make use of the fact that:
0: valid in enum
1: unknown in enum
2: invalid in enum
We designed the namedtuple the way to represent
that. So valid is a pos 3 and so on.
"""
self.metadata_vp[rec.collector][(
elem.peer_asn,
elem.peer_address)][4 + old_elem.type] -= 1
self.metadata_vp[rec.collector][(
elem.peer_asn, elem.peer_address
)][4 + validated.state.value] += 1
else:
if not self.metadata_vp[rec.collector].get(
(elem.peer_asn, elem.peer_address)):
"""Init the metadata-entry if it not exists already"""
self.metadata_vp[rec.collector][(
elem.peer_asn, elem.peer_address)] = [
elem.peer_asn,
elem.peer_address,
rec.collector,
next_timestamp,
0,
0,
0,
]
# Update the VantagePoint Metadate the same way like above.
self.metadata_vp[rec.collector][(
elem.peer_asn,
elem.peer_address)][4 +
validated.state.value] += 1
self.metadata_vp[rec.collector][(
elem.peer_asn,
elem.peer_address)][3] = next_timestamp
self.peers[rec.collector][elem.peer_asn] += 1
if is_v4:
self.prefix4[rec.collector][prefix] += 1
else:
self.prefix6[rec.collector][prefix] += 1
elif elem.type is "W":
if old_elem:
# Reduce the number of IPv4/v6 Addresses for this prefix
if is_v4:
self.prefix4[rec.collector][prefix] -= 1
if self.prefix4[rec.collector][prefix] == 0:
del (self.prefix4[rec.collector][prefix])
else:
self.prefix6[rec.collector][prefix] -= 1
if self.prefix6[rec.collector][prefix] == 0:
del (self.prefix6[rec.collector][prefix])
# Reduce number of prefixes belonging to this ASN
self.peers[rec.collector][elem.peer_asn] -= 1
if self.peers[rec.collector][elem.peer_asn] == 0:
del (self.peers[rec.collector][elem.peer_asn])
# Update the metadata valid/unknown/invalid count
self.metadata_vp[rec.collector][(
elem.peer_asn,
elem.peer_address)][4 + old_elem.type] -= 1
# Update the metadata timestamp
self.metadata_vp[rec.collector][(
elem.peer_asn,
elem.peer_address)][3] = next_timestamp
# Remove the entry from the route_table
self.route_table[rec.collector][(
elem.peer_asn,
elem.peer_address)].pop(prefix, None)
else:
# !!TODO: write log about that!
pass
elem = rec.get_next_elem()
def load_data(start, stop, collectors, window, threshold):
peers = {}
# collectors is a list of the collectors we want to include
# Start and stop define the interval we are looking in the data
# Create a new BGPStream instance and a reusable BGPRecord instance
stream = BGPStream()
rec = BGPRecord()
# Add filter for each collector.
# If no collector is mentioned, it will consider 16 of them
if collectors:
for collector in collectors:
print collector
stream.add_filter('collector', collector)
else:
for i in range(0, 10):
stream.add_filter('collector', 'rrc0' + str(i))
for i in range(10, 16):
stream.add_filter('collector', 'rrc' + str(i))
stream.add_filter('record-type', 'updates')
# Consider the interval from "start" to "stop" in seconds since epoch
stream.add_interval_filter(start, stop)
# Start the stream
stream.start()
# For each record (one record = one second, can have multiple elements for the same second) we handle its updates
while stream.get_next_record(rec):
timestamp = rec.time
if rec.status != "valid":
print rec.project, rec.collector, rec.type, timestamp, rec.status
else:
# Go through all elements of the record
elem = rec.get_next_elem()
while elem:
# Consider only the A and W updates
if elem.type not in ['A', 'W']:
elem = rec.get_next_elem()
continue
peer = elem.peer_address
updatetype = elem.type
prefix = elem.fields['prefix']
if peer not in peers:
peers[peer] = {
'A': [],
'W': []
}
update = {'tst': timestamp, 'prefix': prefix}
if updatetype == 'A':
handleUpdate(peers[peer]['A'], burst2writeA, update, peer, updatetype, timestamp, window, threshold)
saveGraphPoint(peers[peer]['A'], updatetype, peer, timestamp, collectors, threshold)
else:
handleUpdate(peers[peer]['W'], burst2writeW, update, peer, updatetype, timestamp, window, threshold)
saveGraphPoint(peers[peer]['W'], updatetype, peer, timestamp, collectors, threshold)
elem = rec.get_next_elem()
# After processing all records, we write the graph json files with the graph points recorded for each peer
for peer in graph_points:
peer_file_name = peer.replace(':', '_')
if not os.path.exists(peer_file_name):
os.makedirs(peer_file_name)
with open(peer_file_name+'/'+peer_file_name + '-graph.json', 'w') as outfile:
json.dump(graph_points[peer], outfile, indent=2)
# Write the last burst of A updates if there is one left
if burst2writeA:
for peer in burst2writeA:
if burst2writeA[peer]:
for timestamp in burst2writeA[peer]:
writeBurst(peer, burst2writeA, 'A', timestamp)
# Write the last burst of W updates if there is one left
if burst2writeW:
for peer in burst2writeW:
if burst2writeW[peer]:
for timestamp in burst2writeW[peer]:
writeBurst(peer, burst2writeW, 'W', timestamp)
# transform csv names in json file to use getJSON in plotGrap
# step to CSV is used to avoid appending to the end of a json file directly as appending
# to a json file overwrite the whole file
jsonlist = []
with open('csv_peernames-'+'-'.join(collectors)+'.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
jsonlist.append(row[0])
jsondata = json.dumps(jsonlist, indent=2)
fd = open('json_file_names-' + '-'.join(collectors) + '.json', 'w')
fd.write(jsondata)
fd.close()
def build_bgp_graph(project=None, collector=None):
print project, collector
global bgp_graphs
stream = BGPStream()
rec = BGPRecord()
if collector:
stream.add_filter('collector', collector)
else:
stream.add_filter('project', project)
# Consider RIBs dumps only
stream.add_filter('record-type', 'ribs')
# Consider this time interval:
#cur_time = int( time.time() )
cur_time = 1471219200
if collector and 'views' in collector:
interval = 3
else:
interval = 10
#prev_time = int(cur_time - (60 * 60 * interval))
prev_time = 1471132800
print "Starting stream with %d interval" % interval
stream.add_interval_filter(prev_time, cur_time)
stream.start()
ribEntryCount = 0
loopCount = 0
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while (elem):
pdb.set_trace()
ribEntryCount += 1
peer = str(elem.peer_asn)
hops = [k for k, g in groupby(elem.fields['as-path'].split(" "))]
loops = [i for i, x in enumerate(hops) if hops.count(x) > 1]
if loops:
# print( "Routing loop! AS path %s" % elem.fields['as-path'] )
loopCount += 1
elem = rec.get_next_elem()
continue
if '{' in elem.fields['as-path']:
elem = rec.get_next_elem()
continue
if len(hops) > 1 and hops[0] == peer:
# Get the origin ASN, is the destination for traffic
# toward this prefix
origin = hops[-1]
if origin in ixp.IXPs:
print "Origin", origin, "is an IXP, it announced prefix", elem.fields[
'prefix']
elem = rec.get_next_elem()
continue
if origin in bgp_graphs:
as_graph = bgp_graphs[origin]
else:
as_graph = nx.DiGraph()
bgp_graphs[origin] = as_graph
# Add new edges to the NetworkX graph
new_hops = []
for hop in hops:
if hop in ixp.IXPs:
continue
new_hops.append(hop)
if len(new_hops) <= 1:
elem = rec.get_next_elem()
continue
for i in range(0, len(new_hops) - 1):
as_graph.add_edge(new_hops[i], new_hops[i + 1])
# Making sure, root has 0 out degree
#if as_graph.out_degree( origin ) != 0:
# pdb.set_trace()
elem = rec.get_next_elem()
print "Total RIB entries parsed", ribEntryCount, "Number of AS paths with loops", loopCount
def get_data(timestamp):
graph = Graph(password="******")
stream = BGPStream()
rec = BGPRecord()
rec_time = None
# stream.add_filter('prefix', '198.41.0.0/24') # A-root
# stream.add_filter('prefix', '192.228.79.0/24') # B-root, only 1 site
# stream.add_filter('prefix', '192.33.4.0/24') # C-root
# stream.add_filter('prefix', '199.7.91.0/24') # D-root
# stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only
# stream.add_filter('prefix', '192.5.5.0/24') # F-root
# stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only
# stream.add_filter('prefix', '198.97.190.0/24') # H-root
# stream.add_filter('prefix', '192.36.148.0/24') # I-root
# stream.add_filter('prefix', '192.58.128.0/24') # J-root
stream.add_filter('prefix', '193.0.14.0/24') # K-root
# stream.add_filter('prefix', '199.7.83.0/24') # L-root
# stream.add_filter('prefix', '202.12.27.0/24') # M-root
# IPv6
# stream.add_filter('prefix', '2001:503:ba3e::/48') # A
## stream.add_filter('prefix', '2001:500:84::/48') # B, only 1 site
# stream.add_filter('prefix', '2001:500:2::/48') # C
# stream.add_filter('prefix', '2001:500:2d::/48') # D
# stream.add_filter('prefix', '2001:500:2f::/48') # F
# stream.add_filter('prefix', '2001:500:1::/48') # H
# stream.add_filter('prefix', '2001:7fe::/33') # I
# stream.add_filter('prefix', '2001:503:c27::/48') # J
# stream.add_filter('prefix', '2001:7fd::/48') # K
# stream.add_filter('prefix', '2001:500:9f::/48') # L
# stream.add_filter('prefix', '2001:dc3::/32') # M
stream.add_filter('record-type', 'ribs')
# stream.add_filter('collector', 'rrc01')
stream.add_filter('project', 'routeviews')
stream.add_interval_filter(timestamp, timestamp)
stream.start()
result = {}
while stream.get_next_record(rec):
rec_time = rec.time
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields
as_path = elem.fields['as-path'].split()
as_path.reverse()
prefix = elem.fields['prefix']
if prefix not in result:
result[prefix] = []
result[prefix].append(as_path)
elem = rec.get_next_elem()
# get only unique lists in result
for prefix in result:
result[prefix] = [list(x) for x in set(tuple(x) for x in result[prefix])]
print('timestamp {} ==> result: {}'.format(rec_time, result))
for prefix in result:
for path in result[prefix]:
print('path: {}'.format(path))
cur_node = None
prev_node = None
counter_as_prepend = 0
for index, asn in enumerate(path):
searched_node = graph.find('asn', property_key='label', property_value=asn)
try:
cur_node = searched_node.next() # see if the AS node is already in the db or not. If yes, cur_node == prev_node
except StopIteration:
cur_node = Node('asn', label=str(asn)) # if not exists, then create a new one
if index > 0:
if index == len(path) - 1:
cur_node['path'] = path # attach AS path to the last ASN
if cur_node != prev_node:
if counter_as_prepend > 0:
cur_node['prepended'] = counter_as_prepend
counter_as_prepend = 0 # reset
peering = Relationship(cur_node, 'TO', prev_node, time=rec_time, prefix=prefix)
# peering['time'] = rec_time
# peering['prefix'] = prefix
graph.create(peering)
else: # AS prepending
counter_as_prepend += 1
prev_node = cur_node
def readrib(self):
stream = None
rec = None
if self.txtFile is None:
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type ribs"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if not self.asnFilter is None:
# bgprFilter += ' and path %s$' % self.asnFilter
for p in self.includedPeers:
bgprFilter += " and peer %s " % p
for p in self.includedPrefix:
bgprFilter += " and prefix more %s " % p
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" %
(self.startts - 3600, self.startts + 3600))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts - 3600,
self.startts + 3600)
if self.livemode:
stream.set_live_mode()
stream.start()
else:
rec = txtReader.txtReader(self.txtFile)
# for line in p1.stdout:
while (self.txtFile
and not rec.running) or (stream
and stream.get_next_record(rec)):
if rec.status != "valid":
print rec.project, rec.collector, rec.type, rec.time, rec.status
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
sPath = elem.fields["as-path"]
# print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields))
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes (and using IP addresses as spatial
# resolution)
self.peersASN[zOrig].add(zAS)
if len(path) < 2:
# Ignore paths with only one AS
elem = rec.get_next_elem()
continue
node = self.rtree.add(zPfx)
if zOrig in node.data:
# Already read this entry, we should read only one RIB per peer
elem = rec.get_next_elem()
continue
if self.ribQueue is not None:
self.ribQueue.put((zDt, zOrig, zAS, zPfx, path))
node.data[zOrig] = {
"path": set(path),
"count": 0,
"origAS": origAS
}
# print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields)
if self.spatialResolution:
# compute weight for this path
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# assert count >= 0
node.data[zOrig]["count"] = count
# Update above nodes
parent = self.findParent(node, zOrig)
if not parent is None:
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
pcountBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
pOrigAS = parent.data[zOrig]["origAS"]
asns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, pOrigAS, zAS, asns)
self.incTotalCount(pdiff, zOrig, pOrigAS, zAS)
else:
count = 1
node.data[zOrig]["count"] = count
asns = node.data[zOrig]["path"]
self.incTotalCount(count, zOrig, origAS, zAS)
self.incCount(count, zOrig, origAS, zAS, asns)
elem = rec.get_next_elem()
def sendMessageToKafka(col_name, col_data):
# print "Collector-{} Sending Message...".format(col_name)
size_total = 0
num_total = 0
num_ipv4 = 0
stream = BGPStream()
record = BGPRecord()
time_start = int(col_data.get('updates').get('latestDumpTime'))
time_end = time_start + int(col_data.get('updates').get('dumpPeriod'))
# print "Start Time:{}, End Time:{}".format(time_start, time_end)
stream.add_filter('collector', col_name)
stream.add_filter('record-type', 'ribs')
# stream.add_interval_filter(time_start, time_end)
# stream.add_interval_filter(time_start, time_start+300)
stream.add_interval_filter(1503475200, 1503475200 + 7200)
# print "Before Start>>>>>"
stream.start()
print col_name
# print "After Start>>>>>>"
producer = Kafka_producer()
while stream.get_next_record(record):
if record.status == "valid":
elem = record.get_next_elem()
while elem:
if filter(lambda x: ':' in x.peer_address, [elem]):
num_total += 1
elem = record.get_next_elem()
continue
#print "Element:{},{},{}".format(elem.type, elem.peer_address, elem.peer_asn)
field = elem.fields
#print type(field)
prefix = field['prefix'] if field.has_key('prefix') else ''
next_hop = field['next-hop'] if field.has_key(
'next-hop') else ''
as_path = field['as-path'] if field.has_key('as-path') else ''
as_path = as_path.replace(' ', '|')
text = [
elem.type, elem.peer_address,
str(elem.peer_asn), prefix, next_hop, as_path,
str(record.time)
]
text = ','.join(text)
# producer = Kafka_producer()
producer.send_data(col_name, text)
num_total += 1
num_ipv4 += 1
# print "[{}]-{}".format(col_name, num_total)
# size_total += len(text)
#NUM_TOTAL += 1
#print "[{}]-{}-{}-{}-{}".format(col_name, num_total, num_ipv4, size_total, time.ctime(record.time))
#print "No.{} Message Send Success-[{}]".format(num_total, text)
elem = record.get_next_elem()
else:
pass
# print "## Current record not valid!"
# break
# print "One Collector Finished"
else:
# print "-----------------------------"
# print "Collector[{}] And Records Send Finished\nTotal Num:{}, IPv4 Num:{}, Total Size:{}".format(col_name, num_total, num_ipv4, size_total)
# print "-----------------------------"
print "Collector:[{}]".format(col_name)
print "Total Num:{}, IPv4 Num:{}, Total Size:{}".format(
num_total, num_ipv4, size_total)
def readupdates(self):
#TODO implement txt file for update messages?
if self.txtFile:
return
# create a new bgpstream instance
stream = BGPStream()
bgprFilter = "type updates"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
# bgprFilter += " and collector rrc10 "
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if self.asnFilter is not None:
# # TOFIX filter is now deprecated, we need to have both
# # announcements and withdrawals
# bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (self.startts, self.endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts, self.endts)
if self.livemode:
stream.set_live_mode()
stream.start()
# for line in p1.stdout:
# create a reusable bgprecord instance
rec = BGPRecord()
while (stream.get_next_record(rec)):
if rec.status != "valid":
logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" %
(rec.project, rec.collector, rec.type, rec.time,
rec.status))
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
if zOrig not in self.peers:
# no need to update the counts for non-full feed peers
elem = rec.get_next_elem()
continue
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
msgTs = zDt
# set first time bin!
if self.ts is None:
self.slideTimeWindow(msgTs)
elif self.ts + self.timeWindow <= msgTs:
self.slideTimeWindow(msgTs)
elif self.ts > msgTs:
#Old update, ignore this to update the graph
logging.warn(
"Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s"
% (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields)))
elem = rec.get_next_elem()
continue
node = self.rtree.search_exact(zPfx)
if elem.type == "W":
# Withdraw: remove the corresponding node
if not node is None and zOrig in node.data:
origAS = node.data[zOrig]["origAS"]
if self.spatialResolution:
count = node.data[zOrig]["count"]
# Update count for above node
parent = self.findParent(node, zOrig)
if parent is None:
self.incTotalCount(-count, zOrig, origAS, zAS)
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
else:
node.data[zOrig]["count"] = 0
# Add ips to above node and corresponding ASes
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
# oldpCount = parent.data[zOrig]["count"]
# pCount = self.nbIPs(parent.prefixlen) - pcountBelow
# parent.data[zOrig]["count"] = pCount
# pdiff = pCount - oldpCount
# assert pdiff==count
# Update count for origAS and path from the
# parent node
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(count, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(count, zOrig, porigAS, zAS)
# Update count for withdrawn origAS and path
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
self.incTotalCount(-count, zOrig, origAS, zAS)
else:
asns = node.data[zOrig]["path"]
self.incCount(-1, zOrig, origAS, zAS, asns)
self.incTotalCount(-1, zOrig, origAS, zAS)
del node.data[zOrig]
else:
# Announce: update counters
sPath = elem.fields["as-path"]
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes or implicit withdraws
if len(path) < 2:
# Ignoring paths with only one AS
elem = rec.get_next_elem()
continue
if self.announceQueue is not None:
self.announceQueue.put((zDt, zOrig, zAS, zPfx, path))
# Announce:
if node is None or not zOrig in node.data:
# Add a new node
node = self.rtree.add(zPfx)
if self.spatialResolution:
# Compute the exact number of IPs
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
parent = self.findParent(node, zOrig)
if not parent is None:
# Update above nodes
# print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count))
pcountBelow = sum([
n.data[zOrig]["count"] for n in
self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(
parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
# print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount))
# print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ]
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(pdiff, zOrig, porigAS, zAS)
else:
self.incTotalCount(1, zOrig, origAS, zAS)
count = 1
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asn = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
else:
#Update node path and counts
if self.spatialResolution:
count = node.data[zOrig]["count"]
else:
count = 1
porigAS = node.data[zOrig]["origAS"]
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, porigAS, zAS, asns)
self.incTotalCount(-count, zOrig, porigAS, zAS)
node.data[zOrig]["path"] = set(path)
node.data[zOrig]["origAS"] = origAS
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
elem = rec.get_next_elem()
def get_data(timestamp):
graph = Graph(password="******")
stream = BGPStream()
rec = BGPRecord()
rec_time = None
# IPv4
stream.add_filter('prefix', '198.41.0.0/24') # A-root
stream.add_filter('prefix', '192.33.4.0/24') # C-root
stream.add_filter('prefix', '199.7.91.0/24') # D-root
stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only
stream.add_filter('prefix', '192.5.5.0/24') # F-root
stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only
stream.add_filter('prefix', '198.97.190.0/24') # H-root
stream.add_filter('prefix', '192.36.148.0/24') # I-root
stream.add_filter('prefix', '192.58.128.0/24') # J-root
stream.add_filter('prefix', '193.0.14.0/24') # K-root
stream.add_filter('prefix', '199.7.83.0/24') # L-root
stream.add_filter('prefix', '202.12.27.0/24') # M-root
# IPv6
stream.add_filter('prefix', '2001:503:ba3e::/48') # A
stream.add_filter('prefix', '2001:500:2::/48') # C
stream.add_filter('prefix', '2001:500:2d::/48') # D
stream.add_filter('prefix', '2001:500:2f::/48') # F
stream.add_filter('prefix', '2001:500:1::/48') # H
stream.add_filter('prefix', '2001:7fe::/33') # I
stream.add_filter('prefix', '2001:503:c27::/48') # J
stream.add_filter('prefix', '2001:7fd::/48') # K
stream.add_filter('prefix', '2001:500:9f::/48') # L
stream.add_filter('prefix', '2001:dc3::/32') # M
stream.add_filter('record-type', 'ribs')
# stream.add_filter('collector', 'route-views.soxrs')
stream.add_filter('project', 'routeviews')
stream.add_interval_filter(timestamp, timestamp)
stream.start()
result = {}
while stream.get_next_record(rec):
rec_time = rec.time
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
print('{} {} {} {} {}'.format(rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields))
as_path = elem.fields['as-path'].split()
as_path.reverse()
prefix = elem.fields['prefix']
if prefix not in result:
result[prefix] = []
result[prefix].append(as_path)
elem = rec.get_next_elem()
# get only unique lists in result
for prefix in result:
result[prefix] = [list(x) for x in set(tuple(x) for x in result[prefix])]
print('timestamp {} ==> result: {}'.format(rec_time, result))
for prefix in result:
for path in result[prefix]:
print('path: {}'.format(path))
cur_node = None
prev_node = None
counter_as_prepend = 0
for index, asn in enumerate(path):
cur_node = asn
graph.run('MERGE(s:asn{{name:"{0}", label:"{0}"}})'.format(asn)) # create new node if not exist.
if index > 0:
if cur_node != prev_node:
query = 'MATCH (s:asn),(d:asn) ' \
'WHERE s.name="{0}" AND d.name="{1}" ' \
'MERGE (s)-[r:TO {{prefix: "{3}", time: {2}, prepended: {4}}}]->(d)'\
.format(cur_node, prev_node, rec_time, prefix, counter_as_prepend)
graph.run(query)
if counter_as_prepend > 0:
counter_as_prepend = 0 # reset
else: # AS prepending
counter_as_prepend += 1
# else: # origin AS
# graph.run('MATCH (s:asn)'
# 'WHERE s.name="{0}"'
# 'SET s.origin="{1}_{2}"'
# 'RETURN s'.format(asn, prefix, rec_time))
prev_node = cur_node
class BGPTest(object):
def __init__(self, route_collector="rrc00", rpki_validator="rpki-validator.realmv6.org:8282"):
self.rc = route_collector
rpki = rpki_validator.split(":")
self.mgr = RTRManager(rpki[0], rpki[1])
# self._start_rtr_manager()
self.stream = BGPStream()
self.rec = BGPRecord()
def _start_rtr_manager(self):
self.mgr.start()
while not self.mgr.is_synced():
sleep(0.2)
if status.error:
print("Connection error")
exit()
def start_stream(self, start_time=None, end_time=0, route_collector=""):
""" Starts the
"""
if route_collector == "":
route_collector = self.rc
self.stream.add_filter('collector', route_collector)
self.stream.add_filter('record-type', 'ribs')
if (start_time is None) or not isinstance(start_time, datetime):
start_time = datetime.utcnow()
if isinstance(end_time, datetime):
end = int(end_time.strftime("%s"))
else:
end = 0
start = int(datetime.utcnow().strftime("%s"))
print(start)
self.stream.add_interval_filter(start, 0)
# print('Start stream with', start_time, end_time)
self.stream.start()
def get_records(self):
while(self.stream.get_next_record(self.rec)):
# Print the self.record information only if it is not a valid self.record
if self.rec.status != "valid":
pass
else:
elem = self.rec.get_next_elem()
while(elem):
# Print self.record and elem information
print(self.rec.project, self.rec.collector,
self.rec.type, self.rec.time, self.rec.status)
print(elem.type, elem.peer_address,
elem.peer_asn, elem.fields)
# prefix = elem.fields["prefix"].split('/')
# result = mgr.validate((int) elem.fields["as-path"].split(" ")[-1], prefix[0], prefix[1])
elem = self.rec.get_next_elem()
print("done.")
