def calculate_effective_prefixes(self, start_interval=1451692800):
'''
This method calculates the number of effective /24
prefixes that lie underneath a specified route prefix
:param start_interval: the start interval of the data
stream
:return: number of effective prefixes under the route's
root prefix
'''
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', self.collector_name)
stream.add_filter('record-type', 'ribs')
if isinstance(start_interval, datetime.datetime):
interval_start_utc = self.convert_interval_to_utc(start_interval)
stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300)
else:
stream.add_interval_filter(start_interval - 300, start_interval + 300)
stream.start()
print "Starting routing table parsing"
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while elem:
# Get the peer ASN and IP. We then construct a peer_id, since a collector
# can establish multiple connections with the same ASN.
peer_asn = elem.peer_asn
peer_asn_ip = elem.peer_address
# make this an unmodifiable tuple
peer_id = (peer_asn, peer_asn_ip)
peer_route_trie = self.peer_trie_handles[peer_id]
# Do a single level search of the route prefix to
# find the left and right prefix advertisements
single_level_prefixes = \
peer_route_trie.single_level_search(elem.fields['prefix'])
print single_level_prefixes
prefix_count = len(single_level_prefixes)
root_24_prefix_count = 0
lr_24_prefix_count1 = lr_24_prefix_count2 = 0
# The /24 prefixes below the advertised prefix are calculated
# as all the /24 prefixes served by the root - sum of the /24
# prefixes served by root's children
if prefix_count == 1:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
elif prefix_count == 2:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count1 = \
2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<')
.rstrip('>').split('/')[1]))
else:
root_24_prefix_count = \
2 ** (24 - int(str(single_level_prefixes[0]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count1 = \
2 ** (24 - int(str(single_level_prefixes[1]).lstrip('<')
.rstrip('>').split('/')[1]))
lr_24_prefix_count2 = \
2 ** (24 - int(str(single_level_prefixes[2]).lstrip('<')
.rstrip('>').split('/')[1]))
effective_24_prefix_count = \
root_24_prefix_count - (lr_24_prefix_count1 +
lr_24_prefix_count2)
print "Effective Prefix Count : ", \
effective_24_prefix_count
for prefix in single_level_prefixes:
trie_node = peer_route_trie.search_exact(str(prefix))
as_path = trie_node.data['as-path'].split(" ")
as_path_headless = as_path[1:-1]
print "AS-Path : ", as_path
as_headless_len = len(as_path_headless)
if as_headless_len > 1:
for i in range(0, as_headless_len - 1):
print "Headless nodes : ", as_path_headless[i], \
as_path_headless[i + 1]
if as_path_headless[i] in self.aslink_datastore:
self.aslink_datastore[as_path_headless[i]] += 1
else:
self.aslink_datastore[as_path_headless[i]] = 1
elem = rec.get_next_elem()
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# to track the peers that have elems in this record
peer_signatures = set()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
peer_signatures.add(sig)
# if this is the first time we have ever seen this peer, create
# an empty result: (Pfxs_v4_set, Pfxs_v6_set, ASNs_v4_set, ASNs_v6_set)
if sig not in peers_data:
peers_data[sig] =[set(),set(),
set(),set()]
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
if(":" in pfx):
peers_data[sig][1].add(pfx)
else:
peers_data[sig][0].add(pfx)
if('as-path' in elem.fields):
path_split=elem.fields['as-path'].split()
if(":" in pfx):
for ASN in path_split:
peers_data[sig][3].add(ASN)
else:
for ASN in path_split:
peers_data[sig][2].add(ASN)
elem = rec.get_next_elem()
# one peer that was present in this record (allows a true, per-collector
# count of records since each record can contain elems for many peers)
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def __init__(self):
# Create a new bgpstream instance
# and a reusable bgprecord instances
self.stream = BGPStream()
self.rec = BGPRecord()
self.origin_ases = set()
self.bgp_lens = defaultdict(lambda: defaultdict(lambda: None))
def __init__(self,
filters={'collector': ['rrc00']},
rpki_validator="rpki-validator.realmv6.org:8282",
db="metasnap.db"):
self.stream = BGPStream()
self.filters = filters
self.route_table = dict()
self.i = 0
for filter_type, filter_array in filters.items():
for filter_value in filter_array:
self.stream.add_filter(filter_type, filter_value)
for collector in filters['collector']:
self.route_table[collector] = defaultdict(dict)
# self.db = DBConnector(db, read_only=False)
rpki = rpki_validator.split(":")
self.mgr = RTRManager(rpki[0], rpki[1])
self.mgr.start()
self.counter = Counter()
start_timestamp = self.get_push_timestamp(datetime.now(timezone.utc))
# self.start_collecting(start_timestamp, int(datetime.now(timezone.utc).strftime("%s")))
self.start_collecting(start_timestamp, start_timestamp)
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
# if this is the first time we have ever seen this peer, create
# an empty result: (dict(Pfx_v4)=Pfx_origins, dict(Pfx_v6)=Pfx_origins))
if sig not in peers_data:
peers_data[sig] =[{},{}]
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
origin=""
if('as-path' in elem.fields):
path_split=elem.fields['as-path'].split()
if(len(path_split)!=0):
origin=path_split[len(path_split)-1]
if(":" in pfx):
if(pfx not in peers_data[sig][1]):
peers_data[sig][1][pfx]=set()
#discard as origin: AS sets, and ASN=23456 [AS_TRANS]
if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][1][pfx].add(origin)
else:
if(pfx not in peers_data[sig][0]):
peers_data[sig][0][pfx]=set()
#discard as origin: AS sets, and ASN=23456 [AS_TRANS]
if(origin!="" and origin!="23456" and "{" not in origin): peers_data[sig][0][pfx].add(origin)
elem = rec.get_next_elem()
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def __init__(self, route_collector="rrc00", rpki_validator="rpki-validator.realmv6.org:8282"):
self.rc = route_collector
rpki = rpki_validator.split(":")
self.mgr = RTRManager(rpki[0], rpki[1])
# self._start_rtr_manager()
self.stream = BGPStream()
self.rec = BGPRecord()
def get_data_graph(self):
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(self.start, self.end)
stream.start()
as_graph = nx.Graph()
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
# the list is a list, which stores an as-path
as_path = [
k
for k, g in groupby(elem.fields['as-path'].split(" "))
]
peer = str(elem.peer_asn)
# judge whether the as-path is legal
if len(as_path) > 1 and as_path[0] == peer:
if re.match(self.pattern, elem.fields['prefix']):
self.as_prefix[as_path[-1]].add(
elem.fields['prefix'])
# add edges to the graph
for i in range(0, len(as_path) - 1):
as_graph.add_edge(as_path[i], as_path[i + 1])
elem = rec.get_next_elem()
as_graph = self.add_geo_loc(as_graph)
return as_graph
def sendMessageToKafka(self, col_name, col_data):
print "lalalala"
stream = BGPStream()
record = BGPRecord()
time_start = int(col_data.get('ribs').get('latestDumpTime'))
time_end = time_start + int(col_data.get('ribs').get('dumpPeriod'))
stream.add_filter('collector', col_name)
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(time_start, time_end)
print "Before Start"
stream.start()
print "After Start"
while stream.get_next_record(record):
if record.status == "valid":
elem = record.get_next_elem()
while elem:
# print "Record:{}".format(elem)
producer = Kafka_producer()
producer.send_data(col_name, json.dumps(elem))
elem = record.get_next_elem()
else:
print "## Current record not valid!"
print "One Collector Finished"
def downloader(start_date, duration):
"""Download BGP paths from Routeviews and RIPE NCC from a start date for a certain duration."""
# Start of UNIX time
base = int(datetime.datetime.strptime(start_date, '%m/%d/%Y').strftime('%s'))
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
stream.add_interval_filter(base, base + int(duration))
stream.add_filter('record-type', 'ribs')
stream.start()
path_set = set()
f = open('rib.txt', 'w')
while True:
rec = stream.get_next_record()
if rec is None:
return
if rec.status != "valid":
continue
else:
elem = rec.get_next_elem()
while(elem):
path = elem.fields['as-path']
if '{' in path or '(' in path:
elem = rec.get_next_elem()
continue
prefix = elem.fields['prefix']
# Focus on IPv4 prefixes
if ":" not in prefix and path not in path_set:
f.write(path.replace(' ', '|') + '\n')
path_set.add(path)
elem = rec.get_next_elem()
f.close()
def build_sql_db():
conn = sqlite3.connect('bgp_stage.db')
c = conn.cursor()
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector','rrc11')
stream.add_interval_filter(1438417216,1438417316)
stream.start()
while(stream.get_next_record(rec)):
if rec.status != "valid":
continue
else:
elem = rec.get_next_elem()
while(elem):
if elem.type == "AB":
prefix = elem.fields["prefix"]
as_path = elem.fields["as-path"].split(" ")
origin = as_path[-1]
time = elem.time
#IP Prefix database
ip_min, ip_max = calculate_min_max(prefix)
c.execute("SELECT ip_min FROM prefix_as WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin))
row = c.fetchone()
if len(row) != 0:
c.execute("UPDATE prefix_as SET count = count + 1 WHERE ip_min = (?) AND ip_max = (?) AND as_o = (?)", (ip_min, ip_max, origin))
else:
c.execute("INSERT INTO prefix_as VALUES(?,?,?,?,?)", (ip_min, ip_max, origin, 1, time))
#AS link database
for as1,as2 in zip(as_path, as_path[1:]) :
c.execute("SELECT as_o FROM as_link WHERE as_o = (?) AND as_n = (?)",(as1,as2))
row = c.fetchone()
if len(row) != 0:
c.execute("UPDATE as_link SET count = count + 1 WHERE as_o = (?) AND as_n = (?)",
(as1, as2))
else:
c.execute("INSERT INTO as_link VALUES(?,?,?,?)", (as1, as2, 1, 0))
elif elem.type == "WA":
prefix = elem.fields["prefix"]
time = elem.time
#Needs research
print(rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.peer_address, elem.peer_asn, elem.fields)
print(prefix,elem.time, "W")
print(rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.peer_address, elem.peer_asn, elem.fields)
elem = rec.get_next_elem()
conn.commit()
conn.close()
def getBGPStream(recordType, AF, collectors, startts, endts):
stream = BGPStream()
# recordType is supposed to be ribs or updates
bgprFilter = "type " + recordType
if AF == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
for c in collectors:
bgprFilter += " and collector %s " % c
if isinstance(startts, str):
startts = datetime.strptime(startts + "UTC", "%Y-%m-%dT%H:%M:%S%Z")
startts = dt2ts(startts)
if isinstance(endts, str):
endts = datetime.strptime(endts + "UTC", "%Y-%m-%dT%H:%M:%S%Z")
endts = dt2ts(endts)
currentts = dt2ts(datetime.now())
if endts > currentts:
stream.set_live_mode()
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(startts, endts)
return stream
def getBGPStream(self):
logging.info(
f"creating BGPstream {self.collector} {self.record_type} {self.start.year}-{self.start.month} "
)
stream = BGPStream()
stream.add_filter('collector', self.collector)
stream.add_filter('record-type', self.record_type)
if self.record_type == "updates":
stream.add_interval_filter(dt2ts(self.start), dt2ts(self.end))
elif self.record_type == "ribs":
_start = dt2ts(self.start - timedelta(hours=2))
_end = dt2ts(self.start + timedelta(hours=2))
stream.add_interval_filter(_start, _end)
return stream
def download_data():
peer_state = defaultdict(dict)
results = defaultdict(defaultdict_list)
current_bin = 0
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type updates"
bgprFilter += " and project ris "
for prefix in prefixes:
bgprFilter += " and prefix more %s " % prefix
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (startts, endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(startts, endts)
stream.start()
while (stream.get_next_record(rec)):
if rec.status != "valid":
print(rec.project, rec.collector, rec.type, rec.time, rec.status)
# from IPython import embed
# embed()
if current_bin == 0:
current_bin = rec.time
# slide the time window:
if current_bin + bin_size < rec.time:
timebins = range(current_bin, rec.time, bin_size)
for i in timebins[:-1]:
results["other"]["timebin"].append(i)
for pfx, p_s in peer_state.items():
for peeras, state in p_s.items():
results[pfx][peeras].append(state)
current_bin = timebins[-1]
elem = rec.get_next_elem()
while (elem):
# peerip g= elem.peer_address
peeras = elem.peer_asn
prefix = elem.fields["prefix"]
peer_state[prefix][peeras] = elem.type
elem = rec.get_next_elem()
return results
def recv_bgpstream_updates(begin, until, collector, output_queue):
logging.info ("CALL recv_bgpstream_updates")
# wait for first RIB table dump to complete
while (rib_ts < 0):
time.sleep(RIB_TS_WAIT/10)
time.sleep(RIB_TS_WAIT)
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type','updates')
stream.add_interval_filter(rib_ts,until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
logging.info("Record TS: "+str(rec.time))
while (elem):
logging.info(" -- Record Element Type: " + elem.type + ", TS: " + str(elem.time))
bgp_message = BGPmessage(elem.time, 'update')
src_peer = dict()
src_addr = elem.peer_address
src_asn = elem.peer_asn
src_peer['addr'] = src_addr
src_peer['port'] = 0
src_peer['asn'] = src_asn
bgp_message.set_source(src_peer)
if elem.type.upper() == 'A':
bgp_message.add_announce(elem.fields['prefix'])
bgp_message.set_nexthop(elem.fields['next-hop'])
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
output_queue.put(bgp_message)
elif elem.type.upper() == 'W':
bgp_message.add_withdraw(elem.fields['prefix'])
output_queue.put(bgp_message)
elem = rec.get_next_elem()
def getting_BGP_update():
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', 'rrc11')
stream.add_interval_filter(1438417216, 1438417216)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
if rec.status != "valid":
print rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while (elem):
print rec.project, rec.collector, rec.type, rec.time, rec.status,
print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
def generate_stream():
bs = BGPStream()
rec = BGPRecord()
#initialize MySql
db_writer = MysqlWriter()
db_writer.start()
#initialize the publisher in port number 12345
publisher = ZmqPublisher(12345)
bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0)
# bs.add_filter('collector', 'route-views.sg')
bs.add_filter('record-type', 'updates')
bs.start()
print('Beginning to read from stream')
input_id = 0
while bs.get_next_record(rec):
elem = rec.get_next_elem()
while elem is not None:
# sleep until it is time to send this record
'''
now = calendar.timegm(time.gmtime())
sim_time = now - delay
if elem.time > sim_time:
time.sleep(elem.time - sim_time)
'''
if elem.type not in ['A', 'W']:
continue
input_id += 1
msg = elem2bgplay(rec, elem)
msg['type'] = 'A'
msg['id'] = input_id
print(msg)
# Publish the message
publisher.publish(msg)
# Write it to DB
if elem.type == 'A':
db_writer.add(msg)
elif elem.type == 'W':
db_writer.remove(msg)
else:
print "Error: Unknown type: " + elem.type
elem = rec.get_next_elem()
def recv_bgpstream_rib(begin, until, collector, output_queue):
logging.info ("CALL recv_bgpstream_rib")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector', collector)
stream.add_filter('record-type','updates')
stream.add_interval_filter(begin,until)
# Start the stream
stream.start()
while (stream.get_next_record(rec)):
global rib_ts
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping.")
continue
if (rib_ts > 0) and (rec.time > (rib_ts + RIB_TS_INTERVAL/2)):
logging.info("received full RIB table dump.")
break
bgp_message = None
while (elem):
if (elem.type.upper() == 'A') or (elem.type.upper() == 'R'):
rib_ts = elem.time
bgp_message = BGPmessage(elem.time, 'update')
bgp_message.set_nexthop(elem.fields['next-hop'])
src_peer = dict()
src_addr = elem.peer_address
src_asn = elem.peer_asn
src_peer['addr'] = src_addr
src_peer['port'] = 0
src_peer['asn'] = src_asn
bgp_message.set_source(src_peer)
aspath = elem.fields['as-path'].split()
for a in aspath:
if not '{' in a: # ignore AS-SETs
bgp_message.add_as_to_path(a)
bgp_message.add_announce(elem.fields['prefix'])
output_queue.put(bgp_message)
elem = rec.get_next_elem()
def get_stream(self):
logging.debug(
f"[ZombieRecordFinder-{self.collector}] try to create BGPstream")
_start = datetime.datetime(self.year, self.month, 10)
_end = datetime.datetime(self.year, self.month, 20)
stream = BGPStream()
stream.add_interval_filter(dt2ts(_start), dt2ts(_end))
stream.add_filter('collector', self.collector)
for _, p in self.zombies:
stream.add_filter('prefix-exact', p)
return stream
def __init__(
self,
filters={"collector": ["rrc00"]},
rpki_validator="rpki-validator.realmv6.org:8282",
settings_file="../settings.json",
):
self.stream = BGPStream()
self.filters = filters
self.route_table = dict()
self.i = 0
self.metadata_vp = dict()
self.metadata_rc = dict()
self.peers = Counter()
self.prefix4 = Counter()
self.prefix6 = Counter()
start_timestamp = get_push_timestamp(datetime.now(timezone.utc))
for filter_type, filter_array in filters.items():
for filter_value in filter_array:
self.stream.add_filter(filter_type, filter_value)
for collector in filters["collector"]:
self.route_table[collector] = defaultdict(dict)
self.metadata_vp[collector] = defaultdict(list)
self.metadata_rc[collector] = RouteCollectorMeta(None, 0, 0, 0, 0)
self.peers[collector] = defaultdict(int)
self.prefix4[collector] = defaultdict(int)
self.prefix6[collector] = defaultdict(int)
settings = get_settings(settings_file)
settings["db"]["password"] = os.environ["PGPASS"]
self.db = DBConnector(settings["db"])
rpki = rpki_validator.split(":")
self.mgr = RTRManager(rpki[0], rpki[1])
self.mgr.start()
self.start_collecting(start_timestamp)
def generate_stream():
bs = BGPStream()
rec = BGPRecord()
#initialize MySql
a = MySqlDAL()
a.start()
#initialize the publisher in port number 12345
publisher = ZmqPublisher(12345)
bs.add_interval_filter(calendar.timegm(time.gmtime()) - delay, 0)
# bs.add_filter('collector', 'route-views.sg')
bs.add_filter('record-type', 'updates')
bs.start()
print('Beginning to read from stream')
input_id = 0
while bs.get_next_record(rec):
elem = rec.get_next_elem()
while elem is not None:
# sleep until it is time to send this record
'''
now = calendar.timegm(time.gmtime())
sim_time = now - delay
if elem.time > sim_time:
time.sleep(elem.time - sim_time)
'''
if elem.type not in ['A', 'W']:
continue
input_id += 1
msg = elem2bgplay(rec, elem)
msg['type'] = 'A'
msg['id'] = input_id
print(msg)
# Publish the message
publisher.publish(msg)
# Write it to DB
if elem.type == 'A':
a.add(msg)
elif elem.type == 'W':
a.remove(msg)
else:
print "Error: Unknown type: " + elem.type
elem = rec.get_next_elem()
def readUpdates(self):
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type updates"
bgprFilter += " and project ris "
for prefix in self.prefixes:
bgprFilter += " and prefix more %s " % prefix
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (self.startts, self.endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts, self.endts)
stream.start()
while (stream.get_next_record(rec)):
if (rec.status != "valid" and rec.status != "filtered-source"
and rec.status != "empty-source"):
raise Exception(rec.project, rec.collector, rec.type, rec.time,
rec.status)
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
# peerip g= elem.peer_address
peeras = elem.peer_asn
sPrefix = elem.fields["prefix"]
if elem.type == "W":
self.withdraws[sPrefix][peeras] = True
else:
sPath = elem.fields["as-path"]
self.paths[sPrefix][peeras] = sPath
self.withdraws[sPrefix][peeras] = False
elem = rec.get_next_elem()
def _start_stream(self, **filters):
"""Initializes, starts, and returns bgp stream with filters"""
# Create a new bgpstream instance
stream = BGPStream()
# Add filters if params exist
for key, value in filters.items():
if (key in ['peer-asn', 'prefix', 'collector']
and value is not None):
stream.add_filter(key, value)
# Time params must be in epoch
start_epoch = calendar.timegm(filters.get('start').timetuple())
end_epoch = calendar.timegm(filters.get('end').timetuple())
stream.add_interval_filter(start_epoch, end_epoch)
# Start the stream
stream.start()
self.logger.debug("Started stream for caida announcements")
return stream
def main():
parser = argparse.ArgumentParser(description='', epilog='')
parser.add_argument('-b', '--begin',
help='Begin date (inclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-u', '--until',
help='Until date (exclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-c', '--collector',
help='Route collector from RIPE RIS or Route-Views project.',
type=str, required=True)
parser.add_argument('-m', '--mongodb',
help='MongoDB connection parameters.',
type=str, default=None)
parser.add_argument('-l', '--loglevel',
help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].',
type=str, default='WARNING')
args = vars(parser.parse_args())
numeric_level = getattr(logging, args['loglevel'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level,
format='%(asctime)s : %(levelname)s : %(message)s')
ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds())
ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds())
mongodbstr = None
if args['mongodb']:
mongodbstr = args['mongodb'].strip()
# BEGIN
logging.info("START")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector',args['collector'])
stream.add_filter('record-type','ribs')
stream.add_interval_filter(ts_begin,ts_until)
# Start the stream
stream.start()
rib_ts = 0
rib_origins = dict()
while(stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
if rec.time > (rib_ts + RIB_TS_INTERVAL):
rib_ts = rec.time
if mongodbstr:
store_rib_origins(rib_ts, rib_origins, mongodbstr)
else:
print_rib_origins(rib_ts, rib_origins)
rib_origins = dict()
while(elem):
prefix = elem.fields['prefix']
aspath = elem.fields['as-path'].split()
for a in aspath: # remove AS-SETs
if '{' in a:
aspath.remove(a)
origin = aspath[-1]
if prefix not in rib_origins:
rib_origins[prefix] = list()
if origin not in rib_origins[prefix]:
rib_origins[prefix].append(origin)
elem = rec.get_next_elem()
from _pybgpstream import BGPStream, BGPRecord, BGPElem
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
start = 1454800000
end = 1454802000
target_pref = '200.7.6.0/24'
print start, end, target_pref
print target_pref
stream.add_filter('prefix', target_pref)
# Consider RIPE RRC 10 only
# stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
stream.add_interval_filter(start, end)
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# """ Very short period for test """
# stream.add_interval_filter(start, end)
# """ Jan till now """
# stream.add_interval_filter(1451606400,1454785264)
# """ yesterday """
# stream.add_interval_filter(1454630400, 1454716800)
# Start the stream
stream.start()
from _pybgpstream import BGPStream, BGPElem, BGPRecord
from py2neo import Graph, Node, Relationship
graph = Graph(password="******")
stream = BGPStream()
rec = BGPRecord()
# stream.add_filter('prefix', '198.41.0.0/24') # A-root
# stream.add_filter('prefix', '192.33.4.0/24') # C-root
# stream.add_filter('prefix', '199.7.91.0/24') # D-root
# stream.add_filter('prefix', '192.203.230.0/24') # E-root, IPv4 only
# stream.add_filter('prefix', '192.5.5.0/24') # F-root
# stream.add_filter('prefix', '192.112.36.0/24') # G-root, IPv4 only
# stream.add_filter('prefix', '198.97.190.0/24') # H-root
# stream.add_filter('prefix', '192.36.148.0/24') # I-root
# stream.add_filter('prefix', '192.58.128.0/24') # J-root
stream.add_filter('prefix', '193.0.14.0/24') # K-root
stream.add_filter('prefix', '199.7.83.0/24') # L-root
stream.add_filter('prefix', '202.12.27.0/24') # M-root
stream.add_filter('record-type','ribs')
# stream.add_filter('collector','route-views.sg')
stream.add_filter('project','routeviews')
timestamp = 1464739200 # 2016/6/1 00:00
stream.add_interval_filter(timestamp, timestamp) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC); 1464682200
stream.start()
result = {}
while stream.get_next_record(rec):
def main():
(options, args) = getopts()
limit = options.limit
start = options.start_time
end = options.end_time
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
stream.add_filter('prefix', '0.0.0.0/0')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264)
#stream.add_interval_filter(1454630400, 1454716800)
# 1 hour
#1454284800 - 1454288400
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
prefixes_update = defaultdict(int)
prefixes_withdraw = defaultdict(int)
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A':
#print elem.fields['as-path']
prefixes_update[elem.fields['prefix']] += 1
if elem.type == 'W':
prefixes_withdraw[elem.fields['prefix']] += 1
#print rec.project, rec.collector, rec.type, rec.time, rec.status,
#print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
for k in prefixes_update:
if prefixes_update[k] >= limit:
print k + "\t" + str(prefixes_update[k]) + "\t" + str(
prefixes_withdraw[k])
stream_start = int(sys.argv[1])
stream_end = int(sys.argv[2])
out_file_name = sys.argv[3]
except:
print "Usage: %s [start time] [end time] [output file name]" %(sys.argv[0])
exit()
#stream_start = 1454284800
#stream_end = 1454288400
buckets = create_time_buckets(stream_start, stream_end)
prefixList = []
raw_bgp_stream_data = {}
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', 'rrc06')
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(stream_start, stream_end)
stream.start()
while(stream.get_next_record(rec)):
elem = rec.get_next_elem()
while(elem):
prefix = elem.fields.get("prefix", "")
time_stamp = rec.time # unix epoc timestamp 1427846670
def readupdates(self):
#TODO implement txt file for update messages?
if self.txtFile:
return
# create a new bgpstream instance
stream = BGPStream()
bgprFilter = "type updates"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
# bgprFilter += " and collector rrc10 "
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if self.asnFilter is not None:
# # TOFIX filter is now deprecated, we need to have both
# # announcements and withdrawals
# bgprFilter += ' and (path %s$ or elemtype withdrawals)' % self.asnFilter
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" % (self.startts, self.endts))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts, self.endts)
if self.livemode:
stream.set_live_mode()
stream.start()
# for line in p1.stdout:
# create a reusable bgprecord instance
rec = BGPRecord()
while (stream.get_next_record(rec)):
if rec.status != "valid":
logging.warn("Invalid BGP record: %s, %s, %s, %s, %s" %
(rec.project, rec.collector, rec.type, rec.time,
rec.status))
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
if zOrig not in self.peers:
# no need to update the counts for non-full feed peers
elem = rec.get_next_elem()
continue
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
msgTs = zDt
# set first time bin!
if self.ts is None:
self.slideTimeWindow(msgTs)
elif self.ts + self.timeWindow <= msgTs:
self.slideTimeWindow(msgTs)
elif self.ts > msgTs:
#Old update, ignore this to update the graph
logging.warn(
"Ignoring old update (peer IP: %s, timestamp: %s, current time bin: %s): %s"
% (zOrig, zDt, self.ts, (elem.type, zAS, elem.fields)))
elem = rec.get_next_elem()
continue
node = self.rtree.search_exact(zPfx)
if elem.type == "W":
# Withdraw: remove the corresponding node
if not node is None and zOrig in node.data:
origAS = node.data[zOrig]["origAS"]
if self.spatialResolution:
count = node.data[zOrig]["count"]
# Update count for above node
parent = self.findParent(node, zOrig)
if parent is None:
self.incTotalCount(-count, zOrig, origAS, zAS)
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
else:
node.data[zOrig]["count"] = 0
# Add ips to above node and corresponding ASes
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
# oldpCount = parent.data[zOrig]["count"]
# pCount = self.nbIPs(parent.prefixlen) - pcountBelow
# parent.data[zOrig]["count"] = pCount
# pdiff = pCount - oldpCount
# assert pdiff==count
# Update count for origAS and path from the
# parent node
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(count, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(count, zOrig, porigAS, zAS)
# Update count for withdrawn origAS and path
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, origAS, zAS, asns)
self.incTotalCount(-count, zOrig, origAS, zAS)
else:
asns = node.data[zOrig]["path"]
self.incCount(-1, zOrig, origAS, zAS, asns)
self.incTotalCount(-1, zOrig, origAS, zAS)
del node.data[zOrig]
else:
# Announce: update counters
sPath = elem.fields["as-path"]
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes or implicit withdraws
if len(path) < 2:
# Ignoring paths with only one AS
elem = rec.get_next_elem()
continue
if self.announceQueue is not None:
self.announceQueue.put((zDt, zOrig, zAS, zPfx, path))
# Announce:
if node is None or not zOrig in node.data:
# Add a new node
node = self.rtree.add(zPfx)
if self.spatialResolution:
# Compute the exact number of IPs
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
parent = self.findParent(node, zOrig)
if not parent is None:
# Update above nodes
# print("%s: (%s) %s, %s, %s" % (zDt, elem.type, zAS, zPfx, count))
pcountBelow = sum([
n.data[zOrig]["count"] for n in
self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(
parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
# print("parent %s: (%s) %s, %s, %s" % (zDt, zAS, parent.prefix, oldpCount, pCount))
# print [(n.prefix,n.data[zOrig]["count"]) for n in self.rtree.search_covered(parent.prefix) if zOrig in n.data and n!=parent ]
porigAS = parent.data[zOrig]["origAS"]
pasns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, porigAS, zAS,
pasns)
self.incTotalCount(pdiff, zOrig, porigAS, zAS)
else:
self.incTotalCount(1, zOrig, origAS, zAS)
count = 1
# Update the ASes counts
node.data[zOrig] = {
"path": set(path),
"count": count,
"origAS": origAS
}
asn = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
else:
#Update node path and counts
if self.spatialResolution:
count = node.data[zOrig]["count"]
else:
count = 1
porigAS = node.data[zOrig]["origAS"]
asns = node.data[zOrig]["path"]
self.incCount(-count, zOrig, porigAS, zAS, asns)
self.incTotalCount(-count, zOrig, porigAS, zAS)
node.data[zOrig]["path"] = set(path)
node.data[zOrig]["origAS"] = origAS
asns = node.data[zOrig]["path"]
self.incCount(count, zOrig, origAS, zAS, asns)
self.incTotalCount(count, zOrig, origAS, zAS)
elem = rec.get_next_elem()
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
# NB: BGPStream uses inclusive/inclusive intervals, so subtract one off the
# end time since we are using inclusive/exclusive intervals
stream.add_interval_filter(start_time, end_time-1)
stream.add_filter('record-type', data_type)
stream.start()
# per-peer data
peers_data = {}
# loop over all records in the stream
while stream.get_next_record(rec):
elem = rec.get_next_elem()
# loop over all elems in the record
while elem:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
# if this is the first time we have ever seen this peer, create
# an empty result: (Pfxs_v4_set , Pfxs_v6_set,
# dict(Transit_ASN)=Pfxs_v4_set , dict(Transit_ASN)=Pfxs_v6_set)
if sig not in peers_data:
peers_data[sig] =[set(),set(),{},{}]
pfx=""
if('prefix' in elem.fields):
pfx=elem.fields['prefix'];
if(":" in pfx):
peers_data[sig][1].add(pfx)
else:
peers_data[sig][0].add(pfx)
if('as-path' in elem.fields):
#Squash the AS Path to keep only distinct ASNs, i.e., remove prepending
path_split = [k for k, g in groupby(elem.fields['as-path'].split(" "))]
if(len(path_split)!=0):
for i in range(1,len(path_split)-1):
transit=path_split[i]
if(":" in pfx):
if(transit not in peers_data[sig][3]):
peers_data[sig][3][transit]=set()
peers_data[sig][3][transit].add(pfx)
elif(pfx!=""):
if(transit not in peers_data[sig][2]):
peers_data[sig][2][transit]=set()
peers_data[sig][2][transit].add(pfx)
elem = rec.get_next_elem()
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def dump2file(type, name, latestDumpTime, dumpDuration, dumpPeriod):
print type + ' of ' + name + ': '
csv_header = ['type', 'addr', 'as', 'prefix', 'next_hop', 'as_path']
_file = None
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', c_name)
if type == 'ribs':
stream.add_filter('record-type', 'ribs')
_file = open(c_name + '_ribs.csv', 'w+')
elif type == 'updates':
stream.add_filter('record-type', 'updates')
_file = open(c_name + '_updates.csv', 'w+')
stream.add_interval_filter(latestDumpTime, latestDumpTime + dumpPeriod)
stream.start()
count = 0
useless_c = 0
writer = csv.writer(_file)
writer.writerow(csv_header)
# Get next record
while (stream.get_next_record(rec)):
# Print the record information only if it is not a valid record
if rec.status != "valid":
# print rec.project, rec.collector, rec.type, rec.time, rec.status
print 'current rec not valid.'
else:
elem = rec.get_next_elem()
while (elem):
useless_c += 1
if useless_c % 1000 == 0:
print 'Got ' + str(useless_c) + ' elem totally.'
# Print record and elem information
if isIPV6(elem): # ipv6 packet ignored
elem = rec.get_next_elem()
continue
count += 1
# print rec.project, rec.collector, rec.type, rec.time, rec.status,
# print elem.type, elem.peer_address, elem.peer_asn
# elem.fields contains four column: communities, next-hop, prefix, as-path
field = elem.fields
prefix = field['prefix'] if 'prefix' in field.keys() else ''
next_hop = field['next-hop'] if 'next-hop' in field.keys(
) else ''
as_path = field['as-path'] if 'as-path' in field.keys() else ''
as_path = as_path.replace(' ', '|')
writer.writerow([
elem.type, elem.peer_address, elem.peer_asn, prefix,
next_hop, as_path
])
elem = rec.get_next_elem()
_file.close()
print 'count: ' + str(count)
return count
def create_trie_from_bgpstream_info(self, interval_start=1451692800):
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', self.collector_name)
stream.add_filter('record-type', 'ribs')
if isinstance(interval_start, datetime.datetime):
interval_start_utc = self.convert_interval_to_utc(interval_start)
stream.add_interval_filter(interval_start_utc - 300, interval_start_utc + 300)
else:
stream.add_interval_filter(interval_start - 300, interval_start + 300)
stream.start()
while (stream.get_next_record(rec)):
elem = rec.get_next_elem()
while elem:
# Get the peer ASN and IP. We then construct a peer_id, since a collector
# can establish multiple connections with the same ASN.
peer_asn = elem.peer_asn
peer_asn_ip = elem.peer_address
# make this an unmodifiable tuple
peer_id = (peer_asn, peer_asn_ip)
peer_route_trie = self.peer_trie_handles[peer_id]
trie_node = peer_route_trie.add(elem.fields['prefix'])
trie_node.data['as-path'] = elem.fields['as-path']
elem = rec.get_next_elem()
#!/usr/bin/env python
import os, sys, time
from datetime import datetime
import yaml
from _pybgpstream import BGPStream, BGPRecord, BGPElem
stream = BGPStream()
class BGPStreamReader(object):
defaults = {
'mrt_file': None,
'collector': 'rrc00',
'record_type': 'update',
'from_date': int(time.time()) - 3600 * 24 * 7, # back up a week
'until_date': 0,
'prefix_filter': None,
'peer_as_filter': None,
'communities_filter': None,
}
def __init__(self, config={}):
self.config = config
for k, v in self.defaults.items():
self.config.setdefault(k, v)
if self.config['mrt_file'] is None:
stream.add_filter('collector', self.config['collector'])
stream.add_filter('record-type', self.config['record_type'])
stream.add_interval_filter(self.config['from_date'],
self.config['until_date'])
stream.set_live_mode()
def main():
(options, args) = getopts()
limit = options.limit
start = options.start_time
end = options.end_time
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
stream.add_filter('prefix', '0.0.0.0/0')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264)
#stream.add_interval_filter(1454630400, 1454716800)
# 1 hour
#1454284800 - 1454288400
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
prefixes_update = defaultdict(int)
prefixes_withdraw = defaultdict(int)
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A':
#print elem.fields['as-path']
prefixes_update[elem.fields['prefix']] += 1
if elem.type == 'W':
prefixes_withdraw[elem.fields['prefix']] += 1
#print rec.project, rec.collector, rec.type, rec.time, rec.status,
#print elem.type, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
for k in prefixes_update:
if prefixes_update[k] >= limit:
print k + "\t" + str(prefixes_update[k]) + "\t" + str(prefixes_withdraw[k])
import sys
from BlackHoleDB import BlackHoleDb
from _pybgpstream import BGPStream, BGPRecord, BGPElem
data_source = "DE-CIX"
sys.stderr.write("Inserting data from DE-CIX\n")
# get connection to database
bh_db = BlackHoleDb()
# export LD_LIBRARY_PATH="/usr/local/lib"
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RRC12
stream.add_filter('collector','rrc12')
# Consider RIBs dumps only
stream.add_filter('record-type','ribs')
# One rib per day
stream.add_rib_period_filter(3600*24)
# Consider this time interval: May 2016
stream.add_interval_filter(1462060800,1464739200)
# Start the stream
from _pybgpstream import BGPRecord, BGPStream
stop = 1422778200
start = stop - 1200 # 15000 second seems to be the shortest interval to get data from BGPstream
result = []
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('prefix', '192.33.4.0/24')
stream.add_filter('record-type', 'ribs')
stream.add_filter('project', 'ris')
# stream.add_filter('collector', 'router-route-views.routeviews.org.peer-IPV4_route-spews.cbbtier3.att.net')
stream.add_interval_filter(start, stop)
stream.start()
print('start')
# test = stream.get_data_interfaces()
# print('test: {}'.format(test))
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
as_path = elem.fields['as-path'].split()
as_path.append(' ') # for tree creation
result.append(as_path)
elem = rec.get_next_elem()
def _get_data(self, prefix, datetime):
"""
output example: [['15547', '8220', '1853', '1205'],[..another AS path..]]
:param prefix:
:param datetime: end interval
:return: list of AS paths
"""
print('[*] ris.py: _get_data() called')
print('[*] ris.py: _get_data() prefix: {}'.format(prefix))
start = int(datetime) - 20000 # 20000 second seems to be the shortest interval to get data from BGPstream
stop = int(datetime)
result = []
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('prefix', prefix)
stream.add_filter('record-type', 'ribs')
stream.add_filter('project', 'ris')
stream.add_interval_filter(start, stop)
stream.start()
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
as_path = elem.fields['as-path'].split()
as_path.append(' ') # for tree creation
result.append(as_path)
elem = rec.get_next_elem()
print('[*] ris.py: _get_data() finished.')
return result
global peer_id_dict
global next_peer_id
if collector_name not in peer_id_dict:
peer_id_dict[collector_name] = dict()
if peer_asn not in peer_id_dict[collector_name]:
peer_id_dict[collector_name][peer_asn] = dict()
if peer_address not in peer_id_dict[collector_name][peer_asn]:
next_peer_id = next_peer_id
peer_id_dict[collector_name][peer_asn][peer_address] = next_peer_id
next_peer_id += 1
return peer_id_dict[collector_name][peer_asn][peer_address]
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('project','ris')
stream.add_filter('project','routeviews')
# Consider RIBs dumps only
stream.add_filter('record-type','ribs')
# Consider this time interval:
jan_02_2016 = 1451692800
stream.add_interval_filter(jan_02_2016 - 300, jan_02_2016 + 300)
stream.start()
edge_peer_asn = dict()
#!/usr/bin/env python
# Collects snapshot data from a ribs file and adds those nodes and
# connections to a database file. Contains routing data as well as AS connections
from _pybgpstream import BGPStream, BGPRecord, BGPElem
from collections import defaultdict
import string
import sys
# Initialize BGPStream with relevant filters
stream = BGPStream()
rec = BGPRecord()
mode = ""
if len(sys.argv) == 1:
collectors = ['rrc00', 'rrc01', 'rrc03', 'rrc04', 'rrc05']
for collector in collectors:
stream.add_filter('collector', collector)
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(1475310000, 1475350000)
stream.add_filter('prefix', '8.0.0.0/8')
mode = 'ripe'
elif len(sys.argv) == 2:
filename = sys.argv[1]
stream.set_data_interface('singlefile')
stream.set_data_interface_option('singlefile', 'rib-file', filename)
collectors = ['singlefile_ds']
mode = 'file'
stream.start()
#!/usr/bin/env python
# Collects data from ribs files and adds those nodes and
# connections to a database file along with a node representing
# the time
from _pybgpstream import BGPStream, BGPRecord, BGPElem
from collections import defaultdict
import string
import datetime
# Initialize BGPStream with relevant filters
stream = BGPStream()
rec = BGPRecord()
collectors = ['rrc01']
for collector in collectors:
stream.add_filter('collector', collector)
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(1438000000, 1439000000)
stream.add_filter('prefix', '8.8.0.0/16')
stream.start()
# open files for neo4j-import
collector_file = open("csv/collector.csv", 'w')
AS_file = open("csv/AS.csv", 'w')
prefix_file = open("csv/prefix.csv", 'w')
route_file = open("csv/route.csv", 'w')
time_file = open("csv/time.csv", 'w')
connect_rels_file = open("csv/connect_rels.csv", 'w')
def main():
parser = argparse.ArgumentParser()
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.description = textwrap.dedent('''\
a proof-of-concept utility for watching updates from BGPstream
and then printing out if an unexpected update is heard
''')
parser.epilog = textwrap.dedent('''\
Example: watch these route announcements
%(prog)s -f routes.yaml ''')
required = parser.add_argument_group('required arguments')
required.add_argument("-f",
"--file",
required=True,
help="yaml file of prefixes to origin asn")
parser.add_argument("-d",
"--debug",
action='store_true',
help="print out all updates containing these prefixes")
args = parser.parse_args()
routes = pytricia.PyTricia(48) # longest reasonable pfx in dfz
with open(args.file, 'r') as f:
routesfile = yaml.safe_load(f)
for pfx in routesfile:
routes[pfx] = routesfile[pfx]
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(int(time.time()), 0)
stream.set_live_mode()
stream.start()
while (stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
if 'as-path' in elem.fields:
path = elem.fields['as-path'].split()
prefix = elem.fields['prefix']
if prefix in routes and (routes[prefix] != path[-1]
or args.debug):
print('Heard prefix:', elem.fields['prefix'],
'AS-PATH:', elem.fields['as-path'],
' Found by project:', rec.project, 'collector:',
rec.collector, 'type:', rec.type, 'at time:',
rec.time, 'Type:', elem.type, 'Peer:',
elem.peer_address, 'AS', elem.peer_asn)
elem = rec.get_next_elem()
import time
from _pybgpstream import BGPStream, BGPRecord, BGPElem
# read here: http://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
start_time = time.time()
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('prefix','140.78.0.0/16')
stream.add_filter('record-type','updates')
# stream.add_filter('collector','rrc11')
stream.add_filter('project','ris')
stream.add_interval_filter(1462942850, 1462962850)
stream.start()
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
# Print record and elem information
print rec.collector, elem.peer_address, elem.peer_asn, elem.fields
elem = rec.get_next_elem()
# result = []
# while stream.get_next_record(rec):
# if rec.status == "valid":
# elem = rec.get_next_elem()
# while elem:
def readrib(self):
stream = None
rec = None
if self.txtFile is None:
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
bgprFilter = "type ribs"
if self.af == 6:
bgprFilter += " and ipversion 6"
else:
bgprFilter += " and ipversion 4"
for c in self.collectors:
bgprFilter += " and collector %s " % c
# if not self.asnFilter is None:
# bgprFilter += ' and path %s$' % self.asnFilter
for p in self.includedPeers:
bgprFilter += " and peer %s " % p
for p in self.includedPrefix:
bgprFilter += " and prefix more %s " % p
logging.info("Connecting to BGPstream... (%s)" % bgprFilter)
logging.info("Timestamps: %s, %s" %
(self.startts - 3600, self.startts + 3600))
stream.parse_filter_string(bgprFilter)
stream.add_interval_filter(self.startts - 3600,
self.startts + 3600)
if self.livemode:
stream.set_live_mode()
stream.start()
else:
rec = txtReader.txtReader(self.txtFile)
# for line in p1.stdout:
while (self.txtFile
and not rec.running) or (stream
and stream.get_next_record(rec)):
if rec.status != "valid":
print rec.project, rec.collector, rec.type, rec.time, rec.status
zDt = rec.time
elem = rec.get_next_elem()
while (elem):
zOrig = elem.peer_address
zAS = elem.peer_asn
if zAS in self.excludedPeers or (len(
self.includedPeers) and zAS not in self.includedPeers):
elem = rec.get_next_elem()
continue
zPfx = elem.fields["prefix"]
sPath = elem.fields["as-path"]
# print("%s: %s, %s, %s" % (zDt, zAS, zPfx, elem.fields))
if zPfx == "0.0.0.0/0" or zPfx in self.excludedPrefix or (
len(self.includedPrefix)
and zPfx not in self.includedPrefix):
elem = rec.get_next_elem()
continue
path = sPath.split(" ")
origAS = path[-1]
if origAS in self.excludedOriginASN or (
len(self.includedOriginASN)
and origAS not in self.includedOriginASN):
elem = rec.get_next_elem()
continue
# FIXME: this is not going to work in the case of
# delegated prefixes (and using IP addresses as spatial
# resolution)
self.peersASN[zOrig].add(zAS)
if len(path) < 2:
# Ignore paths with only one AS
elem = rec.get_next_elem()
continue
node = self.rtree.add(zPfx)
if zOrig in node.data:
# Already read this entry, we should read only one RIB per peer
elem = rec.get_next_elem()
continue
if self.ribQueue is not None:
self.ribQueue.put((zDt, zOrig, zAS, zPfx, path))
node.data[zOrig] = {
"path": set(path),
"count": 0,
"origAS": origAS
}
# print "%s, %s, %s, %s, %s" % (elem.time, elem.type, elem.peer_address, elem.peer_asn, elem.fields)
if self.spatialResolution:
# compute weight for this path
count = self.nbIPs(node.prefixlen)
countBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(zPfx)
if zOrig in n.data and n != node
])
count -= countBelow
# assert count >= 0
node.data[zOrig]["count"] = count
# Update above nodes
parent = self.findParent(node, zOrig)
if not parent is None:
# pcountBelow = sum([n.data[zOrig]["count"] for n in self.rtree.search_covered(parent.prefix) if n.parent == parent and zOrig in n.data])
pcountBelow = sum([
n.data[zOrig]["count"]
for n in self.rtree.search_covered(parent.prefix)
if zOrig in n.data and n != parent
])
oldpCount = parent.data[zOrig]["count"]
pCount = self.nbIPs(parent.prefixlen) - pcountBelow
pdiff = pCount - oldpCount
parent.data[zOrig]["count"] = pCount
pOrigAS = parent.data[zOrig]["origAS"]
asns = parent.data[zOrig]["path"]
self.incCount(pdiff, zOrig, pOrigAS, zAS, asns)
self.incTotalCount(pdiff, zOrig, pOrigAS, zAS)
else:
count = 1
node.data[zOrig]["count"] = count
asns = node.data[zOrig]["path"]
self.incTotalCount(count, zOrig, origAS, zAS)
self.incCount(count, zOrig, origAS, zAS, asns)
elem = rec.get_next_elem()
def __init__(self):
self.collector_list = []
self.collector_types = ('routeviews', 'ris')
self.stream = BGPStream()
self.record = BGPRecord()
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("exp_name")
parser.add_argument("collector")
parser.add_argument("event_number", type=int)
parser.add_argument(
"--anchors",
help="if set, downloads the anchor, otherwise downloads beacons",
action='store_true')
args = parser.parse_args()
exp_name = args.exp_name
collector = args.collector
event_number = args.event_number
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector)
if args.anchors:
prefixes = anchor_list(exp_name)
else:
prefixes = beacon_list(exp_name)
for prefix in prefixes:
# print('Adding filter for prefix {}'.format(prefix))
stream.add_filter('prefix', prefix)
fn = download_filename(exp_name, collector, args.anchors, event_number)
if os.path.isfile(fn) and os.stat(fn).st_size > 0:
# To debug
from _pybgpstream import BGPStream, BGPRecord, BGPElem
from collections import defaultdict
from itertools import groupby
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RIS RRC 00 only
stream.add_filter('collector','route-views.sfmix')
# Consider RIBs dumps only
stream.add_filter('record-type','ribs')
# Consider this time interval:
jan_02_2016 = 1451692800
stream.add_interval_filter(jan_02_2016 - 300, jan_02_2016 + 300)
stream.start()
tier1_str = "174 209 286 701 1239 1299 2828 2914 3257 3320 3356 5511 6453 6461 6762 7018 12956"
tier1s = tier1_str.split()
edge_pfx = dict()
transit = set()
while(stream.get_next_record(rec)):
elem = rec.get_next_elem()
while(elem):
print buf
print >> f, buf
list_linkset = []
dictup = {}
dictdown = {}
nevents = {}
history = {}
current = start
current_time_short = strftime("%Y/%m/%d-%H:%M:%S", gmtime(current))
current_time_long = strftime("%a, %d %b %Y %H:%M:%S", gmtime(current))
while current <= end:
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('collector', collector_filter)
stream.add_filter('record-type','ribs')
stream.add_interval_filter(current - 10 * 60, current + 10 * 60)
stream.add_filter('peer-asn','3356')
stream.add_filter('peer-asn','174')
stream.add_filter('peer-asn','3257')
stream.add_filter('peer-asn','1299')
stream.add_filter('peer-asn','2914')
stream.add_filter('peer-asn','6453')
stream.add_filter('peer-asn','6762')
stream.add_filter('peer-asn','6939')
stream.add_filter('peer-asn','2828')
stream.add_filter('peer-asn','3549')
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
#
from _pybgpstream import BGPStream, BGPRecord, BGPElem
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
# configure the stream to retrieve RIB records from the RRC06 collector at
# 2015/05/01 00:00 UTC
stream.add_filter('collector', 'rrc06')
stream.add_filter('record-type', 'ribs')
stream.add_interval_filter(1427846400, 1427846700)
# start the stream
stream.start()
as_topology = set()
rib_entries = 0
import time
from _pybgpstream import BGPStream, BGPRecord, BGPElem
start_time = time.time()
stream = BGPStream()
rec = BGPRecord()
stream.add_filter('prefix','140.78.0.0/16')
stream.add_filter('record-type','ribs')
stream.add_filter('collector','rrc11')
# stream.add_filter('project','ris')
stream.add_interval_filter(1464681000, 1464682200) # 1464682200 ==> 05/31/2016 @ 8:10am (UTC)
stream.start()
#
# while(stream.get_next_record(rec)):
# # Print the record information only if it is not a valid record
# if rec.status == "valid":
# elem = rec.get_next_elem()
# while elem:
# # Print record and elem information
# print rec.collector, elem.peer_address, elem.peer_asn, elem.fields['as-path']
# elem = rec.get_next_elem()
result = []
while stream.get_next_record(rec):
if rec.status == "valid":
elem = rec.get_next_elem()
while elem:
print rec.collector, elem.type, elem.peer_address, elem.peer_asn, elem.fields
def main():
parser = argparse.ArgumentParser(description='', epilog='')
parser.add_argument('-b', '--begin',
help='Begin date (inclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-u', '--until',
help='Until date (exclusive), format: yyyy-mm-dd HH:MM',
type=valid_date, required=True)
parser.add_argument('-c', '--collector',
help='Route collector from RIPE RIS or Route-Views project.',
type=str, required=True)
parser.add_argument('-m', '--mongodb',
help='MongoDB connection parameters.',
type=str, default=None)
parser.add_argument('-k', '--keepsnapshots',
help='Keep all snapshots, works only with -s.',
action='store_true')
parser.add_argument('-s', '--snapshot',
help='Enable snapshoting.',
action='store_true')
parser.add_argument('-l', '--loglevel',
help='Set loglevel [DEBUG,INFO,WARNING,ERROR,CRITICAL].',
type=str, default='WARNING')
args = vars(parser.parse_args())
numeric_level = getattr(logging, args['loglevel'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level,
format='#> %(asctime)s : %(levelname)s : %(message)s')
ts_begin = int((args['begin'] - datetime(1970, 1, 1)).total_seconds())
ts_until = int((args['until'] - datetime(1970, 1, 1)).total_seconds())
mongodbstr = None
if args['mongodb']:
mongodbstr = args['mongodb'].strip()
rib_ts = 0
rib_origins = dict()
origins_lt = list()
if args['snapshot']:
rib_ts, rib_origins = load_snapshot(mongodbstr)
if rib_ts > ts_begin:
logging.info ("SKIP, found snapshot with newer ts")
ts_begin = rib_ts - RIB_TS_THRESHOLD
# BEGIN
logging.info("START")
# Create bgpstream
stream = BGPStream()
rec = BGPRecord()
# set filtering
stream.add_filter('collector',args['collector'])
stream.add_filter('record-type','ribs')
stream.add_interval_filter(ts_begin,ts_until)
# Start the stream
stream.start()
while(stream.get_next_record(rec)):
if rec.status == 'valid':
elem = rec.get_next_elem()
else:
logging.warn("stream record invalid, skipping ...")
continue
#end if
if rec.time > (rib_ts + RIB_TS_THRESHOLD):
for p in rib_origins:
for o in rib_origins[p]:
if rib_origins[p][o][1] < (rib_ts - RIB_TS_THRESHOLD):
origins_lt.append( (p,o,rib_origins[p][o][0],rib_origins[p][o][1]) )
#end if
#end for
#end for
if args['snapshot'] and (len(rib_origins.keys()) > 0):
store_snapshot(rec.time, rib_origins, mongodbstr)
if not args['keepsnapshots']:
remove_snapshot(rib_ts, mongodbstr)
# end if keepsnapshots
# end if snapshot
rib_ts = rec.time
logging.info("ts: "+str(rib_ts))
if len(origins_lt) > 0:
if mongodbstr:
store_origins_lt(rib_ts,origins_lt, mongodbstr)
else:
print_origins_lt(rib_ts,origins_lt)
#end if
for l in origins_lt:
del rib_origins[l[0]][l[1]]
#end for
origins_lt = list()
# end if
#end if
while(elem):
prefix = elem.fields['prefix']
aspath = elem.fields['as-path'].split()
for a in aspath: # remove AS-SETs
if '{' in a:
aspath.remove(a)
#end if
#end for
origin = aspath[-1]
if prefix not in rib_origins:
rib_origins[prefix] = dict()
#end if
if origin not in rib_origins[prefix]:
rib_origins[prefix][origin] = (rib_ts,rib_ts)
else:
rib_origins[prefix][origin] = (rib_origins[prefix][origin][0],rib_ts)
#end if
elem = rec.get_next_elem()
#end while
#end while
if args['snapshot']:
print "NOTE: remaining origin lifetimes are stored in latest snapshot (%d)!\n" % rib_ts
if (len(rib_origins.keys()) > 0):
store_snapshot(rib_ts, rib_origins, mongodbstr)
# end if
else:
print "NOTE: output remaining origin lifetimes with current ts (%d)\n" % rib_ts
origins_lt = list()
for p in rib_origins:
for o in rib_origins[p]:
origins_lt.append( (p,o,rib_origins[p][o][0],rib_ts) )
if mongodbstr:
store_origins_lt(rib_ts,origins_lt, mongodbstr)
else:
print_origins_lt(rib_ts,origins_lt)
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <http://www.gnu.org/licenses/>.
#
from _pybgpstream import BGPStream, BGPRecord, BGPElem
# create a new bgpstream instance
stream = BGPStream()
# create a reusable bgprecord instance
rec = BGPRecord()
# configure the stream to retrieve Updates records from the RRC06 collector
# The commented out add_filter lines are the old way, the parse_filter_string
# way is the new method for applying filters
#stream.add_filter('collector', 'rrc06')
#stream.add_filter('record-type', 'updates')
stream.parse_filter_string('collector rrc06 and type updates')
# select the time interval to process:
# Wed Apr 1 00:02:50 UTC 2015 -> Wed Apr 1 00:04:30
stream.add_interval_filter(1427846570, 1427846670)
# profile) before using pybgpstream
# export LD_LIBRARY_PATH="/usr/local/pkg/ioda-tools/lib:$LD_LIBRARY_PATH"
# this tutorial is not complete, please
# refer to this page for a complete documentation
#
# https://staff.caida.org/~alistair/pybgpstream/index.html
from _pybgpstream import BGPStream, BGPRecord, BGPElem
import radix
import sys
import calendar
import time
stream = BGPStream()
rec = BGPRecord()
start_interval= 1401623715
##stream.add_filter('project', 'ris')
stream.add_filter('collector', 'route-views2')
#stream.add_filter('collector', 'rrc04')
stream.add_filter('record-type', 'ribs')
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(start_interval,1406894115 )
stream.add_rib_period_filter(172800)
stream.set_data_interface('broker')
#stream.set_data_interface_option('broker', 'db-host', 'loki-ge')
#stream.set_data_interface_option('broker', 'db-port', '3306')
#stream.set_data_interface_option('broker', 'db-user', 'bgpstream')
print "start bgpstream"
stream.start()
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("collector")
parser.add_argument("--our-servers", default="localhost:9092")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
save_file = "ts-{}".format(args.collector)
stream = BGPStream()
stream.add_filter('project', args.collector)
if os.path.exists(save_file):
with open(save_file, "r") as f:
last_ts = int(float(f.read().strip()))
logger.info("loading timestamp from file: %s",
datetime.utcfromtimestamp(last_ts))
else:
# Consider RIBs dumps only
now = time.time()
last_ts = int(now - now % 3600)
logger.info("loading from: %s", datetime.utcfromtimestamp(last_ts))
stream.add_filter('record-type', 'ribs')
def main():
(options, args) = getopts()
start = options.start_time
end = options.end_time
target_prefs = Set()
with open('./../../atlas/anchor_prefix.txt', 'rb') as br:
for l in br:
target_prefs.add(l.strip())
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
with open('./data/stream_{0}'.format(start), 'wb') as bw:
#for pref in target_prefs:
for ptmp in target_prefs:
stream.add_filter('prefix', ptmp)
# stream.add_filter('prefix','0.0.0.0/0')
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('collector', 'rrc00')
# Consider this time interval:
# Sat Aug 1 08:20:11 UTC 2015
# stream.add_interval_filter(1438417216,1438417216)
# stream.add_interval_filter(1451606400,1454785264
stream.add_interval_filter(start, end)
# Start the stream
stream.start()
# Get next record
cnt = 0
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
pass
# print '*', rec.project, rec.collector, rec.type, rec.time, rec.status
else:
cnt += 1
elem = rec.get_next_elem()
while elem:
if elem.type == 'S':
continue
# Print record and elem information
# print rec.project, rec.collector, rec.type, rec.time, rec.status,
# print elem.type, elem.peer_address, elem.peer_asn, elem.fields, elem.pref
bw.write('{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\n'.format(
rec.project, rec.collector, rec.type, rec.time, rec.status,
elem.type, elem.fields['prefix'], elem.peer_address, elem.peer_asn, elem.fields))
bw.flush()
elem = rec.get_next_elem()
print 'Successful termination; Start time: {0}'.format(start)
def sendMessageToKafka(col_name, col_data):
# print "Collector-{} Sending Message...".format(col_name)
size_total = 0
num_total = 0
num_ipv4 = 0
stream = BGPStream()
record = BGPRecord()
time_start = int(col_data.get('updates').get('latestDumpTime'))
time_end = time_start + int(col_data.get('updates').get('dumpPeriod'))
# print "Start Time:{}, End Time:{}".format(time_start, time_end)
stream.add_filter('collector', col_name)
stream.add_filter('record-type', 'ribs')
# stream.add_interval_filter(time_start, time_end)
# stream.add_interval_filter(time_start, time_start+300)
stream.add_interval_filter(1503475200, 1503475200 + 7200)
# print "Before Start>>>>>"
stream.start()
print col_name
# print "After Start>>>>>>"
producer = Kafka_producer()
while stream.get_next_record(record):
if record.status == "valid":
elem = record.get_next_elem()
while elem:
if filter(lambda x: ':' in x.peer_address, [elem]):
num_total += 1
elem = record.get_next_elem()
continue
#print "Element:{},{},{}".format(elem.type, elem.peer_address, elem.peer_asn)
field = elem.fields
#print type(field)
prefix = field['prefix'] if field.has_key('prefix') else ''
next_hop = field['next-hop'] if field.has_key(
'next-hop') else ''
as_path = field['as-path'] if field.has_key('as-path') else ''
as_path = as_path.replace(' ', '|')
text = [
elem.type, elem.peer_address,
str(elem.peer_asn), prefix, next_hop, as_path,
str(record.time)
]
text = ','.join(text)
# producer = Kafka_producer()
producer.send_data(col_name, text)
num_total += 1
num_ipv4 += 1
# print "[{}]-{}".format(col_name, num_total)
# size_total += len(text)
#NUM_TOTAL += 1
#print "[{}]-{}-{}-{}-{}".format(col_name, num_total, num_ipv4, size_total, time.ctime(record.time))
#print "No.{} Message Send Success-[{}]".format(num_total, text)
elem = record.get_next_elem()
else:
pass
# print "## Current record not valid!"
# break
# print "One Collector Finished"
else:
# print "-----------------------------"
# print "Collector[{}] And Records Send Finished\nTotal Num:{}, IPv4 Num:{}, Total Size:{}".format(col_name, num_total, num_ipv4, size_total)
# print "-----------------------------"
print "Collector:[{}]".format(col_name)
print "Total Num:{}, IPv4 Num:{}, Total Size:{}".format(
num_total, num_ipv4, size_total)
def main(rib, target_prefs):
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
with open('./data/stream_{0}'.format(start), 'wb') as bw:
for ptmp in target_prefs:
stream.add_filter('prefix', ptmp)
# Consider RIPE RRC 10 only
stream.add_filter('record-type', 'updates')
stream.add_filter('record-type', 'ribs')
#stream.add_filter('collector', 'rrc04')
stream.add_filter('project', 'ris')
stream.add_filter('project', 'routeviews')
stream.add_interval_filter(start-60*60*8, start)
stream.add_rib_period_filter(10000000000000)
# Start the stream
stream.start()
while stream.get_next_record(rec):
# Print the record information only if it is not a valid record
if rec.status != "valid":
continue
#if rec.time < start:
elem = rec.get_next_elem()
while elem:
if elem.type == 'A' or elem.type == 'R':
rib.add_to_rib(rec.collector, elem.peer_address, elem.fields['prefix'], elem.time, elem.fields['as-path'])
elem = rec.get_next_elem()
#else:
rib.flush()
print 'Successful termination; Start time: {0}'.format(start)
def load_data(start, stop, collectors, window, threshold):
peers = {}
# collectors is a list of the collectors we want to include
# Start and stop define the interval we are looking in the data
# Create a new BGPStream instance and a reusable BGPRecord instance
stream = BGPStream()
rec = BGPRecord()
# Add filter for each collector.
# If no collector is mentioned, it will consider 16 of them
if collectors:
for collector in collectors:
print collector
stream.add_filter('collector', collector)
else:
for i in range(0, 10):
stream.add_filter('collector', 'rrc0' + str(i))
for i in range(10, 16):
stream.add_filter('collector', 'rrc' + str(i))
stream.add_filter('record-type', 'updates')
# Consider the interval from "start" to "stop" in seconds since epoch
stream.add_interval_filter(start, stop)
# Start the stream
stream.start()
# For each record (one record = one second, can have multiple elements for the same second) we handle its updates
while stream.get_next_record(rec):
timestamp = rec.time
if rec.status != "valid":
print rec.project, rec.collector, rec.type, timestamp, rec.status
else:
# Go through all elements of the record
elem = rec.get_next_elem()
while elem:
# Consider only the A and W updates
if elem.type not in ['A', 'W']:
elem = rec.get_next_elem()
continue
peer = elem.peer_address
updatetype = elem.type
prefix = elem.fields['prefix']
if peer not in peers:
peers[peer] = {
'A': [],
'W': []
}
update = {'tst': timestamp, 'prefix': prefix}
if updatetype == 'A':
handleUpdate(peers[peer]['A'], burst2writeA, update, peer, updatetype, timestamp, window, threshold)
saveGraphPoint(peers[peer]['A'], updatetype, peer, timestamp, collectors, threshold)
else:
handleUpdate(peers[peer]['W'], burst2writeW, update, peer, updatetype, timestamp, window, threshold)
saveGraphPoint(peers[peer]['W'], updatetype, peer, timestamp, collectors, threshold)
elem = rec.get_next_elem()
# After processing all records, we write the graph json files with the graph points recorded for each peer
for peer in graph_points:
peer_file_name = peer.replace(':', '_')
if not os.path.exists(peer_file_name):
os.makedirs(peer_file_name)
with open(peer_file_name+'/'+peer_file_name + '-graph.json', 'w') as outfile:
json.dump(graph_points[peer], outfile, indent=2)
# Write the last burst of A updates if there is one left
if burst2writeA:
for peer in burst2writeA:
if burst2writeA[peer]:
for timestamp in burst2writeA[peer]:
writeBurst(peer, burst2writeA, 'A', timestamp)
# Write the last burst of W updates if there is one left
if burst2writeW:
for peer in burst2writeW:
if burst2writeW[peer]:
for timestamp in burst2writeW[peer]:
writeBurst(peer, burst2writeW, 'W', timestamp)
# transform csv names in json file to use getJSON in plotGrap
# step to CSV is used to avoid appending to the end of a json file directly as appending
# to a json file overwrite the whole file
jsonlist = []
with open('csv_peernames-'+'-'.join(collectors)+'.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
jsonlist.append(row[0])
jsondata = json.dumps(jsonlist, indent=2)
fd = open('json_file_names-' + '-'.join(collectors) + '.json', 'w')
fd.write(jsondata)
fd.close()
# profile) before using pybgpstream
# export LD_LIBRARY_PATH="/usr/local/pkg/ioda-tools/lib:$LD_LIBRARY_PATH"
# this tutorial is not complete, please
# refer to this page for a complete documentation
#
# https://staff.caida.org/~alistair/pybgpstream/index.html
from _pybgpstream import BGPStream, BGPRecord, BGPElem
import radix
import sys
import calendar
import time
stream = BGPStream()
rec = BGPRecord()
start_interval= 1403042654
end_interval= 1403055754
##stream.add_filter('project', 'ris')
stream.add_filter('collector', 'route-views2')
#stream.add_filter('collector', 'rrc04')
stream.add_filter('record-type', 'ribs')
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(start_interval,end_interval )
stream.add_rib_period_filter(172800)
stream.set_data_interface('mysql')
stream.set_data_interface_option('mysql', 'db-host', 'loki-ge')
stream.set_data_interface_option('mysql', 'db-port', '3306')
stream.set_data_interface_option('mysql', 'db-user', 'bgpstream')
def run(self):
stream = BGPStream()
rec = BGPRecord()
if self.upd_file is None:
stream.add_filter('collector', self.collector)
stream.add_filter('record-type', self.record_type)
stream.add_interval_filter(self.from_date, self.to_date)
stream.set_live_mode()
else:
stream.set_data_interface('singlefile')
if self.upd_file:
stream.set_data_interface_option('singlefile', 'upd-file',
self.upd_file)
if self.rib_file:
stream.set_data_interface_option('singlefile', 'rib-file',
self.rib_file)
if self.prefix_filter is not None:
for prefix in self.prefix_filter:
stream.add_filter('prefix', prefix)
if self.peer_as_filter:
for asn in self.peer_as_filter:
stream.add_filter('peer-asn', str(asn))
if self.communities_filter:
for community in self.communities_filter:
stream.add_filter('community', community)
stream.start()
stream.get_next_record(rec)
prev = rec.time
while (stream.get_next_record(rec)):
now = rec.time
if rec.status == 'valid':
elem = rec.get_next_elem()
while (elem):
statement = None
peer_address = elem.peer_address
peer_asn = elem.peer_asn
if peer_asn in self.asn_to_nexthop:
if elem.type == 'A' or elem.type == 'R':
prefix = elem.fields['prefix']
as_path = elem.fields['as-path']
nexthop = elem.fields['next-hop']
if peer_asn in self.asn_to_nexthop:
nexthop = self.asn_to_nexthop[peer_asn]
statement = 'announce route %s next-hop %s as-path' \
' [ %s ]' % (prefix, nexthop, as_path)
elif elem.type == 'W':
prefix = elem.fields['prefix']
statement = 'withdraw route %s' % prefix
if statement:
sys.stdout.write("%s\n" % statement)
sys.stdout.flush()
elem = rec.get_next_elem()
time.sleep(self.delay + now - prev)
prev = now
#!/usr/bin/env python
# docs at bgpstream.caida.org
from _pybgpstream import BGPStream, BGPRecord, BGPElem
from collections import defaultdict
# export LD_LIBRARY_PATH="/usr/local/lib"
# Create a new bgpstream instance and a reusable bgprecord instance
stream = BGPStream()
rec = BGPRecord()
# Consider RRC12
stream.add_filter('collector','rrc12')
# Consider RIBs dumps only
# stream.add_filter('record-type','ribs')
# Consider this time interval:
# Sat, 01 Aug 2015 7:50:00 GMT - 08:10:00 GMT
stream.add_interval_filter(1438415400,1438416600)
# Start the stream
stream.start()
# Get next record
while(stream.get_next_record(rec)):
elem = rec.get_next_elem()
while(elem):
# only consider RIBs entries and Announcement messages
if elem.type in ["R", "A"]:
