def __init__(
self,
from_time=None,
until_time=None,
data_interface=None,
project=None,
projects=None,
collector=None,
collectors=None,
record_type=None,
record_types=None,
filter=None,
):
# create a low-level bgpstream instance
self.stream = _pybgpstream.BGPStream()
# pass along any config options the user asked for
# time interval (accepts string date/times)
from_epoch = self._datestr_to_epoch(from_time)
until_epoch = self._datestr_to_epoch(until_time)
if from_epoch or until_epoch:
self.stream.add_interval_filter(from_epoch, until_epoch)
if data_interface is not None:
self.stream.set_data_interface(data_interface)
self._maybe_add_filter("project", project, projects)
self._maybe_add_filter("collector", collector, collectors)
self._maybe_add_filter("record-type", record_type, record_types)
if filter is not None:
self.stream.parse_filter_string(filter)
self.started = False
def run(self):
self.cont = True
stream = _pybgpstream.BGPStream()
record = _pybgpstream.BGPRecord()
for collector in self.collectors:
stream.add_filter('collector', collector)
for name, value in self.custom_filters:
stream.add_filter(name, value)
stream.add_interval_filter(self.t_start, self.t_end)
stream.start()
# keep announces and withdrawal
msg_type = ['A', 'W']
while self.cont and stream.get_next_record(record):
if record.status == 'valid':
if record.type == 'update':
elem = record.get_next_elem()
while self.cont and elem:
if elem.type not in msg_type:
elem = record.get_next_elem()
continue
elif elem.type == 'A':
msg = self.convertAnnounce(elem)
self.fifo.put(msg)
elif elem.type == 'W':
msg = self.convertWithdrawal(elem)
self.fifo.put(msg)
elem = record.get_next_elem()
def run(self):
# update redis
ping_redis(redis)
redis.set("bgpstreamkafka_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE)
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# set kafka data interface
stream.set_data_interface("kafka")
# set host connection details
stream.set_data_interface_option("kafka", "brokers",
"{}:{}".format(self.host, self.port))
# set topic
stream.set_data_interface_option("kafka", "topic", self.topic)
# filter prefixes
for prefix in self.prefixes:
stream.add_filter("prefix", prefix)
# build monitored prefix tree
prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128)
}
for prefix in self.prefixes:
ip_version = get_ip_version(prefix)
prefix_tree[ip_version].insert(prefix, "")
# filter record type
stream.add_filter("record-type", "updates")
# filter based on timing (if end=0 --> live mode)
# Bypass for https://github.com/FORTH-ICS-INSPIRE/artemis/issues/411#issuecomment-661325802
start_time = int(time.time()) - START_TIME_OFFSET
if "BGPSTREAM_TIMESTAMP_BYPASS" in os.environ:
log.warn(
"Using BGPSTREAM_TIMESTAMP_BYPASS, meaning BMP timestamps are thrown away from BGPStream"
)
start_time = 0
stream.add_interval_filter(start_time, 0)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
# start producing
validator = MformatValidator()
with Producer(self.connection) as producer:
while True:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
if elem.type in {"A", "W"}:
redis.set(
"bgpstreamkafka_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE,
)
this_prefix = str(elem.fields["prefix"])
service = "bgpstreamkafka|{}".format(str(
rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
} for comm in elem.fields["communities"]]
else:
as_path = []
communities = []
timestamp = float(rec.time)
if timestamp == 0:
timestamp = time.time()
log.debug("fixed timestamp: {}".format(timestamp))
peer_asn = elem.peer_asn
ip_version = get_ip_version(this_prefix)
if this_prefix in prefix_tree[ip_version]:
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
try:
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=self.update_exchange,
routing_key="update",
serializer="ujson",
)
else:
log.debug(
"Invalid format message: {}".format(
msg))
except BaseException:
log.exception(
"Error when normalizing BGP message: {}".
format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run_bgpstream(prefixes_file=None, projects=[], start=0, end=0):
"""
Retrieve all records related to a list of prefixes
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefixes_file: <str> input prefix json
:param start: <int> start timestamp in UNIX epochs
:param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode")
:return: -
"""
prefixes = load_json(prefixes_file)
assert prefixes is not None
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# consider collectors from given projects
for project in projects:
stream.add_filter("project", project)
# filter prefixes
for prefix in prefixes:
stream.add_filter("prefix", prefix)
# filter record type
stream.add_filter("record-type", "updates")
# filter based on timing (if end=0 --> live mode)
stream.add_interval_filter(start, end)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
# print('BGPStream started...')
# print('Projects ' + str(projects))
# print('Prefixes ' + str(prefixes))
# print('Start ' + str(start))
# print('End ' + str(end))
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange(
"bgp-update", channel=connection, type="direct", durable=False
)
exchange.declare()
producer = Producer(connection)
validator = mformat_validator()
while True:
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if elem.type in {"A", "W"}:
this_prefix = str(elem.fields["prefix"])
service = "bgpstream|{}|{}".format(
str(rec.project), str(rec.collector)
)
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [
{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
}
for comm in elem.fields["communities"]
]
else:
as_path = []
communities = []
timestamp = float(rec.time)
peer_asn = elem.peer_asn
for prefix in prefixes:
base_ip, mask_length = this_prefix.split("/")
our_prefix = IPNetwork(prefix)
if (
IPAddress(base_ip) in our_prefix
and int(mask_length) >= our_prefix.prefixlen
):
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=exchange,
routing_key="update",
serializer="json",
)
else:
log.warning("Invalid format message: {}".format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run(self):
# update redis
ping_redis(redis)
redis.set("bgpstreamlive_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE)
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# consider collectors from given projects
for project in self.monitor_projects:
# ignore deprecated projects
if project in DEPRECATED_PROJECTS:
continue
# add the classic project sources
stream.add_filter("project", project)
# plus their real-time counterparts
if project in LIVE_PROJECT_MAPPINGS:
stream.add_filter("project", LIVE_PROJECT_MAPPINGS[project])
# filter prefixes
for prefix in self.prefixes:
stream.add_filter("prefix", prefix)
# build monitored prefix tree
prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128)
}
for prefix in self.prefixes:
ip_version = get_ip_version(prefix)
prefix_tree[ip_version].insert(prefix, "")
# filter record type
stream.add_filter("record-type", "updates")
# filter based on timing (if end=0 --> live mode)
stream.add_interval_filter(int(time.time()) - START_TIME_OFFSET, 0)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
# start producing
validator = MformatValidator()
with Producer(self.connection) as producer:
while True:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
if elem.type in {"A", "W"}:
redis.set(
"bgpstreamlive_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE,
)
this_prefix = str(elem.fields["prefix"])
service = "bgpstreamlive|{}|{}".format(
str(rec.project), str(rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
} for comm in elem.fields["communities"]]
else:
as_path = []
communities = []
timestamp = float(rec.time)
peer_asn = elem.peer_asn
ip_version = get_ip_version(this_prefix)
if this_prefix in prefix_tree[ip_version]:
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
try:
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=self.update_exchange,
routing_key="update",
serializer="ujson",
)
else:
log.warning(
"Invalid format message: {}".format(
msg))
except BaseException:
log.exception(
"Error when normalizing BGP message: {}".
format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run_bgpstream(prefixes=[], projects=[], start=0, end=0):
"""
Retrieve all records related to a list of prefixes
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefix: <str> input prefix
:param start: <int> start timestamp in UNIX epochs
:param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode")
:return: -
"""
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# consider collectors from given projects
for project in projects:
stream.add_filter('project', project)
# filter prefixes
for prefix in prefixes:
stream.add_filter('prefix', prefix)
# filter record type
stream.add_filter('record-type', 'updates')
# filter based on timing (if end=0 --> live mode)
stream.add_interval_filter(start, end)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
# print('BGPStream started...')
# print('Projects ' + str(projects))
# print('Prefixes ' + str(prefixes))
# print('Start ' + str(start))
# print('End ' + str(end))
with Connection(RABBITMQ_HOST) as connection:
exchange = Exchange(
'bgp-update',
channel=connection,
type='direct',
durable=False)
exchange.declare()
producer = Producer(connection)
while True:
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if elem.type in ["A", "W"]:
this_prefix = str(elem.fields['prefix'])
service = "bgpstream|{}|{}".format(
str(rec.project), str(rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields['as-path'].split(' ')
communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])}
for comm in elem.fields['communities']]
else:
as_path = []
communities = []
timestamp = float(rec.time)
peer_asn = elem.peer_asn
for prefix in prefixes:
base_ip, mask_length = this_prefix.split('/')
our_prefix = IPNetwork(prefix)
if IPAddress(base_ip) in our_prefix and int(
mask_length) >= our_prefix.prefixlen:
msg = {
'type': type_,
'timestamp': timestamp,
'path': as_path,
'service': service,
'communities': communities,
'prefix': this_prefix,
'peer_asn': peer_asn
}
if mformat_validator(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=exchange,
routing_key='update',
serializer='json'
)
else:
log.warning('Invalid format message: {}'.format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run_bgpstream_beta_bmp(prefixes=[]):
'''
Retrieve all elements related to a list of prefixes
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefix: <str> input prefix
:return: -
'''
# create a new bgpstream instance
stream = _pybgpstream.BGPStream()
# set BMP data interface
stream.set_data_interface('beta-bmp-stream')
# filter prefixes
for prefix in prefixes:
stream.add_filter('prefix', prefix)
# filter record type
stream.add_filter('record-type', 'updates')
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
with Connection(RABBITMQ_HOST) as connection:
exchange = Exchange(
'bgp-update',
channel=connection,
type='direct',
durable=False)
exchange.declare()
producer = Producer(connection)
while True:
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != 'valid') or (rec.type != 'update'):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if elem.type in ['A', 'W']:
this_prefix = str(elem.fields['prefix'])
service = 'betabmp|{}|{}'.format(
str(rec.project), str(rec.collector))
type_ = elem.type
if type_ == 'A':
as_path = elem.fields['as-path'].split(' ')
communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])}
for comm in elem.fields['communities']]
else:
as_path = []
communities = []
timestamp = float(rec.time)
peer_asn = elem.peer_asn
for prefix in prefixes:
base_ip, mask_length = this_prefix.split('/')
our_prefix = IPNetwork(prefix)
if IPAddress(base_ip) in our_prefix and int(
mask_length) >= our_prefix.prefixlen:
msg = {
'type': type_,
'timestamp': timestamp,
'path': as_path,
'service': service,
'communities': communities,
'prefix': this_prefix,
'peer_asn': peer_asn
}
if mformat_validator(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=exchange,
routing_key='update',
serializer='json'
)
else:
log.warning('Invalid format message: {}'.format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run_bgpstream(
prefixes_file=None,
kafka_host=None,
kafka_port=None,
kafka_topic="openbmp.bmp_raw",
start=0,
end=0,
):
"""
Retrieve all records related to a list of prefixes
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefixes_file: <str> input prefix json
:param kafka_host: <str> kafka host
:param kafka_port: <int> kafka_port
:param kafka_topic: <str> kafka topic
:param start: <int> start timestamp in UNIX epochs
:param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode")
:return: -
"""
prefixes = load_json(prefixes_file)
assert prefixes is not None
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# set kafka data interface
stream.set_data_interface("kafka")
# set host connection details
stream.set_data_interface_option("kafka", "brokers",
"{}:{}".format(kafka_host, kafka_port))
# set topic
stream.set_data_interface_option("kafka", "topic", kafka_topic)
# filter prefixes
for prefix in prefixes:
stream.add_filter("prefix", prefix)
# filter record type
stream.add_filter("record-type", "updates")
# filter based on timing (if end=0 --> live mode)
stream.add_interval_filter(start, end)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
producer = Producer(connection)
validator = mformat_validator()
while True:
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if elem.type in {"A", "W"}:
redis.set(
"bgpstreamkafka_seen_bgp_update",
"1",
ex=int(
os.getenv(
"MON_TIMEOUT_LAST_BGP_UPDATE",
DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE,
)),
)
this_prefix = str(elem.fields["prefix"])
service = "bgpstreamkafka|{}".format(str(rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
} for comm in elem.fields["communities"]]
else:
as_path = []
communities = []
timestamp = float(rec.time)
if timestamp == 0:
timestamp = time.time()
log.debug("fixed timestamp: {}".format(timestamp))
peer_asn = elem.peer_asn
for prefix in prefixes:
base_ip, mask_length = this_prefix.split("/")
our_prefix = IPNetwork(prefix)
if (IPAddress(base_ip) in our_prefix
and int(mask_length) >= our_prefix.prefixlen):
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
try:
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=exchange,
routing_key="update",
serializer="ujson",
)
else:
log.warning(
"Invalid format message: {}".format(
msg))
except BaseException:
log.exception(
"Error when normalizing BGP message: {}".
format(msg))
break
try:
elem = rec.get_next_elem()
except BaseException:
continue
def run_bgpstream_beta_bmp(prefixes_file=None):
"""
Retrieve all elements related to a list of prefixes
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefixes_file: <str> input prefix json
:return: -
"""
prefixes = load_json(prefixes_file)
assert prefixes is not None
# create a new bgpstream instance
stream = _pybgpstream.BGPStream()
# set BMP data interface
stream.set_data_interface("beta-bmp-stream")
# filter prefixes
for prefix in prefixes:
stream.add_filter("prefix", prefix)
# filter record type
stream.add_filter("record-type", "updates")
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
producer = Producer(connection)
validator = mformat_validator()
while True:
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if elem.type in {"A", "W"}:
redis.set(
"betabmp_seen_bgp_update",
"1",
ex=int(
os.getenv(
"MON_TIMEOUT_LAST_BGP_UPDATE",
DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE,
)),
)
this_prefix = str(elem.fields["prefix"])
service = "betabmp|{}|{}".format(str(rec.project),
str(rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
} for comm in elem.fields["communities"]]
else:
as_path = []
communities = []
timestamp = float(rec.time)
peer_asn = elem.peer_asn
for prefix in prefixes:
base_ip, mask_length = this_prefix.split("/")
our_prefix = IPNetwork(prefix)
if (IPAddress(base_ip) in our_prefix
and int(mask_length) >= our_prefix.prefixlen):
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=exchange,
routing_key="update",
serializer="json",
)
else:
log.warning(
"Invalid format message: {}".format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
