def __init__( self, from_time=None, until_time=None, data_interface=None, project=None, projects=None, collector=None, collectors=None, record_type=None, record_types=None, filter=None, ): # create a low-level bgpstream instance self.stream = _pybgpstream.BGPStream() # pass along any config options the user asked for # time interval (accepts string date/times) from_epoch = self._datestr_to_epoch(from_time) until_epoch = self._datestr_to_epoch(until_time) if from_epoch or until_epoch: self.stream.add_interval_filter(from_epoch, until_epoch) if data_interface is not None: self.stream.set_data_interface(data_interface) self._maybe_add_filter("project", project, projects) self._maybe_add_filter("collector", collector, collectors) self._maybe_add_filter("record-type", record_type, record_types) if filter is not None: self.stream.parse_filter_string(filter) self.started = False
def run(self): self.cont = True stream = _pybgpstream.BGPStream() record = _pybgpstream.BGPRecord() for collector in self.collectors: stream.add_filter('collector', collector) for name, value in self.custom_filters: stream.add_filter(name, value) stream.add_interval_filter(self.t_start, self.t_end) stream.start() # keep announces and withdrawal msg_type = ['A', 'W'] while self.cont and stream.get_next_record(record): if record.status == 'valid': if record.type == 'update': elem = record.get_next_elem() while self.cont and elem: if elem.type not in msg_type: elem = record.get_next_elem() continue elif elem.type == 'A': msg = self.convertAnnounce(elem) self.fifo.put(msg) elif elem.type == 'W': msg = self.convertWithdrawal(elem) self.fifo.put(msg) elem = record.get_next_elem()
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # set kafka data interface stream.set_data_interface("kafka") # set host connection details stream.set_data_interface_option("kafka", "brokers", "{}:{}".format(self.host, self.port)) # set topic stream.set_data_interface_option("kafka", "topic", self.topic) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) # Bypass for https://github.com/FORTH-ICS-INSPIRE/artemis/issues/411#issuecomment-661325802 start_time = int(time.time()) - START_TIME_OFFSET if "BGPSTREAM_TIMESTAMP_BYPASS" in os.environ: log.warn( "Using BGPSTREAM_TIMESTAMP_BYPASS, meaning BMP timestamps are thrown away from BGPStream" ) start_time = 0 stream.add_interval_filter(start_time, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamkafka|{}".format(str( rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) if timestamp == 0: timestamp = time.time() log.debug("fixed timestamp: {}".format(timestamp)) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.debug( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream(prefixes_file=None, projects=[], start=0, end=0): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in projects: stream.add_filter("project", project) # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() # print('BGPStream started...') # print('Projects ' + str(projects)) # print('Prefixes ' + str(prefixes)) # print('Start ' + str(start)) # print('End ' + str(end)) with Connection(RABBITMQ_URI) as connection: exchange = Exchange( "bgp-update", channel=connection, type="direct", durable=False ) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: this_prefix = str(elem.fields["prefix"]) service = "bgpstream|{}|{}".format( str(rec.project), str(rec.collector) ) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [ { "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"] ] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if ( IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen ): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="json", ) else: log.warning("Invalid format message: {}".format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in self.monitor_projects: # ignore deprecated projects if project in DEPRECATED_PROJECTS: continue # add the classic project sources stream.add_filter("project", project) # plus their real-time counterparts if project in LIVE_PROJECT_MAPPINGS: stream.add_filter("project", LIVE_PROJECT_MAPPINGS[project]) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(int(time.time()) - START_TIME_OFFSET, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamlive|{}|{}".format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream(prefixes=[], projects=[], start=0, end=0): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefix: <str> input prefix :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in projects: stream.add_filter('project', project) # filter prefixes for prefix in prefixes: stream.add_filter('prefix', prefix) # filter record type stream.add_filter('record-type', 'updates') # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() # print('BGPStream started...') # print('Projects ' + str(projects)) # print('Prefixes ' + str(prefixes)) # print('Start ' + str(start)) # print('End ' + str(end)) with Connection(RABBITMQ_HOST) as connection: exchange = Exchange( 'bgp-update', channel=connection, type='direct', durable=False) exchange.declare() producer = Producer(connection) while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in ["A", "W"]: this_prefix = str(elem.fields['prefix']) service = "bgpstream|{}|{}".format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields['as-path'].split(' ') communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])} for comm in elem.fields['communities']] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split('/') our_prefix = IPNetwork(prefix) if IPAddress(base_ip) in our_prefix and int( mask_length) >= our_prefix.prefixlen: msg = { 'type': type_, 'timestamp': timestamp, 'path': as_path, 'service': service, 'communities': communities, 'prefix': this_prefix, 'peer_asn': peer_asn } if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key='update', serializer='json' ) else: log.warning('Invalid format message: {}'.format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream_beta_bmp(prefixes=[]): ''' Retrieve all elements related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefix: <str> input prefix :return: - ''' # create a new bgpstream instance stream = _pybgpstream.BGPStream() # set BMP data interface stream.set_data_interface('beta-bmp-stream') # filter prefixes for prefix in prefixes: stream.add_filter('prefix', prefix) # filter record type stream.add_filter('record-type', 'updates') # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_HOST) as connection: exchange = Exchange( 'bgp-update', channel=connection, type='direct', durable=False) exchange.declare() producer = Producer(connection) while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != 'valid') or (rec.type != 'update'): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in ['A', 'W']: this_prefix = str(elem.fields['prefix']) service = 'betabmp|{}|{}'.format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == 'A': as_path = elem.fields['as-path'].split(' ') communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])} for comm in elem.fields['communities']] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split('/') our_prefix = IPNetwork(prefix) if IPAddress(base_ip) in our_prefix and int( mask_length) >= our_prefix.prefixlen: msg = { 'type': type_, 'timestamp': timestamp, 'path': as_path, 'service': service, 'communities': communities, 'prefix': this_prefix, 'peer_asn': peer_asn } if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key='update', serializer='json' ) else: log.warning('Invalid format message: {}'.format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream( prefixes_file=None, kafka_host=None, kafka_port=None, kafka_topic="openbmp.bmp_raw", start=0, end=0, ): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :param kafka_host: <str> kafka host :param kafka_port: <int> kafka_port :param kafka_topic: <str> kafka topic :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # set kafka data interface stream.set_data_interface("kafka") # set host connection details stream.set_data_interface_option("kafka", "brokers", "{}:{}".format(kafka_host, kafka_port)) # set topic stream.set_data_interface_option("kafka", "topic", kafka_topic) # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_URI) as connection: exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: redis.set( "bgpstreamkafka_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamkafka|{}".format(str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) if timestamp == 0: timestamp = time.time() log.debug("fixed timestamp: {}".format(timestamp)) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) break try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream_beta_bmp(prefixes_file=None): """ Retrieve all elements related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance stream = _pybgpstream.BGPStream() # set BMP data interface stream.set_data_interface("beta-bmp-stream") # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_URI) as connection: exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: redis.set( "betabmp_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) this_prefix = str(elem.fields["prefix"]) service = "betabmp|{}|{}".format(str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="json", ) else: log.warning( "Invalid format message: {}".format(msg)) try: elem = rec.get_next_elem() except BaseException: continue