def exabgp_msg(bgp_message): redis.set( "exabgp_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, ) ), ) msg = { "type": bgp_message["type"], "communities": bgp_message.get("communities", []), "timestamp": float(bgp_message["timestamp"]), "path": bgp_message.get("path", []), "service": "exabgp|{}".format(self.host), "prefix": bgp_message["prefix"], "peer_asn": int(bgp_message["peer_asn"]), } if validator.validate(msg): with Producer(connection) as producer: msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.exchange, routing_key="update", serializer="json", ) else: log.warning("Invalid format message: {}".format(msg))
def gen_implicit_withdrawal(self, monitor_event: Dict) -> NoReturn: """ Checks if a benign BGP update should trigger an implicit withdrawal """ # log.debug('{}'.format(monitor_event['key'])) prefix = monitor_event["prefix"] peer_asn = monitor_event["peer_asn"] if self.redis.exists("prefix_{}_peer_{}_hijacks".format(prefix, peer_asn)): # generate implicit withdrawal withdraw_msg = { "service": "implicit-withdrawal", "type": "W", "prefix": prefix, "path": [], "orig_path": {"triggering_bgp_update": monitor_event}, "communities": [], "timestamp": monitor_event["timestamp"] + 1, "peer_asn": peer_asn, } key_generator(withdraw_msg) self.producer.publish( withdraw_msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", )
def test_key_generator(): ret = key_generator('hello', 'world') assert ret == 'hello,world' ret = key_generator('hello') assert ret == 'hello' ret = key_generator('Hi', 'hello', 'world', delimiter=', ') assert ret == 'Hi, hello, world'
def test_keygen(): key = utils.key_generator() assert len(key) == 6, "Key length doesn't match expected" assert key.isalnum(), "Key characters don't match expected" key = utils.key_generator(size=10) assert len(key) == 10, "Key length parameter doesn't match returned length" assert key.isalnum(), "Key characters don't match expected" key = utils.key_generator(chars=string.digits) assert len(key) == 6, "Key length doesn't match expected" assert key.isdecimal(), "Key characters paramter don't match returned key"
def _build_developer_branch_rel(self, branch): '''Build the relationship between developer and branch DeveloperBranch is the core relationship between developer and branch, and also a core concept to GitView to query and generate report. Developers will created for you if they does not exist in GitView already. ''' branch_name = branch.name remote_branch_name = git.build_remote_branch_name(branch_name) contributors = self.git_repo.get_contributors(remote_branch_name) qs = DeveloperBranch.objects.filter( branch__project=self.project).values('developer__full_name', 'branch__name') existing_rels = set((key_generator(rel['developer__full_name'], rel['branch__name']) for rel in qs.iterator())) get_developer = Developer.objects.get create_developer = Developer.objects.create create_rel = DeveloperBranch.objects.create add_developer_to_project = self.project.developers.add for contributor in contributors: new_rel_key = key_generator(contributor.full_name, branch_name) if new_rel_key in existing_rels: continue try: developer = get_developer(full_name=contributor.full_name, email=contributor.email) except Developer.DoesNotExist: kerb_name = contributor.email.split('@')[0] developer = create_developer(kerb_name=kerb_name, email=contributor.email, full_name=contributor.full_name) add_developer_to_project(developer) # Finally, we have enough things, developer and branch, to # build the relationship. That's great! create_rel(developer=developer, branch=branch) # Remember this newly added relationship, next iterations would # check the existence of relationship between developer and branch existing_rels.add(new_rel_key)
def on_ris_msg(msg): try: producer = Producer(connection) normalize_ripe_ris(msg) if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish(msg, exchange=exchange, routing_key='update', serializer='json') else: log.warning('Invalid format message: {}'.format(msg)) except Exception: log.exception('exception')
def _sync_tags(self): '''Add new tags''' # Useful for getting which branches a tag is related to. # Any good solution? self.git_repo.create_local_branches() logger.info('Local branches are created.') qs = Tag.objects.filter(project=self.project).values('name', 'project') existing_tags = set((key_generator(str(tag['project']), tag['name']) for tag in qs.iterator())) remote_tags = set((key_generator(str(self.project.pk), name) for name in self.git_repo.remote_tag_names())) new_tags = set(remote_tags) - set(existing_tags) if not new_tags: logger.info('No new tags to synchronize') return new_tags = (tag.split(',')[1] for tag in new_tags) create_tag = Tag.objects.create branch_manager = Branch.objects # Start to add new tags for tag in self.git_repo.tags(new_tags): if isinstance(tag, git.InvalidTagError): msg = 'Skip to synchronize invalid tag {0}'.format(tag.data) logger.warning(msg) continue else: logger.info('Synchronize tag {0}'.format(tag.name)) model = create_tag(name=tag.name, submit_date=tag.created_on, project=self.project) for branch_name in tag.branch_names: try: branch = branch_manager.get(project=self.project, name=branch_name) except Branch.DoesNotExist: branch = branch_manager.create(project=self.project, name=branch_name) model.branches.add(branch)
def _prepare_global_data(self): '''Prepare global data used while syncing against a project''' qs = Developer.objects.filter( projects=self.project).values('full_name', 'email') # Text containing contributor's name and email from terminal is encoded # in UTF-8 by default. So, you must have already realized why full_name # and email retrieved from database must be encoded in UTF-8 too. :) self._existing_contributors = set( (key_generator(d['full_name'].encode('utf-8'), d['email'].encode('utf-8')) for d in qs.iterator()))
async def purge(self, ctx, number: typing.Union[discord.Member, int, str] = 10, key=None): """Purges messages from a channel. By default, this will be 10 (including the invoking command)."""\ """ Use 'all' to purge whole channel. Confirmation keys should be tacked on the end, so """\ """`^purge 100 [key]`""" if isinstance(number, discord.Member): await ctx.send("Purging a member's messages not yet supported") elif isinstance(number, int): if number >= 100 and (key is None or key != secure_keys[str(ctx.guild.id)]): rand_key = utils.key_generator() secure_keys[str(ctx.guild.id)] = rand_key await ctx.send(f"Are you sure? If so, re-invoke with {rand_key} on the end.") else: await ctx.channel.purge(limit=number) elif number == "all": if key is None or key != secure_keys[str(ctx.guild.id)]: rand_key = utils.key_generator() secure_keys[str(ctx.guild.id)] = rand_key await ctx.send(f"Are you sure? If so, re-invoke with {rand_key} on the end.") elif key == secure_keys[str(ctx.guild.id)]: await ctx.channel.purge(limit=None) secure_keys[str(ctx.guild.id)] = ""
def exabgp_msg(bgp_message): msg = { 'type': bgp_message['type'], 'communities': bgp_message.get('communities', []), 'timestamp': float(bgp_message['timestamp']), 'path': bgp_message.get('path', []), 'service': 'exabgp|{}'.format(self.host), 'prefix': bgp_message['prefix'], 'peer_asn': int(bgp_message['peer_asn']) } if mformat_validator(msg): with Producer(connection) as producer: msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish(msg, exchange=self.exchange, routing_key='update', serializer='json') else: log.warning('Invalid format message: {}'.format(msg))
def run_bgpstream( prefixes_file=None, kafka_host=None, kafka_port=None, kafka_topic="openbmp.bmp_raw", start=0, end=0, ): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :param kafka_host: <str> kafka host :param kafka_port: <int> kafka_port :param kafka_topic: <str> kafka topic :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # set kafka data interface stream.set_data_interface("kafka") # set host connection details stream.set_data_interface_option("kafka", "brokers", "{}:{}".format(kafka_host, kafka_port)) # set topic stream.set_data_interface_option("kafka", "topic", kafka_topic) # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_URI) as connection: exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: redis.set( "bgpstreamkafka_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamkafka|{}".format(str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) if timestamp == 0: timestamp = time.time() log.debug("fixed timestamp: {}".format(timestamp)) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) break try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream(prefixes=[], projects=[], start=0, end=0): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefix: <str> input prefix :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in projects: stream.add_filter('project', project) # filter prefixes for prefix in prefixes: stream.add_filter('prefix', prefix) # filter record type stream.add_filter('record-type', 'updates') # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() # print('BGPStream started...') # print('Projects ' + str(projects)) # print('Prefixes ' + str(prefixes)) # print('Start ' + str(start)) # print('End ' + str(end)) with Connection(RABBITMQ_HOST) as connection: exchange = Exchange( 'bgp-update', channel=connection, type='direct', durable=False) exchange.declare() producer = Producer(connection) while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in ["A", "W"]: this_prefix = str(elem.fields['prefix']) service = "bgpstream|{}|{}".format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields['as-path'].split(' ') communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])} for comm in elem.fields['communities']] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split('/') our_prefix = IPNetwork(prefix) if IPAddress(base_ip) in our_prefix and int( mask_length) >= our_prefix.prefixlen: msg = { 'type': type_, 'timestamp': timestamp, 'path': as_path, 'service': service, 'communities': communities, 'prefix': this_prefix, 'peer_asn': peer_asn } if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key='update', serializer='json' ) else: log.warning('Invalid format message: {}'.format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def parse_ripe_ris(connection, prefixes_file, hosts): exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() prefixes = load_json(prefixes_file) assert prefixes is not None prefix_tree = radix.Radix() for prefix in prefixes: prefix_tree.add(prefix) ris_suffix = os.getenv("RIS_ID", "my_as") validator = mformat_validator() with Producer(connection) as producer: while True: try: events = requests.get( "https://ris-live.ripe.net/v1/stream/?format=json&client=artemis-{}".format( ris_suffix ), stream=True, timeout=10, ) # http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests iterator = events.iter_lines() next(iterator) for data in iterator: try: parsed = json.loads(data) msg = parsed["data"] if "type" in parsed and parsed["type"] == "ris_error": log.error(msg) # also check if ris host is in the configuration elif ( "type" in msg and msg["type"] == "UPDATE" and (not hosts or msg["host"] in hosts) ): norm_ris_msgs = normalize_ripe_ris(msg, prefix_tree) for norm_ris_msg in norm_ris_msgs: redis.set( "ris_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, ) ), ) if validator.validate(norm_ris_msg): norm_path_msgs = normalize_msg_path(norm_ris_msg) for norm_path_msg in norm_path_msgs: key_generator(norm_path_msg) log.debug(norm_path_msg) producer.publish( norm_path_msg, exchange=exchange, routing_key="update", serializer="json", ) else: log.warning( "Invalid format message: {}".format(msg) ) except json.decoder.JSONDecodeError: log.exception("Message {}".format(data)) except Exception: log.exception("exception") log.warning("Iterator ran out of data; the connection will be retried") except Exception: log.exception("server closed connection") time.sleep(5)
def parse_bgpstreamhist_csvs(prefixes_file=None, input_dir=None): prefixes = load_json(prefixes_file) assert prefixes is not None with Connection(RABBITMQ_URI) as connection: exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() producer = Producer(connection) validator = mformat_validator() for csv_file in glob.glob("{}/*.csv".format(input_dir)): try: with open(csv_file, "r") as f: csv_reader = csv.reader(f, delimiter="|") for row in csv_reader: try: if len(row) != 9: continue if row[0].startswith("#"): continue # example row: 139.91.0.0/16|8522|1403|1403 6461 2603 21320 # 5408 # 8522|routeviews|route-views2|A|"[{""asn"":1403,""value"":6461}]"|1517446677 this_prefix = row[0] if row[6] == "A": as_path = row[3].split(" ") communities = json.loads(row[7]) else: as_path = [] communities = [] service = "historical|{}|{}".format(row[4], row[5]) type_ = row[6] timestamp = float(row[8]) peer_asn = int(row[2]) for prefix in prefixes: try: base_ip, mask_length = this_prefix.split( "/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="json", ) time.sleep(0.1) else: log.warning( "Invalid format message: {}". format(msg)) except Exception: log.exception("prefix") except Exception: log.exception("row") except Exception: log.exception("exception")
def parse_bgpstreamhist_csvs(self): # add /0 if autoconf if self.autoconf: self.prefixes.append("0.0.0.0/0") self.prefixes.append("::/0") with Connection(RABBITMQ_URI) as connection: self.update_exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) self.update_exchange.declare() producer = Producer(connection) validator = mformat_validator() for csv_file in glob.glob("{}/*.csv".format(self.input_dir)): try: with open(csv_file, "r") as f: csv_reader = csv.reader(f, delimiter="|") for row in csv_reader: try: if len(row) != 9: continue if row[0].startswith("#"): continue # example row: 139.91.0.0/16|8522|1403|1403 6461 2603 21320 # 5408 # 8522|routeviews|route-views2|A|"[{""asn"":1403,""value"":6461}]"|1517446677 this_prefix = row[0] if row[6] == "A": as_path = row[3].split(" ") communities = json.loads(row[7]) else: as_path = [] communities = [] service = "historical|{}|{}".format( row[4], row[5]) type_ = row[6] timestamp = float(row[8]) peer_asn = int(row[2]) for prefix in self.prefixes: try: base_ip, mask_length = this_prefix.split( "/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) if self.autoconf: if str(our_prefix) in [ "0.0.0.0/0", "::/0", ]: if msg["type"] == "A": as_path = clean_as_path( msg["path"] ) if len(as_path ) > 1: # ignore, since this is not a self-network origination, but sth transit break elif msg[ "type"] == "W": # ignore irrelevant withdrawals break self.autoconf_goahead = False correlation_id = uuid() callback_queue = Queue( uuid(), durable=False, auto_delete=True, max_priority=4, consumer_arguments={ "x-priority": 4 }, ) producer.publish( msg, exchange="", routing_key= "conf-autoconf-update-queue", reply_to= callback_queue. name, correlation_id= correlation_id, retry=True, declare=[ Queue( "conf-autoconf-update-queue", durable= False, max_priority =4, consumer_arguments ={ "x-priority": 4 }, ), callback_queue, ], priority=4, serializer="ujson", ) with Consumer( connection, on_message=self . handle_autoconf_update_goahead_reply, queues= [callback_queue], accept=[ "ujson" ], ): while (not self. autoconf_goahead ): connection.drain_events( ) producer.publish( msg, exchange=self. update_exchange, routing_key="update", serializer="ujson", ) time.sleep(0.1) else: log.warning( "Invalid format message: {}" .format(msg)) break except Exception: log.exception("prefix") except Exception: log.exception("row") except Exception: log.exception("exception")
def exabgp_msg(bgp_message): redis.set( "exabgp_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) msg = { "type": bgp_message["type"], "communities": bgp_message.get("communities", []), "timestamp": float(bgp_message["timestamp"]), "path": bgp_message.get("path", []), "service": "exabgp|{}".format(self.host), "prefix": bgp_message["prefix"], "peer_asn": int(bgp_message["peer_asn"]), } for prefix in self.prefixes: try: base_ip, mask_length = bgp_message["prefix"].split( "/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): if validator.validate(msg): with Producer(connection) as producer: msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) if self.autoconf: if str(our_prefix) in [ "0.0.0.0/0", "::/0", ]: if msg["type"] == "A": as_path = clean_as_path( msg["path"]) if len(as_path) > 1: # ignore, since this is not a self-network origination, but sth transit break elif msg["type"] == "W": # ignore irrelevant withdrawals break self.autoconf_goahead = False correlation_id = uuid() callback_queue = Queue( uuid(), durable=False, auto_delete=True, max_priority=4, consumer_arguments={ "x-priority": 4 }, ) producer.publish( msg, exchange="", routing_key= "conf-autoconf-update-queue", reply_to=callback_queue. name, correlation_id= correlation_id, retry=True, declare=[ Queue( "conf-autoconf-update-queue", durable=False, max_priority=4, consumer_arguments={ "x-priority": 4 }, ), callback_queue, ], priority=4, serializer="ujson", ) with Consumer( connection, on_message=self. handle_autoconf_update_goahead_reply, queues=[ callback_queue ], accept=["ujson"], ): while not self.autoconf_goahead: connection.drain_events( ) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) break except Exception: log.exception("exception")
def run_bgpstream(prefixes_file=None, projects=[], start=0, end=0): """ Retrieve all records related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :param start: <int> start timestamp in UNIX epochs :param end: <int> end timestamp in UNIX epochs (if 0 --> "live mode") :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in projects: stream.add_filter("project", project) # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(start, end) # set live mode stream.set_live_mode() # start the stream stream.start() # print('BGPStream started...') # print('Projects ' + str(projects)) # print('Prefixes ' + str(prefixes)) # print('Start ' + str(start)) # print('End ' + str(end)) with Connection(RABBITMQ_URI) as connection: exchange = Exchange( "bgp-update", channel=connection, type="direct", durable=False ) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: this_prefix = str(elem.fields["prefix"]) service = "bgpstream|{}|{}".format( str(rec.project), str(rec.collector) ) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [ { "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"] ] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if ( IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen ): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="json", ) else: log.warning("Invalid format message: {}".format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def parse_bgpstreamhist_csvs(prefixes=[], input_dir=None): with Connection(RABBITMQ_HOST) as connection: exchange = Exchange('bgp-update', channel=connection, type='direct', durable=False) exchange.declare() producer = Producer(connection) for csv_file in glob.glob("{}/*.csv".format(input_dir)): try: with open(csv_file, 'r') as f: csv_reader = csv.reader(f, delimiter="|") for row in csv_reader: try: if len(row) != 9: continue if row[0].startswith('#'): continue # example row: 139.91.0.0/16|8522|1403|1403 6461 2603 21320 # 5408 # 8522|routeviews|route-views2|A|"[{""asn"":1403,""value"":6461}]"|1517446677 this_prefix = row[0] if row[6] == 'A': as_path = row[3].split(' ') communities = json.loads(row[7]) else: as_path = [] communities = [] service = "historical|{}|{}".format(row[4], row[5]) type_ = row[6] timestamp = float(row[8]) peer_asn = int(row[2]) for prefix in prefixes: try: base_ip, mask_length = this_prefix.split( '/') our_prefix = IPNetwork(prefix) if IPAddress( base_ip) in our_prefix and int( mask_length ) >= our_prefix.prefixlen: msg = { 'type': type_, 'timestamp': timestamp, 'path': as_path, 'service': service, 'communities': communities, 'prefix': this_prefix, 'peer_asn': peer_asn } if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key='update', serializer='json') else: log.warning( 'Invalid format message: {}'. format(msg)) except Exception: log.exception('prefix') except Exception: log.exception('row') except Exception: log.exception('exception')
def run_bgpstream_beta_bmp(prefixes=[]): ''' Retrieve all elements related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefix: <str> input prefix :return: - ''' # create a new bgpstream instance stream = _pybgpstream.BGPStream() # set BMP data interface stream.set_data_interface('beta-bmp-stream') # filter prefixes for prefix in prefixes: stream.add_filter('prefix', prefix) # filter record type stream.add_filter('record-type', 'updates') # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_HOST) as connection: exchange = Exchange( 'bgp-update', channel=connection, type='direct', durable=False) exchange.declare() producer = Producer(connection) while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != 'valid') or (rec.type != 'update'): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in ['A', 'W']: this_prefix = str(elem.fields['prefix']) service = 'betabmp|{}|{}'.format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == 'A': as_path = elem.fields['as-path'].split(' ') communities = [{'asn': int(comm.split(':')[0]), 'value': int(comm.split(':')[1])} for comm in elem.fields['communities']] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split('/') our_prefix = IPNetwork(prefix) if IPAddress(base_ip) in our_prefix and int( mask_length) >= our_prefix.prefixlen: msg = { 'type': type_, 'timestamp': timestamp, 'path': as_path, 'service': service, 'communities': communities, 'prefix': this_prefix, 'peer_asn': peer_asn } if mformat_validator(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key='update', serializer='json' ) else: log.warning('Invalid format message: {}'.format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def run_bgpstream_beta_bmp(prefixes_file=None): """ Retrieve all elements related to a list of prefixes https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html :param prefixes_file: <str> input prefix json :return: - """ prefixes = load_json(prefixes_file) assert prefixes is not None # create a new bgpstream instance stream = _pybgpstream.BGPStream() # set BMP data interface stream.set_data_interface("beta-bmp-stream") # filter prefixes for prefix in prefixes: stream.add_filter("prefix", prefix) # filter record type stream.add_filter("record-type", "updates") # set live mode stream.set_live_mode() # start the stream stream.start() with Connection(RABBITMQ_URI) as connection: exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() producer = Producer(connection) validator = mformat_validator() while True: # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if elem.type in {"A", "W"}: redis.set( "betabmp_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) this_prefix = str(elem.fields["prefix"]) service = "betabmp|{}|{}".format(str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn for prefix in prefixes: base_ip, mask_length = this_prefix.split("/") our_prefix = IPNetwork(prefix) if (IPAddress(base_ip) in our_prefix and int(mask_length) >= our_prefix.prefixlen): msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=exchange, routing_key="update", serializer="json", ) else: log.warning( "Invalid format message: {}".format(msg)) try: elem = rec.get_next_elem() except BaseException: continue