def handle_mitigation_request(self, message): message.ack() hijack_event = message.payload ip_version = get_ip_version(hijack_event["prefix"]) if hijack_event["prefix"] in self.prefix_tree[ip_version]: prefix_node = self.prefix_tree[ip_version][ hijack_event["prefix"]] mitigation_action = prefix_node["data"]["mitigation"][0] if mitigation_action == "manual": log.info("starting manual mitigation of hijack {}".format( hijack_event)) else: log.info( "starting custom mitigation of hijack {} using '{}' script" .format(hijack_event, mitigation_action)) hijack_event_str = json.dumps(hijack_event) subprocess.Popen( [mitigation_action, "-i", hijack_event_str], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) # do something mit_started = {"key": hijack_event["key"], "time": time.time()} self.producer.publish( mit_started, exchange=self.mitigation_exchange, routing_key="mit-start", priority=2, serializer="ujson", ) else: log.warn("no rule for hijack {}".format(hijack_event))
def init_mitigation(self): log.info("Initiating mitigation...") log.info("Starting building mitigation prefix tree...") self.prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128), } raw_prefix_count = 0 for rule in self.rules: try: for prefix in rule["prefixes"]: for translated_prefix in translate_rfc2622(prefix): ip_version = get_ip_version(translated_prefix) node = { "prefix": translated_prefix, "data": { "mitigation": rule["mitigation"] }, } self.prefix_tree[ip_version].insert( translated_prefix, node) raw_prefix_count += 1 except Exception: log.exception("Exception") log.info( "{} prefixes integrated in mitigation prefix tree in total". format(raw_prefix_count)) log.info("Finished building mitigation prefix tree.") log.info("Mitigation initiated, configured and running.")
def build_prefix_tree(self): log.info("Starting building autoignore prefix tree...") self.prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128), } raw_prefix_count = 0 for key in self.autoignore_rules: try: rule = self.autoignore_rules[key] for prefix in rule["prefixes"]: for translated_prefix in translate_rfc2622(prefix): ip_version = get_ip_version(translated_prefix) if self.prefix_tree[ip_version].has_key(translated_prefix): node = self.prefix_tree[ip_version][translated_prefix] else: node = {"prefix": translated_prefix, "rule_key": key} self.prefix_tree[ip_version].insert( translated_prefix, node ) raw_prefix_count += 1 except Exception: log.exception("Exception") log.info( "{} prefixes integrated in autoignore prefix tree in total".format( raw_prefix_count ) ) log.info("Finished building autoignore prefix tree.")
def start_monitors(self): log.info("Initiating monitor...") for proc_id in self.process_ids: try: proc_id[1].terminate() except ProcessLookupError: log.exception("process terminate") self.process_ids.clear() self.prefixes.clear() log.info("Starting building monitor prefix tree...") self.prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128), } raw_prefix_count = 0 for rule in self.rules: try: for prefix in rule["prefixes"]: for translated_prefix in translate_rfc2622(prefix): ip_version = get_ip_version(translated_prefix) self.prefix_tree[ip_version].insert(translated_prefix, "") raw_prefix_count += 1 except Exception: log.exception("Exception") log.info( "{} prefixes integrated in monitor prefix tree in total".format( raw_prefix_count ) ) log.info("Finished building monitor prefix tree.") # only keep super prefixes for monitors log.info("Calculating monitored prefixes for monitor to supervise...") for ip_version in self.prefix_tree: for prefix in self.prefix_tree[ip_version]: worst_prefix = search_worst_prefix( prefix, self.prefix_tree[ip_version] ) if worst_prefix: self.prefixes.add(worst_prefix) dump_json(list(self.prefixes), self.prefix_file) log.info("Calculated monitored prefixes for monitor to supervise.") log.info("Initiating configured monitoring instances....") self.init_ris_instance() self.init_exabgp_instance() self.init_bgpstreamhist_instance() self.init_bgpstreamlive_instance() self.init_bgpstreamkafka_instance() log.info("All configured monitoring instances initiated.") log.info("Monitor initiated, configured and running.")
def find_prefix_node(self, prefix): ip_version = get_ip_version(prefix) prefix_node = None shared_memory_locks["prefix_tree"].acquire() if ip_version == "v4": size = 32 else: size = 128 # need to turn to pytricia tree since this means that the tree has changed due to re-configuration if self.shared_memory_manager_dict["prefix_tree_recalculate"]: self.prefix_tree[ip_version] = dict_to_pytricia( self.shared_memory_manager_dict["prefix_tree"][ip_version], size ) log.info("{} pytricia tree re-parsed from configuration".format(ip_version)) self.shared_memory_manager_dict["prefix_tree_recalculate"] = False if prefix in self.prefix_tree[ip_version]: prefix_node = self.prefix_tree[ip_version][prefix] shared_memory_locks["prefix_tree"].release() return prefix_node
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # set kafka data interface stream.set_data_interface("kafka") # set host connection details stream.set_data_interface_option("kafka", "brokers", "{}:{}".format(self.host, self.port)) # set topic stream.set_data_interface_option("kafka", "topic", self.topic) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) # Bypass for https://github.com/FORTH-ICS-INSPIRE/artemis/issues/411#issuecomment-661325802 start_time = int(time.time()) - START_TIME_OFFSET if "BGPSTREAM_TIMESTAMP_BYPASS" in os.environ: log.warn( "Using BGPSTREAM_TIMESTAMP_BYPASS, meaning BMP timestamps are thrown away from BGPStream" ) start_time = 0 stream.add_interval_filter(start_time, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamkafka|{}".format(str( rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) if timestamp == 0: timestamp = time.time() log.debug("fixed timestamp: {}".format(timestamp)) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.debug( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def configure_prefixtree(msg, shared_memory_manager_dict): config = msg try: # check newer config config_timestamp = shared_memory_manager_dict["config_timestamp"] if config["timestamp"] > config_timestamp: shared_memory_locks["service_reconfiguring"].acquire() shared_memory_manager_dict["service_reconfiguring"] = True shared_memory_locks["service_reconfiguring"].release() # calculate prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } rules = config.get("rules", []) for rule in rules: rule_translated_origin_asn_set = set() for asn in rule["origin_asns"]: this_translated_asn_list = flatten( translate_asn_range(asn)) rule_translated_origin_asn_set.update( set(this_translated_asn_list)) rule["origin_asns"] = list(rule_translated_origin_asn_set) rule_translated_neighbor_set = set() for asn in rule["neighbors"]: this_translated_asn_list = flatten( translate_asn_range(asn)) rule_translated_neighbor_set.update( set(this_translated_asn_list)) rule["neighbors"] = list(rule_translated_neighbor_set) conf_obj = { "origin_asns": rule["origin_asns"], "neighbors": rule["neighbors"], "prepend_seq": rule.get("prepend_seq", []), "policies": list(set(rule.get("policies", []))), "community_annotations": rule.get("community_annotations", []), "mitigation": rule.get("mitigation", "manual"), } for prefix in rule["prefixes"]: for translated_prefix in translate_rfc2622(prefix): ip_version = get_ip_version(translated_prefix) if prefix_tree[ip_version].has_key(translated_prefix): node = prefix_tree[ip_version][translated_prefix] else: node = { "prefix": translated_prefix, "data": { "confs": [] }, "timestamp": config["timestamp"], } prefix_tree[ip_version].insert( translated_prefix, node) node["data"]["confs"].append(conf_obj) # calculate the monitored and configured prefixes configured_prefix_count = 0 monitored_prefixes = set() for ip_version in prefix_tree: for prefix in prefix_tree[ip_version]: configured_prefix_count += 1 monitored_prefix = search_worst_prefix( prefix, prefix_tree[ip_version]) if monitored_prefix: monitored_prefixes.add(monitored_prefix) # extract autoignore rules autoignore_rules = config.get("autoignore", {}) # calculate autoignore prefix tree autoignore_prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128), } for key in autoignore_rules: rule = autoignore_rules[key] for prefix in rule["prefixes"]: for translated_prefix in translate_rfc2622(prefix): ip_version = get_ip_version(translated_prefix) if not autoignore_prefix_tree[ip_version].has_key( translated_prefix): node = { "prefix": translated_prefix, "rule_key": key } autoignore_prefix_tree[ip_version].insert( translated_prefix, node) # note that the object should be picklable (e.g., dict instead of pytricia tree, # see also: https://github.com/jsommers/pytricia/issues/20) shared_memory_locks["prefix_tree"].acquire() dict_prefix_tree = { "v4": pytricia_to_dict(prefix_tree["v4"]), "v6": pytricia_to_dict(prefix_tree["v6"]), } shared_memory_manager_dict["prefix_tree"] = dict_prefix_tree shared_memory_manager_dict["prefix_tree_recalculate"] = True shared_memory_locks["prefix_tree"].release() shared_memory_locks["monitored_prefixes"].acquire() shared_memory_manager_dict["monitored_prefixes"] = list( monitored_prefixes) shared_memory_locks["monitored_prefixes"].release() shared_memory_locks["configured_prefix_count"].acquire() shared_memory_manager_dict[ "configured_prefix_count"] = configured_prefix_count shared_memory_locks["configured_prefix_count"].release() # note that the object should be picklable (e.g., dict instead of pytricia tree, # see also: https://github.com/jsommers/pytricia/issues/20) dict_autoignore_prefix_tree = { "v4": pytricia_to_dict(autoignore_prefix_tree["v4"]), "v6": pytricia_to_dict(autoignore_prefix_tree["v6"]), } shared_memory_locks["autoignore"].acquire() shared_memory_manager_dict["autoignore_rules"] = autoignore_rules shared_memory_manager_dict[ "autoignore_prefix_tree"] = dict_autoignore_prefix_tree shared_memory_manager_dict["autoignore_recalculate"] = True shared_memory_locks["autoignore"].release() shared_memory_locks["config_timestamp"].acquire() shared_memory_manager_dict["config_timestamp"] = config[ "timestamp"] shared_memory_locks["config_timestamp"].release() shared_memory_locks["service_reconfiguring"].acquire() shared_memory_manager_dict["service_reconfiguring"] = False shared_memory_locks["service_reconfiguring"].release() return {"success": True, "message": "configured"} except Exception: log.exception("exception") shared_memory_locks["service_reconfiguring"].acquire() shared_memory_manager_dict["service_reconfiguring"] = False shared_memory_locks["service_reconfiguring"].release() return { "success": False, "message": "error during service configuration" }
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in self.monitor_projects: # ignore deprecated projects if project in DEPRECATED_PROJECTS: continue # add the classic project sources stream.add_filter("project", project) # plus their real-time counterparts if project in LIVE_PROJECT_MAPPINGS: stream.add_filter("project", LIVE_PROJECT_MAPPINGS[project]) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(int(time.time()) - START_TIME_OFFSET, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamlive|{}|{}".format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def commit_hijack(self, monitor_event: Dict, hijacker: int, hij_dimensions: List[str]) -> NoReturn: """ Commit new or update an existing hijack to the database. It uses redis server to store ongoing hijacks information to not stress the db. """ hij_type = "|".join(hij_dimensions) redis_hijack_key = redis_key(monitor_event["prefix"], hijacker, hij_type) if "hij_key" in monitor_event: monitor_event["final_redis_hijack_key"] = redis_hijack_key hijack_value = { "prefix": monitor_event["prefix"], "hijack_as": hijacker, "type": hij_type, "time_started": monitor_event["timestamp"], "time_last": monitor_event["timestamp"], "peers_seen": {monitor_event["peer_asn"]}, "monitor_keys": {monitor_event["key"]}, "configured_prefix": monitor_event["matched_prefix"], "timestamp_of_config": monitor_event["prefix_node"]["timestamp"], "end_tag": None, "outdated_parent": None, "rpki_status": "NA", } if (RPKI_VALIDATOR_ENABLED == "true" and self.rtrmanager and monitor_event["path"]): try: asn = monitor_event["path"][-1] if "/" in monitor_event["prefix"]: network, netmask = monitor_event["prefix"].split("/") # /32 or /128 else: ip_version = get_ip_version(monitor_event["prefix"]) network = monitor_event["prefix"] netmask = 32 if ip_version == "v6": netmask = 128 redis_rpki_asn_prefix_key = "rpki_as{}_p{}".format( asn, monitor_event["prefix"]) redis_rpki_status = self.redis.get(redis_rpki_asn_prefix_key) if not redis_rpki_status: rpki_status = get_rpki_val_result(self.rtrmanager, asn, network, int(netmask)) else: rpki_status = redis_rpki_status.decode("utf-8") hijack_value["rpki_status"] = rpki_status # the default refresh interval for the RPKI RTR manager is 3600 seconds self.redis.set(redis_rpki_asn_prefix_key, rpki_status, ex=3600) except Exception: log.exception("exception") if ("hij_key" in monitor_event and monitor_event["initial_redis_hijack_key"] != monitor_event["final_redis_hijack_key"]): hijack_value["outdated_parent"] = monitor_event["hij_key"] # identify the number of infected ases hijack_value["asns_inf"] = set() if hij_dimensions[1] in {"0", "1"}: hijack_value["asns_inf"] = set( monitor_event["path"][:-(int(hij_dimensions[1]) + 1)]) elif hij_dimensions[3] == "L": hijack_value["asns_inf"] = set(monitor_event["path"][:-2]) # assume the worst-case scenario of a type-2 hijack elif len(monitor_event["path"]) > 2: hijack_value["asns_inf"] = set(monitor_event["path"][:-3]) # make the following operation atomic using blpop (blocking) # first, make sure that the semaphore is initialized if self.redis.getset("{}token_active".format(redis_hijack_key), 1) != b"1": redis_pipeline = self.redis.pipeline() redis_pipeline.lpush("{}token".format(redis_hijack_key), "token") # lock, by extracting the token (other processes that access # it at the same time will be blocked) # attention: it is important that this command is batched in the # pipeline since the db may async delete # the token redis_pipeline.blpop("{}token".format(redis_hijack_key)) redis_pipeline.execute() else: # lock, by extracting the token (other processes that access it # at the same time will be blocked) token = self.redis.blpop("{}token".format(redis_hijack_key), timeout=60) # if timeout after 60 seconds, return without hijack alert # since this means that sth has been purged in the meanwhile (e.g., due to outdated hijack # in another instance; a detector cannot be stuck for a whole minute in a single hijack BGP update) if not token: log.info( "Monitor event {} encountered redis token timeout and will be cleared as benign for hijack {}" .format(str(monitor_event), redis_hijack_key)) return # proceed now that we have clearance redis_pipeline = self.redis.pipeline() try: result = self.redis.get(redis_hijack_key) if result: result = classic_json.loads(result.decode("utf-8")) result["time_started"] = min(result["time_started"], hijack_value["time_started"]) result["time_last"] = max(result["time_last"], hijack_value["time_last"]) result["peers_seen"] = set(result["peers_seen"]) result["peers_seen"].update(hijack_value["peers_seen"]) result["asns_inf"] = set(result["asns_inf"]) result["asns_inf"].update(hijack_value["asns_inf"]) # no update since db already knows! result["monitor_keys"] = hijack_value["monitor_keys"] self.comm_annotate_hijack(monitor_event, result) result["outdated_parent"] = hijack_value["outdated_parent"] result["bgpupdate_keys"] = set(result["bgpupdate_keys"]) result["bgpupdate_keys"].add(monitor_event["key"]) result["rpki_status"] = hijack_value["rpki_status"] else: hijack_value["time_detected"] = time.time() hijack_value["key"] = get_hash([ monitor_event["prefix"], hijacker, hij_type, "{0:.6f}".format(hijack_value["time_detected"]), ]) hijack_value["bgpupdate_keys"] = {monitor_event["key"]} redis_pipeline.sadd("persistent-keys", hijack_value["key"]) result = hijack_value self.comm_annotate_hijack(monitor_event, result) self.producer.publish( result, exchange=self.hijack_notification_exchange, routing_key="mail-log", retry=False, priority=1, serializer="ujson", ) redis_pipeline.set(redis_hijack_key, json.dumps(result)) # store the origin, neighbor combination for this hijack BGP update origin = None neighbor = None if monitor_event["path"]: origin = monitor_event["path"][-1] if len(monitor_event["path"]) > 1: neighbor = monitor_event["path"][-2] redis_pipeline.sadd( "hij_orig_neighb_{}".format(redis_hijack_key), "{}_{}".format(origin, neighbor), ) # store the prefix and peer ASN for this hijack BGP update redis_pipeline.sadd( "prefix_{}_peer_{}_hijacks".format(monitor_event["prefix"], monitor_event["peer_asn"]), redis_hijack_key, ) redis_pipeline.sadd( "hijack_{}_prefixes_peers".format(redis_hijack_key), "{}_{}".format(monitor_event["prefix"], monitor_event["peer_asn"]), ) except Exception: log.exception("exception") finally: # execute whatever has been accumulated in redis till now redis_pipeline.execute() # publish hijack self.publish_hijack_fun(result, redis_hijack_key) self.producer.publish( result, exchange=self.hijack_notification_exchange, routing_key="hij-log", retry=False, priority=1, serializer="ujson", ) # unlock, by pushing back the token (at most one other process # waiting will be unlocked) redis_pipeline = self.redis.pipeline() redis_pipeline.set("{}token_active".format(redis_hijack_key), 1) redis_pipeline.lpush("{}token".format(redis_hijack_key), "token") redis_pipeline.execute()
def normalize_ripe_ris(msg, prefix_tree): msgs = [] if isinstance(msg, dict): msg["key"] = None # initial placeholder before passing the validator if "community" in msg: msg["communities"] = [{ "asn": comm[0], "value": comm[1] } for comm in msg["community"]] del msg["community"] if "host" in msg: msg["service"] = "ripe-ris|" + msg["host"] del msg["host"] if "peer_asn" in msg: msg["peer_asn"] = int(msg["peer_asn"]) if "path" not in msg: msg["path"] = [] if "timestamp" in msg: msg["timestamp"] = float(msg["timestamp"]) if "type" in msg: del msg["type"] if "raw" in msg: del msg["raw"] if "origin" in msg: del msg["origin"] if "id" in msg: del msg["id"] if "announcements" in msg and "withdrawals" in msg: # need 2 separate messages # one for announcements msg_ann = deepcopy(msg) msg_ann["type"] = update_to_type["announcements"] prefixes = [] for element in msg_ann["announcements"]: if "prefixes" in element: prefixes.extend(element["prefixes"]) for prefix in prefixes: ip_version = get_ip_version(prefix) try: if prefix in prefix_tree[ip_version]: new_msg = deepcopy(msg_ann) new_msg["prefix"] = prefix del new_msg["announcements"] del new_msg["withdrawals"] msgs.append(new_msg) except Exception: log.exception("exception") # one for withdrawals msg_wit = deepcopy(msg) msg_wit["type"] = update_to_type["withdrawals"] msg_wit["path"] = [] msg_wit["communities"] = [] prefixes = msg_wit["withdrawals"] for prefix in prefixes: ip_version = get_ip_version(prefix) try: if prefix in prefix_tree[ip_version]: new_msg = deepcopy(msg_wit) new_msg["prefix"] = prefix del new_msg["announcements"] del new_msg["withdrawals"] msgs.append(new_msg) except Exception: log.exception("exception") else: for update_type in update_types: if update_type in msg: msg["type"] = update_to_type[update_type] prefixes = [] for element in msg[update_type]: if update_type == "announcements": if "prefixes" in element: prefixes.extend(element["prefixes"]) elif update_type == "withdrawals": prefixes.append(element) for prefix in prefixes: ip_version = get_ip_version(prefix) try: if prefix in prefix_tree[ip_version]: new_msg = deepcopy(msg) new_msg["prefix"] = prefix del new_msg[update_type] msgs.append(new_msg) except Exception: log.exception("exception") return msgs
def parse_ripe_ris(connection, prefixes_file, hosts): exchange = Exchange("bgp-update", channel=connection, type="direct", durable=False) exchange.declare() prefixes = load_json(prefixes_file) assert prefixes is not None prefix_tree = {"v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128)} for prefix in prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") ris_suffix = os.getenv("RIS_ID", "my_as") validator = mformat_validator() with Producer(connection) as producer: while True: try: events = requests.get( "https://ris-live.ripe.net/v1/stream/?format=json&client=artemis-{}" .format(ris_suffix), stream=True, timeout=10, ) # http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests iterator = events.iter_lines() next(iterator) for data in iterator: try: parsed = json.loads(data) msg = parsed["data"] if "type" in parsed and parsed["type"] == "ris_error": log.error(msg) # also check if ris host is in the configuration elif ("type" in msg and msg["type"] == "UPDATE" and (not hosts or msg["host"] in hosts)): norm_ris_msgs = normalize_ripe_ris( msg, prefix_tree) for norm_ris_msg in norm_ris_msgs: redis.set( "ris_seen_bgp_update", "1", ex=int( os.getenv( "MON_TIMEOUT_LAST_BGP_UPDATE", DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE, )), ) try: if validator.validate(norm_ris_msg): norm_path_msgs = normalize_msg_path( norm_ris_msg) for norm_path_msg in norm_path_msgs: key_generator(norm_path_msg) log.debug(norm_path_msg) producer.publish( norm_path_msg, exchange=exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}". format(msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}" .format(norm_ris_msg)) except Exception: log.exception("exception message {}".format(data)) log.warning( "Iterator ran out of data; the connection will be retried") except Exception: log.info( "RIPE RIS Server closed connection. Restarting socket in 60seconds.." ) time.sleep(60)
def run(self): # update redis ping_redis(redis) redis.set("ris_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # set RIS suffix on connection ris_suffix = RIS_ID # main loop to process BGP updates validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break try: events = requests.get( "https://ris-live.ripe.net/v1/stream/?format=json&client=artemis-{}" .format(ris_suffix), stream=True, timeout=10, ) # http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests iterator = events.iter_lines() next(iterator) for data in iterator: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break try: parsed = json.loads(data) msg = parsed["data"] if "type" in parsed and parsed[ "type"] == "ris_error": log.error(msg) # also check if ris host is in the configuration elif ( "type" in msg and msg["type"] == "UPDATE" and (not self.hosts or msg["host"] in self.hosts)): norm_ris_msgs = self.normalize_ripe_ris( msg, prefix_tree) for norm_ris_msg in norm_ris_msgs: redis.set( "ris_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) try: if validator.validate(norm_ris_msg): norm_path_msgs = normalize_msg_path( norm_ris_msg) for norm_path_msg in norm_path_msgs: key_generator(norm_path_msg) log.debug(norm_path_msg) producer.publish( norm_path_msg, exchange=self. update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}". format(msg)) except BaseException: log.exception("exception") log.error( "Error when normalizing BGP message: {}" .format(norm_ris_msg)) except Exception: log.exception("exception") log.error("exception message {}".format(data)) log.warning( "Iterator ran out of data; the connection will be retried" ) except Exception: log.exception("exception") log.info( "RIPE RIS Server closed connection. Restarting socket in 10 seconds.." ) time.sleep(10)
def run(self): # build monitored prefix tree prefix_tree = {"v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128)} for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # start producing validator = MformatValidator() with Producer(self.connection) as producer: for csv_file in glob.glob("{}/*.csv".format(self.input_dir)): if not self.shared_memory_manager_dict["data_worker_should_run"]: break try: with open(csv_file, "r") as f: csv_reader = csv.reader(f, delimiter="|") for row in csv_reader: if not self.shared_memory_manager_dict[ "data_worker_should_run" ]: break try: if len(row) != 9: continue if row[0].startswith("#"): continue # example row: 139.91.0.0/16|8522|1403|1403 6461 2603 21320 # 5408 # 8522|routeviews|route-views2|A|"[{""asn"":1403,""value"":6461}]"|1517446677 this_prefix = row[0] if row[6] == "A": as_path = row[3].split(" ") communities = json.loads(row[7]) else: as_path = [] communities = [] service = "historical|{}|{}".format(row[4], row[5]) type_ = row[6] timestamp = float(row[8]) peer_asn = int(row[2]) ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) time.sleep(0.01) else: log.warning( "Invalid format message: {}".format(msg) ) except BaseException: log.exception( "Error when normalizing BGP message: {}".format( msg ) ) except Exception: log.exception("row") except Exception: log.exception("exception") # run until instructed to stop while True: if not self.shared_memory_manager_dict["data_worker_should_run"]: break time.sleep(1)
def handle_exabgp_msg(bgp_message): redis.set("exabgp_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) msg = { "type": bgp_message["type"], "communities": bgp_message.get("communities", []), "timestamp": float(bgp_message["timestamp"]), "path": bgp_message.get("path", []), "service": "exabgp|{}".format(host), "prefix": bgp_message["prefix"], "peer_asn": int(bgp_message["peer_asn"]), } this_prefix = msg["prefix"] ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) if autoconf: try: if learn_neighbors: msg["learn_neighbors"] = True shared_memory_locks[ "autoconf_updates"].acquire() autoconf_updates = self.shared_memory_manager_dict[ "autoconf_updates"] autoconf_updates[msg["key"]] = msg self.shared_memory_manager_dict[ "autoconf_updates"] = autoconf_updates # mark the autoconf BGP updates for configuration # processing in redis redis_pipeline = redis.pipeline() redis_pipeline.sadd( "autoconf-update-keys-to-process", msg["key"], ) redis_pipeline.execute() except Exception: log.exception("exception") finally: shared_memory_locks[ "autoconf_updates"].release() else: with Producer(self.connection) as producer: producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format(msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}".format( msg))
def run_host_sio_process(self, host): def exit_gracefully(signum, frame): if sio is not None: sio.disconnect() log.info("'{}' sio disconnected".format(host)) log.info("'{}' client exited".format(host)) shared_memory_locks["data_worker"].acquire() self.shared_memory_manager_dict["data_worker_should_run"] = False shared_memory_locks["data_worker"].release() # register signal handler signal.signal(signal.SIGTERM, exit_gracefully) signal.signal(signal.SIGINT, exit_gracefully) try: # set autoconf booleans autoconf = False learn_neighbors = False if "autoconf" in self.hosts[host]: autoconf = True if "learn_neighbors" in self.hosts[host]: learn_neighbors = True # build monitored prefix tree if autoconf: prefixes = ["0.0.0.0/0", "::/0"] else: prefixes = self.prefixes prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # set up message validator validator = MformatValidator() def handle_exabgp_msg(bgp_message): redis.set("exabgp_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) msg = { "type": bgp_message["type"], "communities": bgp_message.get("communities", []), "timestamp": float(bgp_message["timestamp"]), "path": bgp_message.get("path", []), "service": "exabgp|{}".format(host), "prefix": bgp_message["prefix"], "peer_asn": int(bgp_message["peer_asn"]), } this_prefix = msg["prefix"] ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) if autoconf: try: if learn_neighbors: msg["learn_neighbors"] = True shared_memory_locks[ "autoconf_updates"].acquire() autoconf_updates = self.shared_memory_manager_dict[ "autoconf_updates"] autoconf_updates[msg["key"]] = msg self.shared_memory_manager_dict[ "autoconf_updates"] = autoconf_updates # mark the autoconf BGP updates for configuration # processing in redis redis_pipeline = redis.pipeline() redis_pipeline.sadd( "autoconf-update-keys-to-process", msg["key"], ) redis_pipeline.execute() except Exception: log.exception("exception") finally: shared_memory_locks[ "autoconf_updates"].release() else: with Producer(self.connection) as producer: producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format(msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}".format( msg)) # set up socket-io client sio = SocketIO("http://" + host, namespace=BaseNamespace) log.info("'{}' client ready to receive sio messages".format(host)) sio.on("exa_message", handle_exabgp_msg) sio.emit("exa_subscribe", {"prefixes": prefixes}) if autoconf: route_refresh_command_v4 = "announce route-refresh ipv4 unicast" sio.emit("route_command", {"command": route_refresh_command_v4}) route_refresh_command_v6 = "announce route-refresh ipv6 unicast" sio.emit("route_command", {"command": route_refresh_command_v6}) sio.wait() except Exception: log.exception("exception")
def find_best_prefix_node(self, prefix): ip_version = get_ip_version(prefix) if prefix in self.prefix_tree[ip_version]: return self.prefix_tree[ip_version][prefix] return None