def __init__(self, connection): self.module_name = "monitor" self.connection = connection self.timestamp = -1 self.prefix_tree = None self.process_ids = [] self.rules = None self.prefixes = set() self.prefix_file = "/root/monitor_prefixes.json" self.monitors = None self.flag = True self.redis = redis.Redis(host=REDIS_HOST, port=REDIS_PORT) ping_redis(self.redis) self.correlation_id = None # EXCHANGES self.config_exchange = create_exchange("config", connection) # QUEUES self.config_queue = create_queue( self.module_name, exchange=self.config_exchange, routing_key="notify", priority=2, ) self.config_request_rpc() # setup Redis monitor listeners self.setup_redis_mon_listeners() log.info("started")
def run(self): # update redis ping_redis(redis) redis.set("exabgp_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) autoconf_running = self.shared_memory_manager_dict["autoconf_running"] if not autoconf_running: log.info("setting up autoconf updater process...") with Connection(RABBITMQ_URI) as connection: self.autoconf_updater = AutoconfUpdater( connection, self.shared_memory_manager_dict) shared_memory_locks["autoconf_updates"].acquire() self.shared_memory_manager_dict["autoconf_running"] = True shared_memory_locks["autoconf_updates"].release() mp.Process(target=self.autoconf_updater.run).start() log.info("autoignore checker set up") # start host processes host_processes = [] for host in self.hosts: host_process = mp.Process(target=self.run_host_sio_process, args=(host, )) host_processes.append(host_process) host_process.start() while True: if not self.shared_memory_manager_dict["data_worker_should_run"]: for host_process in host_processes: host_process.terminate() break time.sleep(1)
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # set kafka data interface stream.set_data_interface("kafka") # set host connection details stream.set_data_interface_option("kafka", "brokers", "{}:{}".format(self.host, self.port)) # set topic stream.set_data_interface_option("kafka", "topic", self.topic) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) # Bypass for https://github.com/FORTH-ICS-INSPIRE/artemis/issues/411#issuecomment-661325802 start_time = int(time.time()) - START_TIME_OFFSET if "BGPSTREAM_TIMESTAMP_BYPASS" in os.environ: log.warn( "Using BGPSTREAM_TIMESTAMP_BYPASS, meaning BMP timestamps are thrown away from BGPStream" ) start_time = 0 stream.add_interval_filter(start_time, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamkafka_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamkafka|{}".format(str( rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) if timestamp == 0: timestamp = time.time() log.debug("fixed timestamp: {}".format(timestamp)) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.debug( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def __init__(self, connection: Connection, shared_memory_manager_dict: Dict) -> NoReturn: self.connection = connection self.redis = redis.Redis(host=REDIS_HOST, port=REDIS_PORT) ping_redis(self.redis) self.shared_memory_manager_dict = shared_memory_manager_dict self.prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } shared_memory_locks["prefix_tree"].acquire() if self.shared_memory_manager_dict["prefix_tree_recalculate"]: for ip_version in ["v4", "v6"]: if ip_version == "v4": size = 32 else: size = 128 self.prefix_tree[ip_version] = dict_to_pytricia( self.shared_memory_manager_dict["prefix_tree"][ip_version], size) log.info("{} pytricia tree parsed from configuration".format( ip_version)) self.shared_memory_manager_dict[ "prefix_tree_recalculate"] = False shared_memory_locks["prefix_tree"].release() self.autoignore_prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128), } shared_memory_locks["autoignore"].acquire() if self.shared_memory_manager_dict["autoignore_recalculate"]: for ip_version in ["v4", "v6"]: if ip_version == "v4": size = 32 else: size = 128 self.autoignore_prefix_tree[ip_version] = dict_to_pytricia( self.shared_memory_manager_dict["autoignore_prefix_tree"] [ip_version], size, ) log.info("{} pytricia tree parsed from configuration".format( ip_version)) self.shared_memory_manager_dict[ "autoignore_recalculate"] = False shared_memory_locks["autoignore"].release() # EXCHANGES self.update_exchange = create_exchange("bgp-update", connection, declare=True) self.hijack_exchange = create_exchange("hijack-update", connection, declare=True) self.autoconf_exchange = create_exchange("autoconf", connection, declare=True) self.pg_amq_bridge = create_exchange("amq.direct", connection) self.mitigation_exchange = create_exchange("mitigation", connection, declare=True) self.autoignore_exchange = create_exchange("autoignore", connection, declare=True) self.command_exchange = create_exchange("command", connection, declare=True) # QUEUES self.update_queue = create_queue( SERVICE_NAME, exchange=self.update_exchange, routing_key="update", priority=1, ) self.hijack_ongoing_queue = create_queue( SERVICE_NAME, exchange=self.hijack_exchange, routing_key="ongoing", priority=1, ) self.pg_amq_update_queue = create_queue( SERVICE_NAME, exchange=self.pg_amq_bridge, routing_key="update-insert", priority=1, ) self.mitigation_request_queue = create_queue( SERVICE_NAME, exchange=self.mitigation_exchange, routing_key="mitigate", priority=2, ) self.unmitigation_request_queue = create_queue( SERVICE_NAME, exchange=self.mitigation_exchange, routing_key="unmitigate", priority=2, ) self.stop_queue = create_queue( "{}-{}".format(SERVICE_NAME, uuid()), exchange=self.command_exchange, routing_key="stop-{}".format(SERVICE_NAME), priority=1, ) self.autoconf_update_queue = create_queue( SERVICE_NAME, exchange=self.autoconf_exchange, routing_key="update", priority=4, random=True, ) self.ongoing_hijack_prefixes_queue = create_queue( SERVICE_NAME, exchange=self.autoignore_exchange, routing_key="ongoing-hijack-prefixes", priority=1, random=True, ) log.info("data worker initiated")
def run(self): # update redis ping_redis(redis) redis.set("bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # create a new bgpstream instance and a reusable bgprecord instance stream = _pybgpstream.BGPStream() # consider collectors from given projects for project in self.monitor_projects: # ignore deprecated projects if project in DEPRECATED_PROJECTS: continue # add the classic project sources stream.add_filter("project", project) # plus their real-time counterparts if project in LIVE_PROJECT_MAPPINGS: stream.add_filter("project", LIVE_PROJECT_MAPPINGS[project]) # filter prefixes for prefix in self.prefixes: stream.add_filter("prefix", prefix) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # filter record type stream.add_filter("record-type", "updates") # filter based on timing (if end=0 --> live mode) stream.add_interval_filter(int(time.time()) - START_TIME_OFFSET, 0) # set live mode stream.set_live_mode() # start the stream stream.start() # start producing validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break # get next record try: rec = stream.get_next_record() except BaseException: continue if (rec.status != "valid") or (rec.type != "update"): continue # get next element try: elem = rec.get_next_elem() except BaseException: continue while elem: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break if elem.type in {"A", "W"}: redis.set( "bgpstreamlive_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) this_prefix = str(elem.fields["prefix"]) service = "bgpstreamlive|{}|{}".format( str(rec.project), str(rec.collector)) type_ = elem.type if type_ == "A": as_path = elem.fields["as-path"].split(" ") communities = [{ "asn": int(comm.split(":")[0]), "value": int(comm.split(":")[1]), } for comm in elem.fields["communities"]] else: as_path = [] communities = [] timestamp = float(rec.time) peer_asn = elem.peer_asn ip_version = get_ip_version(this_prefix) if this_prefix in prefix_tree[ip_version]: msg = { "type": type_, "timestamp": timestamp, "path": as_path, "service": service, "communities": communities, "prefix": this_prefix, "peer_asn": peer_asn, } try: if validator.validate(msg): msgs = normalize_msg_path(msg) for msg in msgs: key_generator(msg) log.debug(msg) producer.publish( msg, exchange=self.update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}".format( msg)) except BaseException: log.exception( "Error when normalizing BGP message: {}". format(msg)) try: elem = rec.get_next_elem() except BaseException: continue
def __init__(self, connection: Connection, shared_memory_manager_dict: Dict) -> NoReturn: self.connection = connection self.shared_memory_manager_dict = shared_memory_manager_dict self.rtrmanager = None # wait for other needed data workers to start wait_data_worker_dependencies(DATA_WORKER_DEPENDENCIES) # EXCHANGES self.update_exchange = create_exchange("bgp-update", connection, declare=True) self.hijack_exchange = create_exchange("hijack-update", connection, declare=True) self.hijack_hashing = create_exchange("hijack-hashing", connection, "x-consistent-hash", declare=True) self.handled_exchange = create_exchange("handled-update", connection, declare=True) self.hijack_notification_exchange = create_exchange( "hijack-notification", connection, declare=True) self.command_exchange = create_exchange("command", connection, declare=True) # QUEUES self.update_queue = create_queue( SERVICE_NAME, exchange=self.update_exchange, routing_key="stored-update-with-prefix-node", priority=1, ) self.hijack_ongoing_queue = create_queue( SERVICE_NAME, exchange=self.hijack_exchange, routing_key="ongoing-with-prefix-node", priority=1, ) self.stop_queue = create_queue( "{}-{}".format(SERVICE_NAME, uuid()), exchange=self.command_exchange, routing_key="stop-{}".format(SERVICE_NAME), priority=1, ) setattr(self, "publish_hijack_fun", self.publish_hijack_result_production) if TEST_ENV == "true": setattr(self, "publish_hijack_fun", self.publish_hijack_result_test) self.redis = redis.Redis(host=REDIS_HOST, port=REDIS_PORT) ping_redis(self.redis) if RPKI_VALIDATOR_ENABLED == "true": from rtrlib import RTRManager while True: try: self.rtrmanager = RTRManager(RPKI_VALIDATOR_HOST, RPKI_VALIDATOR_PORT) self.rtrmanager.start() log.info("Connected to RPKI VALIDATOR '{}:{}'".format( RPKI_VALIDATOR_HOST, RPKI_VALIDATOR_PORT)) break except Exception: log.info( "Could not connect to RPKI VALIDATOR '{}:{}'".format( RPKI_VALIDATOR_HOST, RPKI_VALIDATOR_PORT)) log.info("Retrying RTR connection in 30 seconds...") time.sleep(30) log.info("data worker initiated")
def run(self): # update redis ping_redis(redis) redis.set("ris_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE) # build monitored prefix tree prefix_tree = { "v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128) } for prefix in self.prefixes: ip_version = get_ip_version(prefix) prefix_tree[ip_version].insert(prefix, "") # set RIS suffix on connection ris_suffix = RIS_ID # main loop to process BGP updates validator = MformatValidator() with Producer(self.connection) as producer: while True: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break try: events = requests.get( "https://ris-live.ripe.net/v1/stream/?format=json&client=artemis-{}" .format(ris_suffix), stream=True, timeout=10, ) # http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests iterator = events.iter_lines() next(iterator) for data in iterator: if not self.shared_memory_manager_dict[ "data_worker_should_run"]: break try: parsed = json.loads(data) msg = parsed["data"] if "type" in parsed and parsed[ "type"] == "ris_error": log.error(msg) # also check if ris host is in the configuration elif ( "type" in msg and msg["type"] == "UPDATE" and (not self.hosts or msg["host"] in self.hosts)): norm_ris_msgs = self.normalize_ripe_ris( msg, prefix_tree) for norm_ris_msg in norm_ris_msgs: redis.set( "ris_seen_bgp_update", "1", ex=MON_TIMEOUT_LAST_BGP_UPDATE, ) try: if validator.validate(norm_ris_msg): norm_path_msgs = normalize_msg_path( norm_ris_msg) for norm_path_msg in norm_path_msgs: key_generator(norm_path_msg) log.debug(norm_path_msg) producer.publish( norm_path_msg, exchange=self. update_exchange, routing_key="update", serializer="ujson", ) else: log.warning( "Invalid format message: {}". format(msg)) except BaseException: log.exception("exception") log.error( "Error when normalizing BGP message: {}" .format(norm_ris_msg)) except Exception: log.exception("exception") log.error("exception message {}".format(data)) log.warning( "Iterator ran out of data; the connection will be retried" ) except Exception: log.exception("exception") log.info( "RIPE RIS Server closed connection. Restarting socket in 10 seconds.." ) time.sleep(10)