def run(self): """ Override """ self.LOG = init_mp_logger("bmp_consumer", self._log_queue) # Enable to topics/feeds topics = ['openbmp.parsed.collector', 'openbmp.parsed.router', 'openbmp.parsed.peer', 'openbmp.bmp_raw'] self.LOG.info("Running bmp_consumer") # wait for config to load while not self.stopped(): if self._cfg and 'kafka' in self._cfg: break try: # connect and bind to topics self.LOG.info("Connecting to %s ... takes a minute to load offsets and topics, please wait" % self._cfg['kafka']['servers']) consumer = kafka.KafkaConsumer(*topics, bootstrap_servers=self._cfg['kafka']['servers'], client_id=self._cfg['kafka']['client_id'] + '-' + socket.gethostname(), group_id=self._cfg['kafka']['group_id'], enable_auto_commit=True, auto_commit_interval_ms=1000, auto_offset_reset='latest' if self._cfg['kafka']['offset_reset_largest'] else "smallest") self.LOG.info("Connected, now consuming") prev_ts = time.time() # Loop till stopped while not self.stopped(): # Read messages for m in consumer: if self.stopped(): break self.process_msg(m) except kafka.common.KafkaUnavailableError as err: self.LOG.error("Kafka Error: %s" % str(err)) self._fwd_queue.put('DISCONNECT ALL') except KeyboardInterrupt: pass self.LOG.info("consumer stopped")
def run(self): """ Override """ self.LOG = init_mp_logger("bmp_writer", self._log_queue) self.LOG.info("Running bmp_writer") self.connect() try: # wait for the mapping config to be loaded while not self.stopped(): if self._cfg and 'logging' in self._cfg: break # Read queue while not self.stopped(): # Do not pop any message unless connected msg = self._fwd_queue.get() if msg == 'DISCONNECT ALL': self.disconnect() else: if msg.DEST_CONFIG_NAME not in self._dest_socks: self.connect(msg.DEST_CONFIG_NAME) sock_arr = self._dest_socks[msg.DEST_CONFIG_NAME] if sock_arr[1]: sent = False while not sent: sent = self.send(msg.BMP_MSG, sock_arr[0]) self.LOG.debug("Received bmp message: %s %s %s", msg.COLLECTOR_ADMIN_ID, msg.ROUTER_IP, msg.ROUTER_NAME) else: self.LOG.info("Peer group \'{}\' not connected, attempting to reconnect".format(msg.DEST_CONFIG_NAME)) sock_arr[1] = self.connect(msg.DEST_CONFIG_NAME) except KeyboardInterrupt: pass self.LOG.info("rewrite stopped")
def run(self): """ Override """ self.LOG = init_mp_logger("bmp_writer", self._log_queue) self.LOG.info("Running bmp_writer") self.connect() try: # wait for the mapping config to be loaded while not self.stopped(): if self._cfg and 'logging' in self._cfg: break # Read queue while not self.stopped(): # Do not pop any message unless connected if self._isConnected: msg = self._fwd_queue.get() sent = False while not sent: sent = self.send(msg.BMP_MSG) self.LOG.debug("Received bmp message: %s %s %s", msg.COLLECTOR_ADMIN_ID, msg.ROUTER_IP, msg.ROUTER_NAME) else: self.LOG.info("Not connected, attempting to reconnect") sleep(1) self.connect() except KeyboardInterrupt: pass self.LOG.info("rewrite stopped")