def start_logPublisher(self, publish): log = logging.getLogger() log.info('Activating log publisher on port {}'.format(publish)) zmq_log_handler = PUBHandler("tcp://*:{}".format(publish)) zmq_log_handler.setFormatter( logging.Formatter(fmt='{asctime}|{message}', style='{')) zmq_log_handler.root_topic = self.nodename log.addHandler(zmq_log_handler) if self.camwatcher: handoff = { 'node': self.nodename, 'log': self.publish_log, 'video': self.publish_cam, 'host': socket.gethostname() } msg = "CameraUp|" + json.dumps(handoff) try: with zmq.Context().socket(zmq.REQ) as sock: log.debug('connecting to ' + self.camwatcher) sock.connect(self.camwatcher) sock.send(msg.encode("ascii")) resp = sock.recv().decode("ascii") except Exception as ex: log.exception('Unable to connect with camwatcher:' + ex) sys.exit() log.handlers.remove( log.handlers[0]) # OK, all logging over PUB socket only log.setLevel(logging.INFO) return log
def start_stats(root_topic="snowpear"): context = zmq.Context(cpu_count()) pub = context.socket(zmq.PUB) pub.bind(options.stats_endpoint) handler = PUBHandler(pub) handler.root_topic = root_topic logger = logging.getLogger() logger.addHandler(handler)
def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s" % self.log_url) context = zmq.Context.instance() lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler)
def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s"%self.log_url) context = zmq.Context.instance() lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler)
def start_logging(self): super(IPControllerApp, self).start_logging() if self.master_config.Global.log_url: context = self.factory.context lsock = context.socket(zmq.PUB) lsock.connect(self.master_config.Global.log_url) handler = PUBHandler(lsock) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler)
def zmq_external_logger(host='localhost', port='8899'): ''' This publish logging messages over a zmq.PUB socket ''' context = zmq.Context() socket = context.socket(zmq.PUB) socket.connect('tcp://{0}:{1}'.format(host, port)) handler = PUBHandler(socket) logger = logging.getLogger() logger.addHandler(handler) handler.root_topic = 'logging' return logger
def PublishLogging(self,LoggingName="zmq.auth", root_topic="zmq.auth"): """ Publishes the given python logger to the publishing service :param LoggingName: Name of the python logger service :type LoggingName: str :param root_topic: the topic given with message. is appended with .<LEVEL> :type root_topic: str :return: """ handler = PUBHandler(self.Logger) handler.root_topic = root_topic handler.formatters[logging.DEBUG] = logging.Formatter(fmt='%(asctime)s\t%(levelname)s: %(message)s', datefmt='%H:%M:%S') handler.formatters[logging.INFO] = logging.Formatter(fmt='%(asctime)s\t%(levelname)s: %(message)s', datefmt='%H:%M:%S') l = logging.getLogger(LoggingName) l.addHandler(handler)
def PublishLogging(self, LoggingName="zmq.auth", root_topic="zmq.auth"): """ Publishes the given python logger to the publishing service :param LoggingName: Name of the python logger service :type LoggingName: str :param root_topic: the topic given with message. is appended with .<LEVEL> :type root_topic: str :return: """ handler = PUBHandler(self.Logger) handler.root_topic = root_topic handler.formatters[logging.DEBUG] = logging.Formatter( fmt='%(asctime)s\t%(levelname)s: %(message)s', datefmt='%H:%M:%S') handler.formatters[logging.INFO] = logging.Formatter( fmt='%(asctime)s\t%(levelname)s: %(message)s', datefmt='%H:%M:%S') l = logging.getLogger(LoggingName) l.addHandler(handler)
def register_zmq_handler(cls, zmq_socket): # pragma: no cover handler = PUBHandler(zmq_socket) handler.root_topic = "logger" logger = logging.getLogger() logger.addHandler(handler)
def main(settings): """ The :meth:`main` method for worker processes. Here we will: - create a :class:`ZmqMgmt` instance - create a :class:`Fetcher` instance - initialize and instantiate the extractor chain The `settings` have to be loaded already. """ # create my own identity identity = "worker:%s:%s" % (socket.gethostname(), os.getpid()) ctx = zmq.Context() io_loop = IOLoop.instance() # initialize the logging subsystem log_pub = ctx.socket(zmq.PUB) log_pub.connect(settings.ZEROMQ_LOGGING) zmq_logging_handler = PUBHandler(log_pub) zmq_logging_handler.root_topic = "spyder.worker" logger = logging.getLogger() logger.addHandler(zmq_logging_handler) logger.setLevel(settings.LOG_LEVEL_WORKER) logger.info("process::Starting up another worker") mgmt = create_worker_management(settings, ctx, io_loop) logger.debug("process::Initializing fetcher, extractor and scoper") fetcher = create_worker_fetcher(settings, mgmt, ctx, zmq_logging_handler, io_loop) fetcher.start() extractor = create_worker_extractor(settings, mgmt, ctx, zmq_logging_handler, io_loop) extractor.start() def quit_worker(raw_msg): """ When the worker should quit, stop the io_loop after 2 seconds. """ msg = MgmtMessage(raw_msg) if ZMQ_SPYDER_MGMT_WORKER_QUIT == msg.data: logger.info("process::We have been asked to shutdown, do so") DelayedCallback(io_loop.stop, 2000, io_loop).start() ack = MgmtMessage(topic=ZMQ_SPYDER_MGMT_WORKER, identity=identity, data=ZMQ_SPYDER_MGMT_WORKER_QUIT_ACK) mgmt._out_stream.send_multipart(ack.serialize()) mgmt.add_callback(ZMQ_SPYDER_MGMT_WORKER, quit_worker) mgmt.start() # notify the master that we are online msg = MgmtMessage(topic=ZMQ_SPYDER_MGMT_WORKER, identity=identity, data=ZMQ_SPYDER_MGMT_WORKER_AVAIL) mgmt._out_stream.send_multipart(msg.serialize()) def handle_shutdown_signal(_sig, _frame): """ Called from the os when a shutdown signal is fired. """ msg = MgmtMessage(data=ZMQ_SPYDER_MGMT_WORKER_QUIT) quit_worker(msg.serialize()) # zmq 2.1 stops blocking calls, restart the ioloop io_loop.start() # handle kill signals signal.signal(signal.SIGINT, handle_shutdown_signal) signal.signal(signal.SIGTERM, handle_shutdown_signal) logger.info("process::waiting for action") # this will block until the worker quits try: io_loop.start() except ZMQError: logger.debug("Caught a ZMQError. Hopefully during shutdown") logger.debug(traceback.format_exc()) for mod in [fetcher, extractor, mgmt]: mod.close() logger.info("process::Houston: Worker down") ctx.term()
def main(settings): """ Main method for master processes. """ # create my own identity identity = "master:%s:%s" % (socket.gethostname(), os.getpid()) ctx = zmq.Context() io_loop = IOLoop.instance() # initialize the logging subsystem log_pub = ctx.socket(zmq.PUB) log_pub.connect(settings.ZEROMQ_LOGGING) zmq_logging_handler = PUBHandler(log_pub) zmq_logging_handler.root_topic = "spyder.master" logger = logging.getLogger() logger.addHandler(zmq_logging_handler) logger.setLevel(settings.LOG_LEVEL_MASTER) logger.info("process::Starting up the master") mgmt = create_master_management(settings, ctx, io_loop) frontier = create_frontier(settings, zmq_logging_handler) publishing_socket = ctx.socket(zmq.PUSH) publishing_socket.setsockopt(zmq.HWM, settings.ZEROMQ_MASTER_PUSH_HWM) publishing_socket.bind(settings.ZEROMQ_MASTER_PUSH) receiving_socket = ctx.socket(zmq.SUB) receiving_socket.setsockopt(zmq.SUBSCRIBE, "") receiving_socket.bind(settings.ZEROMQ_MASTER_SUB) master = ZmqMaster(settings, identity, receiving_socket, publishing_socket, mgmt, frontier, zmq_logging_handler, settings.LOG_LEVEL_MASTER, io_loop) def handle_shutdown_signal(_sig, _frame): """ Called from the os when a shutdown signal is fired. """ master.shutdown() # zmq 2.1 stops blocking calls, restart the ioloop io_loop.start() # handle kill signals signal.signal(signal.SIGINT, handle_shutdown_signal) signal.signal(signal.SIGTERM, handle_shutdown_signal) if settings.MASTER_CALLBACK: callback = import_class(settings.MASTER_CALLBACK) callback(settings, ctx, io_loop, frontier) mgmt.start() master.start() # this will block until the master stops try: io_loop.start() except ZMQError: logger.debug("Caught a ZMQError. Hopefully during shutdown") logger.debug(traceback.format_exc()) master.close() mgmt.close() logger.info("process::Master is down.") log_pub.close() ctx.term()