def __init__(self, confPath): """ :param confPath: 配置文件的路径 """ conf = os.path.join(confPath, 'server.json') with open(conf, 'r') as f: conf = json.load(f) self.conf = conf # 子进程通信队列 self.q = Queue() # 订阅子进程中的屏幕日志 for conf in self.conf.values(): subscriber = ZeroMQSubscriber('tcp://{}'.format( conf["logzmqhost"])) self.streamHandler = StreamHandler(sys.stdout, level="DEBUG") subscriber.dispatch_in_background(self.streamHandler) # 期货行情 conf = self.conf["easyctp"] self.easyctpProcess = Process(target=EasyctpServer.process, args=[self.q], kwargs=conf)
def __call__(self): # Find an open port for the logs # (that's a race condition, deal with it) tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmpsock.bind(('localhost', 0)) log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname()) tmpsock.close() setup = logbook.NestedSetup([ logbook.NullHandler(), logbook.StderrHandler(level=logbook.INFO), logbook.Processor(self._process_record), ]) self.subscriber = ZeroMQSubscriber(log_uri, multi=True) self.subscriber.dispatch_in_background(setup=setup) self.process = sh.python( '-m', 'onitu', '--entries', self.entries, '--log-uri', log_uri, _bg=self.bg, ) return self.process
def start_logbook_subscriber(self): """ Run in the background the log receiver. """ subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000', multi=True) self._logbook_controller = subscriber.dispatch_in_background( self._qt_handler)
def start_logbook_subscriber(self): """ Run in the background the log receiver. """ if self._logbook_controller is None: subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000', multi=True) self._logbook_controller = subscriber.dispatch_in_background( self._qt_handler)
class MyZeroMQSubscriber: def __init__(self, socket): self.subscriber = ZeroMQSubscriber(socket) def startSubscriber(self): with TimedRotatingFileHandler('/home/asmi/compute/subscriber1/foo.log', date_format='%Y-%m-%d %H:%M'): self.subscriber.dispatch_forever()
class TriggerSubscriber: def __init__(self): print "subscriber initiated" # multi = True for bind() call on socket self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True) def startSubscriber(self, filepath): print "subscriber started" with GmailHandler("*****@*****.**", "asmi9971026789", "*****@*****.**"): print "log received : ", self.subscriber.recv() self.subscriber.dispatch_forever()
def test_zeromq_handler(self): from logbook.queues import ZeroMQHandler, ZeroMQSubscriber tests = [ u'Logging something', u'Something with umlauts äöü', u'Something else for good measure', ] uri = 'tcp://127.0.0.1:42000' handler = ZeroMQHandler(uri) subscriber = ZeroMQSubscriber(uri) for test in tests: with handler: self.log.warn(test) record = subscriber.recv() self.assertEqual(record.message, test) self.assertEqual(record.channel, self.log.name)
class MyZeroMQSubscriber: def __init__(self): print "subscriber initiated" # multi = True for bind() call on socket try: self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True) except zmq.error.ZMQError as error: print "error while binding to socket :" + Constants.getSocket() raise zmq.error.ZMQError("error while binding to socket :" + Constants.getSocket()) def startSubscriber(self, filepath): print "subscriber started" + `self.subscriber` with TimedRotatingFileHandler(filepath, date_format='%Y-%m-%d %H:%M'): print "log received : " + self.subscriber.recv().message self.subscriber.dispatch_forever()
def __init__(self): print "subscriber initiated" # multi = True for bind() call on socket try: self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True) except zmq.error.ZMQError as error: print "error while binding to socket :" + Constants.getSocket() raise zmq.error.ZMQError("error while binding to socket :" + Constants.getSocket())
def get_logs_dispatcher(uri=None, debug=False): handlers = [] if not debug: handlers.append(NullHandler(level=DEBUG)) handlers.append(ColorizedStderrHandler(level=INFO)) if not uri: # Find an open port. # This is a race condition as the port could be used between # the check and its binding. However, this is probably one of the # best solution without patching Logbook. tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmpsock.bind(('localhost', 0)) uri = 'tcp://{}:{}'.format(*tmpsock.getsockname()) tmpsock.close() subscriber = ZeroMQSubscriber(uri, multi=True) return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))
def test_zeromq_background_thread(self): from logbook.queues import ZeroMQHandler, ZeroMQSubscriber uri = 'tcp://127.0.0.1:42001' handler = ZeroMQHandler(uri) subscriber = ZeroMQSubscriber(uri) test_handler = logbook.TestHandler() controller = subscriber.dispatch_in_background(test_handler) with handler: self.log.warn('This is a warning') self.log.error('This is an error') # stop the controller. This will also stop the loop and join the # background process. Before that we give it a fraction of a second # to get all results time.sleep(0.1) controller.stop() self.assert_(test_handler.has_warning('This is a warning')) self.assert_(test_handler.has_error('This is an error'))
def handlers_subscriber(multi): from logbook.queues import ZeroMQHandler, ZeroMQSubscriber # Get an unused port tempsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tempsock.bind(('127.0.0.1', 0)) host, unused_port = tempsock.getsockname() tempsock.close() # Retrieve the ZeroMQ handler and subscriber uri = 'tcp://%s:%d' % (host, unused_port) if multi: handlers = [ZeroMQHandler(uri, multi=True) for _ in range(3)] else: handlers = [ZeroMQHandler(uri)] subscriber = ZeroMQSubscriber(uri, multi=multi) # Enough time to start time.sleep(0.1) return handlers, subscriber
def __init__(self, socket): self.subscriber = ZeroMQSubscriber(socket)
class Launcher(object): def __init__(self, entries='entries.json', background=True): self.entries = entries self.bg = background self.process = None self.events = {} self.event_triggers = {} def set_event(self, name, triggers, action): event = { 'triggers': list(triggers), 'state': [False] * len(triggers), 'action': action, } self.events[name] = event for trigger in triggers: self.event_triggers[trigger] = event def unset_event(self, name): for trigger in self.events[name]['triggers']: del self.event_triggers[trigger] del self.events[name] def quit(self): self.process.signal(signal.SIGINT) def kill(self): self.process.signal(signal.SIGTERM) def wait(self): self.process.wait() def _process_record(self, record): trigger = (record.channel, record.message) event = self.event_triggers.get(trigger) if not event: return event['state'][event['triggers'].index(trigger)] = True if all(event['state']): event['state'] = [False] * len(event['triggers']) event['action']() def __call__(self): # Find an open port for the logs # (that's a race condition, deal with it) tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmpsock.bind(('localhost', 0)) log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname()) tmpsock.close() setup = logbook.NestedSetup([ logbook.NullHandler(), logbook.StderrHandler(level=logbook.INFO), logbook.Processor(self._process_record), ]) self.subscriber = ZeroMQSubscriber(log_uri, multi=True) self.subscriber.dispatch_in_background(setup=setup) self.process = sh.python( '-m', 'onitu', '--entries', self.entries, '--log-uri', log_uri, _bg=self.bg, ) return self.process def _on_event(self, name): log_triggers = logs[name] def caller(action, **kwargs): triggers = set((channel.format(**kwargs), message.format(**kwargs)) for (channel, message) in log_triggers) self.set_event(name, triggers, action) return caller def __getattr__(self, name): if name.startswith('on_'): return self._on_event(name[3:]) return super(Launcher, self).__getattr__(name)
from logbook.queues import ZeroMQSubscriber from logbook import TimedRotatingFileHandler #from logger.common.logbookRotator import rotator mySocket = 'tcp://127.0.0.1:5000' subscriber = ZeroMQSubscriber(mySocket) #controller = subscriber.dispatch_in_background(TimedRotatingFileHandler('/home/asmi/compute/foo.log', date_format='%Y-%m-%d %H:%M')) #a = 1 #while a > 0: # record = subscriber.recv() # print "**** Log Record **** \n" , record.message #controller.stop() with TimedRotatingFileHandler('/home/asmi/compute/demo_publish/foo.log', date_format='%Y-%m-%d %H:%M'): print subscriber.recv().message subscriber.dispatch_forever()
def __init__(self): print "subscriber initiated" # multi = True for bind() call on socket self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True)
def test_missing_zeromq(): from logbook.queues import ZeroMQHandler, ZeroMQSubscriber with pytest.raises(RuntimeError): ZeroMQHandler('tcp://127.0.0.1:42000') with pytest.raises(RuntimeError): ZeroMQSubscriber('tcp://127.0.0.1:42000')
SERVICE = ConfigReader.getValue(SECTION, "Service") #sys.argv[1] LOGDIR = ConfigReader.getValue(SECTION, "LogDir") #sys.argv[2] FILENAME = ConfigReader.getValue(SECTION, "Filename") #sys.argv[3] SOCKET = ConfigReader.getValue(SECTION, "Socket") #sys.argv[4] HOSTNAME = socket.gethostname() print "SERVICE - ", SERVICE print "LOGDIR - ", LOGDIR print "FILENAME - ", FILENAME print "SOCKET - ", SOCKET print "HOSTNAME - ", HOSTNAME subscriber = None try: subscriber = ZeroMQSubscriber(SOCKET, multi = True) print "Subscriber bind to socket - ", SOCKET except zmq.error.ZMQError as error: print "error in service " + SERVICE + " while binding to socket :" + SOCKET raise zmq.error.ZMQError("error in service " + SERVICE + " while binding to socket :" + SOCKET) path = os.path.join(os.path.join(LOGDIR, HOSTNAME), SERVICE) path_with_filename = os.path.join(path, FILENAME) print "PATH - ", path if not os.path.exists(path): os.makedirs(path) with TimedRotatingFileHandler(path_with_filename, date_format='%Y-%m-%d %H:%M'): print "Log Received - ", subscriber.recv().message subscriber.dispatch_forever()
class Launcher(object): def __init__(self, entries='entries.json', background=True): self.entries = entries self.bg = background self.process = None self.events = {} self.event_triggers = {} def set_event(self, name, triggers, action): event = { 'triggers': list(triggers), 'state': [False] * len(triggers), 'action': action, } self.events[name] = event for trigger in triggers: self.event_triggers[trigger] = event def unset_event(self, name): for trigger in self.events[name]['triggers']: del self.event_triggers[trigger] del self.events[name] def quit(self): self.process.signal(signal.SIGINT) def kill(self): self.process.signal(signal.SIGTERM) def wait(self): self.process.wait() def _process_record(self, record): trigger = (record.channel, record.message) event = self.event_triggers.get(trigger) if not event: return event['state'][event['triggers'].index(trigger)] = True if all(event['state']): event['state'] = [False] * len(event['triggers']) event['action']() def __call__(self): # Find an open port for the logs # (that's a race condition, deal with it) tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tmpsock.bind(('localhost', 0)) log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname()) tmpsock.close() setup = logbook.NestedSetup([ logbook.NullHandler(), logbook.StderrHandler(level=logbook.INFO), logbook.Processor(self._process_record), ]) self.subscriber = ZeroMQSubscriber(log_uri, multi=True) self.subscriber.dispatch_in_background(setup=setup) self.process = sh.python( '-m', 'onitu', '--entries', self.entries, '--log-uri', log_uri, _bg=self.bg, ) return self.process def _on_event(self, name): log_triggers = logs[name] def caller(action, **kwargs): triggers = set( ( channel.format(**kwargs), message.format(**kwargs) ) for (channel, message) in log_triggers ) self.set_event(name, triggers, action) return caller def __getattr__(self, name): if name.startswith('on_'): return self._on_event(name[3:]) return super(Launcher, self).__getattr__(name)
from logbook.queues import ZeroMQSubscriber from logbook import TimedRotatingFileHandler #from logger.common.logbookRotator import rotator mySocket = 'tcp://127.0.0.1:5000' subscriber = ZeroMQSubscriber(mySocket) #controller = subscriber.dispatch_in_background(TimedRotatingFileHandler('/home/asmi/compute/foo.log', date_format='%Y-%m-%d %H:%M')) #a = 1 #while a > 0: # record = subscriber.recv() # print "**** Log Record **** \n" , record.message #controller.stop() with TimedRotatingFileHandler('/home/asmi/compute/subscriber2/foo.log', date_format='%Y-%m-%d %H:%M'): subscriber.dispatch_forever()