def __init__(self, confPath):
"""
:param confPath: 配置文件的路径
"""
conf = os.path.join(confPath, 'server.json')
with open(conf, 'r') as f:
conf = json.load(f)
self.conf = conf
# 子进程通信队列
self.q = Queue()
# 订阅子进程中的屏幕日志
for conf in self.conf.values():
subscriber = ZeroMQSubscriber('tcp://{}'.format(
conf["logzmqhost"]))
self.streamHandler = StreamHandler(sys.stdout, level="DEBUG")
subscriber.dispatch_in_background(self.streamHandler)
# 期货行情
conf = self.conf["easyctp"]
self.easyctpProcess = Process(target=EasyctpServer.process,
args=[self.q],
kwargs=conf)
def __call__(self):
# Find an open port for the logs
# (that's a race condition, deal with it)
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
setup = logbook.NestedSetup([
logbook.NullHandler(),
logbook.StderrHandler(level=logbook.INFO),
logbook.Processor(self._process_record),
])
self.subscriber = ZeroMQSubscriber(log_uri, multi=True)
self.subscriber.dispatch_in_background(setup=setup)
self.process = sh.python(
'-m',
'onitu',
'--entries',
self.entries,
'--log-uri',
log_uri,
_bg=self.bg,
)
return self.process
def start_logbook_subscriber(self):
"""
Run in the background the log receiver.
"""
subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000', multi=True)
self._logbook_controller = subscriber.dispatch_in_background(
self._qt_handler)
def start_logbook_subscriber(self):
"""
Run in the background the log receiver.
"""
if self._logbook_controller is None:
subscriber = ZeroMQSubscriber('tcp://127.0.0.1:5000', multi=True)
self._logbook_controller = subscriber.dispatch_in_background(
self._qt_handler)
class MyZeroMQSubscriber:
def __init__(self, socket):
self.subscriber = ZeroMQSubscriber(socket)
def startSubscriber(self):
with TimedRotatingFileHandler('/home/asmi/compute/subscriber1/foo.log', date_format='%Y-%m-%d %H:%M'):
self.subscriber.dispatch_forever()
class TriggerSubscriber:
def __init__(self):
print "subscriber initiated"
# multi = True for bind() call on socket
self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True)
def startSubscriber(self, filepath):
print "subscriber started"
with GmailHandler("*****@*****.**", "asmi9971026789", "*****@*****.**"):
print "log received : ", self.subscriber.recv()
self.subscriber.dispatch_forever()
def test_zeromq_handler(self):
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
tests = [
u'Logging something',
u'Something with umlauts äöü',
u'Something else for good measure',
]
uri = 'tcp://127.0.0.1:42000'
handler = ZeroMQHandler(uri)
subscriber = ZeroMQSubscriber(uri)
for test in tests:
with handler:
self.log.warn(test)
record = subscriber.recv()
self.assertEqual(record.message, test)
self.assertEqual(record.channel, self.log.name)
class MyZeroMQSubscriber:
def __init__(self):
print "subscriber initiated"
# multi = True for bind() call on socket
try:
self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True)
except zmq.error.ZMQError as error:
print "error while binding to socket :" + Constants.getSocket()
raise zmq.error.ZMQError("error while binding to socket :" + Constants.getSocket())
def startSubscriber(self, filepath):
print "subscriber started" + `self.subscriber`
with TimedRotatingFileHandler(filepath, date_format='%Y-%m-%d %H:%M'):
print "log received : " + self.subscriber.recv().message
self.subscriber.dispatch_forever()
def __init__(self):
print "subscriber initiated"
# multi = True for bind() call on socket
try:
self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True)
except zmq.error.ZMQError as error:
print "error while binding to socket :" + Constants.getSocket()
raise zmq.error.ZMQError("error while binding to socket :" + Constants.getSocket())
def get_logs_dispatcher(uri=None, debug=False):
handlers = []
if not debug:
handlers.append(NullHandler(level=DEBUG))
handlers.append(ColorizedStderrHandler(level=INFO))
if not uri:
# Find an open port.
# This is a race condition as the port could be used between
# the check and its binding. However, this is probably one of the
# best solution without patching Logbook.
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
subscriber = ZeroMQSubscriber(uri, multi=True)
return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))
def test_zeromq_background_thread(self):
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
uri = 'tcp://127.0.0.1:42001'
handler = ZeroMQHandler(uri)
subscriber = ZeroMQSubscriber(uri)
test_handler = logbook.TestHandler()
controller = subscriber.dispatch_in_background(test_handler)
with handler:
self.log.warn('This is a warning')
self.log.error('This is an error')
# stop the controller. This will also stop the loop and join the
# background process. Before that we give it a fraction of a second
# to get all results
time.sleep(0.1)
controller.stop()
self.assert_(test_handler.has_warning('This is a warning'))
self.assert_(test_handler.has_error('This is an error'))
def get_logs_dispatcher(uri=None, debug=False):
handlers = []
if not debug:
handlers.append(NullHandler(level=DEBUG))
handlers.append(ColorizedStderrHandler(level=INFO))
if not uri:
# Find an open port.
# This is a race condition as the port could be used between
# the check and its binding. However, this is probably one of the
# best solution without patching Logbook.
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
subscriber = ZeroMQSubscriber(uri, multi=True)
return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))
def handlers_subscriber(multi):
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
# Get an unused port
tempsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tempsock.bind(('127.0.0.1', 0))
host, unused_port = tempsock.getsockname()
tempsock.close()
# Retrieve the ZeroMQ handler and subscriber
uri = 'tcp://%s:%d' % (host, unused_port)
if multi:
handlers = [ZeroMQHandler(uri, multi=True) for _ in range(3)]
else:
handlers = [ZeroMQHandler(uri)]
subscriber = ZeroMQSubscriber(uri, multi=multi)
# Enough time to start
time.sleep(0.1)
return handlers, subscriber
def __call__(self):
# Find an open port for the logs
# (that's a race condition, deal with it)
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
setup = logbook.NestedSetup([
logbook.NullHandler(),
logbook.StderrHandler(level=logbook.INFO),
logbook.Processor(self._process_record),
])
self.subscriber = ZeroMQSubscriber(log_uri, multi=True)
self.subscriber.dispatch_in_background(setup=setup)
self.process = sh.python(
'-m', 'onitu',
'--entries', self.entries,
'--log-uri', log_uri,
_bg=self.bg,
)
return self.process
def __init__(self, socket): self.subscriber = ZeroMQSubscriber(socket)
class Launcher(object):
def __init__(self, entries='entries.json', background=True):
self.entries = entries
self.bg = background
self.process = None
self.events = {}
self.event_triggers = {}
def set_event(self, name, triggers, action):
event = {
'triggers': list(triggers),
'state': [False] * len(triggers),
'action': action,
}
self.events[name] = event
for trigger in triggers:
self.event_triggers[trigger] = event
def unset_event(self, name):
for trigger in self.events[name]['triggers']:
del self.event_triggers[trigger]
del self.events[name]
def quit(self):
self.process.signal(signal.SIGINT)
def kill(self):
self.process.signal(signal.SIGTERM)
def wait(self):
self.process.wait()
def _process_record(self, record):
trigger = (record.channel, record.message)
event = self.event_triggers.get(trigger)
if not event:
return
event['state'][event['triggers'].index(trigger)] = True
if all(event['state']):
event['state'] = [False] * len(event['triggers'])
event['action']()
def __call__(self):
# Find an open port for the logs
# (that's a race condition, deal with it)
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
setup = logbook.NestedSetup([
logbook.NullHandler(),
logbook.StderrHandler(level=logbook.INFO),
logbook.Processor(self._process_record),
])
self.subscriber = ZeroMQSubscriber(log_uri, multi=True)
self.subscriber.dispatch_in_background(setup=setup)
self.process = sh.python(
'-m',
'onitu',
'--entries',
self.entries,
'--log-uri',
log_uri,
_bg=self.bg,
)
return self.process
def _on_event(self, name):
log_triggers = logs[name]
def caller(action, **kwargs):
triggers = set((channel.format(**kwargs), message.format(**kwargs))
for (channel, message) in log_triggers)
self.set_event(name, triggers, action)
return caller
def __getattr__(self, name):
if name.startswith('on_'):
return self._on_event(name[3:])
return super(Launcher, self).__getattr__(name)
from logbook.queues import ZeroMQSubscriber
from logbook import TimedRotatingFileHandler
#from logger.common.logbookRotator import rotator
mySocket = 'tcp://127.0.0.1:5000'
subscriber = ZeroMQSubscriber(mySocket)
#controller = subscriber.dispatch_in_background(TimedRotatingFileHandler('/home/asmi/compute/foo.log', date_format='%Y-%m-%d %H:%M'))
#a = 1
#while a > 0:
# record = subscriber.recv()
# print "**** Log Record **** \n" , record.message
#controller.stop()
with TimedRotatingFileHandler('/home/asmi/compute/demo_publish/foo.log', date_format='%Y-%m-%d %H:%M'):
print subscriber.recv().message
subscriber.dispatch_forever()
def __init__(self): print "subscriber initiated" # multi = True for bind() call on socket self.subscriber = ZeroMQSubscriber(Constants.getSocket(), multi = True)
def test_missing_zeromq():
from logbook.queues import ZeroMQHandler, ZeroMQSubscriber
with pytest.raises(RuntimeError):
ZeroMQHandler('tcp://127.0.0.1:42000')
with pytest.raises(RuntimeError):
ZeroMQSubscriber('tcp://127.0.0.1:42000')
SERVICE = ConfigReader.getValue(SECTION, "Service") #sys.argv[1]
LOGDIR = ConfigReader.getValue(SECTION, "LogDir") #sys.argv[2]
FILENAME = ConfigReader.getValue(SECTION, "Filename") #sys.argv[3]
SOCKET = ConfigReader.getValue(SECTION, "Socket") #sys.argv[4]
HOSTNAME = socket.gethostname()
print "SERVICE - ", SERVICE
print "LOGDIR - ", LOGDIR
print "FILENAME - ", FILENAME
print "SOCKET - ", SOCKET
print "HOSTNAME - ", HOSTNAME
subscriber = None
try:
subscriber = ZeroMQSubscriber(SOCKET, multi = True)
print "Subscriber bind to socket - ", SOCKET
except zmq.error.ZMQError as error:
print "error in service " + SERVICE + " while binding to socket :" + SOCKET
raise zmq.error.ZMQError("error in service " + SERVICE + " while binding to socket :" + SOCKET)
path = os.path.join(os.path.join(LOGDIR, HOSTNAME), SERVICE)
path_with_filename = os.path.join(path, FILENAME)
print "PATH - ", path
if not os.path.exists(path):
os.makedirs(path)
with TimedRotatingFileHandler(path_with_filename, date_format='%Y-%m-%d %H:%M'):
print "Log Received - ", subscriber.recv().message
subscriber.dispatch_forever()
class Launcher(object):
def __init__(self, entries='entries.json', background=True):
self.entries = entries
self.bg = background
self.process = None
self.events = {}
self.event_triggers = {}
def set_event(self, name, triggers, action):
event = {
'triggers': list(triggers),
'state': [False] * len(triggers),
'action': action,
}
self.events[name] = event
for trigger in triggers:
self.event_triggers[trigger] = event
def unset_event(self, name):
for trigger in self.events[name]['triggers']:
del self.event_triggers[trigger]
del self.events[name]
def quit(self):
self.process.signal(signal.SIGINT)
def kill(self):
self.process.signal(signal.SIGTERM)
def wait(self):
self.process.wait()
def _process_record(self, record):
trigger = (record.channel, record.message)
event = self.event_triggers.get(trigger)
if not event:
return
event['state'][event['triggers'].index(trigger)] = True
if all(event['state']):
event['state'] = [False] * len(event['triggers'])
event['action']()
def __call__(self):
# Find an open port for the logs
# (that's a race condition, deal with it)
tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tmpsock.bind(('localhost', 0))
log_uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
tmpsock.close()
setup = logbook.NestedSetup([
logbook.NullHandler(),
logbook.StderrHandler(level=logbook.INFO),
logbook.Processor(self._process_record),
])
self.subscriber = ZeroMQSubscriber(log_uri, multi=True)
self.subscriber.dispatch_in_background(setup=setup)
self.process = sh.python(
'-m', 'onitu',
'--entries', self.entries,
'--log-uri', log_uri,
_bg=self.bg,
)
return self.process
def _on_event(self, name):
log_triggers = logs[name]
def caller(action, **kwargs):
triggers = set(
(
channel.format(**kwargs),
message.format(**kwargs)
)
for (channel, message) in log_triggers
)
self.set_event(name, triggers, action)
return caller
def __getattr__(self, name):
if name.startswith('on_'):
return self._on_event(name[3:])
return super(Launcher, self).__getattr__(name)
from logbook.queues import ZeroMQSubscriber
from logbook import TimedRotatingFileHandler
#from logger.common.logbookRotator import rotator
mySocket = 'tcp://127.0.0.1:5000'
subscriber = ZeroMQSubscriber(mySocket)
#controller = subscriber.dispatch_in_background(TimedRotatingFileHandler('/home/asmi/compute/foo.log', date_format='%Y-%m-%d %H:%M'))
#a = 1
#while a > 0:
# record = subscriber.recv()
# print "**** Log Record **** \n" , record.message
#controller.stop()
with TimedRotatingFileHandler('/home/asmi/compute/subscriber2/foo.log', date_format='%Y-%m-%d %H:%M'):
subscriber.dispatch_forever()