def minimal_logger(namespace, extra_fields=None, debug=False): """Make and return a minimal console logger. NOTE: this does apparently *not* work with logbook as I first thought, and log handlers will *not* take care of output. If something is to be logged to a file in a module, the logger has to be implemented for that particular purpose. The current function is copied from cement.core.backend. :param namespace: namspace of logger """ config = cf.load_config() log = logbook.Logger(namespace, level=logbook.INFO) s_h = logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True) log.handlers.append(s_h) try: host = config.get('log', 'redis_host') port = config.getint('log', 'redis_port') key = config.get('log', 'redis_key') password = config.get('log', 'redis_password') if not extra_fields: extra_fields = {"program": "pm", "command": namespace} r_h = RedisHandler(host=host, port=port, key=key, password=password, extra_fields=extra_fields, level=logbook.INFO, bubble=True) log.handlers.append(r_h) except: log.debug('Not loading RedisHandler') pass # FIX ME: really don't want to hard check sys.argv like this but # can't figure any better way get logging started (only for debug) # before the app logging is setup. Besides, this will fail for # tests since sys.argv will consist of the test call arguments. if '--debug' in sys.argv or debug: try: #If there was any problem loading the RedisHandler, at this point #the variable r_h will not exist r_h.level = logbook.DEBUG except UnboundLocalError: pass s_h.level = logbook.DEBUG log.level = logbook.DEBUG return log
def test_redis_handler_rpush(): """ Test if rpush stores messages in the right order old items should be first on list """ import redis from logbook.queues import RedisHandler null_handler = logbook.NullHandler() redis_handler = RedisHandler(key='rpushed', push_method='rpush', level=logbook.INFO, bubble=True) with null_handler.applicationbound(): with redis_handler: logbook.info("old item") logbook.info("new item") time.sleep(1.5) r = redis.Redis(decode_responses=True) logs = r.lrange('rpushed', 0, -1) assert logs assert "old item" in logs[0] r.delete('rpushed')
def minimal_logger(namespace, extra_fields=None, debug=False): """Make and return a minimal console logger. NOTE: this does apparently *not* work with logbook as I first thought, and log handlers will *not* take care of output. If something is to be logged to a file in a module, the logger has to be implemented for that particular purpose. The current function is copied from cement.core.backend. :param namespace: namspace of logger """ config = cf.load_config() log = logbook.Logger(namespace, level=logbook.INFO) s_h = logbook.StreamHandler(sys.stdout, level = logbook.INFO, bubble=True) log.handlers.append(s_h) try: host = config.get('log', 'redis_host') port = config.getint('log', 'redis_port') key = config.get('log', 'redis_key') password = config.get('log', 'redis_password') if not extra_fields: extra_fields = {"program": "pm", "command": namespace} r_h = RedisHandler(host=host, port=port, key=key, password=password, extra_fields=extra_fields, level=logbook.INFO, bubble=True) log.handlers.append(r_h) except: log.debug('Not loading RedisHandler') pass # FIX ME: really don't want to hard check sys.argv like this but # can't figure any better way get logging started (only for debug) # before the app logging is setup. Besides, this will fail for # tests since sys.argv will consist of the test call arguments. if '--debug' in sys.argv or debug: try: #If there was any problem loading the RedisHandler, at this point #the variable r_h will not exist r_h.level = logbook.DEBUG except UnboundLocalError: pass s_h.level = logbook.DEBUG log.level = logbook.DEBUG return log
def create_redis_handler(host="127.0.0.1", port=6379, key="ultros", extra_fields=None, flush_threshold=128, flush_time=1, level=0, filter=None, password=False, bubble=True, context=None, push_method="rpush"): if not extra_fields: extra_fields = {} return ThreadedWrapperHandler( RedisHandler(host, port, key, extra_fields, flush_threshold, flush_time, level, filter, password, bubble, context, push_method))
def test_redis_handler(): import redis from logbook.queues import RedisHandler KEY = 'redis' FIELDS = ['message', 'host'] r = redis.Redis(decode_responses=True) redis_handler = RedisHandler(level=logbook.INFO, bubble=True) # We don't want output for the tests, so we can wrap everything in a NullHandler null_handler = logbook.NullHandler() # Check default values with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) # Are all the fields in the record? for field in FIELDS: assert message.find(field) assert key == KEY assert message.find(LETTERS) # Change the key of the handler and check on redis KEY = 'test_another_key' redis_handler.key = KEY with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) assert key == KEY # Check that extra fields are added if specified when creating the handler FIELDS.append('type') extra_fields = {'type': 'test'} del(redis_handler) redis_handler = RedisHandler(key=KEY, level=logbook.INFO, extra_fields=extra_fields, bubble=True) with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) for field in FIELDS: assert message.find(field) assert message.find('test') # And finally, check that fields are correctly added if appended to the # log message FIELDS.append('more_info') with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS, more_info='This works') key, message = r.blpop(KEY) for field in FIELDS: assert message.find(field) assert message.find('This works')
def test_redis_handler(): import redis from logbook.queues import RedisHandler KEY = 'redis' FIELDS = ['message', 'host'] r = redis.Redis(decode_responses=True) redis_handler = RedisHandler(level=logbook.INFO, bubble=True) # We don't want output for the tests, so we can wrap everything in a NullHandler null_handler = logbook.NullHandler() # Check default values with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) # Are all the fields in the record? for field in FIELDS: assert message.find(field) assert key == KEY assert message.find(LETTERS) # Change the key of the handler and check on redis KEY = 'test_another_key' redis_handler.key = KEY with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) assert key == KEY # Check that extra fields are added if specified when creating the handler FIELDS.append('type') extra_fields = {'type': 'test'} del (redis_handler) redis_handler = RedisHandler(key=KEY, level=logbook.INFO, extra_fields=extra_fields, bubble=True) with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS) key, message = r.blpop(KEY) for field in FIELDS: assert message.find(field) assert message.find('test') # And finally, check that fields are correctly added if appended to the # log message FIELDS.append('more_info') with null_handler.applicationbound(): with redis_handler: logbook.info(LETTERS, more_info='This works') key, message = r.blpop(KEY) for field in FIELDS: assert message.find(field) assert message.find('This works')
from time import time import logbook from logbook.queues import RedisHandler from datetime import datetime logbook.set_datetime_format("local") import gevent logger = logbook.Logger('logger') log = logbook.FileHandler('test_debug.log', level='DEBUG') log.push_application() log2 = RedisHandler('127.0.0.1', port='6379', key='logQ') #logbook.FileHandler('test_info.log') log2.push_application() while True: t = time() logger.debug("debug,timestamp:[%s]" % (t)) logger.info("timestamp:[%s]" % (t)) print t gevent.sleep(3)
def create_log_handler(config, batch_records=False): log_dir = config.get("log_dir", None) email = config.get("email", None) rabbitmq = config.get("rabbitmq_logging", None) redis = config.get("redis_handler", None) handlers = [] if log_dir: utils.safe_makedir(log_dir) handlers.append( logbook.FileHandler(os.path.join(log_dir, "%s.log" % LOG_NAME))) else: handlers.append(logbook.StreamHandler(sys.stdout)) if email: smtp_host = config.get("smtp_host", None) smtp_port = config.get("smtp_port", 25) if smtp_host is not None: smtp_host = [smtp_host, smtp_port] email = email.split(",") handlers.append( logbook.MailHandler( email[0], email, server_addr=smtp_host, format_string= u'''Subject: [BCBB pipeline] {record.extra[run]} \n\n {record.message}''', level='INFO', bubble=True)) if rabbitmq: from logbook.queues import RabbitMQHandler handlers.append( RabbitMQHandler(rabbitmq["url"], queue=rabbitmq["log_queue"], bubble=True)) if redis: try: redis_host = config.get('redis_handler').get('host') redis_port = int(config.get('redis_handler').get('port')) redis_key = config.get('redis_handler').get('key') redis_password = config.get('redis_handler').get('password') redis_handler = RedisHandler(host=redis_host, port=redis_port, key=redis_key, password=redis_password) handlers.append(redis_handler) except: logger2.warn("Failed loading Redis handler, please check your \ configuration and the connectivity to your Redis Database") if config.get("debug", False): for handler in handlers: handler.level = logbook.DEBUG logger2.level = logbook.DEBUG else: for handler in handlers: handler.level = logbook.INFO return logbook.NestedSetup(handlers)
from time import time import logbook from logbook.queues import RedisHandler from datetime import datetime logbook.set_datetime_format("local") import gevent logger = logbook.Logger('logger') log = logbook.FileHandler('test_debug.log', level='DEBUG') log.push_application() log2 = RedisHandler('127.0.0.1', port=6379, key='logQ') #logbook.FileHandler('test_info.log') log2.push_application() while True: t = time() logger.debug("debug,timestamp:[%s]" % (t)) logger.info("timestamp:[%s]" % (t)) print t gevent.sleep(3)
#Construct the results dict for buffer_size in buffer_sizes: results['buffering'][str(buffer_size)] = OrderedDict() for rush in rushes: results['no_buffering'][str(rush)] = {} results['buffering'][str(buffer_size)][str(rush)] = {} ######################### # # # Tests with buffering # # # ######################### for buffer_size in buffer_sizes: h = RedisHandler(flush_threshold=buffer_size) for rush in rushes: for execution in range(EXECUTIONS): with h: t_start = time.time() for i in range(rush): l.info(MESSAGE.format(num=str(i))) t_end = time.time() results.get('buffering').get(str(buffer_size)).get(str(rush))[str(execution)] = \ str(t_end - t_start) #Clean up redis while r.keys(): r.blpop(KEY)