def enable_memory_log(self):
from logging.handlers import MemoryHandler
if self._memory_handler is None:
h = MemoryHandler(10 * 1024 * 1024)
h.setFormatter(self._get_formatter())
self._log.addHandler(h)
self._memory_handler = h
def duplicate_handlers(old_handlers, formatter):
log_files = []
new_handlers = []
for handler in old_handlers:
if isinstance(handler, logging.FileHandler):
log_name = handler.baseFilename
if log_name in log_files:
continue
log_files.append(log_name)
task_file_out_handler = logging.FileHandler(log_name)
task_file_out_handler.setFormatter(formatter)
task_file_out_handler.setLevel(handler.level)
new_handlers.append(task_file_out_handler)
if isinstance(handler, MemoryHandler) and isinstance(
handler.target, logging.FileHandler):
log_name = handler.target.baseFilename
if log_name in log_files:
continue
log_files.append(log_name)
task_file_out_handler = logging.FileHandler(log_name, delay=True)
task_file_out_handler.setFormatter(formatter)
task_file_out_handler.setLevel(handler.level)
task_mem_handler = MemoryHandler(handler.capacity,
flushLevel=handler.flushLevel)
task_mem_handler.setFormatter(formatter)
task_mem_handler.setLevel(handler.level)
task_mem_handler.setTarget(task_file_out_handler)
new_handlers.append(task_mem_handler)
return new_handlers
def add_buffer_handler(
logger: logging.Logger,
buffer_size: int = 100,
flush_level: int = 1000,
file_name: str = None,
):
"""
Adds a buffer handler to the logger.
Useful for log files that are written to the output directory if that directory does not exist yet
:param logger: The logger object
:param buffer_size: The number of messages to buffer before a flush is forced
:param flush_level: The minimum level of a message to cause the handler to be flushed early.
Defaults to a value that won't be reached by normal log levels to prevent premature flushing
:param file_name: The name of the log file to be used when it is available
"""
# Flush level should be high enough that flush is not called for regular messages
buffer_handler = MemoryHandler(buffer_size, flushLevel=flush_level)
if logger.name.startswith(THREADED):
buffer_handler.setFormatter(general_threaded_formatter)
else:
buffer_handler.setFormatter(general_formatter)
if file_name is not None:
file_out_handler = logging.FileHandler(file_name, delay=True)
if logger.name.startswith(THREADED):
file_out_handler.setFormatter(general_threaded_formatter)
else:
file_out_handler.setFormatter(general_formatter)
buffer_handler.setTarget(file_out_handler)
logger.addHandler(buffer_handler)
def full_context_error_logger():
""" capture all log specific to a context
:return:
"""
from logging.handlers import MemoryHandler
from StringIO import StringIO
buffer = StringIO()
logLevel = logging.DEBUG
streamhandler = logging.StreamHandler(buffer)
streamhandler.setLevel(logLevel)
streamhandler.setFormatter(formatter)
memory_handler = MemoryHandler(capacity=1024 * 100,
flushLevel=logging.ERROR,
target=streamhandler)
memory_handler.setLevel(logLevel)
memory_handler.setFormatter(formatter)
rootLogger = logging.getLogger()
rootLogger.addHandler(memory_handler)
result = {"error_log": None}
try:
yield result
except:
memory_handler.flush()
buffer.flush()
result["error_log"] = buffer.getvalue() + traceback.format_exc()
finally:
rootLogger.removeHandler(memory_handler)
memory_handler.close()
buffer.close()
def __init__(self):
memory_handler = MemoryHandler(self.MEMORY_HANDLER_SIZE)
memory_handler.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
self.handlers = []
self.handlers.append(memory_handler)
self.registered_loggers = {}
def make_mem_logger(name, to_log, size=8192):
from logging.handlers import MemoryHandler
logger = logging.getLogger(name)
hdlr = MemoryHandler(size, target=to_log)
formatter = logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.memhandler = hdlr
logger.setLevel(logging.DEBUG)
return logger
def make_mem_logger(name,to_log,size=8192):
from logging.handlers import MemoryHandler
logger=logging.getLogger(name)
hdlr=MemoryHandler(size,target=to_log)
formatter=logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.memhandler=hdlr
logger.setLevel(logging.DEBUG)
return logger
def make_mem_logger(name, to_log, size=8192):
"""
Creates a memory logger to avoid writing concurrently to the main logger
"""
from logging.handlers import MemoryHandler
logger = logging.getLogger(name)
hdlr = MemoryHandler(size, target=to_log)
formatter = logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.memhandler = hdlr
logger.setLevel(logging.DEBUG)
return logger
def make_mem_logger(name, to_log, size=8192):
"""
Creates a memory logger to avoid writing concurrently to the main logger
"""
from logging.handlers import MemoryHandler
logger = logging.getLogger(name)
hdlr = MemoryHandler(size, target=to_log)
formatter = logging.Formatter('%(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.memhandler = hdlr
logger.setLevel(logging.DEBUG)
return logger
def add_handler(handler):
"""
Add a new handler, wrapped in a MemoryHandler for buffering.
"""
# Use MemoryHandler as an intermediary for buffering. Essentially: we don't
# want to flush every message, since that presents a severe scalability
# issue.
mem_handler = MemoryHandler(32, target=handler)
mem_handler.setFormatter(logger_formatter)
# Real filtering will be done by the logger itself
mem_handler.setLevel(DEBUG)
MEM_HANDLERS.append(mem_handler)
base_logger.addHandler(mem_handler)
def set_up_logger(log_level=logging.DEBUG):
stream_handler = logging.StreamHandler()
stream_handler.setLevel(log_level)
stream_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT,
datefmt=LOGGING_DATE_FORMAT))
logger.setLevel(log_level)
if CURRENT_ENV != 'Product':
logger.addHandler(stream_handler)
return
memory_handler = MemoryHandler(
capacity=64,
flushLevel=logging.ERROR,
target=stream_handler
)
memory_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT,
datefmt=LOGGING_DATE_FORMAT))
logger.addHandler(memory_handler)
def flush():
memory_handler.flush()
atexit.register(flush)
logger.debug("Logger init")
sys.stdout = sys.__stdout__ #old_stdout
sys.stderr = sys.__stderr__
#adds a stream catcher for display and a memory handler for saving
log_stream = StreamCatch()
logger = getLogger()
display_handler = StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name = "StreamCatch"
logger.addHandler(display_handler)
memory_handler = MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name = "MemoryLog"
logger.addHandler(memory_handler)
log_info("Started logging")
def make_log_file(log_path, mode='a'):
"""Points memory handler at a particular file to save the log."""
file_handler = FileHandler(filename=log_path, mode=mode)
file_handler.setLevel(LOGLEVEL)
file_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.setTarget(file_handler)
def remove_log_file():
else:
sys.stdout=sys.__stdout__ #old_stdout
sys.stderr=sys.__stderr__
#adds a stream catcher for display and a memory handler for saving
log_stream=StreamCatch()
logger=getLogger()
display_handler=StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name="StreamCatch"
logger.addHandler(display_handler)
memory_handler=MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name="MemoryLog"
logger.addHandler(memory_handler)
log_debug("Started logging")
def make_log_file(log_path, mode='a'):
"""Points memory handler at a particular file to save the log."""
file_handler = FileHandler(filename=log_path, mode=mode)
file_handler.setLevel(LOGLEVEL)
file_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.setTarget(file_handler)
def remove_log_file():
"""closes the log file and removes memory_handler from pointing at it"""
if memory_handler.target:
# Original punctuation should be out of the brackets
self.positive("I am hear!", "I am [here]!")
# Drop leading and trailing spaces
self.positive(" their is ", "[there] is")
# Retain double spaces
self.positive("their is", "[there] is")
# Drop periods, commas, and semicolons
self.positive("their is.", "[there] is")
self.positive("their is,", "[there] is")
self.positive("their is;", "[there] is")
def test_wording(self):
"""Verify that wording can be generated without failing"""
self.positive(
"Their is and your don't supposed to! (blah) They think their is.")
logging.debug(self.parser.generate_wording('@@')
.encode('ascii', 'replace'))
# not implemented yet
self.assertRaises(NotImplementedError,
self.parser.generate_wording_long, '')
if __name__ == '__main__':
stream_null = StringIO()
logging.basicConfig(stream=stream_null, level=logging.DEBUG)
handler_stream = StreamHandler(stream=sys.stderr)
handler_mem = MemoryHandler(1024, target=handler_stream)
handler_mem.setLevel(logging.DEBUG)
handler_mem.setFormatter(logging.Formatter())
logging.getLogger().addHandler(handler_mem)
unittest.main()
