def enable_memory_log(self): from logging.handlers import MemoryHandler if self._memory_handler is None: h = MemoryHandler(10 * 1024 * 1024) h.setFormatter(self._get_formatter()) self._log.addHandler(h) self._memory_handler = h
def duplicate_handlers(old_handlers, formatter): log_files = [] new_handlers = [] for handler in old_handlers: if isinstance(handler, logging.FileHandler): log_name = handler.baseFilename if log_name in log_files: continue log_files.append(log_name) task_file_out_handler = logging.FileHandler(log_name) task_file_out_handler.setFormatter(formatter) task_file_out_handler.setLevel(handler.level) new_handlers.append(task_file_out_handler) if isinstance(handler, MemoryHandler) and isinstance( handler.target, logging.FileHandler): log_name = handler.target.baseFilename if log_name in log_files: continue log_files.append(log_name) task_file_out_handler = logging.FileHandler(log_name, delay=True) task_file_out_handler.setFormatter(formatter) task_file_out_handler.setLevel(handler.level) task_mem_handler = MemoryHandler(handler.capacity, flushLevel=handler.flushLevel) task_mem_handler.setFormatter(formatter) task_mem_handler.setLevel(handler.level) task_mem_handler.setTarget(task_file_out_handler) new_handlers.append(task_mem_handler) return new_handlers
def add_buffer_handler( logger: logging.Logger, buffer_size: int = 100, flush_level: int = 1000, file_name: str = None, ): """ Adds a buffer handler to the logger. Useful for log files that are written to the output directory if that directory does not exist yet :param logger: The logger object :param buffer_size: The number of messages to buffer before a flush is forced :param flush_level: The minimum level of a message to cause the handler to be flushed early. Defaults to a value that won't be reached by normal log levels to prevent premature flushing :param file_name: The name of the log file to be used when it is available """ # Flush level should be high enough that flush is not called for regular messages buffer_handler = MemoryHandler(buffer_size, flushLevel=flush_level) if logger.name.startswith(THREADED): buffer_handler.setFormatter(general_threaded_formatter) else: buffer_handler.setFormatter(general_formatter) if file_name is not None: file_out_handler = logging.FileHandler(file_name, delay=True) if logger.name.startswith(THREADED): file_out_handler.setFormatter(general_threaded_formatter) else: file_out_handler.setFormatter(general_formatter) buffer_handler.setTarget(file_out_handler) logger.addHandler(buffer_handler)
def full_context_error_logger(): """ capture all log specific to a context :return: """ from logging.handlers import MemoryHandler from StringIO import StringIO buffer = StringIO() logLevel = logging.DEBUG streamhandler = logging.StreamHandler(buffer) streamhandler.setLevel(logLevel) streamhandler.setFormatter(formatter) memory_handler = MemoryHandler(capacity=1024 * 100, flushLevel=logging.ERROR, target=streamhandler) memory_handler.setLevel(logLevel) memory_handler.setFormatter(formatter) rootLogger = logging.getLogger() rootLogger.addHandler(memory_handler) result = {"error_log": None} try: yield result except: memory_handler.flush() buffer.flush() result["error_log"] = buffer.getvalue() + traceback.format_exc() finally: rootLogger.removeHandler(memory_handler) memory_handler.close() buffer.close()
def __init__(self): memory_handler = MemoryHandler(self.MEMORY_HANDLER_SIZE) memory_handler.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) self.handlers = [] self.handlers.append(memory_handler) self.registered_loggers = {}
def make_mem_logger(name, to_log, size=8192): from logging.handlers import MemoryHandler logger = logging.getLogger(name) hdlr = MemoryHandler(size, target=to_log) formatter = logging.Formatter('%(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.memhandler = hdlr logger.setLevel(logging.DEBUG) return logger
def make_mem_logger(name,to_log,size=8192): from logging.handlers import MemoryHandler logger=logging.getLogger(name) hdlr=MemoryHandler(size,target=to_log) formatter=logging.Formatter('%(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.memhandler=hdlr logger.setLevel(logging.DEBUG) return logger
def make_mem_logger(name, to_log, size=8192): """ Creates a memory logger to avoid writing concurrently to the main logger """ from logging.handlers import MemoryHandler logger = logging.getLogger(name) hdlr = MemoryHandler(size, target=to_log) formatter = logging.Formatter('%(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.memhandler = hdlr logger.setLevel(logging.DEBUG) return logger
def add_handler(handler): """ Add a new handler, wrapped in a MemoryHandler for buffering. """ # Use MemoryHandler as an intermediary for buffering. Essentially: we don't # want to flush every message, since that presents a severe scalability # issue. mem_handler = MemoryHandler(32, target=handler) mem_handler.setFormatter(logger_formatter) # Real filtering will be done by the logger itself mem_handler.setLevel(DEBUG) MEM_HANDLERS.append(mem_handler) base_logger.addHandler(mem_handler)
def set_up_logger(log_level=logging.DEBUG): stream_handler = logging.StreamHandler() stream_handler.setLevel(log_level) stream_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT, datefmt=LOGGING_DATE_FORMAT)) logger.setLevel(log_level) if CURRENT_ENV != 'Product': logger.addHandler(stream_handler) return memory_handler = MemoryHandler( capacity=64, flushLevel=logging.ERROR, target=stream_handler ) memory_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT, datefmt=LOGGING_DATE_FORMAT)) logger.addHandler(memory_handler) def flush(): memory_handler.flush() atexit.register(flush) logger.debug("Logger init")
sys.stdout = sys.__stdout__ #old_stdout sys.stderr = sys.__stderr__ #adds a stream catcher for display and a memory handler for saving log_stream = StreamCatch() logger = getLogger() display_handler = StreamHandler(stream=log_stream) display_handler.setLevel(LOGLEVEL) display_handler.setFormatter(Formatter(LOGFORMATTER)) display_handler.name = "StreamCatch" logger.addHandler(display_handler) memory_handler = MemoryHandler(MEMBUFFER) memory_handler.setLevel(LOGLEVEL) memory_handler.setFormatter(Formatter(LOGFORMATTER)) memory_handler.name = "MemoryLog" logger.addHandler(memory_handler) log_info("Started logging") def make_log_file(log_path, mode='a'): """Points memory handler at a particular file to save the log.""" file_handler = FileHandler(filename=log_path, mode=mode) file_handler.setLevel(LOGLEVEL) file_handler.setFormatter(Formatter(LOGFORMATTER)) memory_handler.setTarget(file_handler) def remove_log_file():
else: sys.stdout=sys.__stdout__ #old_stdout sys.stderr=sys.__stderr__ #adds a stream catcher for display and a memory handler for saving log_stream=StreamCatch() logger=getLogger() display_handler=StreamHandler(stream=log_stream) display_handler.setLevel(LOGLEVEL) display_handler.setFormatter(Formatter(LOGFORMATTER)) display_handler.name="StreamCatch" logger.addHandler(display_handler) memory_handler=MemoryHandler(MEMBUFFER) memory_handler.setLevel(LOGLEVEL) memory_handler.setFormatter(Formatter(LOGFORMATTER)) memory_handler.name="MemoryLog" logger.addHandler(memory_handler) log_debug("Started logging") def make_log_file(log_path, mode='a'): """Points memory handler at a particular file to save the log.""" file_handler = FileHandler(filename=log_path, mode=mode) file_handler.setLevel(LOGLEVEL) file_handler.setFormatter(Formatter(LOGFORMATTER)) memory_handler.setTarget(file_handler) def remove_log_file(): """closes the log file and removes memory_handler from pointing at it""" if memory_handler.target:
# Original punctuation should be out of the brackets self.positive("I am hear!", "I am [here]!") # Drop leading and trailing spaces self.positive(" their is ", "[there] is") # Retain double spaces self.positive("their is", "[there] is") # Drop periods, commas, and semicolons self.positive("their is.", "[there] is") self.positive("their is,", "[there] is") self.positive("their is;", "[there] is") def test_wording(self): """Verify that wording can be generated without failing""" self.positive( "Their is and your don't supposed to! (blah) They think their is.") logging.debug(self.parser.generate_wording('@@') .encode('ascii', 'replace')) # not implemented yet self.assertRaises(NotImplementedError, self.parser.generate_wording_long, '') if __name__ == '__main__': stream_null = StringIO() logging.basicConfig(stream=stream_null, level=logging.DEBUG) handler_stream = StreamHandler(stream=sys.stderr) handler_mem = MemoryHandler(1024, target=handler_stream) handler_mem.setLevel(logging.DEBUG) handler_mem.setFormatter(logging.Formatter()) logging.getLogger().addHandler(handler_mem) unittest.main()