Ejemplo n.º 1
0
 def enable_memory_log(self):
     from logging.handlers import MemoryHandler
     if self._memory_handler is None:
         h = MemoryHandler(10 * 1024 * 1024)
         h.setFormatter(self._get_formatter())
         self._log.addHandler(h)
         self._memory_handler = h
Ejemplo n.º 2
0
def duplicate_handlers(old_handlers, formatter):
    log_files = []
    new_handlers = []
    for handler in old_handlers:
        if isinstance(handler, logging.FileHandler):
            log_name = handler.baseFilename
            if log_name in log_files:
                continue
            log_files.append(log_name)
            task_file_out_handler = logging.FileHandler(log_name)
            task_file_out_handler.setFormatter(formatter)
            task_file_out_handler.setLevel(handler.level)
            new_handlers.append(task_file_out_handler)

        if isinstance(handler, MemoryHandler) and isinstance(
                handler.target, logging.FileHandler):
            log_name = handler.target.baseFilename
            if log_name in log_files:
                continue
            log_files.append(log_name)

            task_file_out_handler = logging.FileHandler(log_name, delay=True)
            task_file_out_handler.setFormatter(formatter)
            task_file_out_handler.setLevel(handler.level)

            task_mem_handler = MemoryHandler(handler.capacity,
                                             flushLevel=handler.flushLevel)
            task_mem_handler.setFormatter(formatter)
            task_mem_handler.setLevel(handler.level)
            task_mem_handler.setTarget(task_file_out_handler)
            new_handlers.append(task_mem_handler)

    return new_handlers
Ejemplo n.º 3
0
def add_buffer_handler(
    logger: logging.Logger,
    buffer_size: int = 100,
    flush_level: int = 1000,
    file_name: str = None,
):
    """
        Adds a buffer handler to the logger.
        Useful for log files that are written to the output directory if that directory does not exist yet
        :param logger: The logger object
        :param buffer_size: The number of messages to buffer before a flush is forced
        :param flush_level: The minimum level of a message to cause the handler to be flushed early.
        Defaults to a value that won't be reached by normal log levels to prevent premature flushing
        :param file_name: The name of the log file to be used when it is available
        """
    # Flush level should be high enough that flush is not called for regular messages
    buffer_handler = MemoryHandler(buffer_size, flushLevel=flush_level)
    if logger.name.startswith(THREADED):
        buffer_handler.setFormatter(general_threaded_formatter)
    else:
        buffer_handler.setFormatter(general_formatter)

    if file_name is not None:
        file_out_handler = logging.FileHandler(file_name, delay=True)
        if logger.name.startswith(THREADED):
            file_out_handler.setFormatter(general_threaded_formatter)
        else:
            file_out_handler.setFormatter(general_formatter)

        buffer_handler.setTarget(file_out_handler)

    logger.addHandler(buffer_handler)
def full_context_error_logger():
    """ capture all log specific to a context
    :return:
    """
    from logging.handlers import MemoryHandler
    from StringIO import StringIO
    buffer = StringIO()
    logLevel = logging.DEBUG
    streamhandler = logging.StreamHandler(buffer)
    streamhandler.setLevel(logLevel)
    streamhandler.setFormatter(formatter)
    memory_handler = MemoryHandler(capacity=1024 * 100,
                                   flushLevel=logging.ERROR,
                                   target=streamhandler)
    memory_handler.setLevel(logLevel)
    memory_handler.setFormatter(formatter)
    rootLogger = logging.getLogger()
    rootLogger.addHandler(memory_handler)
    result = {"error_log": None}
    try:
        yield result
    except:
        memory_handler.flush()
        buffer.flush()
        result["error_log"] = buffer.getvalue() + traceback.format_exc()
    finally:
        rootLogger.removeHandler(memory_handler)
        memory_handler.close()
        buffer.close()
Ejemplo n.º 5
0
    def __init__(self):
        memory_handler = MemoryHandler(self.MEMORY_HANDLER_SIZE)
        memory_handler.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))

        self.handlers = []
        self.handlers.append(memory_handler)

        self.registered_loggers = {}
Ejemplo n.º 6
0
Archivo: Logs.py Proyecto: Mokl/SP4L
def make_mem_logger(name, to_log, size=8192):
    from logging.handlers import MemoryHandler
    logger = logging.getLogger(name)
    hdlr = MemoryHandler(size, target=to_log)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.memhandler = hdlr
    logger.setLevel(logging.DEBUG)
    return logger
Ejemplo n.º 7
0
def make_mem_logger(name,to_log,size=8192):
	from logging.handlers import MemoryHandler
	logger=logging.getLogger(name)
	hdlr=MemoryHandler(size,target=to_log)
	formatter=logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.memhandler=hdlr
	logger.setLevel(logging.DEBUG)
	return logger
Ejemplo n.º 8
0
def make_mem_logger(name, to_log, size=8192):
    """
	Creates a memory logger to avoid writing concurrently to the main logger
	"""
    from logging.handlers import MemoryHandler
    logger = logging.getLogger(name)
    hdlr = MemoryHandler(size, target=to_log)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.memhandler = hdlr
    logger.setLevel(logging.DEBUG)
    return logger
Ejemplo n.º 9
0
Archivo: Logs.py Proyecto: afeldman/waf
def make_mem_logger(name, to_log, size=8192):
	"""
	Creates a memory logger to avoid writing concurrently to the main logger
	"""
	from logging.handlers import MemoryHandler
	logger = logging.getLogger(name)
	hdlr = MemoryHandler(size, target=to_log)
	formatter = logging.Formatter('%(message)s')
	hdlr.setFormatter(formatter)
	logger.addHandler(hdlr)
	logger.memhandler = hdlr
	logger.setLevel(logging.DEBUG)
	return logger
Ejemplo n.º 10
0
def add_handler(handler):
    """
    Add a new handler, wrapped in a MemoryHandler for buffering.
    """
    # Use MemoryHandler as an intermediary for buffering. Essentially: we don't
    # want to flush every message, since that presents a severe scalability
    # issue.
    mem_handler = MemoryHandler(32, target=handler)
    mem_handler.setFormatter(logger_formatter)
    # Real filtering will be done by the logger itself
    mem_handler.setLevel(DEBUG)
    MEM_HANDLERS.append(mem_handler)
    base_logger.addHandler(mem_handler)
Ejemplo n.º 11
0
def set_up_logger(log_level=logging.DEBUG):
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(log_level)
    stream_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT,
                                                  datefmt=LOGGING_DATE_FORMAT))
    logger.setLevel(log_level)
    if CURRENT_ENV != 'Product':
        logger.addHandler(stream_handler)
        return

    memory_handler = MemoryHandler(
        capacity=64,
        flushLevel=logging.ERROR,
        target=stream_handler
    )
    memory_handler.setFormatter(logging.Formatter(fmt=LOGGING_MSG_FORMAT,
                                                  datefmt=LOGGING_DATE_FORMAT))
    logger.addHandler(memory_handler)

    def flush():
        memory_handler.flush()
    atexit.register(flush)
    logger.debug("Logger init")
Ejemplo n.º 12
0
            sys.stdout = sys.__stdout__  #old_stdout
            sys.stderr = sys.__stderr__


#adds a stream catcher for display and a memory handler for saving
log_stream = StreamCatch()
logger = getLogger()
display_handler = StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name = "StreamCatch"
logger.addHandler(display_handler)

memory_handler = MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name = "MemoryLog"
logger.addHandler(memory_handler)

log_info("Started logging")


def make_log_file(log_path, mode='a'):
    """Points memory handler at a particular file to save the log."""
    file_handler = FileHandler(filename=log_path, mode=mode)
    file_handler.setLevel(LOGLEVEL)
    file_handler.setFormatter(Formatter(LOGFORMATTER))
    memory_handler.setTarget(file_handler)


def remove_log_file():
Ejemplo n.º 13
0
        else:
            sys.stdout=sys.__stdout__ #old_stdout
            sys.stderr=sys.__stderr__

#adds a stream catcher for display and a memory handler for saving
log_stream=StreamCatch()
logger=getLogger()
display_handler=StreamHandler(stream=log_stream)
display_handler.setLevel(LOGLEVEL)
display_handler.setFormatter(Formatter(LOGFORMATTER))
display_handler.name="StreamCatch"
logger.addHandler(display_handler)

memory_handler=MemoryHandler(MEMBUFFER)
memory_handler.setLevel(LOGLEVEL)
memory_handler.setFormatter(Formatter(LOGFORMATTER))
memory_handler.name="MemoryLog"
logger.addHandler(memory_handler)

log_debug("Started logging")

def make_log_file(log_path, mode='a'):
    """Points memory handler at a particular file to save the log."""
    file_handler = FileHandler(filename=log_path, mode=mode)
    file_handler.setLevel(LOGLEVEL)
    file_handler.setFormatter(Formatter(LOGFORMATTER))
    memory_handler.setTarget(file_handler)

def remove_log_file():
    """closes the log file and removes memory_handler from pointing at it"""
    if memory_handler.target:
Ejemplo n.º 14
0
        # Original punctuation should be out of the brackets
        self.positive("I am hear!", "I am [here]!")
        # Drop leading and trailing spaces
        self.positive(" their is ", "[there] is")
        # Retain double spaces
        self.positive("their  is", "[there]  is")
        # Drop periods, commas, and semicolons
        self.positive("their is.", "[there] is")
        self.positive("their is,", "[there] is")
        self.positive("their is;", "[there] is")

    def test_wording(self):
        """Verify that wording can be generated without failing"""
        self.positive(
            "Their is and your don't supposed to! (blah) They think their is.")
        logging.debug(self.parser.generate_wording('@@')
                      .encode('ascii', 'replace'))
        # not implemented yet
        self.assertRaises(NotImplementedError,
                          self.parser.generate_wording_long, '')

if __name__ == '__main__':
    stream_null = StringIO()
    logging.basicConfig(stream=stream_null, level=logging.DEBUG)
    handler_stream = StreamHandler(stream=sys.stderr)
    handler_mem = MemoryHandler(1024, target=handler_stream)
    handler_mem.setLevel(logging.DEBUG)
    handler_mem.setFormatter(logging.Formatter())
    logging.getLogger().addHandler(handler_mem)
    unittest.main()