Esempio n. 1
0
def make_middleware(app,
                    global_conf,
                    verbose_log=None,
                    trace_log=None,
                    max_bodylen='3KB',
                    max_logsize='100MB',
                    backup_count='10',
                    keep='100',
                    ):
    """ Paste filter-app converter """
    backup_count = int(backup_count)
    max_bytes = byte_size(max_logsize)
    max_bodylen = byte_size(max_bodylen)
    keep = int(keep)
    from logging import Logger
    from logging.handlers import RotatingFileHandler

    if verbose_log:
        handler = RotatingFileHandler(verbose_log, maxBytes=max_bytes,
                                      backupCount=backup_count)
        verbose_log = Logger('repoze.debug.verboselogger')
        verbose_log.handlers = [handler]

    if trace_log:
        handler = RotatingFileHandler(trace_log, maxBytes=max_bytes,
                                      backupCount=backup_count)
        trace_log = Logger('repoze.debug.tracelogger')
        trace_log.handlers = [handler]

    return ResponseLoggingMiddleware(app, max_bodylen, keep, verbose_log,
                                     trace_log)
Esempio n. 2
0
def make_middleware(
    app,
    global_conf,
    verbose_log=None,
    trace_log=None,
    max_bodylen='3KB',
    max_logsize='100MB',
    backup_count='10',
    keep='100',
):
    """ Paste filter-app converter """
    backup_count = int(backup_count)
    max_bytes = byte_size(max_logsize)
    max_bodylen = byte_size(max_bodylen)
    keep = int(keep)
    from logging import Logger
    from logging.handlers import RotatingFileHandler

    if verbose_log:
        handler = RotatingFileHandler(verbose_log,
                                      maxBytes=max_bytes,
                                      backupCount=backup_count)
        verbose_log = Logger('repoze.debug.verboselogger')
        verbose_log.handlers = [handler]

    if trace_log:
        handler = RotatingFileHandler(trace_log,
                                      maxBytes=max_bytes,
                                      backupCount=backup_count)
        trace_log = Logger('repoze.debug.tracelogger')
        trace_log.handlers = [handler]

    return ResponseLoggingMiddleware(app, max_bodylen, keep, verbose_log,
                                     trace_log)
Esempio n. 3
0
def process_alerts_and_remediations(log: Logger, clusters,
                                    alerts: List[Tuple[str,
                                                       Callable[...,
                                                                BaseAlert]]]):
    for context in clusters:
        client: OpenShift = OpenShift(context=context,
                                      use_internal=environ.get(
                                          'INTERNAL_CLUSTER',
                                          'false') == 'true')
        cluster: str = client.client.configuration.host
        log.info(f'Processing alerts for Cluster: {cluster} ...')
        log.handlers = [logging_config.get_tabbed_formatter()]
        # Iterate over all alerts
        for _, alert_class in alerts:
            alert = alert_class(client)
            alert.process_alerts()
            if environ.get('REMEDIATION', 'false') == 'true':
                alert.process_remediations()
            if alert.failed_alerts:
                if environ.get('SKIP_EMAIL_FOR_SUCCESSFUL_REMEDIATION',
                               'false') == 'true':
                    if not all(
                            failure.get('remediated', False)
                            for failure in alert.failed_alerts):
                        alert.email_results()
                else:
                    alert.email_results()

        log.handlers = [logging_config.get_normal_formatter()]
        log.info(f'Finished processing alerts for Cluster: {cluster}.')
Esempio n. 4
0
def redirect_logging_to_tqdm(logger: logging.Logger = None):
    if logger is None:
        logger = logging.root
    tqdm_handler = TqdmLoggingHandler()
    original_handlers = logger.handlers
    tqdm_handler.setFormatter(_get_console_formatter(original_handlers))
    try:
        logger.handlers = [
            handler for handler in logger.handlers
            if not _is_console_logging_handler(handler)
        ] + [tqdm_handler]
        yield
    finally:
        logger.handlers = original_handlers
Esempio n. 5
0
def configure_logger_for_colour(logger: logging.Logger,
                                level: int = logging.INFO,
                                remove_existing: bool = False,
                                extranames: List[str] = None,
                                with_process_id: bool = False,
                                with_thread_id: bool = False) -> None:
    """
    Applies a preconfigured datetime/colour scheme to a logger.

    Should ONLY be called from the ``if __name__ == 'main'`` script;
    see https://docs.python.org/3.4/howto/logging.html#library-config.

    Args:
        logger: logger to modify
        level: log level to set
        remove_existing: remove existing handlers from logger first?
        extranames: additional names to append to the logger's name
        with_process_id: include the process ID in the logger's name?
        with_thread_id: include the thread ID in the logger's name?
    """
    if remove_existing:
        logger.handlers = []  # http://stackoverflow.com/questions/7484454
    handler = get_colour_handler(extranames,
                                 with_process_id=with_process_id,
                                 with_thread_id=with_thread_id)
    handler.setLevel(level)
    logger.addHandler(handler)
    logger.setLevel(level)
def logging_config(folder: Optional[str] = None,
                   name: Optional[str] = None,
                   logger: logging.Logger = logging.root,
                   level: int = logging.INFO,
                   console_level: int = logging.INFO,
                   console: bool = True,
                   overwrite_handler: bool = False) -> str:
    """Config the logging module. It will set the logger to save to the specified file path.
    Parameters
    ----------
    folder
        The folder to save the log
    name
        Name of the saved
    logger
        The logger
    level
        Logging level
    console_level
        Logging level of the console log
    console
        Whether to also log to console
    overwrite_handler
        Whether to overwrite the existing handlers in the logger
    Returns
    -------
    folder
        The folder to save the log file.
    """
    if name is None:
        name = inspect.stack()[-1][1].split('.')[0]
    if folder is None:
        folder = os.path.join(os.getcwd(), name)
    if not os.path.exists(folder):
        os.makedirs(folder, exist_ok=True)
    need_file_handler = True
    need_console_handler = True
    # Check all loggers.
    if overwrite_handler:
        logger.handlers = []
    else:
        for handler in logger.handlers:
            if isinstance(handler, logging.StreamHandler):
                need_console_handler = False
    logpath = os.path.join(folder, name + ".log")
    print("All Logs will be saved to {}".format(logpath))
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    if need_file_handler:
        logfile = logging.FileHandler(logpath)
        logfile.setLevel(level)
        logfile.setFormatter(formatter)
        logger.addHandler(logfile)
    if console and need_console_handler:
        # Initialze the console logging
        logconsole = logging.StreamHandler()
        logconsole.setLevel(console_level)
        logconsole.setFormatter(formatter)
        logger.addHandler(logconsole)
    return folder
Esempio n. 7
0
def configure_databricks_logger(log_level=logging.INFO,
                                logger: logging.Logger = None,
                                file_path: str = None) -> logging.Logger:
    """Configure logging for databricks.

    Args:
        log_level: the logging level
        logger: the logger to use, default is root logger
        file_path: the path to a file for storing logs to persistent disk if provided
    """
    if logger is None:
        logger = logging.getLogger()
    logger.setLevel(log_level)
    log_handlers = [logging.StreamHandler(sys.stdout)]
    if file_path is not None:
        file_handler = TimedRotatingFileHandler(file_path,
                                                when="midnight",
                                                backupCount=7)
        file_handler.setFormatter(
            logging.Formatter(
                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
        log_handlers.append(file_handler)
    logger.handlers = log_handlers
    logging.getLogger("py4j").setLevel(
        logging.ERROR)  # To remove the unnecessary databricks logging output
    return logger
Esempio n. 8
0
def default_logging_config(logger_: logging.Logger, ) -> logging.Logger:  # pylint: disable=redefined-outer-name
    """Set up the default handler and formatter on the given logger."""
    default_handler = logging.StreamHandler(stream=sys.stdout)
    default_handler.formatter = ColorFormatter()
    logger_.handlers = [default_handler]
    logger_.propagate = True
    return logger_
Esempio n. 9
0
def basic_config(logger: logging.Logger) -> logging.Logger:
    """Configure the logger for reporting to stderr using click."""

    handler = ClickErrHandler()
    handler.formatter = ColorFormatter()

    logger.handlers = [handler]

    return logger
def configure_logger_for_colour(log: logging.Logger,
                                remove_existing: bool = True) -> None:
    """
    Applies a preconfigured datetime/colour scheme to a logger.
    Should ONLY be called from the "if __name__ == 'main'" script:
        https://docs.python.org/3.4/howto/logging.html#library-config
    """
    if remove_existing:
        log.handlers = []  # http://stackoverflow.com/questions/7484454
    log.addHandler(COLOUR_HANDLER)
def configure_logger_for_colour(log: logging.Logger,
                                remove_existing: bool = True) -> None:
    """
    Applies a preconfigured datetime/colour scheme to a logger.
    Should ONLY be called from the "if __name__ == 'main'" script:
        https://docs.python.org/3.4/howto/logging.html#library-config
    """
    if remove_existing:
        log.handlers = []  # http://stackoverflow.com/questions/7484454
    log.addHandler(COLOUR_HANDLER)
def get_logger(name: str) -> Logger:
    logger = _loggers.get(name)
    if logger:
        return logger

    handler = TimedRotatingFileHandler(f'{DATA_PATH}/logs/{name}.log', when='D', backupCount=10)
    handler.setFormatter(JSONLogFormatter())
    logger = Logger(name, LOG_LEVEL)
    logger.handlers = []
    logger.addHandler(handler)

    return logger
Esempio n. 13
0
def configure_logger_for_colour(log: logging.Logger,
                                level: int = logging.INFO,
                                remove_existing: bool = False,
                                extranames: List[str] = None) -> None:
    """
    Applies a preconfigured datetime/colour scheme to a logger.
    Should ONLY be called from the "if __name__ == 'main'" script:
        https://docs.python.org/3.4/howto/logging.html#library-config
    """
    if remove_existing:
        log.handlers = []  # http://stackoverflow.com/questions/7484454
    log.addHandler(get_colour_handler(extranames))
    log.setLevel(level)
Esempio n. 14
0
def configure_logger_for_colour(log: logging.Logger,
                                level: int = logging.INFO,
                                remove_existing: bool = False,
                                extranames: List[str] = None) -> None:
    """
    Applies a preconfigured datetime/colour scheme to a logger.
    Should ONLY be called from the "if __name__ == 'main'" script:
        https://docs.python.org/3.4/howto/logging.html#library-config
    """
    if remove_existing:
        log.handlers = []  # http://stackoverflow.com/questions/7484454
    log.addHandler(get_colour_handler(extranames))
    log.setLevel(level)
Esempio n. 15
0
def set_log_handler(logger: logging.Logger, file_name: str) -> None:
    if not BucketLoader.DEFAULT_LOG_FOLDER.exists():
        BucketLoader.DEFAULT_LOG_FOLDER.mkdir(parents=True)

    logging.root.setLevel(level=logging.NOTSET)

    file_handler = logging.FileHandler(BucketLoader.DEFAULT_LOG_FOLDER /
                                       file_name)
    file_handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        '%m/%d/%Y %I:%M:%S %p')
    file_handler.setFormatter(formatter)
    logger.handlers = [file_handler]

    logger.write = get_write_method(logger)
    logger.flush = lambda: None
Esempio n. 16
0
def setup_logger(logger: logging.Logger) -> None:
    """
    Prepare logging for the provided logger
    :param logger:
    :return:
    """
    log_level = config.get_log_level()
    logger.setLevel(log_level)
    logger.handlers = []
    log_format = config.get_log_format()
    log_formatter = logging.Formatter(fmt=log_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(log_formatter)
    logger.addHandler(stream_handler)

    if config.get_log_to_file():
        add_log_file_to_logger(logger, log_formatter)
def tee_logging_lines_to(
        *line_writers: Callable[[str], None],
        logger: logging.Logger = None,
        formatter: logging.Formatter = None,
        **kwargs):
    if logger is None:
        logger = logging.root
    if formatter is None:
        formatter = get_default_logging_formatter()
    prev_handlers = logger.handlers
    try:
        handler = LineWriterLoggingHandler(*line_writers, **kwargs)
        if formatter is not None:
            handler.setFormatter(formatter)
        logger.addHandler(handler)
        yield logger
    finally:
        flush_logging_handlers(logger.handlers)
        flush_logging_handlers(logging.root.handlers)
        logger.handlers = prev_handlers
Esempio n. 18
0
def _setup_logger(
    logger: logging.Logger,
    fmt: str = '%(levelname)8s %(asctime)s <daemon> %(message)s',
    datefmt: str = '%Y-%m-%d %H:%M:%S',
):
    # Remove default handler
    if logger.handlers:
        logger.handlers = []

    formatter = logging.Formatter(fmt, datefmt=datefmt)

    h_stdout = logging.StreamHandler(sys.stdout)
    h_stdout.setLevel(logging.NOTSET)
    h_stdout.addFilter(lambda record: record.levelno <= logging.WARNING)
    h_stdout.setFormatter(formatter)
    logger.addHandler(h_stdout)

    h_stderr = logging.StreamHandler(sys.stderr)
    h_stderr.setLevel(logging.ERROR)
    h_stderr.setFormatter(formatter)
    logger.addHandler(h_stderr)
Esempio n. 19
0
def setLoggerDefaults(
    logger:logging.Logger, level:int=logging.INFO, logFileName:Optional[str]=None
) -> None:
    logger.setLevel(level)
    logger.handlers = []
    
    formatter = logging.Formatter(
        '[%(asctime)s] %(levelname)s [%(module)s:%(lineno)s] - %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    
    def setHandlerDefaults(handler:Union[logging.StreamHandler, logging.FileHandler]) -> None:
        handler.setLevel(level)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
    
    streamHandler = logging.StreamHandler(sys.stdout)
    setHandlerDefaults(streamHandler)
    
    if logFileName is not None:
        fileHandler = logging.FileHandler(logFileName, 'w')
        setHandlerDefaults(fileHandler)
Esempio n. 20
0
def close_logger(log: logging.Logger):
    for handler in log.handlers:
        handler.close()
    log.handlers = []
Esempio n. 21
0
 def remove_all_handlers(self, logger: logging.Logger) -> None:
     logger.handlers = list()