예제 #1
0
def logging_wrapper(job, f, ip, port):
    """Wrapper to execute user passed functions remotely after
    setting up logging

    ip and port should specify somewhere we can push logging messages
    over zmq and have something useful happen to them
    """
    handler = NestedSetup([
        ZeroMQPushHandler("tcp://" + ip + ":" + port, level="DEBUG"),
        FileHandler(os.path.join(job["workdir"], job["description"]+".log"),
                    level="DEBUG", bubble=True)
    ])
    logger = Logger(job["description"])
    with handler.applicationbound():
        try:
            if job.get("tmpdir"):
                os.chdir(job["tmpdir"])
            else:
                os.chdir(job["workdir"])
            f(job, logger=logger)
        except:
            if job.get("tmpdir"):
                open(os.path.join(job["tmpdir"], ".error"), 'a').close()
            logger.exception("Task failed with traceback:")
            raise
        return job
예제 #2
0
def defLogging():
    global debug
    global logPath
    global loggingLevel
    global logging_setup

    try:
        if debug:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                StreamHandler(
                        sys.stdout,
                        bubble=False,
                        level=loggingLevel
                ),
                TimedRotatingFileHandler(
                        logPath,
                        level=0,
                        backup_count=3,
                        bubble=True,
                        date_format='%Y-%m-%d',
                ),
            ])
        else:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                FingersCrossedHandler(
                        TimedRotatingFileHandler(
                                logPath,
                                level=0,
                                backup_count=3,
                                bubble=False,
                                date_format='%Y-%m-%d',
                        ),
                        action_level=ERROR,
                        buffer_size=1000,
                        # pull_information=True,
                        # reset=False,
                )
            ])
    except (KeyboardInterrupt, SystemExit):
        raise
    except:
        print("Critical error attempting to setup logging. Falling back to console only.")
        logging_setup = NestedSetup([
            # make sure we never bubble up to the stderr handler
            # if we run out of setup handling
            NullHandler(),
            StreamHandler(
                    sys.stdout,
                    bubble=False
            )
        ])
예제 #3
0
def log_handler(args, thread_wrapping=True):
    """
    Return log handler with given config
    """
    if not isinstance(args, dict):
        args = vars(args)
    if args.get("quiet"):
        stderr_handler = ColorizedStderrHandler(level="ERROR")
    elif args.get("verbose"):
        stderr_handler = ColorizedStderrHandler(level="DEBUG")
    else:
        stderr_handler = ColorizedStderrHandler(level=args.get(
            "stderr_level", "NOTICE").upper(),
                                                bubble=True)
    if args.get("log_file"):
        file_handler = FileHandler(args.get("log_file"),
                                   level=args.get("log_file_level",
                                                  "DEBUG").upper(),
                                   bubble=True)
    else:
        file_handler = NullHandler()

    if thread_wrapping:
        file_handler = ThreadedWrapperHandler(file_handler)
        stderr_handler = ThreadedWrapperHandler(stderr_handler)

    return NestedSetup([
        NullHandler(),  # catch everything else
        file_handler,
        stderr_handler
    ])
예제 #4
0
def get_nestedlog(level='DEBUG', filename='quantrade.log', uri=None):
    # Default uri: tcp://127.0.0.1:5540
    if uri is not None:
        log_setup = NestedSetup([
            ZeroMQHandler(uri),
        ])
    else:
        log_setup = NestedSetup([
            logbook.NullHandler(level=logbook.DEBUG, bubble=True),
            logbook.StreamHandler(sys.stdout,
                                  level=logbook.INFO,
                                  format_string=log_format),
            logbook.StreamHandler(sys.stderr,
                                  level=logbook.ERROR,
                                  format_string=log_format),
            logbook.FileHandler('{}/{}'.format(log_destination, filename),
                                level=level),
        ])

    return log_setup
예제 #5
0
def setup(name, path='log', enable_debug=False):
    """
    Prepare a NestedSetup.

    :param name: the channel name
    :param path: the path where the logs will be written
    :param enable_debug: do we want to save the message at the DEBUG level

    :return a nested Setup
    """
    path_tmpl = os.path.join(path, '{name}_{level}.log')
    info = path_tmpl.format(name=name, level='info')
    warn = path_tmpl.format(name=name, level='warn')
    err = path_tmpl.format(name=name, level='err')
    crit = path_tmpl.format(name=name, level='crit')
    # a nested handler setup can be used to configure more complex setups
    setup = [
        # make sure we never bubble up to the stderr handler
        # if we run out of setup handling
        NullHandler(),
        # then write messages that are at least info to to a logfile
        TimedRotatingFileHandler(info, level='INFO', encoding='utf-8',
                                 date_format='%Y-%m-%d'),
        # then write messages that are at least warnings to to a logfile
        TimedRotatingFileHandler(warn, level='WARNING', encoding='utf-8',
                                 date_format='%Y-%m-%d'),
        # then write messages that are at least errors to to a logfile
        TimedRotatingFileHandler(err, level='ERROR', encoding='utf-8',
                                 date_format='%Y-%m-%d'),
        # then write messages that are at least critical errors to to a logfile
        TimedRotatingFileHandler(crit, level='CRITICAL', encoding='utf-8',
                                 date_format='%Y-%m-%d'),
    ]
    if enable_debug:
        debug = path_tmpl.format(name=name, level='debug')
        setup.insert(1, TimedRotatingFileHandler(debug, level='DEBUG',
                     encoding='utf-8', date_format='%Y-%m-%d'))
    if src_server is not None and smtp_server is not None \
            and smtp_port != 0 and len(dest_mails) != 0:
        mail_tmpl = '{name}_error@{src}'
        from_mail = mail_tmpl.format(name=name, src=src_server)
        subject = 'Error in {}'.format(name)
        # errors should then be delivered by mail and also be kept
        # in the application log, so we let them bubble up.
        setup.append(MailHandler(from_mail, dest_mails, subject,
                                 level='ERROR', bubble=True,
                                 server_addr=(smtp_server, smtp_port)))

    return NestedSetup(setup)
예제 #6
0
def main(config_file, **kwargs):
    with open(config_file) as fh:
        config = yaml.load(fh)

    try:
        rmq_settings = config["rabbitmq_logging"]
    except KeyError:
        print("RabbitMQ logging not configured in {}".format(config_file))
        sys.exit()

    handlers = [NullHandler()]
    if not kwargs["quiet"]:
        handlers.append(StderrHandler(bubble=True))

    if kwargs["filename"]:
        handlers.append(FileHandler(kwargs["filename"], bubble=True))

    if kwargs["log_db"]:
        try:
            cdb_settings = config["couchdb_logging"]
        except KeyError:
            print("CouchDB logging not configured in {}".format(config_file))
            sys.exit()

        db_handler = DatabaseHandler(cdb_settings["couchdb_url"],
                                     backend=CouchDBBackend,
                                     db=cdb_settings["database"],
                                     bubble=True)
        handlers.append(db_handler)

    setup = NestedSetup(handlers)

    print("Now waiting for log messages")
    with setup:
        subscriber = RabbitMQSubscriber(rmq_settings["url"],
                                        queue=rmq_settings["log_queue"])
        try:
            subscriber.dispatch_forever()

        except KeyboardInterrupt:
            print("\nLog subscriber shutting down")
            subscriber.close()

        except Exception:
            print("Log subscriber quit (unexpectedly)")
예제 #7
0
def get_logs_dispatcher(uri=None, debug=False):
    handlers = []

    if not debug:
        handlers.append(NullHandler(level=DEBUG))

    handlers.append(ColorizedStderrHandler(level=INFO))

    if not uri:
        # Find an open port.
        # This is a race condition as the port could be used between
        # the check and its binding. However, this is probably one of the
        # best solution without patching Logbook.
        tmpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        tmpsock.bind(('localhost', 0))
        uri = 'tcp://{}:{}'.format(*tmpsock.getsockname())
        tmpsock.close()

    subscriber = ZeroMQSubscriber(uri, multi=True)
    return uri, subscriber.dispatch_in_background(setup=NestedSetup(handlers))
예제 #8
0
파일: pyfa.py 프로젝트: w9jds/Pyfa
    else:
        savePath_filename = "Pyfa.log"

    config.logPath = os.path.join(config.savePath, savePath_filename)

    try:
        if options.debug:
            logging_mode = "Debug"
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                StreamHandler(sys.stdout,
                              bubble=False,
                              level=options.logginglevel),
                TimedRotatingFileHandler(
                    config.logPath,
                    level=0,
                    backup_count=3,
                    bubble=True,
                    date_format='%Y-%m-%d',
                ),
            ])
        else:
            logging_mode = "User"
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                FingersCrossedHandler(
                    TimedRotatingFileHandler(
예제 #9
0
파일: config.py 프로젝트: thauser/Pyfa
def defLogging():
    global debug
    global logPath
    global loggingLevel
    global logging_setup

    try:
        if debug:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                StreamHandler(sys.stdout, bubble=False, level=loggingLevel),
                TimedRotatingFileHandler(
                    logPath,
                    level=0,
                    backup_count=3,
                    bubble=True,
                    date_format='%Y-%m-%d',
                ),
            ])
        else:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                FingersCrossedHandler(
                    TimedRotatingFileHandler(
                        logPath,
                        level=0,
                        backup_count=3,
                        bubble=False,
                        date_format='%Y-%m-%d',
                    ),
                    action_level=ERROR,
                    buffer_size=1000,
                    # pull_information=True,
                    # reset=False,
                )
            ])
    except:
        print(
            "Critical error attempting to setup logging. Falling back to console only."
        )
        logging_setup = NestedSetup([
            # make sure we never bubble up to the stderr handler
            # if we run out of setup handling
            NullHandler(),
            StreamHandler(sys.stdout, bubble=False)
        ])

    with logging_setup.threadbound():

        # Output all stdout (print) messages as warnings
        try:
            sys.stdout = LoggerWriter(pyfalog.warning)
        except:
            pyfalog.critical(
                "Cannot redirect.  Continuing without writing stdout to log.")

        # Output all stderr (stacktrace) messages as critical
        try:
            sys.stderr = LoggerWriter(pyfalog.critical)
        except:
            pyfalog.critical(
                "Cannot redirect.  Continuing without writing stderr to log.")
예제 #10
0
    servers = [Server(i) for i in range(1, NUM_SERVERS + 1)]

    start_servers(servers)

    time.sleep(10)
    try:
        return test(servers)
    except Exception, e:
        logger.exception('Test failed: %s' % e)
        return 1
    finally:
        logger.info('Stopping')
        stop(servers)

if __name__ == '__main__':
    format = '[{record.time}] {record.level_name:>5} [{record.extra[worker_id]}] {record.message}'

    logging_setup = NestedSetup([
        NullHandler(),
        FileHandler(
            filename=os.path.join(os.path.dirname(__file__), 'log/client.log'),
            format_string=format,
            bubble=True,
        ),
        StderrHandler(level=logbook.INFO, format_string=format, bubble=True),
    ])

    with logging_setup.applicationbound():
        sys.exit(main())
예제 #11
0
    servers = [Server(i) for i in range(1, NUM_SERVERS + 1)]

    start_servers(servers)

    time.sleep(10)
    try:
        return test(servers)
    except Exception, e:
        logger.exception('Test failed: %s' % e)
        return 1
    finally:
        logger.info('Stopping')
        stop(servers)


if __name__ == '__main__':
    format = '[{record.time}] {record.level_name:>5} [{record.extra[worker_id]}] {record.message}'

    logging_setup = NestedSetup([
        NullHandler(),
        FileHandler(
            filename=os.path.join(os.path.dirname(__file__), 'log/client.log'),
            format_string=format,
            bubble=True,
        ),
        StderrHandler(level=logbook.INFO, format_string=format, bubble=True),
    ])

    with logging_setup.applicationbound():
        sys.exit(main())
예제 #12
0
파일: config.py 프로젝트: Sectoid/Pyfa
def defLogging():
    global debug
    global logPath
    global loggingLevel
    global logging_setup

    try:
        if debug:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                StreamHandler(
                        sys.stdout,
                        bubble=False,
                        level=loggingLevel
                ),
                TimedRotatingFileHandler(
                        logPath,
                        level=0,
                        backup_count=3,
                        bubble=True,
                        date_format='%Y-%m-%d',
                ),
            ])
        else:
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                FingersCrossedHandler(
                        TimedRotatingFileHandler(
                                logPath,
                                level=0,
                                backup_count=3,
                                bubble=False,
                                date_format='%Y-%m-%d',
                        ),
                        action_level=ERROR,
                        buffer_size=1000,
                        # pull_information=True,
                        # reset=False,
                )
            ])
    except:
        print("Critical error attempting to setup logging. Falling back to console only.")
        logging_setup = NestedSetup([
            # make sure we never bubble up to the stderr handler
            # if we run out of setup handling
            NullHandler(),
            StreamHandler(
                    sys.stdout,
                    bubble=False
            )
        ])

    with logging_setup.threadbound():

        # Output all stdout (print) messages as warnings
        try:
            sys.stdout = LoggerWriter(pyfalog.warning)
        except:
            pyfalog.critical("Cannot redirect.  Continuing without writing stdout to log.")

        # Output all stderr (stacktrace) messages as critical
        try:
            sys.stderr = LoggerWriter(pyfalog.critical)
        except:
            pyfalog.critical("Cannot redirect.  Continuing without writing stderr to log.")
예제 #13
0
import sys
from logbook import Logger, NestedSetup, StreamHandler, FileHandler, StringFormatterHandlerMixin, NullHandler

format_string = '[{record.time:%y%m%d %H:%M}] {record.level_name}: snakepot {record.channel}:  {record.message}'

NestedSetup([
    FileHandler('logfile.log', format_string=format_string, level='DEBUG'),
    StreamHandler(sys.stderr, format_string=format_string, bubble=True)
]).push_application()
예제 #14
0
from logbook.more import ColorizingStreamHandlerMixin, ColorizedStderrHandler
from logbook import NestedSetup, FileHandler, Processor, StderrHandler, StreamHandler

from utils import get_ip


def inject_information(record):
    record.extra['ip'] = get_ip()

log_format = u'[{record.time:%Y-%m-%d %H:%M}] {record.channel} - {record.level_name}: {record.message} \t({record.extra[ip]})'

# a nested handler setup can be used to configure more complex setups
setup = NestedSetup([
    #StderrHandler(format_string=u'[{record.time:%Y-%m-%d %H:%M}] {record.channel} - {record.level_name}: {record.message} \t({record.extra[ip]})'),
    StreamHandler(sys.stdout, format_string=log_format),
    # then write messages that are at least warnings to to a logfile
    FileHandler(os.environ['QTRADE_LOG'], level='WARNING'),
    Processor(inject_information)
])

color_setup = NestedSetup([
    StreamHandler(sys.stdout, format_string=log_format),
    ColorizedStderrHandler(format_string=log_format, level='NOTICE'),
    Processor(inject_information)
])

remote_setup = NestedSetup([
    ZeroMQHandler('tcp://127.0.0.1:5540'),
    Processor(inject_information)
])
예제 #15
0
        savePath_filename = "Pyfa.log"

    config.logPath = os.path.join(config.savePath, savePath_filename)

    try:
        if options.debug:
            logging_mode = "Debug"
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                StreamHandler(
                        sys.stdout,
                        bubble=False,
                        level=options.logginglevel
                ),
                TimedRotatingFileHandler(
                        config.logPath,
                        level=0,
                        backup_count=3,
                        bubble=True,
                        date_format='%Y-%m-%d',
                ),
            ])
        else:
            logging_mode = "User"
            logging_setup = NestedSetup([
                # make sure we never bubble up to the stderr handler
                # if we run out of setup handling
                NullHandler(),
                FingersCrossedHandler(
예제 #16
0
            logbook.StreamHandler(sys.stderr,
                                  level=logbook.ERROR,
                                  format_string=log_format),
            logbook.FileHandler('{}/{}'.format(log_destination, filename),
                                level=level),
        ])

    return log_setup


# a nested handler setup can be used to configure more complex setups
setup = NestedSetup([
    #StderrHandler(format_string=u'[{record.time:%Y-%m-%d %H:%M}] {record.channel} - {record.level_name}: {record.message} \t({record.extra[ip]})'),
    logbook.StreamHandler(sys.stdout, format_string=log_format),
    # then write messages that are at least warnings to to a logfile
    #FIXME FileHandler(os.environ['QTRADE_LOG'], level='WARNING'),
    logbook.FileHandler('{}/{}'.format(log_destination, 'quantrade.log'),
                        level='WARNING'),
    #Processor(inject_information)
])

color_setup = NestedSetup([
    StreamHandler(sys.stdout, format_string=log_format),
    ColorizedStderrHandler(format_string=log_format, level='NOTICE'),
    #Processor(inject_information)
])

#remote_setup = NestedSetup([
#ZeroMQHandler('tcp://127.0.0.1:56540'),
##Processor(inject_information)
#])