Ejemplo n.º 1
0
def get_logger():
    global hd_log
    if hd_log is None:
        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        print('base_dir', BASE_DIR)
        log_file_path = os.path.join(BASE_DIR, 'Log/python-web.log')
        err_log_file_path = os.path.join(BASE_DIR, 'Log/python-web-err.log')
        logger.add(log_file_path, rotation="50 MB",
                   encoding='utf-8')  # Automatically rotate too big file
        logger.add(err_log_file_path,
                   rotation="50 MB",
                   encoding='utf-8',
                   level='ERROR')  # Automatically rotate too big file
        # logstash
        logstash_ip = get_config('logstash', 'ip')
        if logstash_ip:
            logstash_port = get_config('logstash', 'port')
            logstash_handler = AsynchronousLogstashHandler(logstash_ip,
                                                           int(logstash_port),
                                                           database_path=None)
            logstash_formatter = LogstashFormatter(
                message_type='python-logstash',
                extra_prefix='',
                extra=dict(app_name='python-web'))
            logstash_handler.setFormatter(logstash_formatter)
            logger.add(sink=logstash_handler)
        hd_log = logger
    return hd_log
Ejemplo n.º 2
0
    def test_format(self):
        file_handler = ExceptionCatchingFileHandler(os.devnull)
        file_handler.setFormatter(LogstashFormatter(ensure_ascii=False))
        file_handler.emit(makeLogRecord({"msg": u"тест"}))
        file_handler.close()

        self.assertIsNone(file_handler.exception)
Ejemplo n.º 3
0
    def initLogger(appConfig: dict):
        # formatting for log stash
        logstashFormatter = LogstashFormatter(
            message_type='python-logstash',
            extra=dict(application='mis_weekly_report_gen_service'))

        # set app logger name and minimum logging level
        appLogger = logging.getLogger('python-logstash-logger')
        appLogger.setLevel(logging.INFO)

        # configure console logging
        streamHandler = logging.StreamHandler()
        # streamHandler.setFormatter(logstashFormatter)
        appLogger.addHandler(streamHandler)

        # configure logstash logging
        host = appConfig["logstashHost"]
        port = appConfig["logstashPort"]
        if not (pd.isna(host)) and not (pd.isna(port)):
            logstashHandler = AsynchronousLogstashHandler(
                host, port, database_path='logstash.db')
            logstashHandler.setFormatter(logstashFormatter)
            appLogger.addHandler(logstashHandler)
        AppLogger.__instance = appLogger
Ejemplo n.º 4
0
 def _create_formatter_if_necessary(self):
     if self.formatter is None:
         self.formatter = LogstashFormatter()
Ejemplo n.º 5
0
import logging
import sys
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
from config import getConfig
import json

# logging configuration
appConfig = getConfig()
host: str = appConfig["logstash_host"]
port: int = appConfig["logstash_port"]

# formatting for log stash
logstash_formatter = LogstashFormatter(message_type='python-logstash',
                                       extra_prefix='dev',
                                       extra=dict(application='example-app',
                                                  environment='production'))

test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)

streamHandler = logging.StreamHandler()
# streamHandler.setFormatter(logstash_formatter)
test_logger.addHandler(streamHandler)

logstashHandler = AsynchronousLogstashHandler(host,
                                              port,
                                              database_path='logstash.db')
logstashHandler.setFormatter(logstash_formatter)
test_logger.addHandler(logstashHandler)
Ejemplo n.º 6
0
import logging

from flask import Flask
app = Flask(__name__)


from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter

# Create the logger and set it's logging level
logger = logging.getLogger("logstash")
logger.setLevel(logging.ERROR)        

# Create the handler
handler = AsynchronousLogstashHandler(
    host='172.18.0.3', 
    port=5042, 
    ssl_enable=True, 
    ssl_verify=False,
    database_path='python-elk-logstash.db')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter()
handler.setFormatter(formatter)

# Assign handler to the logger
logger.addHandler(handler)

@app.route('/')
def hello_world():  
    logger.info("Hello there")
    return 'Hello, World!'
Ejemplo n.º 7
0
def setup_bot(backend_name: str, logger, config, restore=None) -> ErrBot:
    # from here the environment is supposed to be set (daemon / non daemon,
    # config.py in the python path )

    bot_config_defaults(config)

    if hasattr(config, 'BOT_LOG_FORMATTER'):
        format_logs(formatter=config.BOT_LOG_FORMATTER)
    else:
        format_logs(theme_color=config.TEXT_COLOR_THEME)

        if hasattr(config, 'BOT_LOG_FILE') and config.BOT_LOG_FILE:
            hdlr = logging.FileHandler(config.BOT_LOG_FILE)
            hdlr.setFormatter(
                logging.Formatter(
                    "%(asctime)s %(levelname)-8s %(name)-25s %(message)s"))
            logger.addHandler(hdlr)

    if hasattr(config, 'BOT_LOG_LOGSTASH') and config.BOT_LOG_LOGSTASH:
        try:
            from logstash_async.handler import AsynchronousLogstashHandler
            from logstash_async.formatter import LogstashFormatter
        except ImportError:
            log.exception(
                "You have BOT_LOG_LOGSTASH enabled, but I couldn't import modules "
                "needed for Logstash integration. Did you install python-logstash-async? "
                "(See https://python-logstash-async.readthedocs.io/en/latest/installation.html for installation instructions)"
            )
            exit(-1)

        logger.addHandler(
            AsynchronousLogstashHandler(config.BOT_LOG_LOGSTASH_HOST,
                                        int(config.BOT_LOG_LOGSTASH_PORT),
                                        database_path=None))
        hdlr = logger.handlers[-1]
        hdlr.setFormatter(
            LogstashFormatter(
                extra=dict(application=config.BOT_LOG_LOGSTASH_APP,
                           environment=config.BOT_LOG_LOGSTASH_ENV)))

    if hasattr(config, 'BOT_LOG_SENTRY') and config.BOT_LOG_SENTRY:
        sentry_integrations = []

        try:
            import sentry_sdk
            from sentry_sdk.integrations.logging import LoggingIntegration

        except ImportError:
            log.exception(
                "You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
                "needed for Sentry integration. Did you install sentry-sdk? "
                "(See https://docs.sentry.io/platforms/python for installation instructions)"
            )
            exit(-1)

        sentry_logging = LoggingIntegration(
            level=config.SENTRY_LOGLEVEL, event_level=config.SENTRY_EVENTLEVEL)

        sentry_integrations.append(sentry_logging)

        if hasattr(config,
                   'BOT_LOG_SENTRY_FLASK') and config.BOT_LOG_SENTRY_FLASK:
            try:
                from sentry_sdk.integrations.flask import FlaskIntegration
            except ImportError:
                log.exception(
                    "You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
                    "needed for Sentry integration. Did you install sentry-sdk[flask]? "
                    "(See https://docs.sentry.io/platforms/python/flask for installation instructions)"
                )
                exit(-1)

            sentry_integrations.append(FlaskIntegration())

        try:
            if hasattr(config, 'SENTRY_TRANSPORT') and isinstance(
                    config.SENTRY_TRANSPORT, tuple):
                mod = importlib.import_module(config.SENTRY_TRANSPORT[1])
                transport = getattr(mod, config.SENTRY_TRANSPORT[0])

                sentry_sdk.init(dsn=config.SENTRY_DSN,
                                integrations=sentry_integrations,
                                transport=transport)
            else:
                sentry_sdk.init(dsn=config.SENTRY_DSN,
                                integrations=sentry_integrations)
        except ImportError:
            log.exception(
                f'Unable to import selected SENTRY_TRANSPORT - {config.SENTRY_TRANSPORT}'
            )
            exit(-1)

    logger.setLevel(config.BOT_LOG_LEVEL)

    storage_plugin = get_storage_plugin(config)

    # init the botplugin manager
    botplugins_dir = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
    if not path.exists(botplugins_dir):
        makedirs(botplugins_dir, mode=0o755)

    plugin_indexes = getattr(config, 'BOT_PLUGIN_INDEXES',
                             (PLUGIN_DEFAULT_INDEX, ))
    if isinstance(plugin_indexes, str):
        plugin_indexes = (plugin_indexes, )

    # Extra backend is expected to be a list type, convert string to list.
    extra_backend = getattr(config, 'BOT_EXTRA_BACKEND_DIR', [])
    if isinstance(extra_backend, str):
        extra_backend = [extra_backend]

    backendpm = BackendPluginManager(config, 'errbot.backends', backend_name,
                                     ErrBot, CORE_BACKENDS, extra_backend)

    log.info(f'Found Backend plugin: {backendpm.plugin_info.name}')

    repo_manager = BotRepoManager(storage_plugin, botplugins_dir,
                                  plugin_indexes)

    try:
        bot = backendpm.load_plugin()
        botpm = BotPluginManager(
            storage_plugin,
            config.BOT_EXTRA_PLUGIN_DIR, config.AUTOINSTALL_DEPS,
            getattr(config, 'CORE_PLUGINS',
                    None), lambda name, clazz: clazz(bot, name),
            getattr(config, 'PLUGINS_CALLBACK_ORDER', (None, )))
        bot.attach_storage_plugin(storage_plugin)
        bot.attach_repo_manager(repo_manager)
        bot.attach_plugin_manager(botpm)
        bot.initialize_backend_storage()

        # restore the bot from the restore script
        if restore:
            # Prepare the context for the restore script
            if 'repos' in bot:
                log.fatal('You cannot restore onto a non empty bot.')
                sys.exit(-1)
            log.info(f'**** RESTORING the bot from {restore}')
            restore_bot_from_backup(restore, bot=bot, log=log)
            print('Restore complete. You can restart the bot normally')
            sys.exit(0)

        errors = bot.plugin_manager.update_plugin_places(
            repo_manager.get_all_repos_paths())
        if errors:
            log.error('Some plugins failed to load:\n' +
                      '\n'.join(errors.values()))
            bot._plugin_errors_during_startup = "\n".join(errors.values())
        return bot
    except Exception:
        log.exception("Unable to load or configure the backend.")
        exit(-1)
 def test_format_timestamp_microsecond_2(self):
     formatter = LogstashFormatter()
     # 2021-10-24 13:32:15.024747
     test_time_microsecond2 = 1635082335.024747
     result = formatter._format_timestamp(test_time_microsecond2)
     self.assertEqual(result, '2021-10-24T13:32:15.024Z')
 def test_format_timestamp_microsecond_1(self):
     formatter = LogstashFormatter()
     # 2021-10-24 13:32:15.000024
     test_time_microsecond1 = 1635082335.000024
     result = formatter._format_timestamp(test_time_microsecond1)
     self.assertEqual(result, '2021-10-24T13:32:15.000Z')
 def test_format_timestamp_millisecond(self):
     formatter = LogstashFormatter()
     # 2021-10-24 13:32:15.024000
     test_time_millisecond = 1635082335.024000
     result = formatter._format_timestamp(test_time_millisecond)
     self.assertEqual(result, '2021-10-24T13:32:15.024Z')
 def test_format_timestamp_no_millisecond(self):
     formatter = LogstashFormatter()
     # 2021-10-24 13:32:15
     test_time_simple = 1635082335
     result = formatter._format_timestamp(test_time_simple)
     self.assertEqual(result, '2021-10-24T13:32:15.000Z')