def get_logger(): global hd_log if hd_log is None: BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) print('base_dir', BASE_DIR) log_file_path = os.path.join(BASE_DIR, 'Log/python-web.log') err_log_file_path = os.path.join(BASE_DIR, 'Log/python-web-err.log') logger.add(log_file_path, rotation="50 MB", encoding='utf-8') # Automatically rotate too big file logger.add(err_log_file_path, rotation="50 MB", encoding='utf-8', level='ERROR') # Automatically rotate too big file # logstash logstash_ip = get_config('logstash', 'ip') if logstash_ip: logstash_port = get_config('logstash', 'port') logstash_handler = AsynchronousLogstashHandler(logstash_ip, int(logstash_port), database_path=None) logstash_formatter = LogstashFormatter( message_type='python-logstash', extra_prefix='', extra=dict(app_name='python-web')) logstash_handler.setFormatter(logstash_formatter) logger.add(sink=logstash_handler) hd_log = logger return hd_log
def test_format(self): file_handler = ExceptionCatchingFileHandler(os.devnull) file_handler.setFormatter(LogstashFormatter(ensure_ascii=False)) file_handler.emit(makeLogRecord({"msg": u"тест"})) file_handler.close() self.assertIsNone(file_handler.exception)
def initLogger(appConfig: dict): # formatting for log stash logstashFormatter = LogstashFormatter( message_type='python-logstash', extra=dict(application='mis_weekly_report_gen_service')) # set app logger name and minimum logging level appLogger = logging.getLogger('python-logstash-logger') appLogger.setLevel(logging.INFO) # configure console logging streamHandler = logging.StreamHandler() # streamHandler.setFormatter(logstashFormatter) appLogger.addHandler(streamHandler) # configure logstash logging host = appConfig["logstashHost"] port = appConfig["logstashPort"] if not (pd.isna(host)) and not (pd.isna(port)): logstashHandler = AsynchronousLogstashHandler( host, port, database_path='logstash.db') logstashHandler.setFormatter(logstashFormatter) appLogger.addHandler(logstashHandler) AppLogger.__instance = appLogger
def _create_formatter_if_necessary(self): if self.formatter is None: self.formatter = LogstashFormatter()
import logging import sys from logstash_async.handler import AsynchronousLogstashHandler from logstash_async.formatter import LogstashFormatter from config import getConfig import json # logging configuration appConfig = getConfig() host: str = appConfig["logstash_host"] port: int = appConfig["logstash_port"] # formatting for log stash logstash_formatter = LogstashFormatter(message_type='python-logstash', extra_prefix='dev', extra=dict(application='example-app', environment='production')) test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) streamHandler = logging.StreamHandler() # streamHandler.setFormatter(logstash_formatter) test_logger.addHandler(streamHandler) logstashHandler = AsynchronousLogstashHandler(host, port, database_path='logstash.db') logstashHandler.setFormatter(logstash_formatter) test_logger.addHandler(logstashHandler)
import logging from flask import Flask app = Flask(__name__) from logstash_async.handler import AsynchronousLogstashHandler from logstash_async.formatter import LogstashFormatter # Create the logger and set it's logging level logger = logging.getLogger("logstash") logger.setLevel(logging.ERROR) # Create the handler handler = AsynchronousLogstashHandler( host='172.18.0.3', port=5042, ssl_enable=True, ssl_verify=False, database_path='python-elk-logstash.db') # Here you can specify additional formatting on your log record/message formatter = LogstashFormatter() handler.setFormatter(formatter) # Assign handler to the logger logger.addHandler(handler) @app.route('/') def hello_world(): logger.info("Hello there") return 'Hello, World!'
def setup_bot(backend_name: str, logger, config, restore=None) -> ErrBot: # from here the environment is supposed to be set (daemon / non daemon, # config.py in the python path ) bot_config_defaults(config) if hasattr(config, 'BOT_LOG_FORMATTER'): format_logs(formatter=config.BOT_LOG_FORMATTER) else: format_logs(theme_color=config.TEXT_COLOR_THEME) if hasattr(config, 'BOT_LOG_FILE') and config.BOT_LOG_FILE: hdlr = logging.FileHandler(config.BOT_LOG_FILE) hdlr.setFormatter( logging.Formatter( "%(asctime)s %(levelname)-8s %(name)-25s %(message)s")) logger.addHandler(hdlr) if hasattr(config, 'BOT_LOG_LOGSTASH') and config.BOT_LOG_LOGSTASH: try: from logstash_async.handler import AsynchronousLogstashHandler from logstash_async.formatter import LogstashFormatter except ImportError: log.exception( "You have BOT_LOG_LOGSTASH enabled, but I couldn't import modules " "needed for Logstash integration. Did you install python-logstash-async? " "(See https://python-logstash-async.readthedocs.io/en/latest/installation.html for installation instructions)" ) exit(-1) logger.addHandler( AsynchronousLogstashHandler(config.BOT_LOG_LOGSTASH_HOST, int(config.BOT_LOG_LOGSTASH_PORT), database_path=None)) hdlr = logger.handlers[-1] hdlr.setFormatter( LogstashFormatter( extra=dict(application=config.BOT_LOG_LOGSTASH_APP, environment=config.BOT_LOG_LOGSTASH_ENV))) if hasattr(config, 'BOT_LOG_SENTRY') and config.BOT_LOG_SENTRY: sentry_integrations = [] try: import sentry_sdk from sentry_sdk.integrations.logging import LoggingIntegration except ImportError: log.exception( "You have BOT_LOG_SENTRY enabled, but I couldn't import modules " "needed for Sentry integration. Did you install sentry-sdk? " "(See https://docs.sentry.io/platforms/python for installation instructions)" ) exit(-1) sentry_logging = LoggingIntegration( level=config.SENTRY_LOGLEVEL, event_level=config.SENTRY_EVENTLEVEL) sentry_integrations.append(sentry_logging) if hasattr(config, 'BOT_LOG_SENTRY_FLASK') and config.BOT_LOG_SENTRY_FLASK: try: from sentry_sdk.integrations.flask import FlaskIntegration except ImportError: log.exception( "You have BOT_LOG_SENTRY enabled, but I couldn't import modules " "needed for Sentry integration. Did you install sentry-sdk[flask]? " "(See https://docs.sentry.io/platforms/python/flask for installation instructions)" ) exit(-1) sentry_integrations.append(FlaskIntegration()) try: if hasattr(config, 'SENTRY_TRANSPORT') and isinstance( config.SENTRY_TRANSPORT, tuple): mod = importlib.import_module(config.SENTRY_TRANSPORT[1]) transport = getattr(mod, config.SENTRY_TRANSPORT[0]) sentry_sdk.init(dsn=config.SENTRY_DSN, integrations=sentry_integrations, transport=transport) else: sentry_sdk.init(dsn=config.SENTRY_DSN, integrations=sentry_integrations) except ImportError: log.exception( f'Unable to import selected SENTRY_TRANSPORT - {config.SENTRY_TRANSPORT}' ) exit(-1) logger.setLevel(config.BOT_LOG_LEVEL) storage_plugin = get_storage_plugin(config) # init the botplugin manager botplugins_dir = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR) if not path.exists(botplugins_dir): makedirs(botplugins_dir, mode=0o755) plugin_indexes = getattr(config, 'BOT_PLUGIN_INDEXES', (PLUGIN_DEFAULT_INDEX, )) if isinstance(plugin_indexes, str): plugin_indexes = (plugin_indexes, ) # Extra backend is expected to be a list type, convert string to list. extra_backend = getattr(config, 'BOT_EXTRA_BACKEND_DIR', []) if isinstance(extra_backend, str): extra_backend = [extra_backend] backendpm = BackendPluginManager(config, 'errbot.backends', backend_name, ErrBot, CORE_BACKENDS, extra_backend) log.info(f'Found Backend plugin: {backendpm.plugin_info.name}') repo_manager = BotRepoManager(storage_plugin, botplugins_dir, plugin_indexes) try: bot = backendpm.load_plugin() botpm = BotPluginManager( storage_plugin, config.BOT_EXTRA_PLUGIN_DIR, config.AUTOINSTALL_DEPS, getattr(config, 'CORE_PLUGINS', None), lambda name, clazz: clazz(bot, name), getattr(config, 'PLUGINS_CALLBACK_ORDER', (None, ))) bot.attach_storage_plugin(storage_plugin) bot.attach_repo_manager(repo_manager) bot.attach_plugin_manager(botpm) bot.initialize_backend_storage() # restore the bot from the restore script if restore: # Prepare the context for the restore script if 'repos' in bot: log.fatal('You cannot restore onto a non empty bot.') sys.exit(-1) log.info(f'**** RESTORING the bot from {restore}') restore_bot_from_backup(restore, bot=bot, log=log) print('Restore complete. You can restart the bot normally') sys.exit(0) errors = bot.plugin_manager.update_plugin_places( repo_manager.get_all_repos_paths()) if errors: log.error('Some plugins failed to load:\n' + '\n'.join(errors.values())) bot._plugin_errors_during_startup = "\n".join(errors.values()) return bot except Exception: log.exception("Unable to load or configure the backend.") exit(-1)
def test_format_timestamp_microsecond_2(self): formatter = LogstashFormatter() # 2021-10-24 13:32:15.024747 test_time_microsecond2 = 1635082335.024747 result = formatter._format_timestamp(test_time_microsecond2) self.assertEqual(result, '2021-10-24T13:32:15.024Z')
def test_format_timestamp_microsecond_1(self): formatter = LogstashFormatter() # 2021-10-24 13:32:15.000024 test_time_microsecond1 = 1635082335.000024 result = formatter._format_timestamp(test_time_microsecond1) self.assertEqual(result, '2021-10-24T13:32:15.000Z')
def test_format_timestamp_millisecond(self): formatter = LogstashFormatter() # 2021-10-24 13:32:15.024000 test_time_millisecond = 1635082335.024000 result = formatter._format_timestamp(test_time_millisecond) self.assertEqual(result, '2021-10-24T13:32:15.024Z')
def test_format_timestamp_no_millisecond(self): formatter = LogstashFormatter() # 2021-10-24 13:32:15 test_time_simple = 1635082335 result = formatter._format_timestamp(test_time_simple) self.assertEqual(result, '2021-10-24T13:32:15.000Z')