def get_logger():
global hd_log
if hd_log is None:
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
print('base_dir', BASE_DIR)
log_file_path = os.path.join(BASE_DIR, 'Log/python-web.log')
err_log_file_path = os.path.join(BASE_DIR, 'Log/python-web-err.log')
logger.add(log_file_path, rotation="50 MB",
encoding='utf-8') # Automatically rotate too big file
logger.add(err_log_file_path,
rotation="50 MB",
encoding='utf-8',
level='ERROR') # Automatically rotate too big file
# logstash
logstash_ip = get_config('logstash', 'ip')
if logstash_ip:
logstash_port = get_config('logstash', 'port')
logstash_handler = AsynchronousLogstashHandler(logstash_ip,
int(logstash_port),
database_path=None)
logstash_formatter = LogstashFormatter(
message_type='python-logstash',
extra_prefix='',
extra=dict(app_name='python-web'))
logstash_handler.setFormatter(logstash_formatter)
logger.add(sink=logstash_handler)
hd_log = logger
return hd_log
def test_format(self):
file_handler = ExceptionCatchingFileHandler(os.devnull)
file_handler.setFormatter(LogstashFormatter(ensure_ascii=False))
file_handler.emit(makeLogRecord({"msg": u"тест"}))
file_handler.close()
self.assertIsNone(file_handler.exception)
def initLogger(appConfig: dict):
# formatting for log stash
logstashFormatter = LogstashFormatter(
message_type='python-logstash',
extra=dict(application='mis_weekly_report_gen_service'))
# set app logger name and minimum logging level
appLogger = logging.getLogger('python-logstash-logger')
appLogger.setLevel(logging.INFO)
# configure console logging
streamHandler = logging.StreamHandler()
# streamHandler.setFormatter(logstashFormatter)
appLogger.addHandler(streamHandler)
# configure logstash logging
host = appConfig["logstashHost"]
port = appConfig["logstashPort"]
if not (pd.isna(host)) and not (pd.isna(port)):
logstashHandler = AsynchronousLogstashHandler(
host, port, database_path='logstash.db')
logstashHandler.setFormatter(logstashFormatter)
appLogger.addHandler(logstashHandler)
AppLogger.__instance = appLogger
def _create_formatter_if_necessary(self):
if self.formatter is None:
self.formatter = LogstashFormatter()
import logging
import sys
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
from config import getConfig
import json
# logging configuration
appConfig = getConfig()
host: str = appConfig["logstash_host"]
port: int = appConfig["logstash_port"]
# formatting for log stash
logstash_formatter = LogstashFormatter(message_type='python-logstash',
extra_prefix='dev',
extra=dict(application='example-app',
environment='production'))
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
streamHandler = logging.StreamHandler()
# streamHandler.setFormatter(logstash_formatter)
test_logger.addHandler(streamHandler)
logstashHandler = AsynchronousLogstashHandler(host,
port,
database_path='logstash.db')
logstashHandler.setFormatter(logstash_formatter)
test_logger.addHandler(logstashHandler)
import logging
from flask import Flask
app = Flask(__name__)
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
# Create the logger and set it's logging level
logger = logging.getLogger("logstash")
logger.setLevel(logging.ERROR)
# Create the handler
handler = AsynchronousLogstashHandler(
host='172.18.0.3',
port=5042,
ssl_enable=True,
ssl_verify=False,
database_path='python-elk-logstash.db')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter()
handler.setFormatter(formatter)
# Assign handler to the logger
logger.addHandler(handler)
@app.route('/')
def hello_world():
logger.info("Hello there")
return 'Hello, World!'
def setup_bot(backend_name: str, logger, config, restore=None) -> ErrBot:
# from here the environment is supposed to be set (daemon / non daemon,
# config.py in the python path )
bot_config_defaults(config)
if hasattr(config, 'BOT_LOG_FORMATTER'):
format_logs(formatter=config.BOT_LOG_FORMATTER)
else:
format_logs(theme_color=config.TEXT_COLOR_THEME)
if hasattr(config, 'BOT_LOG_FILE') and config.BOT_LOG_FILE:
hdlr = logging.FileHandler(config.BOT_LOG_FILE)
hdlr.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)-8s %(name)-25s %(message)s"))
logger.addHandler(hdlr)
if hasattr(config, 'BOT_LOG_LOGSTASH') and config.BOT_LOG_LOGSTASH:
try:
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
except ImportError:
log.exception(
"You have BOT_LOG_LOGSTASH enabled, but I couldn't import modules "
"needed for Logstash integration. Did you install python-logstash-async? "
"(See https://python-logstash-async.readthedocs.io/en/latest/installation.html for installation instructions)"
)
exit(-1)
logger.addHandler(
AsynchronousLogstashHandler(config.BOT_LOG_LOGSTASH_HOST,
int(config.BOT_LOG_LOGSTASH_PORT),
database_path=None))
hdlr = logger.handlers[-1]
hdlr.setFormatter(
LogstashFormatter(
extra=dict(application=config.BOT_LOG_LOGSTASH_APP,
environment=config.BOT_LOG_LOGSTASH_ENV)))
if hasattr(config, 'BOT_LOG_SENTRY') and config.BOT_LOG_SENTRY:
sentry_integrations = []
try:
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
except ImportError:
log.exception(
"You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
"needed for Sentry integration. Did you install sentry-sdk? "
"(See https://docs.sentry.io/platforms/python for installation instructions)"
)
exit(-1)
sentry_logging = LoggingIntegration(
level=config.SENTRY_LOGLEVEL, event_level=config.SENTRY_EVENTLEVEL)
sentry_integrations.append(sentry_logging)
if hasattr(config,
'BOT_LOG_SENTRY_FLASK') and config.BOT_LOG_SENTRY_FLASK:
try:
from sentry_sdk.integrations.flask import FlaskIntegration
except ImportError:
log.exception(
"You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
"needed for Sentry integration. Did you install sentry-sdk[flask]? "
"(See https://docs.sentry.io/platforms/python/flask for installation instructions)"
)
exit(-1)
sentry_integrations.append(FlaskIntegration())
try:
if hasattr(config, 'SENTRY_TRANSPORT') and isinstance(
config.SENTRY_TRANSPORT, tuple):
mod = importlib.import_module(config.SENTRY_TRANSPORT[1])
transport = getattr(mod, config.SENTRY_TRANSPORT[0])
sentry_sdk.init(dsn=config.SENTRY_DSN,
integrations=sentry_integrations,
transport=transport)
else:
sentry_sdk.init(dsn=config.SENTRY_DSN,
integrations=sentry_integrations)
except ImportError:
log.exception(
f'Unable to import selected SENTRY_TRANSPORT - {config.SENTRY_TRANSPORT}'
)
exit(-1)
logger.setLevel(config.BOT_LOG_LEVEL)
storage_plugin = get_storage_plugin(config)
# init the botplugin manager
botplugins_dir = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
if not path.exists(botplugins_dir):
makedirs(botplugins_dir, mode=0o755)
plugin_indexes = getattr(config, 'BOT_PLUGIN_INDEXES',
(PLUGIN_DEFAULT_INDEX, ))
if isinstance(plugin_indexes, str):
plugin_indexes = (plugin_indexes, )
# Extra backend is expected to be a list type, convert string to list.
extra_backend = getattr(config, 'BOT_EXTRA_BACKEND_DIR', [])
if isinstance(extra_backend, str):
extra_backend = [extra_backend]
backendpm = BackendPluginManager(config, 'errbot.backends', backend_name,
ErrBot, CORE_BACKENDS, extra_backend)
log.info(f'Found Backend plugin: {backendpm.plugin_info.name}')
repo_manager = BotRepoManager(storage_plugin, botplugins_dir,
plugin_indexes)
try:
bot = backendpm.load_plugin()
botpm = BotPluginManager(
storage_plugin,
config.BOT_EXTRA_PLUGIN_DIR, config.AUTOINSTALL_DEPS,
getattr(config, 'CORE_PLUGINS',
None), lambda name, clazz: clazz(bot, name),
getattr(config, 'PLUGINS_CALLBACK_ORDER', (None, )))
bot.attach_storage_plugin(storage_plugin)
bot.attach_repo_manager(repo_manager)
bot.attach_plugin_manager(botpm)
bot.initialize_backend_storage()
# restore the bot from the restore script
if restore:
# Prepare the context for the restore script
if 'repos' in bot:
log.fatal('You cannot restore onto a non empty bot.')
sys.exit(-1)
log.info(f'**** RESTORING the bot from {restore}')
restore_bot_from_backup(restore, bot=bot, log=log)
print('Restore complete. You can restart the bot normally')
sys.exit(0)
errors = bot.plugin_manager.update_plugin_places(
repo_manager.get_all_repos_paths())
if errors:
log.error('Some plugins failed to load:\n' +
'\n'.join(errors.values()))
bot._plugin_errors_during_startup = "\n".join(errors.values())
return bot
except Exception:
log.exception("Unable to load or configure the backend.")
exit(-1)
def test_format_timestamp_microsecond_2(self):
formatter = LogstashFormatter()
# 2021-10-24 13:32:15.024747
test_time_microsecond2 = 1635082335.024747
result = formatter._format_timestamp(test_time_microsecond2)
self.assertEqual(result, '2021-10-24T13:32:15.024Z')
def test_format_timestamp_microsecond_1(self):
formatter = LogstashFormatter()
# 2021-10-24 13:32:15.000024
test_time_microsecond1 = 1635082335.000024
result = formatter._format_timestamp(test_time_microsecond1)
self.assertEqual(result, '2021-10-24T13:32:15.000Z')
def test_format_timestamp_millisecond(self):
formatter = LogstashFormatter()
# 2021-10-24 13:32:15.024000
test_time_millisecond = 1635082335.024000
result = formatter._format_timestamp(test_time_millisecond)
self.assertEqual(result, '2021-10-24T13:32:15.024Z')
def test_format_timestamp_no_millisecond(self):
formatter = LogstashFormatter()
# 2021-10-24 13:32:15
test_time_simple = 1635082335
result = formatter._format_timestamp(test_time_simple)
self.assertEqual(result, '2021-10-24T13:32:15.000Z')
