def basic_logging_config(
filename=None,
log_level=logging.INFO,
console_stream=sys.stderr,
console_formatter_name="formatter_colorlog",
file_formatter_name="formatter_full",
):
# type: (...) -> Optional[dict]
config = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"formatter_full": {
"format": FORMAT_FULL
},
"formatter_simple": {
"format": FORMAT_SIMPLE
},
"formatter_colorlog": {
"()": "dbnd._vendor.colorlog.ColoredFormatter",
"format": FORMAT_COLORLOG,
"reset": True,
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"stream": console_stream,
"formatter": console_formatter_name,
}
},
"root": {
"handlers": ["console"],
"level": log_level
},
}
if filename:
setup_log_file(filename)
config["handlers"]["file"] = {
"class": "logging.FileHandler",
"formatter": file_formatter_name,
"filename": filename,
"encoding": "utf-8",
}
config["root"]["handlers"].append("file")
sentry_url = os.environ.get(ENV_SENTRY_URL)
if sentry_url:
sentry_env = os.environ.get(ENV_SENTRY_ENV) or "dev"
config["handlers"]["sentry"] = get_sentry_logging_config(
sentry_url=sentry_url, sentry_env=sentry_env)
config["root"]["handlers"].append("sentry")
return config
def get_task_log_file_handler(self, log_file):
if not self.task_log_file_formatter:
config = self.get_dbnd_logging_config()
configurator = DictConfigurator(config)
file_formatter_config = configurator.config.get("formatters").get(
self.file_formatter_name)
self.task_log_file_formatter = configurator.configure_formatter(
file_formatter_config)
# "formatter": log_settings.file_formatter,
log_file = str(log_file)
setup_log_file(log_file)
handler = logging.FileHandler(filename=log_file, encoding="utf-8")
handler.setFormatter(self.task_log_file_formatter)
handler.setLevel(self.level)
return handler
def get_dbnd_logging_config_base(self, filename=None):
# type: (LoggingConfig, Optional[str]) -> Optional[dict]
log_settings = self
log_level = log_settings.level
# we want to have "real" output, so nothing can catch our handler
# in opposite to what airflow is doing
console_stream = (sys.__stdout__
if log_settings.stream_stdout else sys.__stderr__)
if "ipykernel" in sys.modules and self.support_jupyter:
# we can not use __stdout__ or __stderr__ as it will not be printed into jupyter web UI
# at the same time using sys.stdout when airflow is active is very dangerous
# as it can create dangerous loop from airflow redirection into root logger
console_stream = sys.stdout if log_settings.stream_stdout else sys.stderr
# dummy path, we will not write to this file
task_file_handler_file = databand_system_path("logs", "task.log")
setup_log_file(task_file_handler_file)
config = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"task_context_filter": {
"()": "dbnd._core.log.logging_utils.TaskContextFilter"
}
},
"formatters": {
"formatter": {
"format": log_settings.formatter
},
"formatter_simple": {
"format": log_settings.formatter_simple
},
"formatter_colorlog": {
"()": "dbnd._vendor.colorlog.ColoredFormatter",
"format": log_settings.formatter_colorlog,
"reset": True,
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"stream": console_stream,
"formatter": log_settings.console_formatter_name,
"filters": ["task_context_filter"],
}
},
"root": {
"handlers": ["console"],
"level": log_level
},
}
if filename:
setup_log_file(filename)
config["handlers"]["file"] = {
"class": "logging.FileHandler",
"formatter": log_settings.file_formatter_name,
"filename": filename,
"encoding": "utf-8",
}
config["root"]["handlers"].append("file")
loggers = config.setdefault("loggers", {})
for logger_warn in log_settings.at_warn:
loggers[logger_warn] = {
"level": logging.WARNING,
"propagate": True
}
for logger_debug in log_settings.at_debug:
loggers[logger_debug] = {"level": logging.DEBUG, "propagate": True}
if log_settings.sqlalchemy_print:
loggers["sqlalchemy.engine"] = {
"level": logging.INFO,
"propagate": True
}
if log_settings.sentry_url:
config["handlers"]["sentry"] = get_sentry_logging_config(
sentry_url=log_settings.sentry_url,
sentry_env=log_settings.sentry_env)
config["root"]["handlers"].append("sentry")
return config
