def setup_logger(logger):
supported_keys = [
'asctime', 'levelname', 'lineno', 'module', 'message', 'name'
]
custom_format = ' '.join(
(lambda x: ['%({0:s})'.format(i) for i in x])(supported_keys))
formatter = jsonlogger.JsonFormatter(custom_format)
logger.setLevel(logging.DEBUG)
h1 = RotatingFileHandler('debug.log', maxBytes=1000000, backupCount=1)
h1.setLevel(logging.DEBUG)
h1.addFilter(InfoFilter())
h1.setFormatter(formatter)
h2 = RotatingFileHandler('warning.log', maxBytes=1000000, backupCount=1)
h2.setLevel(logging.WARNING)
h2.setFormatter(formatter)
h3 = RotatingFileHandler('error.log', maxBytes=1000000, backupCount=1)
h3.setLevel(logging.ERROR)
h3.setFormatter(formatter)
logger.addHandler(h1)
logger.addHandler(h2)
logger.addHandler(h3)
return logger
def register_logger(app):
# Formatter
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(process)d %(thread)d '
'%(pathname)s %(lineno)s %(message)s')
class InfoFilter(logging.Filter):
def filter(self, record):
if logging.INFO <= record.levelno < logging.ERROR:
return super().filter(record)
else:
return 0
# FileHandler Info
file_handler_info = RotatingFileHandler(filename='info.log')
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
# FileHandler Error
file_handler_error = RotatingFileHandler(filename='error.log')
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
app.logger.addHandler(file_handler_info)
def _get_handler():
# we only need one global handler
global handler
if handler is not None:
return handler
path = '/var/log/rhsm/rhsm.log'
try:
if not os.path.isdir("/var/log/rhsm"):
os.mkdir("/var/log/rhsm")
except Exception:
pass
# Try to write to /var/log, fallback on console logging:
try:
handler = RotatingFileHandler(path,
maxBytes=0x100000,
backupCount=5,
encoding='utf-8')
except IOError:
handler = logging.StreamHandler()
except Exception:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
handler.setLevel(LOG_LEVEL)
handler.addFilter(ContextLoggingFilter(name=""))
return handler
def register_logging(app):
# TODO https://blog.csdn.net/zwxiaoliu/article/details/80890136
# email errors to the administrators
import logging
from logging.handlers import RotatingFileHandler
# Formatter
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(pathname)s %(lineno)s %(module)s.%(funcName)s %(message)s'
)
# log dir
if not os.path.exists(app.config['LOG_PATH']):
os.makedirs(app.config['LOG_PATH'])
# FileHandler Info
file_handler_info = RotatingFileHandler(
filename=app.config['LOG_PATH_INFO'])
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(
filename=app.config['LOG_PATH_ERROR'])
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def make_logger():
logger = logging.getLogger('DownloaderForReddit')
logger.setLevel(logging.DEBUG)
stream_formatter = JsonStreamFormatter(
'%(asctime)s: %(levelname)s : %(name)s : %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
json_formatter = jsonlogger.JsonFormatter(
fmt=
'%(levelname) %(version) %(platform) %(name) %(filename) %(module) %(funcName) %(lineno) %(message) '
'%(asctime)',
datefmt='%m/%d/%Y %I:%M:%S %p',
json_indent=4,
json_ensure_ascii=True)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(stream_formatter)
log_path = os.path.join(system_util.get_data_directory(),
'DownloaderForReddit.log')
file_handler = RotatingFileHandler(log_path,
maxBytes=1024 * 1024,
backupCount=2)
file_handler.addFilter(LogFilter())
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(json_formatter)
logger.addHandler(stream_handler)
logger.addHandler(file_handler)
def _get_handler(settings):
""" Return a log handler object according to settings """
filename = settings.get('LOG_FILE')
if filename:
encoding = settings.get('LOG_ENCODING')
if settings.get("LOG_ROTATING") is True:
max_bytes = settings.get('LOG_MAX_BYTES', 0)
log_backup_count = settings.get('LOG_BACKUP_COUNT', 0)
handler = RotatingFileHandler(filename,
maxBytes=max_bytes,
backupCount=log_backup_count,
encoding=encoding)
else:
handler = logging.FileHandler(filename, encoding=encoding)
elif settings.getbool('LOG_ENABLED'):
handler = logging.StreamHandler()
else:
handler = logging.NullHandler()
formatter = logging.Formatter(fmt=settings.get('LOG_FORMAT'),
datefmt=settings.get('LOG_DATEFORMAT'))
handler.setFormatter(formatter)
handler.setLevel(settings.get('LOG_LEVEL'))
if settings.getbool('LOG_SHORT_NAMES'):
handler.addFilter(TopLevelFormatter(['scrapy']))
return handler
def init_app(cls, app):
# email errors to the administrators
import logging
from logging.handlers import RotatingFileHandler
try:
os.mkdir(cls.LOG_PATH)
except FileExistsError:
pass
# Formatter
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(process)d %(thread)d '
'%(pathname)s %(lineno)s %(message)s')
# FileHandler Info
file_handler_info = RotatingFileHandler(
filename=cls.LOG_PATH_INFO,
maxBytes=cls.LOG_FILE_MAX_BYTES,
backupCount=cls.LOG_FILE_BACKUP_COUNT)
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(
filename=cls.LOG_PATH_ERROR,
maxBytes=cls.LOG_FILE_MAX_BYTES,
backupCount=cls.LOG_FILE_BACKUP_COUNT)
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def _get_handler():
# we only need one global handler
global handler
if handler is not None:
return handler
path = '/var/log/rhsm/rhsm.log'
try:
if not os.path.isdir("/var/log/rhsm"):
os.mkdir("/var/log/rhsm")
except Exception:
pass
# Try to write to /var/log, fallback on console logging:
try:
handler = RotatingFileHandler(path, maxBytes=0x100000, backupCount=5, encoding='utf-8')
except IOError:
handler = logging.StreamHandler()
except Exception:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
handler.setLevel(LOG_LEVEL)
handler.addFilter(ContextLoggingFilter(name=""))
return handler
def init_logger(level):
"""Add custom handler and set logger basic config
Args:
level (str): Logger level
"""
logging.basicConfig(
level=level,
format=
"%(asctime)s [%(levelname)-8s] %(lineno)-4s <%(name)s@%(funcName)s> - %(message)s",
datefmt="%H:%M:%S",
)
root = logging.getLogger()
file_handler = RotatingFileHandler(LOG_PATH,
maxBytes=10 * 1024**2,
backupCount=3,
mode='w')
file_handler.setLevel(level)
file_formatter = logging.Formatter(
fmt=
"%(duplicates)s%(asctime)s [%(levelname)-8s] %(lineno)-4s <%(name)s@%(funcName)s> - %(message)s",
datefmt="%Y/%m/%d %H:%M:%S",
)
file_handler.setFormatter(file_formatter)
file_handler.addFilter(DuplicateFilter())
root.addHandler(file_handler)
def get_logger(logger_name):
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
log_formatter = logging.Formatter(
fmt=
"%(asctime)s, %(levelname)s, %(filename)s[line: %(lineno)d], %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
for level, name in logging._levelToName.items():
if level in (logging.NOTSET, logging.CRITICAL):
continue
log_dir = _create_log_dir(utils.settings.PROJECT_DIR, "logs",
name.lower())
log_handler = RotatingFileHandler(os.path.join(
log_dir, "{}.log".format(name.lower())),
maxBytes=1024 * 1000 * 10,
backupCount=10,
encoding="utf-8")
log_handler.setLevel(level)
log_handler.setFormatter(log_formatter)
log_handler.addFilter(_LoggingFilter(level=level))
logger.addHandler(log_handler)
return logger
def start_thread_logging():
"""
Add a log handler to separate file for current thread
"""
thread_name = threading.Thread.getName(threading.current_thread())
log_file = '{}.log'.format(thread_name)
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=5 * 1024 * 1024,
backupCount=2,
encoding=None,
delay=0)
log_handler.setLevel(Config.LOGGING_LEVEL)
formatter = logging.Formatter("%(asctime)-15s"
"| %(threadName)-11s"
"| %(levelname)-5s"
"| %(message)s")
log_handler.setFormatter(formatter)
log_filter = ThreadLogFilter(thread_name)
log_handler.addFilter(log_filter)
logger = logging.getLogger()
logger.addHandler(log_handler)
return log_handler
def __init__(self) :
"""
Define 3 differents log :
activity.log -> all activity server
warning.log -> only warning server (including illegal acess)
error.log' -> error server
Write all message on terminal too
"""
logging.addLevelName(lvl.SUCCESS, "SUCCESS")
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)-15s :: %(levelname)s :: %(message)s')
file_handler = RotatingFileHandler('log/activity.log', 'a', 1000000, 1)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
file_handler_warning = RotatingFileHandler('log/warning.log', 'a', 1000000, 1)
f1 = SingleLevelFilter(logging.WARNING, False)
file_handler_warning.addFilter(f1)
file_handler_warning.setFormatter(formatter)
self.logger.addHandler(file_handler_warning)
file_handler_error = RotatingFileHandler('log/error.log', 'a', 1000000, 1)
file_handler_error.setLevel(logging.ERROR)
file_handler_error.setFormatter(formatter)
self.logger.addHandler(file_handler_error)
steam_handler = logging.StreamHandler()
steam_handler.setLevel(logging.NOTSET)
self.logger.addHandler(steam_handler)
def register_logging(app):
# TODO https://blog.csdn.net/zwxiaoliu/article/details/80890136
# email errors to the administrators
import logging
from logging.handlers import RotatingFileHandler
# Formatter
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(pathname)s %(lineno)s %(module)s.%(funcName)s %(message)s')
# log dir
if not os.path.exists(app.config['LOG_PATH']):
os.makedirs(app.config['LOG_PATH'])
# FileHandler Info
file_handler_info = RotatingFileHandler(filename=app.config['LOG_PATH_INFO'])
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(filename=app.config['LOG_PATH_ERROR'])
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def init_file_logging(app):
class LogLevelFilter(object):
""" Filter for log messages of a specific level. """
def __init__(self, level):
self.__level = level
def filter(self, logRecord):
return logRecord.levelno == self.__level
# Add a file INFO logging handler.
file_handler = RotatingFileHandler(app.config['LOG_FILENAME_INFO'],
backupCount=10)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setLevel(logging.INFO)
file_handler.addFilter(LogLevelFilter(logging.INFO))
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
# Add a file ERROR logging handler.
file_handler = RotatingFileHandler(app.config['LOG_FILENAME_ERROR'],
backupCount=10)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setLevel(logging.ERROR)
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
def common(path_list):
print(path_list)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s '
'[in %(pathname)s:%(lineno)d]')
debug_log = path_list['debug']
not_exist_makedirs(os.path.dirname(debug_log))
debug_file_handler = RotatingFileHandler(debug_log,
maxBytes=1000000,
backupCount=10)
debug_file_handler.addFilter(LevelFilter(logging.DEBUG))
#debug_file_handler.setLevel(logging.DEBUG)
debug_file_handler.setFormatter(formatter)
app.logger.addHandler(debug_file_handler)
error_log = path_list['error']
not_exist_makedirs(os.path.dirname(error_log))
error_file_handler = RotatingFileHandler(error_log,
maxBytes=1000000,
backupCount=10)
error_file_handler.addFilter(LevelFilter(logging.ERROR))
#error_file_handler.setLevel(logging.ERROR)
error_file_handler.setFormatter(formatter)
app.logger.addHandler(error_file_handler)
app.logger.propagate = False
app.logger.setLevel(logging.DEBUG) ##
def logger_create(service_name, app):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
log_path = os.path.join(dirname(dirname(abspath(__file__))), 'apps',
service_name, 'log')
if not os.path.exists(log_path):
os.mkdir(log_path)
formatter = TloggerFormatter(
f"[%(asctime)s][%(process)d][{service_name}][%(filename)s %(lineno)d %(levelname)s]: %(message)s"
)
logfile = f'{log_path}/{service_name}.log'
fh = RotatingFileHandler(logfile, maxBytes=10 * 1024 * 1024, backupCount=5)
fh.setLevel(logging.INFO)
info_filter = InfoFilter()
fh.addFilter(info_filter)
fh.setFormatter(formatter)
error_logfile = f'{log_path}/{service_name}_error.log'
fh_error = RotatingFileHandler(error_logfile,
maxBytes=10 * 1024 * 1024,
backupCount=5)
fh_error.setLevel(logging.ERROR)
fh_error.setFormatter(formatter)
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG)
sh.setFormatter(formatter)
app.logger.removeHandler(default_handler)
app.logger.addHandler(sh)
app.logger.addHandler(fh)
app.logger.addHandler(fh_error)
def setup_logging():
max_bytes = 32 * 1024 * 1024 # 32 MiB
logging.getLogger("centauri").setLevel(logging.INFO)
logging.getLogger("centauri-debug").setLevel(logging.DEBUG)
log = logging.getLogger()
log.setLevel(logging.DEBUG)
handler = RotatingFileHandler(
filename="centauri.log",
encoding="utf-8",
mode="w",
maxBytes=max_bytes,
backupCount=5,
)
handler2 = RotatingFileHandler(
filename="centauri-debug.log",
encoding="utf-8",
mode="w",
maxBytes=max_bytes,
backupCount=5,
)
dt_fmt = "%Y-%m-%d %H:%M:%S"
fmt = logging.Formatter("[{asctime}] [{levelname:<7}] {name}: {message}",
dt_fmt,
style="{")
handler.setFormatter(fmt)
handler2.setFormatter(fmt)
handler2.addFilter(DebugFilter())
log.addHandler(handler)
log.addHandler(handler2)
def init_log(filter=None):
log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(FORMAT)
# add file rotation handler
file_handler = RotatingFileHandler(
filename='/var/log/mediaplat/mediaplat.log',
maxBytes=1024 * 1024,
backupCount=5,
mode='a+')
stream_handler = logging.StreamHandler()
if filter:
file_handler.addFilter(filter)
stream_handler.addFilter(filter)
# log to file
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
log.addHandler(file_handler)
if DEBUG:
# duplicate log to stdout with color
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(ColoredFormatter())
log.addHandler(stream_handler)
def init_log(filter=None):
log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(FORMAT)
# add file rotation handler
file_handler = RotatingFileHandler(
filename="/var/log/mediaplat/mediaplat.log", maxBytes=1024 * 1024, backupCount=5, mode="a+"
)
stream_handler = logging.StreamHandler()
if filter:
file_handler.addFilter(filter)
stream_handler.addFilter(filter)
# log to file
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
log.addHandler(file_handler)
if DEBUG:
# duplicate log to stdout with color
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(ColoredFormatter())
log.addHandler(stream_handler)
def handler_file_info(self):
if not self._handler_file_info:
file_handler_info = RotatingFileHandler(filename=self.log_path_info)
file_handler_info.setFormatter(self.log_formatter_info)
file_handler_info.setLevel(self.level_info)
file_handler_info.addFilter(self.filter_info)
self._handler_file_info = file_handler_info
return self._handler_file_info
def get_logging_handler(self, log_level=None):
logfile = os.path.join(self.application_home, 'hydropick.log')
# make handler
from logging.handlers import RotatingFileHandler
handler = RotatingFileHandler(logfile, backupCount=5)
handler.doRollover()
handler.setFormatter(self.get_formatter(log_level))
handler.addFilter(self.get_filter(log_level))
return handler
def set_pkt_logging(logger, cc_stdout=False, **kwargs) -> None:
"""Create/configure handlers, formatters, etc.
Parameters:
- backup_count: keep this many copies, and rotate at midnight unless...
- max_bytes: rotate log files when log > rotate_size
"""
file_name = kwargs.get(LOG_FILE_NAME, 0)
backup_count = kwargs.get(LOG_ROTATE_COUNT, 0)
max_bytes = kwargs.get(LOG_ROTATE_BYTES, None)
logger.propagate = False
if _use_color_:
cons_fmt = colorlog.ColoredFormatter(
f"%(log_color)s{CONSOLE_FMT + COLOR_SUFFIX}",
reset=True,
log_colors=LOG_COLOURS,
)
else:
cons_fmt = logging.Formatter(fmt=CONSOLE_FMT + BANDW_SUFFIX)
handler = logging.StreamHandler(stream=sys.stderr)
handler.setFormatter(cons_fmt)
handler.setLevel(logging.WARNING)
handler.addFilter(StdErrFilter())
logger.addHandler(handler)
if cc_stdout:
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(cons_fmt)
handler.setLevel(logging.DEBUG)
handler.addFilter(StdOutFilter())
logger.addHandler(handler)
if not file_name:
return
if max_bytes:
backup_count = backup_count if backup_count else 2
handler = RotatingFileHandler(file_name,
maxBytes=max_bytes,
backupCount=backup_count)
elif backup_count:
handler = TimedRotatingFileHandler(file_name,
when="midnight",
backupCount=backup_count)
else:
handler = logging.FileHandler(file_name)
handler.setFormatter(logging.Formatter(fmt=PKT_LOG_FMT + BANDW_SUFFIX))
handler.setLevel(logging.INFO) # INFO (usually), or DEBUG
handler.addFilter(FileFilter())
logger.addHandler(handler)
def start_log():
log.start_syslog('openxcap')
if Logging.directory:
if not os.path.exists(Logging.directory):
os.mkdir(Logging.directory)
handler = RotatingFileHandler(os.path.join(Logging.directory, 'access.log'), 'a', 2*1024*1024, 5)
handler.addFilter(IsAccessLog())
log.logger.addHandler(handler)
for handler in log.logger.handlers:
if isinstance(handler, log.SyslogHandler):
handler.addFilter(IsNotAccessLog())
def config_logging(stdout_file, stderr_file):
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG')
max_bytes = 1024 * 1024 * 10
stdout_handler = RotatingFileHandler(stdout_file, maxBytes=max_bytes, backupCount=100)
stdout_handler.addFilter(NoErrorFilter())
stderr_handler = RotatingFileHandler(stderr_file, maxBytes=max_bytes, backupCount=100)
stderr_handler.setLevel('ERROR')
console_handler = logging.StreamHandler(sys.stdout)
logging.basicConfig(format='[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d] %(message)s',
datefmt='%Y%m%d-%H:%M:%S', handlers=[stdout_handler, console_handler, stderr_handler],
level=LOG_LEVEL)
def get_logger(name,
log_dir=None,
level=logging.INFO,
split_error=False,
fmt=default_fmt,
use_tqdm=False):
logger = logging.getLogger(name)
if getattr(logger, 'initialized', False):
return logger
logger.initialized = True
logger.setLevel(parse_level(level))
del logger.handlers[:]
if log_dir:
log_dir = Path(log_dir).expanduser()
log_dir.mkdir(parents=True, exist_ok=True)
log_file = log_dir / default_log
file_handler = RotatingFileHandler(str(log_file),
maxBytes=log_max_bytes,
backupCount=log_backup_count)
file_handler.setFormatter(logging.Formatter(fmt))
logger.addHandler(file_handler)
if split_error:
file_handler.addFilter(InfoFilter())
error_log_file = log_dir / default_error_log
error_file_handler = RotatingFileHandler(
error_log_file,
maxBytes=log_max_bytes,
backupCount=log_backup_count)
error_file_handler.formatter = logging.Formatter(fmt)
error_file_handler.addFilter(ErrorFilter())
logger.addHandler(error_file_handler)
if use_tqdm:
from tqdm import tqdm
class TqdmHandler(logging.StreamHandler):
def emit(self, record):
msg = self.format(record)
tqdm.write(msg)
stream_handler = TqdmHandler()
else:
stream_handler = logging.StreamHandler()
stream_handler.formatter = ColorFormatter(fmt)
logger.addHandler(stream_handler)
return logger
def init_logging():
global logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = RotatingFileHandler(log_path, maxBytes=10000000, backupCount=5)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(hostname)s GNS3-Hostname-Grabber: [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
handler.setFormatter(formatter)
f = ContextFilter()
handler.addFilter(f)
logger.addHandler(handler)
def init_app(cls, app):
# FileHandler Info
file_handler_info = RotatingFileHandler(filename=cls.LOG_PATH_INFO)
file_handler_info.setFormatter(cls.LOGGING_FORMATTER)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(filename=cls.LOG_PATH_ERROR)
file_handler_error.setFormatter(cls.LOGGING_FORMATTER)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def configure_logger(log_file_suffix=""):
suffix = log_file_suffix if not log_file_suffix else "." + log_file_suffix
app_handler = RotatingFileHandler(
paths.logs_path(f"{APP_LOG_FILE}{suffix}.log"), mode="w")
app_handler.setFormatter(formatter)
app_handler.setLevel(logging.DEBUG)
app_handler.addFilter(BOOKWORM_FILTER)
logger.addHandler(app_handler)
error_handler = logging.FileHandler(
paths.logs_path(f"{ERROR_LOG_FILE}{suffix}.log"), mode="w")
error_handler.setFormatter(formatter)
error_handler.setLevel(logging.ERROR)
logger.addHandler(error_handler)
def configure_log(name):
"""
Configure a logger, for logging DEBUG, INFO, and WARN message to one file, and ERR, and CRIT to a different
file
:param name: Name of the logger object, we usually just pass in the name of the calling file
:return: ERROR on failure, logger object on success
"""
try:
test_and_mkdir(LOGGING_PATH)
logger = logging.getLogger(name)
# Create a rotating file handler that rotates every 50MB, and stores only the last 5 logs, aka 250MB of data
error_handler = RotatingFileHandler(filename=LOGGING_PATH.joinpath(
ERROR_LOG_FILE + "_{}.log".format(get_current_date_time())),
mode="w+",
maxBytes=52428800,
backupCount=5,
encoding="ISO8859-1")
standard_handler = RotatingFileHandler(
filename=LOGGING_PATH.joinpath(
STANDARD_LOG_FILE +
"_{}.log".format(get_current_date_time())),
mode="w+",
maxBytes=52428800,
backupCount=5,
encoding="ISO8859-1")
# Specify the format of the log files
error_handler.setFormatter(LOG_FILE_FORMAT)
standard_handler.setFormatter(LOG_FILE_FORMAT)
# Add filters to our handlers so only DEBUG, INFO, and WARN go to the standard log file, while ERR, and CRIT go to
# the error file
error_handler.addFilter(ErrorFilter())
standard_handler.addFilter(StandardFilter())
# Add the handlers to our logger so they actually get used
logger.addHandler(error_handler)
logger.addHandler(standard_handler)
# Override the default logging level of WARN, so we will get INFO messages out
logger.setLevel(logging.INFO)
# Set the class field to our new logger
Logger.LOGGER = logger
except Exception as err:
print(err)
return ERROR
def configure_logging(app):
"""
Configure logging
:param app: :py:class:`Flask`
"""
from logging.handlers import RotatingFileHandler
class MyFilter(object):
def __init__(self, level):
self.__level = level
def filter(self, logRecord):
return logRecord.levelno == self.__level
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s")
accesslogger = logging.getLogger('werkzeug')
if app.debug:
accesslogger.propagate = False
file_handler = RotatingFileHandler('{}{}-access.log'.format(app.config.get('PATH_DATA'), app.name), maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setFormatter(formatter)
accesslogger.addHandler(file_handler)
app.logger.addHandler(accesslogger)
# set debug mode to all other loggers of emonitor, use loglevel definition of emonitor.cfg
for l in [l for l in logging.Logger.manager.loggerDict if l.startswith(app.name.lower())]:
lo = logging.getLogger(l)
if lo.level > logging.DEBUG:
lo.setLevel(app.config.get('LOGLEVEL', logging.DEBUG))
else:
accesslogger.setLevel(logging.ERROR)
for l in [l for l in logging.Logger.manager.loggerDict if l.startswith(app.name.lower())]:
logging.getLogger(l).setLevel(app.config.get('LOGLEVEL', logging.ERROR))
logger = logging.getLogger('alembic.runtime.migration')
logger.setLevel(logging.ERROR)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
file_handler = RotatingFileHandler('%s%s.log' % (app.config.get('PATH_DATA'), app.name), maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.INFO)
logger.addHandler(file_handler)
file_handler = RotatingFileHandler('%s%s-error.log' % (app.config.get('PATH_DATA'), app.name), maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setFormatter(formatter)
file_handler.addFilter(MyFilter(logging.ERROR))
file_handler.setLevel(logging.ERROR)
logger.addHandler(file_handler)
def create_rotating_file_handler(filename) -> RotatingFileHandler:
file_handler = RotatingFileHandler(filename,
maxBytes=1024**2,
backupCount=1,
encoding="utf-8")
formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] (%(filename)s).%(funcName)s(%(lineno)d): "
"%(message)s")
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.DEBUG)
file_handler.addFilter(lambda record: record.levelno != logging.INFO)
return file_handler
def init_app(cls, app):
BaseConfig.init_app(app)
# email errors to the administrators
import logging
from logging.handlers import RotatingFileHandler
# Formatter
# formatter = logging.Formatter(
# '%(asctime)s %(levelname)s %(process)d %(thread)d'
# '%(module)s %(lineno)s %(message)s')
formatter = RequestFormatter(
'[%(asctime)s] %(remote_addr)s requested %(url)s'
' --- %(levelname)s in %(module)s: %(message)s'
)
# FileHandler Info
file_handler_info = RotatingFileHandler(filename=cls.LOG_PATH_INFO)
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(filename=cls.LOG_PATH_ERROR)
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
# email errors to the administrators
import logging
from logging.handlers import SMTPHandler
credentials = None
secure = None
if getattr(cls, 'MAIL_USERNAME', None) is not None:
credentials = (cls.MAIL_USERNAME, cls.MAIL_PASSWORD)
if getattr(cls, 'MAIL_USE_TLS', None):
secure = ()
mail_handler = SMTPHandler(
mailhost=(cls.MAIL_SERVER, cls.MAIL_PORT),
fromaddr=cls.CIRCULATE_MAIL_SENDER,
toaddrs=[cls.CIRCULATE_ADMIN],
subject=cls.CIRCULATE_MAIL_SUBJECT_PREFIX + ' Application Error',
credentials=credentials,
secure=secure)
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
def register_logger(app):
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(process)d %(thread)d '
'%(pathname)s %(lineno)s %(message)s')
# FileHandler Info
file_handler_info = RotatingFileHandler(filename=app.config.get('LOG_PATH_INFO'), maxBytes=10*1024*1024, backupCount=10)
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
# FileHandler Error
file_handler_error = RotatingFileHandler(filename=app.config.get('LOG_PATH_ERROR'), maxBytes=10*1024*1024, backupCount=10)
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def init_app(cls, app):
import logging
from logging.handlers import RotatingFileHandler
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(process)d %(thread)d %(funcName)s %(pathname)s %(lineno)s %(message)s'
)
file_handler_info = RotatingFileHandler(filename=cls.LOG_PATH_INFO)
file_handler_info.setFormatter(formatter)
file_handler_info.setLevel(logging.INFO)
info_filter = InfoFilter()
file_handler_info.addFilter(info_filter)
app.logger.addHandler(file_handler_info)
file_handler_error = RotatingFileHandler(filename=cls.LOG_PATH_ERROR)
file_handler_error.setFormatter(formatter)
file_handler_error.setLevel(logging.ERROR)
app.logger.addHandler(file_handler_error)
def __init__(self, class_name):
"""Constructor"""
'''default root console logging'''
self.logger = logging.getLogger(class_name)
self.logger.setLevel(logging.DEBUG)
'''console file logging'''
debug_file_handler = logging.StreamHandler(sys.stdout)
debug_file_handler.setLevel(logging.DEBUG)
debug_file_handler.addFilter(
type(
'', (logging.Filter, ),
{'filter': staticmethod(lambda r: r.levelno <= logging.DEBUG)
}))
'''error file logging'''
error_file_handler = RotatingFileHandler(config['error_log_file'],
mode='a',
maxBytes=5 * 1024 * 1024,
backupCount=100,
encoding='utf-8',
delay=0)
error_file_handler.setLevel(logging.ERROR)
error_file_handler.addFilter(
type(
'', (logging.Filter, ),
{'filter': staticmethod(lambda r: r.levelno <= logging.ERROR)
}))
'''tracking file logging'''
tracking_file_handler = RotatingFileHandler(
config['tracking_log_file'],
mode='a',
maxBytes=5 * 1024 * 1024,
backupCount=100,
encoding='utf-8',
delay=0)
tracking_file_handler.setLevel(logging.INFO)
tracking_file_handler.addFilter(
type(
'', (logging.Filter, ),
{'filter': staticmethod(lambda r: r.levelno <= logging.INFO)}))
self.logger.addHandler(debug_file_handler)
self.logger.addHandler(error_file_handler)
self.logger.addHandler(tracking_file_handler)
def __init__(self, config=None):
if config is None:
config = Config()
# Create a logger instance. NOTE: the level set in the logger
# determines which severity of messages it will pass to it's
# handlers. We want to send everything to the handlers and
# let them decide what to do (aka this must pass everything
# along).
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
# Create file handler for WARNING and above
if not config.log is None:
fmt = ('%(asctime)s, '
'%(clientip)s, '
'%(levelname)s, '
'pid:%(process)d, '
'%(name)s, '
'%(filename)s:%(lineno)d, '
'%(message)s'
)
fh = RotatingFileHandler(config.log,
maxBytes=104857600,
backupCount=5)
fh.addFilter(logging.Filter(ROOT))
fh.setLevel(config.log_level)
fh.setFormatter(logging.Formatter(fmt))
logger.addHandler(fh)
# Create file handler for DEBUG and above
if config.debug and config.log:
fmt = ('%(levelname)-9s'
'%(name)-35s'
'%(filename)-15s'
'%(lineno)-5d'
'%(message)s'
)
fh = RotatingFileHandler(config.log + '.debug',
maxBytes=104857600,
backupCount=5)
fh.setLevel(config.log_level - 20)
fh.setFormatter(logging.Formatter(fmt))
logger.addHandler(fh)
def get_logger(module, with_more_info=True):
if module in frappe.loggers:
return frappe.loggers[module]
formatter = logging.Formatter("[%(levelname)s] %(asctime)s | %(pathname)s:\n%(message)s")
# handler = logging.StreamHandler()
handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100000, backupCount=20)
handler.setFormatter(formatter)
if with_more_info:
handler.addFilter(SiteContextFilter())
logger = logging.getLogger(module)
logger.setLevel(frappe.log_level or default_log_level)
logger.addHandler(handler)
logger.propagate = False
frappe.loggers[module] = logger
return logger
def init_file_logging(app):
class LogLevelFilter(object):
""" Filter for log messages of a specific level. """
def __init__(self, level):
self.__level = level
def filter(self, logRecord):
return logRecord.levelno == self.__level
# Add a file INFO logging handler.
file_handler = RotatingFileHandler(app.config['LOG_FILENAME_INFO'],
backupCount=10)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setLevel(logging.INFO)
file_handler.addFilter(LogLevelFilter(logging.INFO))
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
# Add a file ERROR logging handler.
file_handler = RotatingFileHandler(app.config['LOG_FILENAME_ERROR'],
backupCount=10)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setLevel(logging.ERROR)
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
def _configure_logger(l, config=None, level=None):
verbose = 0
log_file = None
quiet = False
if config:
verbose = config.verbose
log_file = config.log_file
quiet = config.quiet
if level:
log_level = level
else:
if verbose == 0:
log_level = logging.ERROR
if verbose == 1:
log_level = logging.WARN
if verbose == 2:
log_level = logging.INFO
if verbose > 2:
log_level = logging.DEBUG
if log_file:
max_size = 1 * 1024 * 1024 # 1MB
filehandler = RotatingFileHandler(log_file, maxBytes=max_size,
backupCount=5)
filehandler.addFilter(context_filter)
l.addHandler(filehandler)
if os.path.getsize(log_file) > max_size/2:
filehandler.doRollover()
if quiet:
l.removeHandler(consolehandler)
l.setLevel(log_level)
logger.info('Finished setting up logger %s %s %s', l.name, log_level,
log_file)
def setup_logging(level, logfile=None, maxsize=None, maxfiles=None):
# Quiet down some of the libraries that we use.
logging.getLogger("paramiko").setLevel(logging.WARN)
logging.getLogger("requests").setLevel(logging.WARN)
logging.getLogger("bzrest").setLevel(logging.INFO)
if logfile:
handler = RotatingFileHandler(logfile, maxBytes=maxsize, backupCount=maxfiles)
else:
handler = logging.StreamHandler()
class SlaveLogFilter(logging.Filter):
def filter(self, record):
record.slave = getattr(log_data, "slave", "-=-")
return True
fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(slave)s - %(message)s")
handler.addFilter(SlaveLogFilter())
handler.setFormatter(fmt)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(level)
class MyFilter(object):
def __init__(self, level):
self.__level = level
def filter(self, logRecord):
return logRecord.levelno == self.__level
app = Flask(__name__)
json = FlaskJSON(app)
app.config['JSON_ADD_STATUS'] = False
cors = CORS(app, resources={r"/*": {"origins": config.ALLOW_CORS}})
# logging for warnings
warning_handler = RotatingFileHandler('./logs/warnings.log', maxBytes=10*1024*1024, backupCount=1)
warning_handler.setLevel(logging.WARNING)
warning_handler.addFilter(MyFilter(logging.WARNING))
app.logger.addHandler(warning_handler)
# logging for errors and critical
error_handler = RotatingFileHandler('./logs/errors.log', maxBytes=10*1024*1024, backupCount=1)
error_handler.setLevel(logging.ERROR)
app.logger.addHandler(error_handler)
#logging for web requests
access_handler = RotatingFileHandler('./logs/access.log', maxBytes=10*1024*1024, backupCount=1)
access_logger = logging.getLogger('werkzeug')
access_logger.addHandler(access_handler)
# logging for database queries
peewee_handler = RotatingFileHandler('./logs/database.log', maxBytes=10*1024*1024, backupCount=1)
peewee_logger = logging.getLogger('peewee')
def create_handlers_from_settings(settings):
"""
used internally to setup logging for the settings.logs
"""
# clear any previously setup handlers
clear_settings_handlers()
if not settings.logs.enabled:
return
# have to set the root logger lower than WARN (the default) or our
# application logs will never be seen
logging.root.setLevel(APPLICATION)
if settings.logs.errors.enabled:
format_str = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
formatter = logging.Formatter(format_str)
error_handler = RotatingFileHandler(
path.join(settings.dirs.logs, 'errors.log'),
maxBytes=settings.logs.max_bytes,
backupCount=settings.logs.backup_count,
)
error_handler._from_blazeweb_settings = True
error_handler.setLevel(logging.WARN)
error_handler.setFormatter(formatter)
logging.root.addHandler(error_handler)
if settings.logs.application.enabled:
class OnlyLevel25(logging.Filter):
def filter(self, record):
return record.levelno == 25
format_str = "%(asctime)s - %(name)s - %(message)s"
formatter = logging.Formatter(format_str)
app_handler = RotatingFileHandler(
path.join(settings.dirs.logs, 'application.log'),
maxBytes=settings.logs.max_bytes,
backupCount=settings.logs.backup_count,
)
app_handler._from_blazeweb_settings = True
app_handler.setLevel(APPLICATION)
app_handler.setFormatter(formatter)
app_handler.addFilter(OnlyLevel25())
logging.root.addHandler(app_handler)
if settings.logs.email.enabled:
format_str = "%(asctime)s - %(message)s"
formatter = logging.Formatter(format_str)
app_handler = RotatingFileHandler(
path.join(settings.dirs.logs, 'email.log'),
maxBytes=settings.logs.max_bytes,
backupCount=settings.logs.backup_count,
)
app_handler._from_blazeweb_settings = True
app_handler.setLevel(logging.INFO)
app_handler.setFormatter(formatter)
logging.getLogger('blazeweb.mail').addHandler(app_handler)
logging.getLogger('blazeweb.mail').setLevel(logging.INFO)
def get_script_logger(debug=True, quiet=False, config=None):
"""Creates a script logger to log to files / streams."""
log_file_format = (config
and config['LOG_FILE_FORMAT']
or '%(levelname)s %(asctime)s %(message)s')
log_file_datefmt = (config
and config['LOG_FILE_DATEFMT']
or '[%Y-%m-%d %H:%M:%S]')
script_log_info_file_handler = None
if config:
script_log_info_file_handler = RotatingFileHandler(
path.join(config['LOG_FOLDER'], 'script_info.log'),
maxBytes=100000, backupCount=5)
script_log_info_file_handler.setLevel(logging.INFO)
script_log_info_file_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_file_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_file_handler = None
if config:
script_log_error_file_handler = RotatingFileHandler(
path.join(config['LOG_FOLDER'], 'script_error.log'),
maxBytes=100000, backupCount=5)
script_log_error_file_handler.setLevel(logging.ERROR)
script_log_error_file_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stdout)
script_log_info_stream_handler.setLevel(logging.INFO)
script_log_info_stream_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_stream_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stderr)
script_log_error_stream_handler.setLevel(logging.ERROR)
script_log_error_stream_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_logger = logging.getLogger('script')
script_logger.setLevel(logging.INFO)
if config:
script_logger.addHandler(script_log_info_file_handler)
script_logger.addHandler(script_log_error_file_handler)
script_logger.addHandler(script_log_info_stream_handler)
script_logger.addHandler(script_log_error_stream_handler)
if (not debug) and config:
mail_handler = get_mail_handler(config)
script_logger.addHandler(mail_handler)
return script_logger
def make_logger(
name,
level=level,
fmt=_format,
max_bytes=max_bytes,
backup_count=backup_count,
delay=delay,
propagate=propagate,
):
"""
_function_: `mast.logging.make_logger(name, level=level, fmt=_format, filename=None, when=unit, interval=interval, propagate=propagate, backup_count=backup_count)`
Returns an instance of logging.Logger configured with
a [logging.handlers.TimedRotatingFileHandler](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
handler.
Arguments passed to this function determine the format,
level, filename, time unit, interval, backup count and
whether to propagate messages to parent loggers (defined
by dot seperated heirarchy ie in `mast.datapower`,
`datapower` is a logger with a parent logger of mast).
Parameters:
* `name`: Required. the name of the logger instance. This follows
conventions mentioned [here](https://docs.python.org/2/library/logging.html#logger-objects)
* `level`: The logging level to listen for. Accepts an `int` or
one of the logging modules convenience constants defined
[here](https://docs.python.org/2/library/logging.html#logging-levels)
* `fmt`: The format of the log message, see
[here](https://docs.python.org/2/library/logging.html#formatter-objects)
and [here](https://docs.python.org/2/library/logging.html#logrecord-attributes)
for more details
* `filename`: The filename to log to. Defaults to the name of the logger
appended with `.log` in the `$MAST_HOME/var/log` directory or
`$MAST_HOME/var/log/mastd` directory if running as `mastd`
* `when`: The time unit to use for rolling over the log file
as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
* `interval`: The number of time units to wait before rolling the
log files as detailed [here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
* `propagate`: Whether to propagate log messages up the ancestry chain
(ie. if you have a logger `mast.datapower`, and propagate is set to
`False` messages sent to the this logger will not be propagated to the
`mast` logger). See [here](https://docs.python.org/2/library/logging.html#logging.Logger.propagate)
for more details.
* `backup_count`: The number of "rolled" log files to keep, see
[here](https://docs.python.org/2/library/logging.handlers.html#timedrotatingfilehandler)
for more details.
Usage:
:::python
from mast.logging import make_logger
logger = make_logger("my_module")
logger.info("informational message")
logger.debug("debug message")
"""
global t
_logger = logging.getLogger(name)
if _logger.handlers:
if len(_logger.handlers) >= 1:
return _logger
_logger.setLevel(level)
_formatter = logging.Formatter(fmt)
pid = os.getpid()
directory = os.path.join(
mast_home,
"var",
"log",
getpass.getuser(),
"{}-{}".format(t.timestamp, str(pid)),
)
filename = os.path.join(
directory,
"{}-{}.log".format(name, t.timestamp),
)
if not os.path.exists(directory):
os.makedirs(directory)
_handler = RotatingFileHandler(
filename,
maxBytes=max_bytes,
backupCount=backup_count,
delay=delay,
)
_handler.setFormatter(_formatter)
_handler.setLevel(level)
_handler.addFilter(RedactingFilter())
_logger.addHandler(_handler)
_logger.propagate = propagate
return _logger
