def set_log_level(level):
"""
Adapting the log level for information display through process.
:param level: str, [debug/info/warning/error]
"""
# Ensuring the global logger is overwritten
global logger
level_table = {
'debug': logging.DEBUG,
'warn': logging.WARNING,
'warning': logging.WARNING,
'info': logging.INFO,
'error': logging.ERROR
}
log_level = level_table[level.lower()]
if log_level == logging.DEBUG:
formatter = logzero.LogFormatter(
fmt=
"%(color)s[%(levelname)1.1s %(module)s:%(lineno)d]%(end_color)s %(message)s"
)
else:
formatter = logzero.LogFormatter(
fmt="%(color)s[%(levelname)1.1s]%(end_color)s %(message)s")
logger = logzero.setup_logger(formatter=formatter, level=log_level)
def getLogger(
name: str,
level: int = logzero.logging.DEBUG,
formatter: Optional[logzero.LogFormatter] = logzero.LogFormatter(
fmt=LOG_FORMAT),
) -> logzero.logger:
"""Formats and sets up the logger instance.
Args:
name (str): The name of the Logger.
level (int): The default level (logzero.logging.INFO = 20) of the logger.
formatter (:obj:, optional): The format of the log message. Defaults to the default logzero format.
Returns:
An instance of a logger.
Examples:
>>> from edgePy.util import getLogger
>>> log = getLogger(name="script")
>>> log.info('This is your DGElist.')
...
Notes:
1. See https://docs.python.org/3/library/logging.html#levels for more information about logging levels.
"""
log_formatter = (logzero.LogFormatter(
fmt=logzero.LogFormatter.DEFAULT_FORMAT)
if formatter is None else formatter)
logger = logzero.setup_logger(name=name,
level=level,
formatter=log_formatter)
return logger
def setup_logger(logfile=None,
backup_count=20,
log_level=logging.INFO,
include_caller=True):
"""
Setup logzero logger. if logfile is specified, create additional file logger
:param logfile: path to log file destination
:param backup_count: number of rotating files
:param log_level: min. log level FOR FILE LOGGING
:param include_caller: whether to include the caller in the log output to STDOUT, e.g. [script:123]
"""
formatter = logzero.LogFormatter(fmt=_set_log_format(
color=True, include_caller=include_caller),
datefmt='%Y-%m-%d %H:%M:%S')
logzero.setup_default_logger(formatter=formatter)
if logfile:
formatter = logzero.LogFormatter(fmt=_set_log_format(
color=False, include_caller=True),
datefmt='%Y-%m-%d %H:%M:%S')
logzero.logfile(logfile,
formatter=formatter,
loglevel=log_level,
maxBytes=int(1e7),
backupCount=backup_count)
def setup_logger(log_path):
# Default logger
log_format = '%(color)s* %(levelname)1s%(end_color)s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(logging.INFO)
# Log file
log_format_no_color = '* %(levelname)1s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter_no_color = logzero.LogFormatter(fmt=log_format_no_color)
# Log rotation of 20 files for 10MB each
logzero.logfile(log_path, formatter=formatter_no_color, loglevel=logging.INFO, maxBytes=int(1e7), backupCount=20)
def setup_logzero(path, level):
Path(path).parent.mkdir(parents=True, exist_ok=True)
log_fmt = '%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s'
if level == 'debug':
level = logging.DEBUG
log_fmt = (
'%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]'
'%(end_color)s %(message)s')
elif level == 'info':
level = logging.INFO
elif level == 'warning':
level = logging.WARNING
elif level == 'error':
level = logging.ERROR
elif level == 'critical':
level = logging.CRITICAL
formatter = logzero.LogFormatter(fmt=log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(level)
logzero.logfile(path,
loglevel=level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def setup_system_logger(out_log_fp, pdir, logger):
"""fn: setup logger for various package modules
Params
------
out_log_fp: str
log file fp name doesn't include extension fn will add it
logger: logzero logger object
Returns
-------
logger: logzero logger instance
"""
# find .env automagically by walking up directories until it's found
dotenv_path = find_dotenv()
# load up the entries as environment variables
load_dotenv(dotenv_path)
log_format = os.environ.get("LOG_FORMAT")
log_date_format = os.environ.get("LOG_DATE_FORMAT")
now = pd.to_datetime('now', utc=True)
file_ = out_log_fp + f'_{now.date()}.log'
logfile = Path(pdir / 'logs' / file_).as_posix()
check_path(logfile)
formatter = logzero.LogFormatter(fmt=log_format, datefmt=log_date_format)
logzero.setup_default_logger(logfile=logfile, formatter=formatter)
return logger
def syslog(name='app', level=logging.DEBUG, fmt=DEFAULT_FORMAT):
formatter = logzero.LogFormatter(fmt=fmt)
return logzero.setup_logger(
name=name,
level=level,
formatter=formatter
)
def cli(ctx: click.Context, verbose: bool=False, no_version_check: bool=False,
change_dir: str=None, no_log_file: bool=False,
log_file: str="chaostoolkit.log"):
if verbose:
logzero.loglevel(logging.DEBUG, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
else:
logzero.loglevel(logging.INFO, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
if not no_log_file:
# let's ensure we log at DEBUG level
logger.setLevel(logging.DEBUG)
logzero.logfile(
click.format_filename(log_file), mode='a',
loglevel=logging.DEBUG)
logzero.formatter(
formatter=logzero.LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S"),
update_custom_handlers=False)
subcommand = ctx.invoked_subcommand
# make it nicer for going through the log file
logger.debug("#" * 79)
logger.debug("Running command '{}'".format(subcommand))
if not no_version_check:
check_newer_version(command=subcommand)
if change_dir:
logger.warning("Moving to {d}".format(d=change_dir))
os.chdir(change_dir)
def setup_logger(logger: logging.Logger, level: LevelIsh) -> None:
lvl = mklevel(level)
try:
import logzero # type: ignore[import]
formatter = logzero.LogFormatter(
fmt=FORMAT_COLOR,
datefmt=DATEFMT,
)
use_logzero = True
except ModuleNotFoundError:
warnings.warn(
"You might want to install 'logzero' for nice colored logs!")
formatter = logging.Formatter(fmt=FORMAT_NOCOLOR, datefmt=DATEFMT)
use_logzero = False
if use_logzero and not COLLAPSE_DEBUG_LOGS: # all set, nothing to do
# 'simple' setup
logzero.setup_logger(logger.name, level=lvl, formatter=formatter)
return
h = CollapseDebugHandler(
) if COLLAPSE_DEBUG_LOGS else logging.StreamHandler()
logger.setLevel(lvl)
h.setLevel(lvl)
h.setFormatter(formatter)
logger.addHandler(h)
logger.propagate = False # ugh. otherwise it duplicates log messages
def test_default_logger(disableStdErrorLogger=False):
"""
Default logger should work and be able to be reconfigured.
"""
logzero.reset_default_logger()
temp = tempfile.NamedTemporaryFile()
try:
logzero.setup_default_logger(logfile=temp.name, disableStderrLogger=disableStdErrorLogger)
logzero.logger.debug("debug1") # will be logged
# Reconfigure with loglevel INFO
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO, disableStderrLogger=disableStdErrorLogger)
logzero.logger.debug("debug2") # will not be logged
logzero.logger.info("info1") # will be logged
# Reconfigure with a different formatter
log_format = '%(color)s[xxx]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO, formatter=formatter, disableStderrLogger=disableStdErrorLogger)
logzero.logger.info("info2") # will be logged with new formatter
logzero.logger.debug("debug3") # will not be logged
with open(temp.name) as f:
content = f.read()
_test_default_logger_output(content)
finally:
temp.close()
def run_click(debug):
logzero.loglevel(logging.DEBUG if debug else logging.INFO)
logzero.formatter(
logzero.LogFormatter(
fmt=
'%(asctime)s %(color)s[%(levelname).1s]%(end_color)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'))
def setup_logzero(level="info", path="logs/clix.log"):
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
if level == "debug":
level = logging.DEBUG
log_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
elif level == "info":
level = logging.INFO
elif level == "warning":
level = logging.WARNING
elif level == "error":
level = logging.ERROR
elif level == "critical":
level = logging.CRITICAL
# create the directory if it doesn't exist
# https://github.com/metachris/logzero/issues/129
Path(path).parent.mkdir(parents=True, exist_ok=True)
formatter = logzero.LogFormatter(fmt=log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(level)
logzero.logfile(path,
loglevel=level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def __init__(self, file, backCnt):
log_format = '%(color)s[%(asctime)s] [%(levelname)s]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
self.logger = logzero.setup_default_logger(file,
disableStderrLogger=True,
formatter=formatter,
maxBytes=100000,
backupCount=backCnt)
def attach(self, logger):
global loger_initialized
if not loger_initialized:
slack_handler = SlackHanlder()
slack_handler.setLevel(logging.WARNING)
slack_handler.setFormatter(logzero.LogFormatter(color=False))
logger.addHandler(slack_handler)
loger_initialized = True
def setup_logger(working_dir):
color_formatter = logzero.LogFormatter()
monochrome_formatter = logzero.LogFormatter(color=False)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(color_formatter)
color_file_handler = logging.FileHandler(
os.path.join(working_dir, 'log.color'))
color_file_handler.setFormatter(color_formatter)
txt_file_handler = logging.FileHandler(os.path.join(
working_dir, 'log.txt'))
txt_file_handler.setFormatter(monochrome_formatter)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.addHandler(stream_handler)
logger.addHandler(color_file_handler)
logger.addHandler(txt_file_handler)
def test_setup_logger_reconfiguration():
"""
Should be able to reconfigure without loosing custom handlers
"""
logzero.reset_default_logger()
temp = tempfile.NamedTemporaryFile()
temp2 = tempfile.NamedTemporaryFile()
try:
logzero.setup_default_logger(logfile=temp.name)
# Add a custom file handler
filehandler = logging.FileHandler(temp2.name)
filehandler.setLevel(logging.DEBUG)
filehandler.setFormatter(logzero.LogFormatter(color=False))
logzero.logger.addHandler(filehandler)
# First debug message goes to both files
logzero.logger.debug("debug1")
# Reconfigure logger to remove logfile
logzero.setup_default_logger()
logzero.logger.debug("debug2")
# Reconfigure logger to add logfile
logzero.setup_default_logger(logfile=temp.name)
logzero.logger.debug("debug3")
# Reconfigure logger to set minimum loglevel to INFO
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO)
logzero.logger.debug("debug4")
logzero.logger.info("info1")
# Reconfigure logger to set minimum loglevel back to DEBUG
logzero.setup_default_logger(logfile=temp.name, level=logging.DEBUG)
logzero.logger.debug("debug5")
with open(temp.name) as f:
content = f.read()
assert "] debug1" in content
assert "] debug2" not in content
assert "] debug3" in content
assert "] debug4" not in content
assert "] info1" in content
assert "] debug5" in content
with open(temp2.name) as f:
content = f.read()
assert "] debug1" in content
assert "] debug2" in content
assert "] debug3" in content
assert "] debug4" not in content
assert "] info1" in content
assert "] debug5" in content
finally:
temp.close()
def get_logger():
# Set a custom formatter
log_format = '%(color)s[%(levelname)1.1s ' \
'%(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' \
'%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.logfile('logzero.log', maxBytes=1e6, backupCount=3)
return logzero.logger
def init_logger():
console_logformat = (
'%(color)s%(levelname)-8s %(asctime)s %(module)s:%(lineno)d%(end_color)s> '
'%(message)s')
# remove colors on logfile
file_logformat = re.sub(r'%\((end_)?color\)s', '', console_logformat)
console_formatter = logzero.LogFormatter(fmt=console_logformat)
file_formatter = logzero.LogFormatter(fmt=file_logformat)
logzero.setup_default_logger(formatter=console_formatter)
logzero.logfile(
settings.LOGFILE,
maxBytes=settings.LOGFILE_SIZE,
backupCount=settings.LOGFILE_BACKUP_COUNT,
formatter=file_formatter,
)
logger = logzero.logger
logger.setLevel(logzero.DEBUG)
return logger
def cli(verbose: bool = False):
if verbose:
logzero.loglevel(logging.DEBUG)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
else:
logzero.loglevel(logging.INFO)
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
logzero.setup_default_logger(
formatter=logzero.LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S"))
def __init__(self, name, **kwargs):
self.model = LoggerModel(name=name, **kwargs)
self.logger: Logger = logzero.setup_logger(
name=self.model.name,
logfile=self.model.file,
level=self.model.level,
maxBytes=self.model.max_bytes,
backupCount=self.model.backup_count,
fileLoglevel=self.model.file_level,
formatter=logzero.LogFormatter(fmt=self.model.format),
)
def main(argv=None):
"""Main entry point before parsing command line arguments."""
# Setup command line parser.
parser, subparsers = setup_argparse()
# Actually parse command line arguments.
args = parser.parse_args(argv)
# Setup logging incl. verbosity.
if args.verbose: # pragma: no cover
level = logging.DEBUG
else:
# Remove module name and line number if not running in debug mode.s
formatter = logzero.LogFormatter(
fmt=
"%(color)s[%(levelname)1.1s %(asctime)s]%(end_color)s %(message)s")
logzero.formatter(formatter)
level = logging.INFO
logzero.loglevel(level=level)
# Load configuration, if any.
if args.config:
config_paths = (args.config, )
else:
config_paths = GLOBAL_CONFIG_PATHS
for config_path in config_paths:
config_path = os.path.expanduser(os.path.expandvars(config_path))
if os.path.exists(config_path):
with open(config_path, "rt") as tomlf:
toml_config = toml.load(tomlf)
break
else:
toml_config = None
logger.info("Could not find any of the global configuration files %s.",
config_paths)
# Merge configuration from command line/environment args and configuration file.
config = CommonConfig.create(args, toml_config)
# Handle the actual command line.
cmds = {
None: run_nocmd,
"project": run_project,
"landingzone": run_landingzone,
"samplesheet": run_samplesheet,
}
res = cmds[args.cmd](config, toml_config, args, parser,
subparsers.choices[args.cmd] if args.cmd else None)
if not res:
logger.info("All done. Have a nice day!")
else: # pragma: nocover
logger.error("Something did not work out correctly.")
return res
def get_logzero(*args, **kwargs):
import logzero # type: ignore
# ugh, apparently if you get one instance with formatter and another without specifying it, you will end up with default format :(
formatter = logzero.LogFormatter(
fmt=
'%(color)s[%(levelname)s %(name)s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
)
return logzero.setup_logger(
*args,
**kwargs,
formatter=formatter,
)
def setup_logzero(logger, *args, **kwargs):
import logzero # type: ignore
formatter = logzero.LogFormatter(
fmt=
'%(color)s[%(levelname)s %(name)s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
)
logzero.setup_logger(
*args,
**kwargs,
name=logger.name,
formatter=formatter,
)
def setup(self):
# Default logger
log_format = '%(color)s* %(levelname)1s%(end_color)s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(self.log_level)
logzero.loglevel(self.log_level)
# Log file
log_format_no_color = '* %(levelname)1s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter_no_color = logzero.LogFormatter(fmt=log_format_no_color)
# Log rotation of 20 files for 10MB each
logzero.logfile(self.log_file,
formatter=formatter_no_color,
loglevel=self.log_level,
maxBytes=int(1e7),
backupCount=20)
logger.info("smartvadhis2 v.{}".format(__version__))
def setup_logzero(level="info", path="logs/broker.log", silent=False):
path = BROKER_DIRECTORY.joinpath(path)
path.parent.mkdir(parents=True, exist_ok=True)
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
debug_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
log_level = getattr(logging, level.upper(), logging.INFO)
# formatter for terminal
formatter = logzero.LogFormatter(
fmt=debug_fmt if log_level is logging.DEBUG else log_fmt)
logzero.setup_default_logger(formatter=formatter,
disableStderrLogger=silent)
logzero.loglevel(log_level)
# formatter for file
formatter = logzero.LogFormatter(
fmt=debug_fmt if log_level is logging.DEBUG else log_fmt, color=False)
logzero.logfile(path,
loglevel=log_level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def filelog(name='app', file_name='app.log', level=logging.DEBUG, fmt=DEFAULT_FORMAT, max_bytes=1000, backup_count=3,
disable_stderr=False):
formatter = logzero.LogFormatter(fmt=fmt)
return logzero.setup_logger(
name=name,
logfile=file_name,
level=level,
formatter=formatter,
maxBytes=max_bytes,
backupCount=backup_count,
fileLoglevel=level,
disableStderrLogger=disable_stderr
)
def setupLogging(options):
# Set a custom formatter
log_format = '%(color)s[%(levelname)s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
# Set a minimum log level
logzero.loglevel(logging.INFO)
# Set a logfile (all future log messages are also saved there)
logzero.logfile(os.path.abspath(options.folder) + "/submitter.log",
formatter=formatter)
def get_logger():
"""Configures and returns a logzero client"""
config = load_config()
# Set a custom formatter
log_format = '%(color)s[%(levelname)1.1s ' \
'%(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' \
'%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.logfile(config.get('logging', 'location'),
maxBytes=float(config.get('logging', 'max_bytes')),
backupCount=int(config.get('logging', 'backup_count')))
return logzero.logger
def _logger():
"""Add the multiprocessing module's logger.
:return: Returns a multiprocessing logger.
"""
multiprocess_handler = get_logger()
multiprocess_handler = logging.StreamHandler()
multiprocess_handler.setLevel(logging.ERROR)
multiprocess_handler.setFormatter(logzero.LogFormatter(color=True))
# Attach it to the logzero default logger
logzero.logger.addHandler(multiprocess_handler)
logger = logzero.logger
return logger
def cli(verbose, config_file, satellite_server=None):
global log
global cfg
log_level = DEBUG if verbose else INFO
formatter = logzero.LogFormatter(datefmt='%Y-%m-%d %H:%M:%S')
logzero.setup_default_logger(level=log_level,
formatter=formatter,
logfile="sat.log")
cfg = Config(config_file)
cfg.read_config(satellite_server)
logzero.setup_default_logger(level=log_level, formatter=formatter)
# We want to log debug to file, but info to screen, unless -v
logzero.logfile(f"{cfg.satellite['name']}.log", loglevel=DEBUG)
log = logzero.logger
pass
