def setup_logger(logfile=None,
backup_count=20,
log_level=logging.INFO,
include_caller=True):
"""
Setup logzero logger. if logfile is specified, create additional file logger
:param logfile: path to log file destination
:param backup_count: number of rotating files
:param log_level: min. log level FOR FILE LOGGING
:param include_caller: whether to include the caller in the log output to STDOUT, e.g. [script:123]
"""
formatter = logzero.LogFormatter(fmt=_set_log_format(
color=True, include_caller=include_caller),
datefmt='%Y-%m-%d %H:%M:%S')
logzero.setup_default_logger(formatter=formatter)
if logfile:
formatter = logzero.LogFormatter(fmt=_set_log_format(
color=False, include_caller=True),
datefmt='%Y-%m-%d %H:%M:%S')
logzero.logfile(logfile,
formatter=formatter,
loglevel=log_level,
maxBytes=int(1e7),
backupCount=backup_count)
def setup_logzero(path, level):
Path(path).parent.mkdir(parents=True, exist_ok=True)
log_fmt = '%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s'
if level == 'debug':
level = logging.DEBUG
log_fmt = (
'%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]'
'%(end_color)s %(message)s')
elif level == 'info':
level = logging.INFO
elif level == 'warning':
level = logging.WARNING
elif level == 'error':
level = logging.ERROR
elif level == 'critical':
level = logging.CRITICAL
formatter = logzero.LogFormatter(fmt=log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(level)
logzero.logfile(path,
loglevel=level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def setup_system_logger(out_log_fp, pdir, logger):
"""fn: setup logger for various package modules
Params
------
out_log_fp: str
log file fp name doesn't include extension fn will add it
logger: logzero logger object
Returns
-------
logger: logzero logger instance
"""
# find .env automagically by walking up directories until it's found
dotenv_path = find_dotenv()
# load up the entries as environment variables
load_dotenv(dotenv_path)
log_format = os.environ.get("LOG_FORMAT")
log_date_format = os.environ.get("LOG_DATE_FORMAT")
now = pd.to_datetime('now', utc=True)
file_ = out_log_fp + f'_{now.date()}.log'
logfile = Path(pdir / 'logs' / file_).as_posix()
check_path(logfile)
formatter = logzero.LogFormatter(fmt=log_format, datefmt=log_date_format)
logzero.setup_default_logger(logfile=logfile, formatter=formatter)
return logger
def setup_logzero(level="info", path="logs/clix.log"):
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
if level == "debug":
level = logging.DEBUG
log_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
elif level == "info":
level = logging.INFO
elif level == "warning":
level = logging.WARNING
elif level == "error":
level = logging.ERROR
elif level == "critical":
level = logging.CRITICAL
# create the directory if it doesn't exist
# https://github.com/metachris/logzero/issues/129
Path(path).parent.mkdir(parents=True, exist_ok=True)
formatter = logzero.LogFormatter(fmt=log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(level)
logzero.logfile(path,
loglevel=level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def setup_logger(__name__: str,
file_path: str = log_location,
level: int = 10) -> logzero.logger:
# todo: this should be able to write to lambda/local logs without code change
logzero.setup_default_logger()
logzero.logfile(file_path, maxBytes=int(1e6))
logzero.loglevel(level)
return logzero.logger
def test_setup_logger_reconfiguration():
"""
Should be able to reconfigure without loosing custom handlers
"""
logzero.reset_default_logger()
temp = tempfile.NamedTemporaryFile()
temp2 = tempfile.NamedTemporaryFile()
try:
logzero.setup_default_logger(logfile=temp.name)
# Add a custom file handler
filehandler = logging.FileHandler(temp2.name)
filehandler.setLevel(logging.DEBUG)
filehandler.setFormatter(logzero.LogFormatter(color=False))
logzero.logger.addHandler(filehandler)
# First debug message goes to both files
logzero.logger.debug("debug1")
# Reconfigure logger to remove logfile
logzero.setup_default_logger()
logzero.logger.debug("debug2")
# Reconfigure logger to add logfile
logzero.setup_default_logger(logfile=temp.name)
logzero.logger.debug("debug3")
# Reconfigure logger to set minimum loglevel to INFO
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO)
logzero.logger.debug("debug4")
logzero.logger.info("info1")
# Reconfigure logger to set minimum loglevel back to DEBUG
logzero.setup_default_logger(logfile=temp.name, level=logging.DEBUG)
logzero.logger.debug("debug5")
with open(temp.name) as f:
content = f.read()
assert "] debug1" in content
assert "] debug2" not in content
assert "] debug3" in content
assert "] debug4" not in content
assert "] info1" in content
assert "] debug5" in content
with open(temp2.name) as f:
content = f.read()
assert "] debug1" in content
assert "] debug2" in content
assert "] debug3" in content
assert "] debug4" not in content
assert "] info1" in content
assert "] debug5" in content
finally:
temp.close()
def get_logger():
# Set a custom formatter
log_format = '%(color)s[%(levelname)1.1s ' \
'%(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' \
'%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.logfile('logzero.log', maxBytes=1e6, backupCount=3)
return logzero.logger
def setup_logging_level(verbose_level):
"""Define logging level.
By default ``logging.WARNING`` level is enabled. ``-v`` arguments can be
used to increase logging level. Two levels are supported:
``logging.INFO`` and ``logging.DEBUG``.
"""
if verbose_level == 1:
logzero.setup_default_logger(formatter=FORMATTER, level=logging.INFO)
elif verbose_level >= 2:
logzero.setup_default_logger(formatter=FORMATTER, level=logging.DEBUG)
def cli(verbose: bool = False):
if verbose:
logzero.loglevel(logging.DEBUG)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
else:
logzero.loglevel(logging.INFO)
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
logzero.setup_default_logger(
formatter=logzero.LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S"))
def setup_logger(log_path):
# Default logger
log_format = '%(color)s* %(levelname)1s%(end_color)s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(logging.INFO)
# Log file
log_format_no_color = '* %(levelname)1s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter_no_color = logzero.LogFormatter(fmt=log_format_no_color)
# Log rotation of 20 files for 10MB each
logzero.logfile(log_path, formatter=formatter_no_color, loglevel=logging.INFO, maxBytes=int(1e7), backupCount=20)
def setupLogging(options):
# Set a custom formatter
log_format = '%(color)s[%(levelname)s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
# Set a minimum log level
logzero.loglevel(logging.INFO)
# Set a logfile (all future log messages are also saved there)
logzero.logfile(os.path.abspath(options.folder) + "/submitter.log",
formatter=formatter)
def get_logger():
"""Configures and returns a logzero client"""
config = load_config()
# Set a custom formatter
log_format = '%(color)s[%(levelname)1.1s ' \
'%(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' \
'%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.logfile(config.get('logging', 'location'),
maxBytes=float(config.get('logging', 'max_bytes')),
backupCount=int(config.get('logging', 'backup_count')))
return logzero.logger
def configure_logger():
"""
Logging was updated to give time values with milliseconds
"""
log_format = '%(color)s[%(levelname)1.1s ' \
'%(asctime)s.%(msecs)03d %(module)s:%(lineno)d]' \
'%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.logfile(
ApplicationConfig.get("environment", "log_location"),
maxBytes=5e6,
backupCount=5,
loglevel=logging.INFO,
)
def cli(verbose, config_file, satellite_server=None):
global log
global cfg
log_level = DEBUG if verbose else INFO
formatter = logzero.LogFormatter(datefmt='%Y-%m-%d %H:%M:%S')
logzero.setup_default_logger(level=log_level,
formatter=formatter,
logfile="sat.log")
cfg = Config(config_file)
cfg.read_config(satellite_server)
logzero.setup_default_logger(level=log_level, formatter=formatter)
# We want to log debug to file, but info to screen, unless -v
logzero.logfile(f"{cfg.satellite['name']}.log", loglevel=DEBUG)
log = logzero.logger
pass
def __init__(self, args, train=True):
self.args = args
self.start_time = datetime.datetime.now()
self.logger = logzero.setup_default_logger()
# test=False and train=False
if not args.test and train:
self.logger.warn('Test option is {}'.format(args.test))
# setup experiment directory
self.output_dir = self._setup_output_dir()
self.log_dir = self._setup_log_dir()
if train:
self.fig_dir = self._setup_fig_dir()
log_filename = '{}_train.log'.format(self.sdtime)
else:
log_filename = '{}_inference.log'.format(self.sdtime)
log_name = self.log_dir / log_filename
logzero.logfile(str(log_name), loglevel=logging.INFO)
self.log_name = log_name
self.logger.info('Log filename: {}'.format(str(log_name)))
self.logger.info('Server name: {}'.format(socket.gethostname()))
self.dump_common_info()
def setup_logzero(level="info", path="logs/broker.log"):
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
debug_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
log_level = getattr(logging, level.upper(), logging.INFO)
formatter = logzero.LogFormatter(
fmt=debug_fmt if log_level is logging.DEBUG else log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(log_level)
logzero.logfile(path,
loglevel=log_level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def configure_logger(verbose: bool = False,
log_format: str = "string",
log_file: str = None,
logger_name: str = "chaostoolkit",
context_id: str = None):
"""
Configure the chaostoolkit logger.
By default logs as strings to stdout and the given file. When `log_format`
is `"json"`, records are set to the console as JSON strings but remain
as strings in the log file. The rationale is that the log file is mostly
for grepping purpose while records written to the console can be forwarded
out of band to anywhere else.
"""
log_level = logging.INFO
# we define colors ourselves as critical is missing in default ones
colors = {
logging.DEBUG: ForegroundColors.CYAN,
logging.INFO: ForegroundColors.GREEN,
logging.WARNING: ForegroundColors.YELLOW,
logging.ERROR: ForegroundColors.RED,
logging.CRITICAL: ForegroundColors.RED
}
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
if verbose:
log_level = logging.DEBUG
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
formatter = LogFormatter(fmt=fmt,
datefmt="%Y-%m-%d %H:%M:%S",
colors=colors)
if log_format == 'json':
fmt = "(process) (asctime) (levelname) (module) (lineno) (message)"
if context_id:
fmt = "(context_id) {}".format(fmt)
formatter = jsonlogger.JsonFormatter(fmt,
json_default=encoder,
timestamp=True)
# sadly, no other way to specify the name of the default logger publicly
LOGZERO_DEFAULT_LOGGER = logger_name
logger = setup_default_logger(level=log_level, formatter=formatter)
if context_id:
logger.addFilter(ChaosToolkitContextFilter(logger_name, context_id))
if log_file:
# always everything as strings in the log file
logger.setLevel(logging.DEBUG)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
formatter = LogFormatter(fmt=fmt,
datefmt="%Y-%m-%d %H:%M:%S",
colors=colors)
logzero.logfile(log_file,
formatter=formatter,
mode='a',
loglevel=logging.DEBUG)
def main():
const.args = handle.get_arguments()
if const.args.version:
print("spotdl {version}".format(version=__version__))
sys.exit()
internals.filter_path(const.args.folder)
youtube_tools.set_api_key()
logzero.setup_default_logger(formatter=const._formatter,
level=const.args.log_level)
log.debug("Python version: {}".format(sys.version))
log.debug("Platform: {}".format(platform.platform()))
log.debug(pprint.pformat(const.args.__dict__))
try:
if const.args.song:
download_single(raw_song=const.args.song)
elif const.args.list:
if const.args.write_m3u:
youtube_tools.generate_m3u(track_file=const.args.list)
else:
download_list(
tracks_file=const.args.list,
skip_file=const.args.skip,
write_successful_file=const.args.write_successful,
)
elif const.args.playlist:
spotify_tools.write_playlist(playlist_url=const.args.playlist)
elif const.args.album:
spotify_tools.write_album(album_url=const.args.album)
elif const.args.all_albums:
spotify_tools.write_all_albums_from_artist(
artist_url=const.args.all_albums)
elif const.args.username:
spotify_tools.write_user_playlist(username=const.args.username)
# actually we don't necessarily need this, but yeah...
# explicit is better than implicit!
sys.exit(0)
except KeyboardInterrupt as e:
log.exception(e)
sys.exit(3)
def main():
const.args = handle.get_arguments()
internals.filter_path(const.args.folder)
youtube_tools.set_api_key()
logzero.setup_default_logger(formatter=const._formatter, level=const.args.log_level)
try:
match_args()
# actually we don't necessarily need this, but yeah...
# explicit is better than implicit!
sys.exit(0)
except KeyboardInterrupt as e:
log.exception(e)
sys.exit(3)
def __init__(self, file, backCnt):
log_format = '%(color)s[%(asctime)s] [%(levelname)s]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
self.logger = logzero.setup_default_logger(file,
disableStderrLogger=True,
formatter=formatter,
maxBytes=100000,
backupCount=backCnt)
def main_func_caller():
# Todo try new implementation
internals.filter_path(const.args.folder)
youtube_tools.set_api_key()
logzero.setup_default_logger(formatter=const._formatter,
level=const.args.log_level)
try:
operation, text_file, tracks_url = match_args()
if operation is not 'list':
list_downloader(operation, text_file, tracks_url)
# I don't need this type of exception, I'll remove another time
except Exception as e:
print(e)
operation = False
log.exception(e)
sys.exit(3)
def setup_logzero(level="info", path="logs/broker.log", silent=False):
path = BROKER_DIRECTORY.joinpath(path)
path.parent.mkdir(parents=True, exist_ok=True)
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
debug_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
log_level = getattr(logging, level.upper(), logging.INFO)
formatter = logzero.LogFormatter(
fmt=debug_fmt if log_level is logging.DEBUG else log_fmt)
logzero.setup_default_logger(formatter=formatter,
disableStderrLogger=silent)
logzero.loglevel(log_level)
logzero.logfile(path,
loglevel=log_level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
def init_logger():
console_logformat = (
'%(color)s%(levelname)-8s %(asctime)s %(module)s:%(lineno)d%(end_color)s> '
'%(message)s')
# remove colors on logfile
file_logformat = re.sub(r'%\((end_)?color\)s', '', console_logformat)
console_formatter = logzero.LogFormatter(fmt=console_logformat)
file_formatter = logzero.LogFormatter(fmt=file_logformat)
logzero.setup_default_logger(formatter=console_formatter)
logzero.logfile(
settings.LOGFILE,
maxBytes=settings.LOGFILE_SIZE,
backupCount=settings.LOGFILE_BACKUP_COUNT,
formatter=file_formatter,
)
logger = logzero.logger
logger.setLevel(logzero.DEBUG)
return logger
def setup_system_logger(out_log_fp, pdir, logger):
"""fn: setup logger for various package modules
Params
------
out_log_fp: str
log file fp name doesn't include extension fn will add it
logger: logzero logger object
Returns
-------
logger: logzero logger instance
"""
now = pd.to_datetime('now', utc=True)
file_ = out_log_fp+f'_{now.date()}.log'
logfile = Path(pdir/'logs'/file_).as_posix()
check_path(logfile)
formatter = logzero.LogFormatter(fmt=LOG_FORMAT, datefmt=LOG_DATE_FORMAT)
logzero.setup_default_logger(logfile=logfile, formatter=formatter)
return logger
def setup(self):
# Default logger
log_format = '%(color)s* %(levelname)1s%(end_color)s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(self.log_level)
logzero.loglevel(self.log_level)
# Log file
log_format_no_color = '* %(levelname)1s %(asctime)s %(message)s [%(module)s:%(lineno)d]'
formatter_no_color = logzero.LogFormatter(fmt=log_format_no_color)
# Log rotation of 20 files for 10MB each
logzero.logfile(self.log_file,
formatter=formatter_no_color,
loglevel=self.log_level,
maxBytes=int(1e7),
backupCount=20)
logger.info("smartvadhis2 v.{}".format(__version__))
def log(*msgs, STYLE=DEF_STYLE):
log_format = '%(color)s%(message)s%(end_color)s'
formatter = l0.LogFormatter(fmt=log_format)
l0.setup_default_logger(formatter=formatter)
msg = ""
for m in msgs:
msg += m
if (STYLE == "GREEN" or STYLE == "SUCCESS"):
l0.logger.info(str(msg))
elif (STYLE == "BLUE" or STYLE == "INFO"):
l0.logger.debug(str(msg))
elif (STYLE == "YELLOW" or STYLE == "WARNING"):
l0.logger.warning(str(msg))
elif (STYLE == "RED" or STYLE == "ERROR"):
l0.logger.error(str(msg))
else:
l0.logger.error(str("INVALID TYPE"))
print
def test_default_logger(disableStdErrorLogger=False):
"""
Default logger should work and be able to be reconfigured.
"""
logzero.reset_default_logger()
temp = tempfile.NamedTemporaryFile()
try:
logzero.setup_default_logger(logfile=temp.name, disableStderrLogger=disableStdErrorLogger)
logzero.logger.debug("debug1") # will be logged
# Reconfigure with loglevel INFO
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO, disableStderrLogger=disableStdErrorLogger)
logzero.logger.debug("debug2") # will not be logged
logzero.logger.info("info1") # will be logged
# Reconfigure with a different formatter
log_format = '%(color)s[xxx]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(logfile=temp.name, level=logging.INFO, formatter=formatter, disableStderrLogger=disableStdErrorLogger)
logzero.logger.info("info2") # will be logged with new formatter
logzero.logger.debug("debug3") # will not be logged
with open(temp.name) as f:
content = f.read()
_test_default_logger_output(content)
finally:
temp.close()
def setup_logzero(level="info", path="logs/apix.log"):
log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s"
if level == "debug":
level = logging.DEBUG
log_fmt = (
"%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]"
"%(end_color)s %(message)s")
elif level == "info":
level = logging.INFO
elif level == "warning":
level = logging.WARNING
elif level == "error":
level = logging.ERROR
elif level == "critical":
level = logging.CRITICAL
formatter = logzero.LogFormatter(fmt=log_fmt)
logzero.setup_default_logger(formatter=formatter)
logzero.loglevel(level)
logzero.logfile(path,
loglevel=level,
maxBytes=1e9,
backupCount=3,
formatter=formatter)
MCSPASS = ''
LOCKUSER = ''
LOCKPASS = ''
ARCHIVEDAY = 1
TDAY = date.today()
BACKDAY = TDAY.strftime('%Y%m%d')
# Declare
## SSH
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
## Logger
log_format = '%(color)s[%(asctime)s] [%(levelname)s]%(end_color)s %(message)s'
formatter = logzero.LogFormatter(fmt=log_format)
logzero.setup_default_logger(formatter=formatter)
logzero.formatter(formatter=formatter)
_filename = LOGDIR + "/" + str(BACKDAY) + '_mcsbackup.log'
logzero.logfile(_filename, disableStderrLogger=True)
## Backup Function
def backupEXC(module_nm, target_ip):
# Declare
ssh_cli = paramiko.SSHClient()
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connection
try:
if not SSHPASS:
ssh_cli.connect(target_ip,
def set_logger() -> None:
log_format = '%(color)s[%(levelname)1.1s process:%(process)d %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
formatter = LogFormatter(fmt=log_format)
setup_default_logger(formatter=formatter)
