def setup_console():
global err_handler, out_handler, cache_handler
if err_handler is not None:
return
logger = logging.getLogger()
err_handler = StreamHandler(sys.stderr)
err_handler.setLevel(logging.WARNING)
fmt = LogFormatter(fmt="%(levelname)s: %(message)s",
output_markers=("", ""))
fmt.format_exceptions = False
err_handler.setFormatter(fmt)
logger.addHandler(err_handler)
out_handler = StreamHandler(sys.stdout)
out_handler.setLevel(logging.INFO)
out_handler.setFormatter(LogFormatter(fmt="%(message)s"))
out_handler.addFilter(MaxFilter(logging.INFO))
add_common_filters(out_handler)
logger.addHandler(out_handler)
cache_handler = CachingHandler()
logger.addHandler(cache_handler)
logger.setLevel(logging.INFO)
logging.captureWarnings(True)
logging.getLogger("py.warnings").addFilter(LevelDemoteFilter(DEBUG))
def run():
out = StringIO()
handler = StreamHandler(out)
handler.addFilter(DisableFilter())
log.addHandler(handler)
for x in xrange(500):
log.warning('this is not handled')
def cli_logger_factory(name, out, err):
"""
Setup basic logging and return logger instance
:return:
logger instance with following setup:
- DEBUG and INFO will be printed to `out`.
- WARN/WARNING, ERROR, and CRITICAL will be printed to `err`.
:rtype: logging.Logger
"""
logger_ = getLogger(name)
logger_.setLevel(DEBUG)
# handles DEBUG and INFO
handler_out = StreamHandler(out)
level_filter = AllowedLevelsFilter([DEBUG, INFO])
handler_out.addFilter(level_filter)
logger_.addHandler(handler_out)
# handles WARNING, ERROR, and CRITICAL
handler_err = StreamHandler(err)
level_filter = AllowedLevelsFilter([WARNING, ERROR, CRITICAL])
handler_err.addFilter(level_filter)
logger_.addHandler(handler_err)
return logger_
def setup_console():
global err_handler, out_handler, cache_handler
if err_handler is not None:
return
logger = logging.getLogger()
err_handler = StreamHandler(sys.stderr)
err_handler.setLevel(logging.WARNING)
fmt = LogFormatter(fmt="%(levelname)s: %(message)s",
output_markers=("", ""))
fmt.format_exceptions = False
err_handler.setFormatter(fmt)
logger.addHandler(err_handler)
out_handler = StreamHandler(sys.stdout)
out_handler.setLevel(logging.INFO)
out_handler.setFormatter(LogFormatter(fmt="%(message)s"))
out_handler.addFilter(MaxFilter(logging.INFO))
add_common_filters(out_handler)
logger.addHandler(out_handler)
cache_handler = CachingHandler()
logger.addHandler(cache_handler)
logger.setLevel(logging.INFO)
logging.captureWarnings(True)
logging.getLogger("py.warnings").addFilter(LevelDemoteFilter(DEBUG))
def setup_logging(log_level=INFO, log_file=None, use_colors=None):
# type: (int, str, bool) -> None
global _logging_initialized
if _logging_initialized:
return
logger = getLogger()
if logger.handlers:
return
addLevelName(NOTICE, 'NOTICE')
from .this_package_metadata import package_metadata
colors_feature = package_metadata.extra_features['color']
if use_colors and not colors_feature.is_installed:
colors_feature.raise_not_installed('Unable to use colors with logging')
if use_colors is None and package_metadata.extra_features.is_installed(
'color'):
use_colors = True
else:
use_colors = False
default_formatter, notice_formatter = _create_color_formatters(
) if use_colors else _create_no_color_formatters()
level_formatter = LevelFormatter(default_formatter=default_formatter)
level_formatter.add_formatter_for_level(INFO, Formatter('%(message)s'))
level_formatter.add_formatter_for_level(NOTICE, notice_formatter)
console_formatter = level_formatter
from sys import stdout, stderr
stdout_handler = StreamHandler(stdout)
stdout_handler.setLevel(DEBUG)
stdout_handler.addFilter(LevelFilter(NOTICE))
stdout_handler.setFormatter(console_formatter)
stderr_handler = StreamHandler(stderr)
stderr_handler.setLevel(WARNING)
stderr_handler.setFormatter(console_formatter)
logger.setLevel(log_level)
logger.addHandler(stdout_handler)
logger.addHandler(stderr_handler)
_logging_initialized = True
if not log_file:
return
file_formatter = Formatter(
'%(name)-20s - %(levelname)-10s - %(asctime)-30s: %(message)s')
log_file_handler = FileHandler(log_file)
log_file_handler.setLevel(DEBUG)
log_file_handler.setFormatter(file_formatter)
logger.addHandler(log_file_handler)
def add_stream_handler(self, level=None, formatter=None, filters=None):
handler = StreamHandler()
handler.setLevel(level or self.level)
handler.setFormatter(formatter or self.formatter)
if filters is None:
filters = []
for f in filters:
handler.addFilter(f)
self.addHandler(handler)
return self
def update(self, raw):
self.raw = raw
log = StringIO()
handler = StreamHandler(log)
handler.addFilter(ExceptionFilter())
handler.setFormatter(Formatter('<span class="level">%(levelname)s</span>: <span class="message">%(message)s</span><br />'))
scss.log.addHandler(handler)
self.compressed = scss.Scss().compile(self.raw)
scss.log.removeHandler(handler)
handler.flush()
self.put()
return log.getvalue()
def log(name: str, version: str, **options):
"""
log configuration setup, logging all goes to standard out by default
:param str name: endpoint name
:param str version: endpoint version
:param options: TODO other options for other chat notifications (sms,
:return:
"""
# DO NOT 'log' anything here, logging is being set up so infinite loops happen
print(f'entered log({locals()})')
log_format = '{asctime} [{app_host} | {app_name} | {app_vers} | {process}] [{levelname}] {filename}:{lineno} - {message}'
try:
host = socket.gethostname()
except:
host = "?"
def filter_factory():
class LoggingFilter(logging.Filter):
def filter(self, record):
try:
record.app_host = host
except:
record.app_host = '?host?'
try:
record.app_name = name
except:
record.app_name = '?name?'
try:
record.app_vers = version
except:
record.app_vers = '?vers?'
return True
return LoggingFilter
formatter = logging.Formatter(log_format, style="{")
LogFilter = filter_factory()
new_logger = logging.root
new_logger.setLevel(logging.DEBUG)
# standard out
log_handler_s = StreamHandler(sys.stdout)
log_handler_s.setLevel(logging.DEBUG)
log_handler_s.setFormatter(formatter)
log_handler_s.addFilter(LogFilter())
new_logger.addHandler(log_handler_s)
return new_logger
def log_init():
out = StreamHandler(stdout)
out.addFilter(lambda record: record.levelno <= INFO)
out.setLevel(INFO)
err = StreamHandler(stderr)
err.setLevel(ERROR)
basicConfig(
level=INFO,
format="%(message)s",
handlers=[out, err],
)
def configure_root_logger(self):
logger = getLogger('')
logger.setLevel(INFO)
stdout_handler = StreamHandler(sys.stdout)
stdout_handler.addFilter(MaxLevelFilter(WARNING))
stdout_handler.setLevel(INFO)
logger.addHandler(stdout_handler)
stderr_handler = StreamHandler(sys.stderr)
stderr_handler.setLevel(WARNING)
logger.addHandler(stderr_handler)
return logger
def add_console_handler(logger):
stdout_handler = StreamHandler(sys.stdout)
stderr_handler = StreamHandler(sys.stderr)
log_filter = LogFilter(logging.WARNING)
stdout_handler.addFilter(log_filter)
stdout_handler.setLevel(logging.INFO)
stdout_handler.setFormatter(SIMPLE_FORMATTER)
stderr_handler.setLevel(max(MIN_LOG_LEVEL, logging.WARNING))
stderr_handler.setFormatter(SIMPLE_FORMATTER)
logger.addHandler(stdout_handler)
logger.addHandler(stderr_handler)
def add_console_handler(logger):
stdout_handler = StreamHandler(sys.stdout)
stderr_handler = StreamHandler(sys.stderr)
log_filter = LogFilter(logging.WARNING)
stdout_handler.addFilter(log_filter)
stdout_handler.setLevel(logging.INFO)
stdout_handler.setFormatter(SIMPLE_FORMATTER)
stderr_handler.setLevel(max(MIN_LOG_LEVEL, logging.WARNING))
stderr_handler.setFormatter(SIMPLE_FORMATTER)
logger.addHandler(stdout_handler)
logger.addHandler(stderr_handler)
def init_logger():
"""
Initializes the logger settings.
"""
formatter = logging.Formatter(CALENDAR_LOG_FMT)
calendar_log = logging.getLogger("attendance_management_bot")
file_handler = StreamHandler()
file_handler.setFormatter(formatter)
calendar_log.setLevel(CALENDAR_LOG_LEVEL)
file_handler.addFilter(attendance_management_bot.contextlog.RequestContextFilter())
calendar_log.addHandler(file_handler)
logging.getLogger("tornado.application").addHandler(file_handler)
logging.getLogger("tornado.general").addHandler(file_handler)
def update(self, raw):
self.raw = raw
log = StringIO()
handler = StreamHandler(log)
handler.addFilter(ExceptionFilter())
handler.setFormatter(
Formatter(
'<span class="level">%(levelname)s</span>: <span class="message">%(message)s</span><br />'
))
scss.log.addHandler(handler)
self.compressed = scss.Scss().compile(self.raw)
scss.log.removeHandler(handler)
handler.flush()
self.put()
return log.getvalue()
def init_logger():
"""
Initializes the root logger settings.
"""
formatter = logging.Formatter(CALENDAR_LOG_FMT)
root_logger = logging.getLogger()
file_handler = StreamHandler()
file_handler.setFormatter(formatter)
root_logger.setLevel(CALENDAR_LOG_LEVEL)
file_handler.addFilter(faq_bot.contextlog.RequestContextFilter())
root_logger.addHandler(file_handler)
logging.getLogger("tornado.application").addHandler(file_handler)
logging.getLogger("tornado.general").addHandler(file_handler)
def init_logger():
"""
init logger setting
"""
formatter = logging.Formatter(CALENDAR_LOG_FMT)
calendar_log = logging.getLogger("calendar_bot")
file_handler = StreamHandler()
file_handler.setFormatter(formatter)
calendar_log.setLevel(CALENDAR_LOG_LEVEL)
file_handler.addFilter(calendar_bot.contextlog.RequestContextFilter())
calendar_log.addHandler(file_handler)
# add app/gen ERROR log
logging.getLogger("tornado.application").addHandler(file_handler)
logging.getLogger("tornado.general").addHandler(file_handler)
def configure_logging(self, filename, filter, format, level):
"""Used for configuring logging from the command line or aspen.conf.
"""
# Handler
# =======
# sys.stdout or rotated file
if filename is None:
handler = StreamHandler(sys.stdout)
else:
# @@: Handle absolute paths on Windows
# http://sluggo.scrapping.cc/python/unipath/Unipath-current/unipath/abstractpath.py
# http://docs.python.org/library/os.path.html#os.path.splitunc
if not filename.startswith("/"):
filename = join(self.paths.root, filename)
filename = realpath(filename)
logdir = dirname(filename)
if not isdir(logdir):
os.makedirs(logdir, 0755)
handler = TimedRotatingFileHandler(filename=filename, when="midnight", backupCount=7)
# Filter
# ======
if filter is not None:
filter = logging.Filter(filter)
handler.addFilter(filter)
# Format
# ======
formatter = logging.Formatter(fmt=format)
handler.setFormatter(formatter)
# Level
# =====
handler.setLevel(level)
# Installation
# ============
root_logger = logging.getLogger()
root_logger.addHandler(handler)
root_logger.setLevel(level) # bah
def get_logger() -> Logger:
logger = getLogger("{{cookiecutter.project_slug}}")
logger.setLevel(DEBUG if not settings.python_env.startswith("prod") else INFO)
std_handler = StreamHandler()
std_handler.setLevel(INFO)
std_handler.setFormatter(
Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
)
debug_handler = StreamHandler()
debug_handler.addFilter(lambda record: record.levelno == DEBUG)
debug_handler.setFormatter(
Formatter(
"%(asctime)s %(name)s %(levelname)s %(pathname)s:%(lineno)d %(message)s"
)
)
logger.handlers = [std_handler, debug_handler]
return logger
def main(args):
logger = getLogger(__name__)
sh = StreamHandler()
sh.addFilter(Filter('HealthPlanetDataTool'))
basicConfig(handlers=[sh])
if args.verbose:
getLogger('HealthPlanetDataTool').setLevel(DEBUG)
else:
getLogger('HealthPlanetDataTool').setLevel(INFO)
passwd_client = Password(prompt='Please enter your password: ')
passwd = passwd_client.launch()
client = HealthPlanetExport(client_id=args.client_id,
client_secret=args.client_secret,
login_id=args.login_id,
login_pass=passwd)
client.get_auth()
client.get_token()
client.get_data(args.from_date, args.to_date)
client.save(args.out_file)
def set_level(self, level: int):
self.setLevel(level)
for chosen_level in self._choices:
if level == chosen_level[0]:
# reset handlers
self.remove_handlers()
stream_handler = StreamHandler(sys.stdout)
stream_handler.setLevel(level)
file_name = f"logs/{self.room_name}/{chosen_level[1]}.log"
if not os.path.exists(f"logs/{self.room_name}/"):
os.mkdir(f"logs/{self.room_name}/")
file_handler = FileHandler(
filename=f"logs/{self.room_name}/{chosen_level[1]}.log")
file_handler.setLevel(level)
file_handler.setFormatter(file_formatter)
# secondary log file that contains only messages.
if self.level == self.CHAT:
self.add_chat_handler()
stream_handler.addFilter(ChatFilter())
stream_handler.setFormatter(terminal_formatter)
else:
stream_handler.addFilter(DebugFilter())
stream_handler.setFormatter(terminal_formatter)
if self.chat_handler_enabled:
self.add_chat_handler()
self.addHandler(file_handler)
# log to the terminal.
self.addHandler(stream_handler)
return True
# level was not set
return False
def init_logging():
# create formatter
formatter = jsonlogger.JsonFormatter(format_str)
# create stdout handler
stdout_handler = StreamHandler(sys.stdout)
stdout_handler.setFormatter(formatter)
stdout_handler.setLevel(DEBUG)
stdout_handler.addFilter(lambda record: record.levelno in {DEBUG, INFO})
# create stderr handler
stderr_handler = StreamHandler(sys.stderr)
stderr_handler.setFormatter(formatter)
stderr_handler.setLevel(WARNING)
# configure logger
app_logger.addHandler(stdout_handler)
app_logger.addHandler(stderr_handler)
for level in log_levels:
if '=' in level:
config = level.strip().split('=')
if config[0] == logger_name:
app_logger.setLevel(config[1].upper())
getLogger(config[0]).setLevel(config[1].upper())
def create_logger(name=None):
# create logger
log = getLogger(name)
log.setLevel(DEBUG)
# create formatter and add it to the handlers
log_format = Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create console handler with a higher log level
info_handler = StreamHandler(stdout)
info_handler.setLevel(INFO)
info_handler.setFormatter(log_format)
log.addHandler(info_handler)
# create console handler with a higher log level
debug_handler = StreamHandler(stderr)
debug_handler.setLevel(DEBUG)
debug_handler.setFormatter(log_format)
debug_handler.addFilter(LogLevelFilter(DEBUG))
log.addHandler(debug_handler)
return log
def __init__(
self,
name,
level: str = "INFO",
tag: Optional[str] = None,
file: Optional[Path] = None,
clear: bool = False,
) -> None:
"""
Initializes an object of type 'Log'.
Parameters
----------
name : str
The name of the logger, usually '__name__' is best praxis.
level : str
The level to log on (DEBUG, INFO, WARNING, ERROR or CRITICAL).
tag : str, optional
Tag of the logging object (e.g. local or remote), by default None.
file : Path, optional
Path to the log file, by default None.
clear : bool, optional
Clear the provided log file, by default False.
"""
self.name = name
self.level = level
self.tag = tag
self.file = file
# Clear log file if exists
if clear:
self.clear_log_file()
# Logger config
numeric_level = self._level_from_str(level)
level = level.upper()
formatter = Formatter(LOG_FORMAT, LOG_DATETIME_FORMAT)
logger = getLogger(name)
if logger.hasHandlers():
logger.debug(self._format("Clear handlers"))
logger.handlers.clear()
logger.setLevel(numeric_level)
# Stream handler
stream_handler_stdout = StreamHandler(stdout)
stream_handler_stdout.setFormatter(formatter)
stream_handler_stdout.setLevel(DEBUG)
stream_handler_stdout.addFilter(lambda record: record.levelno <= INFO)
stream_handler_stderr = StreamHandler(stderr)
stream_handler_stderr.setFormatter(formatter)
stream_handler_stderr.setLevel(WARNING)
logger.addHandler(stream_handler_stdout)
logger.addHandler(stream_handler_stderr)
# File handler
if file is not None:
file_handler = RotatingFileHandler(Path(file),
maxBytes=512,
backupCount=0)
file_handler.setFormatter(formatter)
file_handler.setLevel(DEBUG)
logger.addHandler(file_handler)
logger.debug(self._format(f"Set file handler to '{file}'"))
# Complete initialization
logger.debug(self._format("Set stream handler to 'STDOUT/STDERR'"))
logger.debug(self._format(f"Set log level to '{level}'"))
self.logger = logger
self.logger.debug(self._format("Logger initialized"))
def get_logger_settings(
env_name,
log_dir,
log_file_name,
application_log_level='DEBUG',
logstash_listner_ip=None,
logstash_listner_port=None,
logstash_tags=[],
cloudwatch_logging_enabled=False,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_region_name=None,
cloudwatch_log_group=None,
cloud_watch_log_stream=None,
sentry_logging_enabled=False,
):
boto3_session = Session(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region_name)
# Formatters
verbose_formatter = logging.Formatter(
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(pathname)s:%(lineno)d] %(message)s'
.format(env_name=env_name),
datefmt='%Y-%m-%d %H:%M:%S')
console_handler = StreamHandler()
console_handler.addFilter(RequireDebugTrue())
console_handler.addFilter(TimestampFilter())
console_handler.setFormatter(verbose_formatter)
mail_admins_handler = AdminEmailHandler()
mail_admins_handler.include_html = True
mail_admins_handler.setLevel(logging.ERROR)
mail_admins_handler.addFilter(RequireDebugFalse())
mail_admins_handler.addFilter(TimestampFilter())
mail_admins_handler.setFormatter(verbose_formatter)
file_handler = RotatingFileHandler(filename=log_dir + '/' + log_file_name,
maxBytes=20 * 1024 * 1024,
backupCount=7)
file_handler.setLevel(logging.DEBUG)
file_handler.addFilter(TimestampFilter())
file_handler.setFormatter(verbose_formatter)
socket_handler = None
if logstash_listner_ip is not None and logstash_listner_port is not None:
socket_handler = SocketLogstashHandler(logstash_listner_ip,
logstash_listner_port)
socket_handler.setLevel(logging.ERROR)
socket_handler.addFilter(RequireDebugTrue())
socket_handler.addFilter(TimestampFilter())
socket_handler.setFormatter(verbose_formatter)
socket_handler.tags = logstash_tags
logging_dict = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
},
'require_debug_true': {
'()': 'django.utils.log.RequireDebugTrue'
},
'execpath': {
'()': ExecpathFilter,
},
'timestamp': {
'()': TimestampFilter,
},
},
'formatters': {
'simple': {
'format': '[%(asctime)s] %(levelname)s %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
},
'verbose': {
'format':
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(pathname)s:%(lineno)d] %(message)s'
.format(env_name=env_name),
'datefmt':
'%Y-%m-%d %H:%M:%S'
},
'default': {
# To be used with default handler only.
'format':
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(execpath)s:%(execline)d] %(execmsg)s'
.format(env_name=env_name),
'datefmt':
'%Y-%m-%d %H:%M:%S'
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'filters': ['timestamp', 'execpath'],
'class': 'logging.FileHandler',
'filename': log_dir + '/' + log_file_name,
'formatter': 'default'
},
'console': {
'filters': ['require_debug_true', 'timestamp'],
'class': 'logging.StreamHandler',
'formatter': 'verbose'
},
'mail_admins': {
'filters': [
'require_debug_false',
],
'class': 'django.utils.log.AdminEmailHandler',
'include_html': True,
'level': 'ERROR',
},
'file_error': {
'class': 'logging.FileHandler',
'filters': ['timestamp'],
'filename': log_dir + '/' + log_file_name,
'formatter': 'verbose',
},
'queue_handler': {
'class': 'logging.handlers.QueueHandler',
'filters': ['timestamp'],
'formatter': 'verbose',
'queue': log_queue
},
},
'loggers': {
'django.request': {
'handlers': [
'default',
'mail_admins',
],
'level': 'ERROR',
'propagate': True
},
'django.security.DisallowedHost': {
'level': 'ERROR',
'handlers': [
'file_error',
'console',
'mail_admins',
],
'propagate': True
},
'application': {
'handlers': ['queue_handler'],
'level': application_log_level,
'propagate': True
},
},
}
if sentry_logging_enabled:
logging_dict['handlers']['sentry'] = {
'level':
'ERROR', # To capture more than ERROR, change to WARNING, INFO, etc.
'class':
'raven.contrib.django.raven_compat.handlers.SentryHandler',
'tags': {},
}
logging_dict['loggers']['application']['handlers'].append('sentry')
if cloudwatch_logging_enabled:
logging_dict['handlers']['watchtower'] = {
'level': 'DEBUG',
'class': 'watchtower.CloudWatchLogHandler',
'boto3_session': boto3_session,
'log_group': cloudwatch_log_group,
'stream_name': cloud_watch_log_stream,
'formatter': 'verbose',
}
logging_dict['loggers']['application']['handlers'].append('watchtower')
queue_listner.handlers = [
console_handler, mail_admins_handler, file_handler
]
if socket_handler:
queue_listner.handlers.append(socket_handler)
return logging_dict
from email.mime.multipart import MIMEMultipart # Used for email notification
from pathlib import Path # Used for specifying a generic path
import logging # Used in logging
from logging import StreamHandler, Formatter # Used in logging
#region ~~~~~~~~~~ Global variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Looking for a better way to do this ...
normalfont = ('Helvetica', '10', 'normal')
boldfont = ('Helvetica', '10', 'bold')
EOL = '\r' # Constant end of line character. This must match with Arduino starter code.
#endregion
#region ~~~~~~~~~~ Code to including error report via logging ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
stdout = StreamHandler(sys.stdout)
stdout.addFilter(lambda r: logging.INFO <= r.levelno < logging.WARNING)
stdout.setFormatter(Formatter()) # Don't take the default format we set later
stderr = StreamHandler(sys.stderr)
stderr.addFilter(lambda r: r.levelno >= logging.WARNING)
stderr.setFormatter(Formatter()) # Don't take the default format we set later
debug_output = StreamHandler(sys.stdout)
debug_output.addFilter(lambda r: logging.debug <= r.levelno < logging.INFO)
# Configure logging
logging.basicConfig(
format="%(asctime)s -- %(name)s/ -- %(levelname)s :: %(message)s",
datefmt="%c",
level=logging.
DEBUG, # if debugging level disabled above, effectively sets level to logging.INFO
drophour = 16
headless = False #True
debug = 1
class Filter:
def filter(event):
return event.name not in (
'requests.packages.urllib3.connectionpool',
'selenium.webdriver.remote.remote_connection')
h = StreamHandler()
h.addFilter(Filter)
ll = WARNING
#ll=INFO
ll = DEBUG
basicConfig(format='{asctime} {threadName:11s}: {message}',
datefmt="%H:%M:%S",
style='{',
level=ll,
handlers=[h])
lg = getLogger(__name__)
accounts = []
with open('100 Gmail.csv') as f:
def get_script_logger(debug=True, quiet=False, config=None):
"""Creates a script logger to log to files / streams."""
log_file_format = (config and config['LOG_FILE_FORMAT']
or '%(levelname)s %(asctime)s %(message)s')
log_file_datefmt = (config and config['LOG_FILE_DATEFMT']
or '[%Y-%m-%d %H:%M:%S]')
script_log_info_file_handler = None
if config:
script_log_info_file_handler = RotatingFileHandler(path.join(
config['LOG_FOLDER'], 'script_info.log'),
maxBytes=100000,
backupCount=5)
script_log_info_file_handler.setLevel(logging.INFO)
script_log_info_file_handler.setFormatter(
Formatter(log_file_format, log_file_datefmt))
script_log_info_file_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_file_handler = None
if config:
script_log_error_file_handler = RotatingFileHandler(path.join(
config['LOG_FOLDER'], 'script_error.log'),
maxBytes=100000,
backupCount=5)
script_log_error_file_handler.setLevel(logging.ERROR)
script_log_error_file_handler.setFormatter(
Formatter(log_file_format, log_file_datefmt))
script_log_info_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stdout)
script_log_info_stream_handler.setLevel(logging.INFO)
script_log_info_stream_handler.setFormatter(
Formatter(log_file_format, log_file_datefmt))
script_log_info_stream_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stderr)
script_log_error_stream_handler.setLevel(logging.ERROR)
script_log_error_stream_handler.setFormatter(
Formatter(log_file_format, log_file_datefmt))
script_logger = logging.getLogger('script')
script_logger.setLevel(logging.INFO)
if config:
script_logger.addHandler(script_log_info_file_handler)
script_logger.addHandler(script_log_error_file_handler)
script_logger.addHandler(script_log_info_stream_handler)
script_logger.addHandler(script_log_error_stream_handler)
if (not debug) and config:
mail_handler = get_mail_handler(config)
script_logger.addHandler(mail_handler)
return script_logger
fmt = f"{prefixedMessage}\n{sourceInfo}"
utcFormatter = UtcFormatter(fmt=fmt, datefmt=datetimeFormat, style=style)
# ---------------------------- initialize logging -----------------------------
rootLogger = getLogger()
# Unset log level on the root Logger instance (`WARNING` by default).
# - Let individual handlers filter on the basis of level, depending on
# (a) their responsibility and (b) the currently set cmdline args.
# - Adjust on using class' initialization.
rootLogger.setLevel(LogLevel.NotSet)
# Let `Warning` (numeric value 30) be the threshold between stdout & stderr.
logLevelThresholdOutErr = LogLevel.Warning
stderrHandler = StreamHandler(stderr)
stdoutHandler = StreamHandler(stdout)
stderrHandler.setLevel(logLevelThresholdOutErr)
stdoutHandler.setLevel(LogLevel.NotSet)
stdoutHandler.addFilter(lambda record: 1
if record.levelno < logLevelThresholdOutErr else 0)
stderrHandler.setFormatter(utcFormatter)
stdoutHandler.setFormatter(utcFormatter)
rootLogger.addHandler(stderrHandler)
rootLogger.addHandler(stdoutHandler)
def setup_syslog(verbosity, quiet=False, logdest="syslog"):
'''
TBD
'''
try:
_fmsg = ""
_status = 100
_my_uuid, _oscp, _mscp, _lscp = get_stores_parms()
# HACK ALERT - A very crude "syslog facility selector"
_syslog_selector = {}
_syslog_selector["16"] = SysLogHandler.LOG_LOCAL0
_syslog_selector["17"] = SysLogHandler.LOG_LOCAL1
_syslog_selector["18"] = SysLogHandler.LOG_LOCAL2
_syslog_selector["19"] = SysLogHandler.LOG_LOCAL3
_syslog_selector["20"] = SysLogHandler.LOG_LOCAL4
_syslog_selector["21"] = SysLogHandler.LOG_LOCAL5
_syslog_selector["22"] = SysLogHandler.LOG_LOCAL6
_syslog_selector["23"] = SysLogHandler.LOG_LOCAL7
_verbosity = int(verbosity)
logger = getLogger()
# Reset the logging handlers
while len(logger.handlers) != 0:
logger.removeHandler(logger.handlers[0])
if logdest == "console" or (not _lscp["hostname"]
or not _lscp["port"]):
hdlr = StreamHandler(stdout)
else:
_facility = int(21)
if _facility > 23 or _facility < 16:
_facility = 23
hdlr = SysLogHandler(address = (_lscp["hostname"], \
int(_lscp["port"])), \
facility=_syslog_selector[str(_facility)])
formatter = Formatter("[%(asctime)s] [%(levelname)s] %(message)s")
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
if _verbosity:
if int(_verbosity) >= 6:
logger.setLevel(DEBUG)
elif int(_verbosity) >= 5:
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 4:
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 3:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
hdlr.addFilter(MsgFilter("Exit point"))
hdlr.addFilter(MsgFilter("Entry point"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 2:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) == 1:
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("clouds"))
logger.setLevel(DEBUG)
else:
logger.setLevel(INFO)
if quiet:
logger.setLevel(ERROR)
_status = 0
except Exception, e:
_status = 23
_fmsg = str(e)
def get_logger_settings(env_name,
log_dir,
log_file_name,
application_log_level='DEBUG',
logstash_listner_ip=None,
logstash_listner_port=None,
logstash_tags=[]):
# Formatters
verbose_formatter = logging.Formatter(
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(pathname)s:%(lineno)d] %(message)s'
.format(env_name=env_name),
datefmt='%Y-%m-%d %H:%M:%S')
console_handler = StreamHandler()
console_handler.addFilter(RequireDebugTrue())
console_handler.addFilter(TimestampFilter())
console_handler.setFormatter(verbose_formatter)
mail_admins_handler = AdminEmailHandler()
mail_admins_handler.include_html = True
mail_admins_handler.setLevel(logging.ERROR)
mail_admins_handler.addFilter(RequireDebugFalse())
mail_admins_handler.addFilter(TimestampFilter())
mail_admins_handler.setFormatter(verbose_formatter)
file_handler = RotatingFileHandler(filename=log_dir + '/' + log_file_name,
maxBytes=20 * 1024 * 1024,
backupCount=7)
file_handler.setLevel(logging.DEBUG)
file_handler.addFilter(TimestampFilter())
file_handler.setFormatter(verbose_formatter)
socket_handler = None
if logstash_listner_ip is not None and logstash_listner_port is not None:
socket_handler = SocketLogstashHandler(logstash_listner_ip,
logstash_listner_port)
socket_handler.setLevel(logging.ERROR)
socket_handler.addFilter(RequireDebugTrue())
socket_handler.addFilter(TimestampFilter())
socket_handler.setFormatter(verbose_formatter)
socket_handler.tags = logstash_tags
logging_dict = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
},
'require_debug_true': {
'()': 'django.utils.log.RequireDebugTrue'
},
'execpath': {
'()': ExecpathFilter,
},
'timestamp': {
'()': TimestampFilter,
},
},
'formatters': {
'simple': {
'format': '[%(asctime)s] %(levelname)s %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
},
'verbose': {
'format':
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(pathname)s:%(lineno)d] %(message)s'
.format(env_name=env_name),
'datefmt':
'%Y-%m-%d %H:%M:%S'
},
'default': {
# To be used with default handler only.
'format':
'[%(timestamp)s] [{env_name}] [%(levelname)s] [%(execpath)s:%(execline)d] %(execmsg)s'
.format(env_name=env_name),
'datefmt':
'%Y-%m-%d %H:%M:%S'
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'filters': ['timestamp', 'execpath'],
'class': 'logging.FileHandler',
'filename': log_dir + '/' + log_file_name,
'formatter': 'default'
},
'console': {
'filters': ['require_debug_true', 'timestamp'],
'class': 'logging.StreamHandler',
'formatter': 'verbose'
},
'mail_admins': {
'filters': [
'require_debug_false',
],
'class': 'django.utils.log.AdminEmailHandler',
'include_html': True,
'level': 'ERROR',
},
'file_error': {
'class': 'logging.FileHandler',
'filters': ['timestamp'],
'filename': log_dir + '/' + log_file_name,
'formatter': 'verbose',
},
'queue_handler': {
'class': 'logging.handlers.QueueHandler',
'filters': ['timestamp'],
'formatter': 'verbose',
'queue': log_queue
}
},
'loggers': {
'django.request': {
'handlers': [
'default',
'mail_admins',
],
'level': 'ERROR',
'propagate': True
},
'django.security.DisallowedHost': {
'level': 'ERROR',
'handlers': [
'file_error',
'console',
'mail_admins',
],
'propagate': True
},
'application': {
'handlers': ['queue_handler'],
'level': application_log_level,
'propagate': True
},
},
}
queue_listner.handlers = [
console_handler, mail_admins_handler, file_handler
]
if socket_handler:
queue_listner.handlers.append(socket_handler)
return logging_dict
class LogConf(object):
'''
Application-specific logging configuration.
This configuration defines sane default filters, formatters, and handlers
for the root logger, which callers may customize (e.g., according to
user-defined settings) by setting various properties of this configuration.
Caveats
----------
**The :meth:`__init__` method may raise exceptions.** Hence, this class
should be instantiated at application startup by an explicit call to the
:func:`betse.util.io.log.conf.logconf.init` function *after* establishing
default exception handling.
**The :meth:`deinit` method must be called at application shutdown.** Doing
so closes the logfile handle opened by the :meth:`__init__` method. Failure
to do so will reliably raise non-fatal warnings (hidden by default, but
visible while running tests) resembling:
C:\\projects\\betse\\betse\\util\\io\\log\\conf\\logconf.py:44:
ResourceWarning: unclosed file <_io.TextIOWrapper
name='C:\\Users\\appveyor\\AppData\\Roaming\\betse\\betse.log' mode='a'
encoding='utf-8'>
_log_conf = LogConf()
Default Settings
----------
All loggers will implicitly propagate messages to the root logger
configured by this class, whose output will be:
* Formatted in a timestamped manner detailing the point of origin (e.g.,
``[2016-04-03 22:02:47] betse ERROR (util.py:50): File not found.``).
* Labelled as the current logger's name, defaulting to `root`. Since this
is *not* a terribly descriptive name, callers are encouraged to replace
this by an application-specific name.
* Printed to standard error if the logging level for this output is either
``WARNING``, ``ERROR``, or ``CRITICAL``.
* Printed to standard output if the logging level for this output is
``INFO``. Together with the prior item, this suggests that output with a
logging level of ``DEBUG`` will *not* be printed by default.
* Appended to the user-specific file defined by the
:meth:`app_meta.log_default_filename` property, whose:
* Level defaults to :data:`logger.ALL`. Hence, *all* messages will be
logged by default, including low-level debug messages. (This is
helpful for debugging client-side errors.)
* Contents will be automatically rotated on exceeding a sensible filesize
(e.g., 16Kb).
If the default log levels are undesirable, consider subsequently calling
this logger's :meth:`Logger.setLevel` method. Since a desired log level is
typically unavailable until after parsing CLI arguments and/or
configuration file settings *and* since a logger is required before this
level becomes available, this class provides a sane interim default.
Attributes
----------
_filename : str
Absolute or relative path of the file logged to by the file handler,
defaulting to :meth:`app_meta.log_default_filename`.
_logger_root : Logger
Root logger.
_logger_root_handler_file : Handler
Root logger handler appending to the current logfile.
_logger_root_handler_stderr : Handler
Root logger handler printing to standard error.
_logger_root_handler_stdout : Handler
Root logger handler printing to standard output.
'''
# ..................{ INITIALIZERS }..................
def __init__(self):
'''
Initialize this logging configuration.
Specifically, this method associates the root logger with a rotating
logfile whose handle remains open until the :meth:`deinit` method is
subsequently called at application shutdown.
'''
# Avoid circular import dependencies.
from betse.util.test import tsttest
# Initialize the superclass.
super().__init__()
# Initialize all non-property attributes to sane defaults. To avoid
# chicken-and-egg issues, properties should *NOT* be set here.
self._deinit_vars()
# Initialize the root logger.
self._init_logger_root()
# Initialize root logger handlers *AFTER* the root logger, as the
# former explicitly add themselves to the latter.
self._init_logger_root_handler_std()
self._init_logger_root_handler_file()
# Redirect all warnings through the logging framewark *AFTER*
# successfully performing the above initialization.
logging.captureWarnings(True)
# If the active Python interpreter is running an automated test suite,
# manually increase logging verbosity as soon as feasible: i.e., here.
# While deferring this configuration to elsewhere (namely, the
# "betse_test.fixture.initter" submodule) is also feasible, doing so
# would horrifyingly squelch all early-time debug messages.
if tsttest.is_testing():
self.is_verbose = True
def _init_logger_root(self) -> None:
'''
Initialize the root logger.
For safety, this function removes all previously initialized handlers
from this logger.
'''
# Avoid circular import dependencies.
from betse.util.app.meta import appmetaone
# Root logger.
self._logger_root = logging.getLogger()
# For uniqueness, change the name of the root logger to that of our
# top-level package "betse" from its ambiguous default "root".
self._logger_root.name = appmetaone.get_app_meta().package_name
# Instruct this logger to entertain all log requests, ensuring these
# requests will be delegated to the handlers defined below. By default,
# this logger ignores all log requests with level less than "WARNING",
# preventing handlers from receiving these requests.
self._logger_root.setLevel(LogLevel.ALL)
# Safely remove all existing handlers from the root logger *BEFORE*
# adding new handlers to this logger.
#
# The root logger should have no handlers under conventional use cases.
# The root logger only has handlers for functional tests *NOT*
# parallelized by "xdist" and hence running in the same Python process.
self._deinit_logger_root_handlers()
def _init_logger_root_handler_std(self) -> None:
'''
Initialize root logger handlers redirecting log messages to the
standard stdout and stderr file handles.
'''
# Avoid circular import dependencies.
from betse.util.io.log.logfilter import (LogFilterThirdPartyDebug,
LogFilterMoreThanInfo)
from betse.util.io.log.conf.logconfformat import LogFormatterWrap
from betse.util.os.command import cmds
# Initialize the stdout handler to:
#
# * Log only informational messages by default.
# * Unconditionally ignore all warning and error messages, which the
# stderr handler already logs.
#
# Sadly, the "StreamHandler" constructor does *NOT* accept the
# customary "level" attribute accepted by its superclass constructor.
self._logger_root_handler_stdout = StreamHandler(sys.stdout)
self._logger_root_handler_stdout.setLevel(LogLevel.INFO)
self._logger_root_handler_stdout.addFilter(LogFilterMoreThanInfo())
# Initialize the stderr handler to:
#
# * Log only warning and error messages by default.
# * Unconditionally ignore all informational and debug messages, which
# the stdout handler already logs.
self._logger_root_handler_stderr = StreamHandler(sys.stderr)
self._logger_root_handler_stderr.setLevel(LogLevel.WARNING)
# Avoid printing third-party debug messages to the terminal.
self._logger_root_handler_stdout.addFilter(LogFilterThirdPartyDebug())
self._logger_root_handler_stderr.addFilter(LogFilterThirdPartyDebug())
#FIXME: Consider colourizing this format string.
# Format standard output and error in the conventional way. For a list
# of all available log record attributes, see:
#
# https://docs.python.org/3/library/logging.html#logrecord-attributes
#
# Note that "{{" and "}}" substrings in format() strings escape literal
# "{" and "}" characters, respectively.
stream_format = '[{}] {{message}}'.format(cmds.get_current_basename())
# Formatter for this format.
stream_formatter = LogFormatterWrap(fmt=stream_format, style='{')
# Assign these formatters to these handlers.
self._logger_root_handler_stdout.setFormatter(stream_formatter)
self._logger_root_handler_stderr.setFormatter(stream_formatter)
# Register these handlers with the root logger.
self._logger_root.addHandler(self._logger_root_handler_stdout)
self._logger_root.addHandler(self._logger_root_handler_stderr)
def _init_logger_root_handler_file(self) -> None:
'''
Initialize the root logger handler appending log messages to the
currently open logfile file handle.
This method is designed to be called multiple times, permitting the
filename associated with this handler to be modified at runtime.
'''
# Avoid circular import dependencies.
from betse.util.io.log.logfilter import LogFilterThirdPartyDebug
from betse.util.io.log.conf.logconfformat import LogFormatterWrap
from betse.util.io.log.conf.logconfhandle import (
LogHandlerFileRotateSafe)
from betse.util.path import pathnames
from betse.util.os.command import cmds
from betse.util.type.numeric import ints
# Absolute or relative path of the directory containing this file.
file_dirname = pathnames.get_dirname(self._filename)
# Minimum level of messages to be log to disk, defaulting to "INFO".
file_level = LogLevel.INFO
# If this handler has already been created...
if self._logger_root_handler_file is not None:
# Preserve the previously set minimum level of messages to log.
file_level = self._logger_root_handler_file.level
# If the root logger has also already been created, remove this
# handler from this root logger.
if self._logger_root is not None:
self._logger_root.removeHandler(self._logger_root_handler_file)
# If the dirname of the directory containing this file is non-empty,
# create this directory if needed. Note this dirname is empty when this
# filename is a pure basename (e.g., when the "--log-file=my.log"
# option is passed).
#
# For safety, this directory is created with standard low-level Python
# functionality rather than our custom higher-level
# dirs.make_parent_unless_dir() function. The latter logs this
# creation. Since the root logger is *NOT* fully configured yet,
# calling that function here would induce subtle errors or exceptions.
if file_dirname:
os.makedirs(file_dirname, exist_ok=True)
# Root logger file handler, preconfigured as documented above.
self._logger_root_handler_file = LogHandlerFileRotateSafe(
filename=self._filename,
# Append rather than overwrite this file.
mode='a',
# Defer opening this file in a just-in-time manner (i.e., until the
# first call to this handler's emit() method is called to write the
# first log via this handler). Why? Because (in no particular
# order):
#
# * If the end user requests that *NO* messages be logged to disk
# (e.g., by passing the "--log-level=none" option), this file
# should *NEVER* be opened and hence created. The simplest means
# of doing so is simply to indefinitely defer opening this file.
# * Doing so slightly reduces the likelihood (but *NOT* eliminate
# the possibility) of race conditions between multiple BETSE
# processes attempting to concurrently rotate the same logfile.
delay=True,
# Encode this file's contents as UTF-8.
encoding='utf-8',
# Maximum filesize in bytes at which to rotate this file,
# equivalent to 1MB.
maxBytes=ints.MiB,
# Maximum number of rotated logfiles to maintain.
backupCount=8,
)
# Initialize this handler's level to the previously established level.
self._logger_root_handler_file.setLevel(file_level)
# Prevent third-party debug messages from being logged to disk.
self._logger_root_handler_file.addFilter(LogFilterThirdPartyDebug())
# Linux-style logfile format.
#
# Note that the "processName" attribute appears to *ALWAYS* expand to
# "MainProcess", which is not terribly descriptive. Hence, the name of
# the current process is manually embedded in this format.
file_format = ('[{{asctime}}] '
'{} {{levelname}} '
'({{module}}.py:{{funcName}}():{{lineno}}) '
'<PID {{process}}>:\n'
' {{message}}'.format(cmds.get_current_basename()))
# Format this file according to this format.
file_formatter = LogFormatterWrap(fmt=file_format, style='{')
self._logger_root_handler_file.setFormatter(file_formatter)
# Register this handler with the root logger.
self._logger_root.addHandler(self._logger_root_handler_file)
# ..................{ DEINITIALIZERS }..................
def deinit(self) -> None:
'''
Deinitialize this logging configuration.
See Also
----------
:func:`_deinit_logger_root_handlers`
:func:`_deinit_vars`
Further details.
'''
# Deinitialize all root logger handlers.
self._deinit_logger_root_handlers()
# Deinitialize all instance variables.
self._deinit_vars()
def _deinit_logger_root_handlers(self) -> None:
'''
Deinitialize all root logger handlers.
Specifically, this method iterates over all handlers previously added
to the root logger and, for each such handler (in order):
#. Closes all open file handles associated with that handler, including
the logfile handle opened by the :meth:`__init__` method.
#. Removes that handler from the root logger.
'''
# For each handler previously added to the root logger...
#
# For safety, a shallow copy of the list of handlers to be removed
# rather than the actual list being modified here is iterated over.
for root_handler in tuple(self._logger_root.handlers):
# Close all open file handles associated with this handler.
root_handler.close()
# Remove this handler from the root logger.
self._logger_root.removeHandler(root_handler)
def _deinit_vars(self) -> None:
'''
Deinitialize all instance variables underlying this logging
configuration to sane defaults.
To circumvent chicken-and-egg issues, this method intentionally avoids
deinitializing (i.e., setting) settable properties.
'''
# Avoid circular import dependencies.
from betse.util.app.meta import appmetaone
# Revert all non-property attributes to sane defaults.
self._filename = appmetaone.get_app_meta().log_default_filename
self._logger_root = None
self._logger_root_handler_file = None
self._logger_root_handler_stderr = None
self._logger_root_handler_stdout = None
# ..................{ PROPERTIES ~ logger }..................
# Read-only properties prohibiting write access to external callers.
@property
def logger_root(self) -> RootLogger:
'''
**Root logger** (i.e., transitive parent of all other loggers).
'''
return self._logger_root
# ..................{ PROPERTIES ~ handler }..................
@property
def handler_file(self) -> Handler:
'''
Root logger handler appending to the current logfile if file logging is
enabled *or* ``None`` otherwise.
'''
return self._logger_root_handler_file
@property
def handler_stderr(self) -> Handler:
'''
Root logger handler printing to standard error.
'''
return self._logger_root_handler_stderr
@property
def handler_stdout(self) -> Handler:
'''
Root logger handler printing to standard output.
'''
return self._logger_root_handler_stdout
# ..................{ PROPERTIES ~ level }..................
@property
def file_level(self) -> LogLevel:
'''
Minimum level of messages to log to the file handler.
'''
return self._logger_root_handler_file.level
@file_level.setter
@type_check
def file_level(self, file_level: LogLevel) -> None:
'''
Set the minimum level of messages to log to the file handler.
'''
self._logger_root_handler_file.setLevel(file_level)
# ..................{ PROPERTIES ~ level : verbose }..................
@property
def is_verbose(self) -> bool:
'''
``True`` only if *all* messages are to be unconditionally logged to the
stdout handler (and hence printed to stdout).
Equivalently, this method returns ``True`` only if the logging level
for the stdout handler is :attr:`LogLevel.ALL`.
Note that this logging level is publicly retrievable by accessing the
:attr:`handler_stdout.level` property.
'''
return self._logger_root_handler_stdout.level == LogLevel.ALL
@is_verbose.setter
@type_check
def is_verbose(self, is_verbose: bool) -> None:
'''
Set the verbosity of the stdout handler.
This method sets this handler's logging level to:
* If the passed boolean is ``True``, :attr:`LogLevel.ALL` .
* If the passed boolean is ``False``, :attr:`LogLevel.INFO`.
'''
# Avoid circular import dependencies.
from betse.util.io.log import logs
# Convert the passed boolean to a logging level for the stdout handler.
self._logger_root_handler_stdout.setLevel(
LogLevel.ALL if is_verbose else LogLevel.INFO)
# If increasing stdout verbosity, log this fact *AFTER* doing so.
#
# Note that reversing this order of statements would silently squelch
# this message -- which would quite defeat the purpose.
if is_verbose:
logs.log_debug('Standard output verbosity enabled.')
# ..................{ PROPERTIES ~ path }..................
@property
def filename(self) -> str:
'''
Absolute or relative path of the file logged to by the file handler.
'''
return self._filename
@filename.setter
@type_check
def filename(self, filename: str) -> None:
'''
Set the absolute or relative path of the file logged to by the file
handler.
Due to flaws in the upstream :mod:`logging` API, this method
necessarily destroys and recreates the current file handler.
'''
# If the passed filename is the same as the current filename, avoid
# unnecessarily destroying and recreating the file handler. This is
# technically a negligible optimization, but every little bit helps.
if self._filename == filename:
return
# Classify this filename *BEFORE* recreating the file handler, which
# accesses this variable.
self._filename = filename
# Destroy and recreate the file handler.
self._init_logger_root_handler_file()
def setup_syslog(verbosity, quiet = False, logdest = "syslog") :
'''
TBD
'''
try :
_fmsg = ""
_status = 100
_my_uuid, _oscp, _mscp, _lscp = get_stores_parms()
# HACK ALERT - A very crude "syslog facility selector"
_syslog_selector = {}
_syslog_selector["16"] = SysLogHandler.LOG_LOCAL0
_syslog_selector["17"] = SysLogHandler.LOG_LOCAL1
_syslog_selector["18"] = SysLogHandler.LOG_LOCAL2
_syslog_selector["19"] = SysLogHandler.LOG_LOCAL3
_syslog_selector["20"] = SysLogHandler.LOG_LOCAL4
_syslog_selector["21"] = SysLogHandler.LOG_LOCAL5
_syslog_selector["22"] = SysLogHandler.LOG_LOCAL6
_syslog_selector["23"] = SysLogHandler.LOG_LOCAL7
_verbosity = int(verbosity)
logger = getLogger()
# Reset the logging handlers
while len(logger.handlers) != 0 :
logger.removeHandler(logger.handlers[0])
if logdest == "console" or (not _lscp["hostname"] or not _lscp["port"]) :
hdlr = StreamHandler(stdout)
else :
_facility = int(21)
if _facility > 23 or _facility < 16 :
_facility = 23
hdlr = SysLogHandler(address = (_lscp["hostname"], \
int(_lscp["port"])), \
facility=_syslog_selector[str(_facility)])
formatter = Formatter("[%(asctime)s] [%(levelname)s] %(message)s")
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
if _verbosity :
if int(_verbosity) >= 6 :
logger.setLevel(DEBUG)
elif int(_verbosity) >= 5 :
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 4 :
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 3 :
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
hdlr.addFilter(MsgFilter("Exit point"))
hdlr.addFilter(MsgFilter("Entry point"))
logger.setLevel(DEBUG)
elif int(_verbosity) >= 2 :
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
logger.setLevel(DEBUG)
elif int(_verbosity) == 1 :
# Filter out gmetad logging statements
hdlr.addFilter(VerbosityFilter("gmetad"))
# Used to filter out all function calls from the "auxiliary"
# subdirectory.
hdlr.addFilter(VerbosityFilter("auxiliary"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("stores"))
hdlr.addFilter(VerbosityFilter("datastore"))
# Used to filter out all function calls from all modules in the
# "collectors" subdirectory.
hdlr.addFilter(VerbosityFilter("collectors"))
# Used to filter out all function calls from the "remote"
# subdirectory.
hdlr.addFilter(VerbosityFilter("remote"))
# Used to filter out all function calls from all modules in the
# "stores" subdirectory.
hdlr.addFilter(VerbosityFilter("clouds"))
logger.setLevel(DEBUG)
else :
logger.setLevel(INFO)
if quiet :
logger.setLevel(ERROR)
_status = 0
except Exception, e :
_status = 23
_fmsg = str(e)
record.url = ''
record.remote_addr = ''
record.username = ''
record.method = ''
record.data = ''
return super().format(record)
formatter = PCFFormatter(
'[%(levelname)s] ' +
'%(remote_addr)s%(username)s%(method)s%(url)s%(data)s' +
'[%(pathname)s:%(lineno)d] %(message)s')
sysout_handler = StreamHandler(stdout)
sysout_handler.setFormatter(formatter)
sysout_handler.addFilter(lambda record: record.levelno <= INFO)
sysout_handler.setLevel(DEBUG)
syserr_handler = StreamHandler()
syserr_handler.setFormatter(formatter)
syserr_handler.setLevel(WARNING)
pcf_logger = getLogger(__name__)
pcf_logger.addHandler(sysout_handler)
pcf_logger.addHandler(syserr_handler)
pcf_logger.setLevel(DEBUG)
gunicorn_logger = getLogger('gunicorn.error')
pcf_logger.setLevel(max(gunicorn_logger.level, DEBUG))
gunicorn_logger.handlers = pcf_logger.handlers
if not found:
return 0
for f in self.disabled_funcs:
if record.funcName.endswith(f):
return 0
return 1
enabled_mods = ["__main__", "console", "insulatedshell", "printhooks", "matplotlib_figure_Figure"]
disabled_funcs = ["router._send", "router._recv", "__init__"]
filt = Filter(enabled_modules=enabled_mods, disabled_funcs=disabled_funcs)
formatter = logging.Formatter(
fmt="%(name)s (PID %(process)d ): %(levelname)s %(filename)s:%(lineno)d:%(funcName)s: %(message)s"
)
handler = StreamHandler(stream=sys.__stderr__)
handler.addFilter(filt)
handler.setFormatter(formatter)
handler.setLevel(logging.DEBUG)
root_logger = logging.getLogger()
root_logger.addHandler(handler)
root_logger.setLevel(logging.DEBUG)
# logging.basicConfig( stream = sys.__stderr__, format = "%(name)s (PID %(process)d ): %(levelname)s %(module)s.%(funcName)s: %(message)s" )
logger = logging.getLogger(__name__)
def msg(*args):
ret = ""
for a in args:
try:
Filter.__init__(self, patterns)
self.patterns = patterns
def filter(self, record):
if not self.patterns:
return True
for p in self.patterns:
if match(record.name, p):
return True
return False
root = getLogger()
handler = StreamHandler(sys.stdout)
filter = PatternFilter(*config.log_categories)
handler.addFilter(filter)
handler.setFormatter(Formatter("%(asctime)s %(levelname)s %(message)s"))
root.addHandler(handler)
root.setLevel(WARN)
log = getLogger("proton.test")
PASS = "******"
SKIP = "skip"
FAIL = "fail"
class Runner:
def __init__(self):
self.exception = None
self.exception_phase_name = None
def get_script_logger(debug=True, quiet=False, config=None):
"""Creates a script logger to log to files / streams."""
log_file_format = (config
and config['LOG_FILE_FORMAT']
or '%(levelname)s %(asctime)s %(message)s')
log_file_datefmt = (config
and config['LOG_FILE_DATEFMT']
or '[%Y-%m-%d %H:%M:%S]')
script_log_info_file_handler = None
if config:
script_log_info_file_handler = RotatingFileHandler(
path.join(config['LOG_FOLDER'], 'script_info.log'),
maxBytes=100000, backupCount=5)
script_log_info_file_handler.setLevel(logging.INFO)
script_log_info_file_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_file_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_file_handler = None
if config:
script_log_error_file_handler = RotatingFileHandler(
path.join(config['LOG_FOLDER'], 'script_error.log'),
maxBytes=100000, backupCount=5)
script_log_error_file_handler.setLevel(logging.ERROR)
script_log_error_file_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stdout)
script_log_info_stream_handler.setLevel(logging.INFO)
script_log_info_stream_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_log_info_stream_handler.addFilter(
LevelSpecificLogFilter(logging.WARNING))
script_log_error_stream_handler = StreamHandler(
quiet and open(os.devnull, 'a') or sys.stderr)
script_log_error_stream_handler.setLevel(logging.ERROR)
script_log_error_stream_handler.setFormatter(Formatter(
log_file_format, log_file_datefmt))
script_logger = logging.getLogger('script')
script_logger.setLevel(logging.INFO)
if config:
script_logger.addHandler(script_log_info_file_handler)
script_logger.addHandler(script_log_error_file_handler)
script_logger.addHandler(script_log_info_stream_handler)
script_logger.addHandler(script_log_error_stream_handler)
if (not debug) and config:
mail_handler = get_mail_handler(config)
script_logger.addHandler(mail_handler)
return script_logger
