def filter(self, record: logging.LogRecord) -> bool:
if isinstance(record.args, collections.abc.Mapping):
record.args = {
self.redact(k): self.redact(v)
for k, v in record.args.items() # type: ignore
}
else:
if record.args is not None:
record.args = tuple(self.redact(a) for a in record.args)
return True
def _color_record_args(self, record: LogRecord) -> LogRecord:
if isinstance(record.args, (tuple, list)):
record.args = tuple(self._color_arg(arg) for arg in record.args)
elif isinstance(record.args, dict):
# Case of logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2})
record.args = {
key: self._color_arg(value) for key, value in record.args.items()
}
elif isinstance(record.args, str):
record.args = self._color_arg(record.args)
return record
def _format_args(self, record: logging.LogRecord) -> None:
if isinstance(record.args, Mapping):
# logger.log(severity, "msg %(foo)s", foo=303)
record.args = {
k: self._format_arg(v)
for k, v in record.args.items()
}
else:
if not isinstance(record.args, tuple):
# logger.log(severity, "msg %s", foo)
record.args = (record.args, )
# logger.log(severity, "msg %s", ('foo',))
record.args = tuple(self._format_arg(arg) for arg in record.args)
def _color_record_args(self, record: LogRecord) -> LogRecord:
if isinstance(record.args, (tuple, list)):
record.args = tuple(self._color_arg(arg) for arg in record.args)
elif isinstance(record.args, dict):
if self._count_number_of_arguments_in_message(record) > 1:
# Case of logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2})
record.args = {
key: self._color_arg(value) for key, value in record.args.items()
}
else:
# Case of single dict passed to formatted string
record.args = self._color_arg(record.args) # type: ignore
elif isinstance(record.args, str):
record.args = self._color_arg(record.args)
return record
def _format_args(self, record: logging.LogRecord) -> None:
if isinstance(record.args, Mapping):
# logger.log(severity, "msg %(foo)s", foo=303)
record.args = {
k: self._format_arg(v)
for k, v in record.args.items()
}
else:
if not isinstance(record.args, tuple):
# logger.log(severity, "msg %s", foo)
# mypy thinks this is unreachable as record is
# always Tuple
record.args = (record.args, ) # type: ignore
# logger.log(severity, "msg %s", ('foo',))
record.args = tuple(self._format_arg(arg) for arg in record.args)
def filter(self, record: LogRecord) -> bool:
if record.args == ():
record.args = {}
elif isinstance(record.args, dict):
record.args.update({"sensor": self.sensorID})
return True
def format(self, record: logging.LogRecord) -> str:
"""Custom formating for the log message.
If the record contains an exception, then add that to the 'message'. Heroku
splits the message according to newline and thus the color format will
disappear, so add the color format after every newline as well.
"""
custom_fmt = self.LOG_LEVEL_FORMAT.get(record.levelname,
self.default_fmt)
# Inject custom formating to the base class which `self.formatMessage` calls.
self._style._fmt = custom_fmt
if record.args and isinstance(record.args, MutableMapping):
record.args = format_args(record.args, record.levelname)
msg = record.getMessage()
if record.exc_info:
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
if msg[-1:] != "\n":
msg += "\n"
msg += record.exc_text
c = getattr(AnsiColor, record.levelname)
msg = msg.replace("\n", f"\n{c}")
if record.stack_info:
if msg[-1:] != "\n":
msg += "\n"
msg += self.formatStack(record.stack_info)
record.message = msg
return self.formatMessage(record)
def filter(self, record: logging.LogRecord) -> bool:
"""Pass the record's message to the monitor, add colors to the message."""
message = record.getMessage()
self._monitor.add_event_sync(message)
if self._color:
record.msg = colors.color(message, fg=self._color)
record.args = ()
return True
def format(self, record: logging.LogRecord) -> str:
"""
Extract ``structlog``'s `event_dict` from ``record.msg`` and format it.
*record* has been patched by `wrap_for_formatter` first though, so the
type isn't quite right.
"""
# Make a shallow copy of the record to let other handlers/formatters
# process the original one
record = logging.makeLogRecord(record.__dict__)
logger = getattr(record, "_logger", _SENTINEL)
meth_name = getattr(record, "_name", _SENTINEL)
if logger is not _SENTINEL and meth_name is not _SENTINEL:
# Both attached by wrap_for_formatter
if self.logger is not None:
logger = self.logger
meth_name = record._name # type: ignore
# We need to copy because it's possible that the same record gets
# processed by multiple logging formatters. LogRecord.getMessage
# would transform our dict into a str.
ed = record.msg.copy() # type: ignore
else:
logger = self.logger
meth_name = record.levelname.lower()
ed = {"event": record.getMessage(), "_record": record}
if self.pass_foreign_args:
ed["positional_args"] = record.args
record.args = ()
# Add stack-related attributes to event_dict and unset them
# on the record copy so that the base implementation wouldn't
# append stacktraces to the output.
if record.exc_info:
ed["exc_info"] = record.exc_info
if record.stack_info:
ed["stack_info"] = record.stack_info
if not self.keep_exc_info:
record.exc_text = None
record.exc_info = None
if not self.keep_stack_info:
record.stack_info = None
# Non-structlog allows to run through a chain to prepare it for the
# final processor (e.g. adding timestamps and log levels).
for proc in self.foreign_pre_chain or ():
ed = proc(logger, meth_name, ed)
del ed["_record"]
record.msg = self.processor(logger, meth_name, ed) # type: ignore
return super().format(record)
def filter(self, record: LogRecord) -> bool:
if isinstance(record.args, tuple):
record.args = {}
clientInfo = self.server.tcp.transport.client
record.args.update({
"src_ip": clientInfo[0],
"src_port": clientInfo[1],
"session": self.sessionId
})
return True
def _format_record(self, record: logging.LogRecord=None) -> logging.LogRecord:
"""
Formatted the LogRecord
:param record: LogRecord with the message info
:return: LogRecord with formatted message
"""
# ensure that exc_info and args
# have been stringified. Removes any chance of
# unpickleable things inside and possibly reduces
# message size sent over the pipe.
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
self.format(record)
record.exc_info = None
return record
def _format_record(self,
record: logging.LogRecord = None) -> logging.LogRecord:
"""
Formatted the LogRecord
:param record: LogRecord with the message info
:return: LogRecord with formatted message
"""
# ensure that exc_info and args
# have been stringified. Removes any chance of
# unpickleable things inside and possibly reduces
# message size sent over the pipe.
if record.args:
record.msg = record.msg % record.args
record.args = None
if record.exc_info:
self.format(record)
record.exc_info = None
return record
def filter(self, record: logging.LogRecord) -> int:
if hasattr(record, 'msg'):
for secret in self._secrets:
if secret:
record.msg = record.msg.replace(str(secret), "***")
if isinstance(record.args, dict):
for key in record.args:
for secret in self._secrets:
if secret:
record.args[key] = record.args[key].replace(
str(secret), "***")
else:
nargs = []
for i in range(len(record.args)):
for secret in self._secrets:
if secret:
nargs.append(record.args[i].replace(
str(secret), "***"))
record.args = tuple(nargs)
return True
def _convert_record(self, record: logging.LogRecord) -> logging.LogRecord:
# we store the originating DagsterEvent in the DAGSTER_META_KEY field, if applicable
dagster_meta = getattr(record, DAGSTER_META_KEY, None)
# generate some properties for this specific record
dagster_message_props = DagsterMessageProps(
orig_message=record.getMessage(), dagster_event=dagster_meta
)
# set the dagster meta info for the record
setattr(
record,
DAGSTER_META_KEY,
get_dagster_meta_dict(self._logging_metadata, dagster_message_props),
)
# update the message to be formatted like other dagster logs
record.msg = construct_log_string(self._logging_metadata, dagster_message_props)
record.args = ()
return record
def rewrite_record(record: logging.LogRecord):
if not BraceFormatStyleFormatter.is_brace_format_style(record):
return
msg = record.msg
msg = msg.replace("{", "_{{")
msg = msg.replace("}", "_}}")
placeholder_count = 0
# add ANSI escape code for next alternating color before each formatting parameter
# and reset color after it.
while True:
if "_{{" not in msg:
break
color_index = placeholder_count % len(ColorizedArgsFormatter.arg_colors)
color = ColorizedArgsFormatter.arg_colors[color_index]
msg = msg.replace("_{{", color + "{", 1)
msg = msg.replace("_}}", "}" + ColorCodes.reset, 1)
placeholder_count += 1
record.msg = msg.format(*record.args)
record.args = []
def _format_line(record: logging.LogRecord, msg: str) -> str:
record = copy(record)
record.msg = msg
record.args = ()
return _super_format(record)
def format(self, record: logging.LogRecord) -> str:
""" Format message Object (via attributes of LogRecord) of unknown type into printable and easily readable string
The *Parameters* are fed in through `record`
Parameters
----------
msg
actual output, could be object, string
heading
title is gerated from this
Notes
-----
`record.getMessage()` will ALWAYS return a `str()`
for the actual object the user provided use `record.msg`
See Also
--------
https://docs.python.org/3/library/logging.html#logging.LogRecord
https://docs.python.org/3/library/logging.html#logrecord-attributes
"""
if record.args is None: record.args = []
level_print = str()
prepend = str()
prependtime = str()
output = record.msg if hasattr(record, "msg") else str()
location = str()
highlight_color = str()
heading = record.args[
'heading'] if 'heading' in record.args and record.args[
'heading'] is not False else False
title = heading if heading is not False and type(
heading) is str else str()
# if there are newlines in the output, make it a block.
try:
if repr(output).count("\n") > 1:
heading = True
except Exception as e:
logging_error(
f"Failure in {self.__class__.__name__} when requesting repr of {output.__class__.__name__} with exception:\n\t{e}"
)
# typically closing is done in the same log message as heading
# but this is disabled when in context
closing = heading
context_marker = str()
context = record.args[
'context'] if 'context' in record.args and record.args[
'context'] in (LogContextStatus.OPENING,
LogContextStatus.CURRENT,
LogContextStatus.CLOSING) else False
clear = self.clear
eol = self.eol
emphasis = record.args[
"emphasis"] if 'emphasis' in record.args else False
relatime = record.args[
"relatime"] if 'relatime' in record.args else False
stack_trace = record.args[
'stack_trace'] if "stack_trace" in record.args and type(
record.args['stack_trace']) is list else False
flag_location = record.args[
"location"] if 'location' in record.args else False
table = record.args["table"] if 'table' in record.args else False
# flag_location = True # TODO: KILL
# try:
if isinstance(output, str):
pass # not a table; this is a string, no further processing
elif not isinstance(output, type) and hasattr(
output, "to_string") and callable(output.to_string):
# mostly useful for pandas.DataFrame
from lib.shared.util import FormatDataFrame
with FormatDataFrame(max_colwidth=0):
output = output.to_string()
# elif table or isinstance(output, (Mapping, list)):
elif table:
# if table and title, set title to heading
iterate_source = output
output = str()
title = title if title else str(type(iterate_source))
if isinstance(iterate_source, Mapping):
iterate_source = iterate_source.items()
elif isinstance(iterate_source, list):
iterate_source = {k: v
for k, v in enumerate(iterate_source)
}.items()
try:
iterate_source = iter(iterate_source)
# determine the width of what is added as a table row.
# Don't exceed console width
max_len = 0
for d in iterate_source:
_row = self.make_row(*d)
max_len = len(_row) if len(_row) > max_len else max_len
# output += f" Len({len(self.make_row(*d))}) "
output += _row
# print(f"title={title}\nheading={heading}\nmaxlen({max_len})+lentitle({len(title)})+lenheading({len(heading)})")
# if ( max_len + len(title) + len(record.args['title']) ) > self.console_width:
# output = "\n" + output
except TypeError:
output += repr(output)
raise TypeError(
"Best handled elsewhere: you requested a table, but this isn't iterable"
)
else:
# when the object is not a string
try:
output = pprint.pformat(output,
width=self.output_width,
indent=4)
except Exception as e:
raise LogFormatException(
f"This is neither a table, nor a string.\nError encountered while using fallback pformat method:\n{e}"
)
if record.levelno == logging.WARNING: # add levelname
level_print = f"{record.levelname} "
if record.levelno == logging.CRITICAL: # add levelname
level_print = f"{record.levelname} "
highlight_color = self.COLOR_HIGHLIGHT_CRITICAL if self.color else ""
if logging.getLogger(__name__).getEffectiveLevel() <= Level.DEBUG:
# NOTE: THIS IS VERY LENGTHY LOGGING
print(f"args={record.args}")
print(f"exc_info={record.exc_info}")
print(f"filename={record.filename}")
print(f"funcName={record.funcName}")
print(f"lineno={record.lineno}")
print(f"pathname={record.pathname}")
print(f"stack_info={record.stack_info}")
print(f"flag_location={flag_location}")
print(f"---- exc_info[3] {type(record.exc_info[2])}--->")
print(traceback.extract_tb(record.exc_info[2]))
# https://docs.python.org/3/library/traceback.html#module-traceback
if record.levelno == logging.CRITICAL and type(
record.exc_info) is tuple and type(
record.exc_info[2]) is not None:
trace = traceback.extract_tb(record.exc_info[2])
last = trace[len(trace) - 1]
if logging.getLogger(__name__).getEffectiveLevel() <= Level.DEBUG:
# NOTE: THIS IS VERY LENGTHY LOGGING
print(f"lineno={last.lineno}")
print(f"filename={last.filename}")
print(f"name={last.name}")
record.lineno = last.lineno
record.pathname = last.filename
record.funcName = last.name
title = output
relatime = True
output = f"{record.exc_info[1]} {repr(record.exc_info[0])}"
stack_trace = trace
if stack_trace is not False:
output += f"\n{clear}" + self.formatStack(stack_trace).replace(
"\n", f"\n{clear}{' '*len(prependtime)}").rstrip(
f"\n{clear}") + f"{eol}{clear}"
if relatime is True: # add relative timestamp
s = record.relativeCreated // 1000
m = f"{int(s // 60 % 60):0>2}m" if s >= 60 else ""
h = str(int(s // 3600)) + "h" if s >= 3600 else ""
s = str(int(s % 60)) + "s"
prependtime = f"[{h:>4} {m:>3} {s:0>3}] "
""" handle context """
if context:
if context == LogContextStatus.OPENING:
context_marker = self.COLOR_CONTEXT_MARKER + f" START {title} ".center(
self.context_marker_width, ">") + f"{clear}"
closing = heading = False
elif context == LogContextStatus.CURRENT:
# NOTE(erichiller) UNSURE OF WHAT TO DO WITH PREPEND RIGHT NOW
# prepend = "\t"
# prepend = "↳"
closing = heading = False
elif context == LogContextStatus.CLOSING:
closing = heading = False
context_marker = self.COLOR_CONTEXT_MARKER + f" END {title} ".center(
self.context_marker_width, "<") + f"{clear}"
if record.msg is None:
return context_marker
if 'title' in record.args and record.args['title'] not in (
None, False): # add user provided title
title = f" {record.args['title']} {title}"
if closing:
output = (
f"{output}" + "\n" + # add an END OF BLOCK marker
f" END {title} ".center(self.column_name_width, "<"))
if heading is not False or table is not False: # this has been marked as a heading so give it some flourish
if len(title) in (0, 1, 2) and table is True:
title = " Table "
title = (title if len(title) > 0 and title[0] == " " else
(f" {title} " if len(title.strip()) > 0 else "")).center(
self.column_name_width - len(prepend), '>') + "\n"
# determine color level, Python has no switch statement
if record.levelno >= Level.CRITICAL: # set color
color = self.COLOR_CRITICAL
flag_location = True
elif record.levelno >= Level.WARNING: # set color
color = self.COLOR_WARNING
flag_location = True
elif record.levelno >= Level.NOTICE:
color = self.COLOR_NOTICE
elif record.levelno == Level.DEBUG:
color = self.COLOR_DEBUG
elif record.levelno == Level.TRACE:
color = self.COLOR_TRACE
else:
color = self.COLOR_DEFAULT
if level_print != "":
title = (level_print + title).ljust( # add level
self.column_name_width - len(prepend) - len(prependtime))
if title != "" and title[-1] != "\n":
title = title + ": "
if len(title) > 0 and title[0].isspace(
): # strip any whitespace on left
title = title.lstrip()
if ":" in title: # run a check just to see if it should be added back in
title = "".join(title.rsplit(
":", 1)).ljust(self.column_name_width - len(prepend) -
len(prependtime))
title = title + ": "
if flag_location:
location_color = self.COLOR_LOCATION if self.color else ""
location = location_color + self.caller_location_string(
record) + f"{eol}\n{clear}"
if self.color is False: eol = color = clear = ""
highlight_color = color if highlight_color == "" else highlight_color
# used below llne purely for debugging what was my logs and what wasn't
# else: color = f"{self.ANSI_BG_MAGENTA}{color}"
return ((
context_marker +
((f"{self.COLOR_EMPHASIS if self.color else ''}" +
('▼' * self.console_width) + clear) if emphasis else '') +
f"{color}{prependtime}{location}{highlight_color}{prepend}{title}{clear}{color}{output}{eol}{clear}"
).replace(f"\n{clear}", f"{eol}{clear}\n").replace(
"\n", f"\n{' ' * len(prependtime)}") +
(('\n' + f"{self.COLOR_EMPHASIS if self.color else ''}" +
('▲' * self.console_width) + clear) if emphasis else ''))
def filter(self, record: logging.LogRecord) -> bool:
"""Filter the record"""
# have to sniff the msg and args values of the LogRecord
record.msg = self.replace(record.getMessage())
record.args = {}
return True
def rewrite_record(record: logging.LogRecord):
if not BraceFormatStyleFormatter.is_brace_format_style(record):
return
record.msg = record.msg.format(*record.args)
record.args = []
def format(self, record: LogRecord) -> str:
record.name = self._shorten_module_name(record.name)
record.args = self._redact_logging_args(record.args)
return super().format(record)
