def format(self, rec):
if (self.colored):
c = list(filter(lambda r: rec.levelno in r, list(COLORS)))
color = COLORS[c[0]]
return color + _Formatter.format(self, rec) + RESET
else:
return _Formatter.format(self, rec)
def assert_logger_warnings_and_errors(caplog, expected_warnings=0, expected_errors=0):
if caplog is None:
return
import logging
counted_warnings = 0
counted_errors = 0
records = caplog.get_records("call")
for record in records:
if record.levelno == logging.WARNING:
counted_warnings += 1
elif record.levelno == logging.ERROR:
counted_errors += 1
# the exception info dict can hold references core and model objects
# -> so the caplog does not allow gc to collect them
if hasattr(record, 'exc_info'):
record.exc_info = None
formatter = Formatter("%(name)s: %(message)s")
if counted_warnings != expected_warnings:
warnings = [formatter.format(record) for record in records if record.levelno == logging.WARNING]
pytest.fail("{} == counted_warnings != expected_warnings == {}\n\n"
"Occured warnings:\n{}".format(counted_warnings, expected_warnings, "\n".join(warnings)))
if counted_errors != expected_errors:
errors = [formatter.format(record) for record in records if record.levelno == logging.ERROR]
pytest.fail("{} == counted_errors == expected_errors == {}\n\n"
"Occured errors:\n{}".format(counted_errors, expected_errors, "\n".join(errors)))
def test_log_formatter_scrapy_2():
middleware = get_test_middleware()
logformatter = CrawleraFetchLogFormatter()
formatter = Formatter()
spider = Spider("foo")
for case in deepcopy(test_requests):
original = case["original"]
response = Response(original.url)
processed = middleware.process_request(original, spider)
crawlera_meta = original.meta.get("crawlera_fetch") or {}
if crawlera_meta.get("skip"):
assert processed is None
continue
# crawled
result = logformatter.crawled(processed, response, spider)
assert result["args"]["request"] == str(original)
record = LogRecord(name="logger",
pathname="n/a",
lineno=1,
exc_info=None,
**result)
logstr = formatter.format(record)
assert logstr == "Crawled (200) %s (referer: None)" % str(original)
# spider_error
result = logformatter.spider_error(Failure(Exception("exc")),
processed, response, spider)
assert result["args"]["request"] == str(original)
record = LogRecord(name="logger",
pathname="n/a",
lineno=1,
exc_info=None,
**result)
logstr = formatter.format(record)
assert logstr == "Spider error processing %s (referer: None)" % str(
original)
# download_error
result = logformatter.download_error(Failure(Exception("exc")),
processed, spider, "foo")
assert result["args"]["request"] == str(original)
record = LogRecord(name="logger",
pathname="n/a",
lineno=2,
exc_info=None,
**result)
logstr = formatter.format(record)
assert logstr == "Error downloading %s: foo" % str(original)
def format(self, record):
colored_record = copy.copy(record)
#levelname = colored_record.levelname
#color = MAPPING.get(levelname, 'white')
#colored_levelname = self.colorer(levelname, color)
#colored_record.levelname = 'MyLevel' #colored_levelname
#print colored_record.levelname
# Markup specialstrings
colored_record.msg = colored_record.msg.replace(
'[OK]', colorize('[OK]', 'light_green'))
colored_record.msg = colored_record.msg.replace(
'[FAIL]', colorize('[FAIL]', 'light_red'))
#Colorize according to error level
if colored_record.levelno == LL_WARNING:
fmessage = colorize(colored_record.msg, 'orange_red_1')
elif colored_record.levelno == LL_ERROR:
fmessage = colorize(colored_record.msg, 'light_red')
elif colored_record.levelno == LL_CRITICAL:
fmessage = colorize(colored_record.msg, 'white', 'red')
else:
fmessage = colored_record.msg
colored_record.msg = fmessage
# Markup tag
if not colored_record.tag == '':
colored_record.tag = tag(colored_record.tag) + ' '
return Formatter.format(self, colored_record)
def format (self, record):
# Add the color codes to the record
record.__dict__.update(escape_codes)
# If we recognise the level name,
# add the levels color as `log_color`
if record.levelname in self.log_colors:
color = self.log_colors[record.levelname]
record.log_color = escape_codes[color]
else:
record.log_color = ""
#NOTE: caller_name is very slow
#record.__dict__['caller_name'] = caller_name(9)
record.__dict__['abbr_levelname'] = record.levelname[0]
record.__dict__['pathname2'] = '.'.join(record.pathname.split('/')[-2:]).rstrip('.py')
# Format the message
if version_info > (2, 7):
message = super(ColoredFormatter, self).format(record)
else:
message = Formatter.format(self, record)
# Add a reset code to the end of the message (if it wasn't explicitly added in format str)
if self.reset and not message.endswith(escape_codes['reset']):
message += escape_codes['reset']
return message
def format(self, record):
if record.args and hasattr(record, 'output_limit'):
# Truncate all args to the set limit.
record.args = tuple([
truncate(arg, limit=record.output_limit) for arg in record.args
])
return Formatter.format(self, record)
class AccessLogHandler(Handler):
"""
追加记录日志
"""
ACCESS_LOG_FORMATTER = '%(ip)s %(levelname)s %(asctime)s %(pathname)s %(module)s [%(funcName)s:%(lineno)d]: %(' \
'message)s'
def __init__(self, filename, **kwargs):
super().__init__()
self.fmt = Formatter(self.ACCESS_LOG_FORMATTER)
self.filename = filename
def emit(self, record) -> None:
request = record.request
record.ip = self.get_request_ip_address(request)
log_content = self.fmt.format(record) + os.linesep
console.write(log_content)
with open(self.filename, 'a') as file:
file.write(log_content)
@staticmethod
def get_request_ip_address(request):
get_peer_fuc = getattr(request, "getpeername", None)
try:
if callable(get_peer_fuc):
return request.getpeername()[0]
else:
return None
except OSError:
return None
def test_log_formatter_scrapy_1():
middleware = get_test_middleware()
logformatter = CrawleraFetchLogFormatter()
formatter = Formatter()
for case in get_test_requests():
original = case["original"]
response = Response(original.url)
processed = middleware.process_request(original, foo_spider)
crawlera_meta = original.meta.get("crawlera_fetch") or {}
if crawlera_meta.get("skip"):
assert processed is None
continue
# crawled
result = logformatter.crawled(processed, response, foo_spider)
assert result["args"]["request"] == str(original)
record = LogRecord(name="logger",
pathname="n/a",
lineno=1,
exc_info=None,
**result)
logstr = formatter.format(record)
expected = "Crawled (200) {request} ['original url: {url}'] (referer: None)".format(
request=original, url=original.url)
assert logstr == expected
def format(self, record):
if not getattr(record, 'session', None):
record.session = ''
record.sessionLS = ' %s' % record.session
record.sessionRS = '%s ' % record.session
else:
record.sessionLS = ' %s' % record.session
record.sessionRS = '%s ' % record.session
if not getattr(record, 'classname', None):
record.classname = ''
if not getattr(record, 'method', None):
record.method = record.funcName
if not getattr(record, 'classAndmethod', None):
if record.method and record.classname:
record.classAndMethod = '%s::%s' % (record.classname, record.method)
else:
record.classAndMethod = ''
created = localtime(record.created)
record.date = strftime('%Y-%m-%d', created)
record.time = strftime('%H:%M:%S', created)
record.asctimeshort = '%s %s' % (record.date, record.time)
#return Formatter.format(self, record)
## splits the record and formats every line
msg = record.getMessage()
res = []
for line in msg.split("\n"):
record.msg, record.args = line, None
res.append(Formatter.format(self, record))
return "\n".join(res)
def format(self, record):
levelname = record.levelname
seq = MAPPING.get(levelname, 37) # default white
colored_levelname = ('{0}{1}m{2}{3}').format(PREFIX, seq, levelname,
SUFFIX)
record.levelname = colored_levelname
return Formatter.format(self, record)
def format(self, record):
formatted_string = Formatter.format(self, record)
(header, _) = formatted_string.split(record.message)
formatted_string = formatted_string.replace('\n',
'\n' + ' ' * len(header))
return formatted_string
def format(self, record):
record = deepcopy(record)
if self.usesColor:
color_end = '\033[00m'
try:
color_begin= {
logging.DEBUG : '',
logging.INFO : '\033[01;32m',
logging.WARNING : '\033[01;33m',
logging.ERROR : '\033[01;31m',
logging.CRITICAL : '\033[01;31m',
logging.FATAL : '\033[00;31m'
}[record.levelno]
except:
color_begin = ''
color_end = ''
record.levelname = color_begin+'*'+color_end
#record.levelno == logging.ERROR or
if record.levelno == logging.CRITICAL or record.levelno == logging.FATAL:
record.msg = color_begin+record.msg+color_end
if self.useIdents:
ident = len(inspect.stack())
if ident<self.minIdent:
self.minIdent = ident
record.msg=' '*(ident-self.minIdent)+record.msg
return Formatter.format(self, record)
def format(self, record):
levelname = record.levelname
if self.use_color and levelname in COLORS:
levelname_color = (COLOR_SEQ % (30 + COLORS[levelname]) +
levelname + RESET_SEQ)
record.levelname = levelname_color
return Formatter.format(self, record)
def format(self, record):
colored_record = copy(record)
levelname = colored_record.levelname
seq = MAPPING.get(levelname, 37) # default white
colored_levelname = (f'{PREFIX}{seq}m{levelname}{SUFFIX}')
colored_record.levelname = colored_levelname
return Formatter.format(self, colored_record)
def format(self, record):
colored_record = copy(record)
levelname = colored_record.levelname
seq = self.__mapping.get(levelname, Fore.WHITE)
colored_levelname = '%s[%s]%s' % (seq, levelname, Style.RESET_ALL)
colored_record.levelname = colored_levelname
return Formatter.format(self, colored_record)
def format(self, record):
try:
msg = super(ANSIFormatter, self).format(record)
except:
# for python2.6
# Formatter is old-style class in python2.6 and type is classobj
# another trick: http://stackoverflow.com/a/18392639/1276501
msg = Formatter.format(self, record)
lvl2color = {
"DEBUG": "blue",
"INFO": "green",
"WARNING": "yellow",
"ERROR": "red",
"CRITICAL": "bgred"
}
rln = record.levelname
if rln in lvl2color:
return "[{0}]: {1}".format(
utils.color_msg(lvl2color[rln], rln),
msg
)
else:
return msg
def format(self, record):
fmt_orig = self._style._fmt
ocelog_record = copy(record)
if _log_colored:
seq = _MAPPING.get(ocelog_record.levelname, '0') # default
ocelog_record.msg = ('{0}{1}m{2}{3}').format(_PREFIX, seq, ocelog_record.msg, _SUFFIX)
# ocelog_record.levelname = ('{0}{1}m{2}{3}').format(_PREFIX, seq, ocelog_record.levelname, _SUFFIX)
if _log_indented:
# print('stack ',len(inspect.stack()))
# print('_indent0 ', _indent0)
# if hasattr(ocelog, 'indent0'):
# print('ocelog.indent0', ocelog.indent0)
# print('logging.indent0_before = ' + str(logging.indent0))
indent = len(inspect.stack())
if indent < ocelog.indent0:
ocelog.indent0 = indent
# print('indent = ' + str(indent - logging.indent0))
# print('logging.indent0_after = ' + str(logging.indent0))
ind_space = ind_str * (indent - ocelog.indent0)
ocelog_record.msg = ind_space + ocelog_record.msg
if _log_debugging:
if ocelog_record.levelname != 'INFO':
self._style._fmt += ' \033[37m(%(filename)s:%(lineno)d)\033[0m'
result = Formatter.format(self, ocelog_record)
self._style._fmt = fmt_orig
return result
def format(self, record: LogRecord) -> str:
_s = self.info(record.name)
record.calledfilename = path.basename(_s.filename)
record.calledfunction = _s.function
record.calledlineno = _s.lineno
record.levelstyle = record.levelname.lower()
return DefaultFormatter.format(self, record)
def format(self, record):
if sys.version_info > (2, 7):
s = super(BaseColorFormatter, self).format(record)
else:
s = Formatter.format(self, record)
color, on_color, attribute = self._parseColor(record)
return colored(s, color, on_color, attrs=attribute)
def format(self, record):
levelname = record.levelname
if self.use_color and levelname in self.COLORS:
fore_color = 30 + self.COLORS[levelname]
levelname_color = self.COLOR_SEQ % fore_color + levelname + self.RESET_SEQ
record.levelname = levelname_color
return Formatter.format(self, record)
def format(self, record):
plain_result = Formatter.format(self, record)
for severity_threshold, severity_color in self.SEVERITY_COLOR_THRESHOLDS:
if record.levelno >= severity_threshold:
color = severity_color
else:
break
return colorize(plain_result, color=color)
def format(self, record):
colored_record = copy(record)
levelname = colored_record.levelname
seq = MAPPING.get(levelname, 37) # 0xdf11f630 white
colored_levelname = ('{0}{1}m{2}{3}') \
.format(PREFIX, seq, levelname, SUFFIX)
colored_record.levelname = colored_levelname
return Formatter.format(self, colored_record)
def format(self, record):
colored_record = record # copy(record)
levelName = colored_record.levelname
seq = COLORED_LOG_MAPPING.get(levelName, 37) # default white
colored_levelname = '{0}{1}m{2}{3}'.format(COLOR_PREFIX, seq,
levelName[:4], COLOR_SUFFIX)
colored_record.levelname = colored_levelname
return Formatter.format(self, colored_record)
def format(self, record):
colored_record = copy(record)
levelname = colored_record.levelname
seq = ColoredFormatter.MAPPING.get(levelname, 37) # default white
colored_levelname = ('{0}{1}m{2}{3}') \
.format(ColoredFormatter.PREFIX, seq, levelname, ColoredFormatter.SUFFIX)
colored_record.levelname = colored_levelname
return Formatter.format(self, colored_record)
class QTextHandler(QtGui.QTextEdit):
def __init__(self, parent=None):
super(QTextHandler, self).__init__(parent)
def setFormatter(self, fmt=None):
self.formatter = Formatter(fmt)
def handle(self, record):
self.append(self.formatter.format(record))
def init_logger_experiment() -> None:
"""
Initialize logger for `nni.experiment.Experiment`.
This function will get invoked after `init_logger()`.
"""
colorful_formatter = Formatter(log_format, time_format)
colorful_formatter.format = _colorful_format
handlers['_default_'].setFormatter(colorful_formatter)
class CustomSMTPHandler(SMTPHandler):
def __init__(self, mailhost, fromaddr, toaddrs, subject):
super(CustomSMTPHandler, self).__init__(
mailhost, fromaddr, toaddrs, subject
)
self.subject_formatter = Formatter(subject)
def getSubject(self, record): # noqa: N802
return self.subject_formatter.format(record).replace('\n', '')
def format(self, record):
# type: (LogRecord, ) -> str
for k, v in self.map.items():
try:
setattr(record, v, getattr(record, k))
except AttributeError:
pass
return Formatter.format(self, record)
def format(self, record):
levelname = record.levelname
msg = Formatter.format(self, record)
if levelname in COLORS:
if stdout.isatty():
msg = COLORS[levelname] % msg
else:
print ("*" * 100, levelname, "(%s)" % type(levelname), "not in",
COLORS.keys())
return msg
def format(self, record):
"""
This is mostly the same as logging.Formatter.format except for adding spaces in front
of the multiline messages.
"""
s = Formatter.format(self, record)
if '\n' in s:
return '\n '.join(s.split('\n'))
else:
return s
def format(self, record):
rv = []
length = 0
for piece in _ws_re.split(Formatter.format(self, record)):
length += len(piece)
if length > 140:
if length - len(piece) < 140:
rv.append(u'…')
break
rv.append(piece)
return u''.join(rv)
def format(self, record):
try:
msg = super(NonANSIFormatter, self).format(record)
except:
# for python2.6
# Formatter is old-style class in python2.6 and type is classobj
# another trick: http://stackoverflow.com/a/18392639/1276501
msg = Formatter.format(self, record)
rln = record.levelname
return "[{0}]: {1}".format(rln, msg)
def format(self, record: LogRecord):
style = self._styles[record.levelname.lower()]
copy = ColoredFormatter.Empty()
copy.__class__ = record.__class__
copy.__dict__.update(record.__dict__)
msg = record.msg if isinstance(record.msg, str) else str(record.msg)
copy.msg = style + msg + colorama.Style.RESET_ALL
record = copy
# Delegate the remaining formatting to the base formatter.
return Formatter.format(self, record)
def format(self, record):
try:
msg = super(NonANSIFormatter, self).format(record)
except:
# for python2.6
# Formatter is old-style class in python2.6 and type is classobj
# another trick: http://stackoverflow.com/a/18392639/1276501
msg = Formatter.format(self, record)
rln = record.levelname
return "[{0}]: {1}".format(rln, msg)
class ContextualLogFormatter(object):
def __init__(self, internal=False):
prefix = ''.join([
'%(asctime)s level=%(levelname)s pid=%(process)d tid=%(threadName)s ',
'logger=%(name)s pos=%(filename)s:%(funcName)s:%(lineno)s '
])
if internal:
prefix = ''.join([
'%(levelname)s logger=%(name)s pid=%(process)d tid=%(threadName)s ',
'logger=%(name)s pos=%(filename)s:%(funcName)s:%(lineno)s '
])
kv = prefix + '| %(_context)s | message="%(message)s" %(_kwargs)s'
raw = prefix + '| %(message)s'
self._kv = Formatter(kv)
self._raw = Formatter(raw)
def format(self, record):
if hasattr(record, '_context') and hasattr(record, '_kwargs'):
return self._kv.format(record)
return self._raw.format(record)
def format(self, record):
rv = []
length = 0
for piece in _ws_re.split(Formatter.format(self, record)):
length += len(piece)
if length > 140:
if length - len(piece) < 140:
rv.append(u'…')
break
rv.append(piece)
return u''.join(rv)
def format(self, record):
colored_record = copy(record)
# Format Level Name
levelname = colored_record.levelname
seq = MAPPING.get(levelname, DEFAULT_COLOR)
colored_levelname = set_as_color(levelname, seq)
colored_record.levelname = colored_levelname
# Format Time if applicable, to be yellow
if self.usesTime():
self.datefmt = set_as_color(self.datefmt, 33)
return Formatter.format(self, colored_record)
def format(self, record):
""" Inject color codes & color resets into log record messages. """
message = Formatter.format(self, record)
fmt_str = '\x1b[%dm'
try:
color = fmt_str % (COLORS[self.colormap[record.levelno]])
message = color + message + self.reset
except:
print('Error in ColorFormatter')
return message
def format(self, record):
"""Format the record with colors."""
color = self.color_seq % (30 + self.colors[record.levelname])
message = Formatter.format(self, record)
message = (message.replace("$RESET", self.reset_seq).replace(
"$BOLD", self.bold_seq).replace("$COLOR", color))
for color, value in self.colors.items():
message = (message.replace(
"$" + color, self.color_seq % (value + 30)).replace(
"$BG" + color, self.color_seq % (value + 40)).replace(
"$BG-" + color, self.color_seq % (value + 40)))
return message + self.reset_seq
def format(self, record):
"""Format the record with colors."""
color = self.color_seq % (30 + self.colors[record.levelname])
message = Formatter.format(self, record)
message = message.replace('$RESET', self.reset_seq)\
.replace('$BOLD', self.bold_seq)\
.replace('$COLOR', color)
for color, value in self.colors.items():
message = message.replace(
'$' + color, self.color_seq % (value + 30))\
.replace('$BG' + color, self.color_seq % (value + 40))\
.replace('$BG-' + color, self.color_seq % (value + 40))
return message + self.reset_seq
def format(self, record):
colored_record = copy(record)
# Add colors to levelname
levelname = colored_record.levelname
color = MAPPING.get(levelname, 'white')
colored_record.levelname = COLORIZE(levelname, color)
# Add colors to tagged message text
msg = colored_record.getMessage()
plain_msg, color_msg = self.color_words(msg)
record.msg = plain_msg
colored_record.msg = color_msg
return Formatter.format(self, colored_record)
class SysLogHandler(_SysLogHandler):
""" 扩展标准库的SysLogHandler，使用loggername作为tag """
def __init__(self, tag=None, withpid=False, **kwargs):
super(SysLogHandler, self).__init__(**kwargs)
# 产生tag的formatter
fmt = tag or "%(name)s"
if withpid:
fmt += "[%(process)d]"
self.tag_formatter = Formatter(fmt)
def format(self, record):
msg = super(SysLogHandler, self).format(record)
tag = self.tag_formatter.format(record)
return "%s: %s" % (tag, msg)
def format(self, record):
try:
message = super(SecureFormatter, self).format(record)
except TypeError:
# In pyhton 2.6 the Formatter does not seem to
# be defined as
# class Formatter(object)
# Using it in the super-statement this will raise a TypeError
message = Formatter.format(self, record)
secured = False
s = ""
for c in message:
if c in string.printable:
s += c
else:
s += '.'
secured = True
if secured:
s = "!!!Log Entry Secured by SecureFormatter!!! " + s
return s
def format (self, record): # Add the color codes to the record record.__dict__.update(escape_codes) # If we recognise the level name, # add the levels color as `log_color` if record.levelname in self.log_colors: color = self.log_colors[record.levelname] record.log_color = escape_codes[color] else: record.log_color = "" # Format the message if version_info > (2, 7): message = super(ColoredFormatter, self).format(record) else: message = Formatter.format(self, record) # Add a reset code to the end of the message (if it wasn't explicitly added in format str) if self.reset and not message.endswith(escape_codes['reset']): message += escape_codes['reset'] return message
class UserFormatter(Formatter):
formatter = None
def __init__(self, fmt=None, datefmt=None):
self.formatter = Formatter(fmt, datefmt)
def format(self, record):
from auditor import AUDIT_SESSION_ID
from common.middleware import threadlocal
user = threadlocal.get_current_user()
if user and user.is_authenticated():
username = user.username
else:
username = '******'
request = threadlocal.get_current_request()
if request and getattr(request, AUDIT_SESSION_ID):
sessionid = getattr(request, AUDIT_SESSION_ID)
else:
sessionid = "None"
setattr(record, 'username', username)
setattr(record, 'sessionid', sessionid)
return self.formatter.format(record)
def format(self, record):
s = Formatter.format(self, record)
return re.sub(r"\d[ \d-]{15,22}", "XXXX-XXXX-XXXX-XXXX", s)
def format(self, record):
"""Overridden to choose format based on record contents."""
self._set_context(record)
self._strip_logger_prefix(record)
return Formatter.format(self, record)
def format(self, record):
lines = iter(Formatter.format(self, record).splitlines())
prefix = lines.next()
return '\n'.join(prefix + line for line in lines)
def format(self, record):
text = Formatter.format(self, record)
return PREFIX_REGEX.sub('', text)
def format(self, record):
return self.color_fmt[record.levelno] % Formatter.format(self, record)
def format(self, record):
if record.args and hasattr(record, 'output_limit'):
# Truncate all args to the set limit.
record.args = tuple([truncate(arg, limit=record.output_limit)
for arg in record.args])
return Formatter.format(self, record)
def format(self, record):
try:
return Formatter.format(self, record)
except Exception, e:
return Formatter.format(self, record).encode('utf8')
def format(self, record):
head, tail = self.FORMATS.get(record.levelno, NO_STYLE)
return head + Formatter.format(self, record) + tail
def format(self, record):
return ("%s %s[%d] " % (str(datetime.datetime.utcnow()),
_programName, os.getpid())) \
+ Formatter.format(self, record)
def format(self, record):
levelname = record.levelname
msg = Formatter.format(self, record)
if levelname in COLORS:
msg = COLORS[levelname] % msg
return msg
def format(self, record):
""" Alter log message level name """
record.level = self.LEVEL_MAP[record.levelno]
record.colour = self.COLOUR_MAP[record.levelno]
record.end = self.COLOUR_END
return Formatter.format(self, record)
def format(self, record):
s = Formatter.format(self, record)
return re.sub(r'\d[ \d-]{15,22}', 'XXXX-XXXX-XXXX-XXXX', s)