def test_format_raises(self, safe_str):
x = ColorFormatter()
def on_safe_str(s):
try:
raise ValueError('foo')
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
@python_2_unicode_compatible
class Record(object):
levelname = 'ERROR'
msg = 'HELLO'
exc_info = 1
exc_text = 'error text'
stack_info = None
def __str__(self):
return on_safe_str('')
def getMessage(self):
return self.msg
record = Record()
safe_str.return_value = record
msg = x.format(record)
assert '<Unrepresentable' in msg
assert safe_str.call_count == 1
def test_formatException_bytes(self, safe_str, fe):
x = ColorFormatter()
fe.return_value = b'HELLO'
try:
raise Exception()
except Exception:
assert x.formatException(sys.exc_info())
def test_format_raises(self, safe_str):
x = ColorFormatter()
def on_safe_str(s):
try:
raise ValueError("foo")
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
@python_2_unicode_compatible
class Record(object):
levelname = "ERROR"
msg = "HELLO"
exc_info = 1
exc_text = "error text"
stack_info = None
def __str__(self):
return on_safe_str("")
def getMessage(self):
return self.msg
record = Record()
safe_str.return_value = record
msg = x.format(record)
self.assertIn("<Unrepresentable", msg)
self.assertEqual(safe_str.call_count, 1)
def test_formatException_not_string(self, fe, safe_str):
x = ColorFormatter()
value = KeyError()
fe.return_value = value
assert x.formatException(value) is value
fe.assert_called()
safe_str.assert_not_called()
def test_formatException_not_string(self, fe, safe_str):
x = ColorFormatter()
value = KeyError()
fe.return_value = value
self.assertIs(x.formatException(value), value)
fe.assert_called()
safe_str.assert_not_called()
def test_format_object(self, _format):
x = ColorFormatter()
x.use_color = True
record = Mock()
record.levelname = 'ERROR'
record.msg = object()
self.assertTrue(x.format(record))
def test_format_raises(self, safe_str):
x = ColorFormatter("HELLO")
def on_safe_str(s):
try:
raise ValueError("foo")
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
class Record(object):
levelname = "ERROR"
msg = "HELLO"
exc_text = "error text"
stack_info = None
def __str__(self):
return on_safe_str("")
def getMessage(self):
return self.msg
record = Record()
safe_str.return_value = record
x.format(record)
self.assertIn("<Unrepresentable", record.msg)
self.assertEqual(safe_str.call_count, 2)
def test_format_raises(self, safe_str):
x = ColorFormatter()
def on_safe_str(s):
try:
raise ValueError('foo')
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
class Record(object):
levelname = 'ERROR'
msg = 'HELLO'
exc_info = 1
exc_text = 'error text'
stack_info = None
def __str__(self):
return on_safe_str('')
def getMessage(self):
return self.msg
record = Record()
safe_str.return_value = record
msg = x.format(record)
self.assertIn('<Unrepresentable', msg)
self.assertEqual(safe_str.call_count, 1)
def test_formatException_not_string(self, fe, safe_str):
x = ColorFormatter()
value = KeyError()
fe.return_value = value
self.assertIs(x.formatException(value), value)
self.assertTrue(fe.called)
self.assertFalse(safe_str.called)
def test_formatException_not_string(self, fe, safe_str):
x = ColorFormatter('HELLO')
value = KeyError()
fe.return_value = value
self.assertIs(x.formatException(value), value)
self.assertTrue(fe.called)
self.assertFalse(safe_str.called)
def test_format_raises(self, safe_str):
x = ColorFormatter('HELLO')
def on_safe_str(s):
try:
raise ValueError('foo')
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
class Record(object):
levelname = 'ERROR'
msg = 'HELLO'
exc_text = 'error text'
stack_info = None
def __str__(self):
return on_safe_str('')
def getMessage(self):
return self.msg
record = Record()
safe_str.return_value = record
x.format(record)
self.assertIn('<Unrepresentable', record.msg)
self.assertEqual(safe_str.call_count, 2)
def test_format_raises_no_color(self, safe_str):
x = ColorFormatter("HELLO", False)
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str):
x = ColorFormatter(use_color=False)
record = Mock()
record.levelname = 'ERROR'
record.msg = 'HELLO'
record.exc_text = 'error text'
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str):
x = ColorFormatter("HELLO", False)
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_formatException_bytes(self, safe_str, fe):
x = ColorFormatter()
fe.return_value = b'HELLO'
try:
raise Exception()
except Exception:
self.assertTrue(x.formatException(sys.exc_info()))
if sys.version_info[0] == 2:
self.assertTrue(safe_str.called)
def test_format_raises_no_color(self, safe_str):
if sys.version_info[0] == 3:
raise SkipTest("py3k")
x = ColorFormatter("HELLO", False)
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str):
if sys.version_info[0] == 3:
raise SkipTest("py3k")
x = ColorFormatter("HELLO", False)
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str):
if sys.version_info[0] == 3:
raise SkipTest('py3k')
x = ColorFormatter('HELLO', False)
record = Mock()
record.levelname = 'ERROR'
record.msg = 'HELLO'
record.exc_text = 'error text'
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str):
if sys.version_info[0] == 3:
raise SkipTest('py3k')
x = ColorFormatter('HELLO', False)
record = Mock()
record.levelname = 'ERROR'
record.msg = 'HELLO'
record.exc_text = 'error text'
x.format(record)
self.assertEqual(safe_str.call_count, 1)
def setup_task_logging_customization(task_id, task, **kwargs):
"""
Customize the logging for the task.
This adds a filter that sets ``request_id`` on the log record. If the request ID
cannot be determined, "unknown" is set instead. This also sets the log format to
the ``cachito_task_log_format`` config.
:param str task_id: the task ID
:param class task: the class of the task being executed
"""
conf = get_worker_config()
request_id = _get_function_arg_value("request_id", task, kwargs["args"],
kwargs["kwargs"])
log_filter = AddRequestIDFilter(str(request_id) or "unknown")
root_logger = logging.getLogger()
for handler in root_logger.handlers:
if not isinstance(handler, logging.StreamHandler):
continue
handler.addFilter(log_filter)
if isinstance(handler.formatter, ColorFormatter):
formatter = ColorFormatter(conf.cachito_task_log_format,
use_color=handler.formatter.use_color)
else:
formatter = logging.Formatter(conf.cachito_task_log_format)
handler.setFormatter(formatter)
def format(self, record):
task = get_current_task()
if task and task.request:
name = task.name.split('.')[-1]
record.__dict__.update(task_id=task.request.id, task_name=name)
else:
record.__dict__.update(task_id='xxx', task_name='xxx')
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task and task.request:
record.__dict__.update(task_id=task.request.id, task_name=task.name)
else:
record.__dict__.setdefault("task_name", "???")
record.__dict__.setdefault("task_id", "???")
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault("task_name", "???")
record.__dict__.setdefault("task_id", "???")
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task and task.request:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return ColorFormatter.format(self, record)
def test_format_raises(self, safe_str):
x = ColorFormatter("HELLO")
def on_safe_str(s):
try:
raise ValueError("foo")
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
safe_str.return_value = record
x.format(record)
self.assertIn("<Unrepresentable", record.msg)
self.assertEqual(safe_str.call_count, 2)
def test_format_raises(self, safe_str):
x = ColorFormatter("HELLO")
def on_safe_str(s):
try:
raise ValueError("foo")
finally:
safe_str.side_effect = None
safe_str.side_effect = on_safe_str
record = Mock()
record.levelname = "ERROR"
record.msg = "HELLO"
record.exc_text = "error text"
safe_str.return_value = record
x.format(record)
self.assertIn("<Unrepresentable", record.msg)
self.assertEqual(safe_str.call_count, 2)
def format(self, record):
self.use_color = False
task = get_current_task()
if task and task.request:
record.__dict__.update(root_id=task.request.root_id,
task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', None)
record.__dict__.setdefault('task_id', None)
record.__dict__.setdefault('root_id', None)
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task and task.request:
# For gevent pool, task_id will be something like `7ab44cb4-aacf-444e-bc20-4cbaa2a7b082`. For logs
# is better to get it short
task_id = task.request.id[:8]
# Task name usually has all the package, better cut the first part for logging
task_name = task.name.split('.')[-1]
record.__dict__.update(task_id=task_id, task_name=task_name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return ColorFormatter.format(self, record)
def setup_logging_handler(loglevel, logfile, format, colorize, **kwargs):
if logfile:
if os.name == 'nt':
logfile = logfile.format(os.getpid())
handler = logging.handlers.RotatingFileHandler(
logfile,
maxBytes=LOGFILE_SIZE_BYTES,
backupCount=LOGFILE_BACKUP_COUNT)
handler.setFormatter(logging.Formatter(fmt=format))
else:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(ColorFormatter(fmt=format, use_color=colorize))
handler.setLevel(loglevel)
logger = logging.getLogger()
logger.handlers = []
logger.addHandler(handler)
logger.setLevel(loglevel)
def format(self, record):
"""Append task-related values to the record upon format."""
task = get_current_task()
if task and task.request:
record.__dict__.update(
task_id=task.request.id,
task_name=task.name,
task_root_id=task.request.root_id,
task_parent_id=task.request.parent_id,
)
else:
record.__dict__.setdefault("task_name", "None")
record.__dict__.setdefault("task_id", "None")
record.__dict__.setdefault("task_root_id", "None")
record.__dict__.setdefault("task_parent_id", "None")
return ColorFormatter.format(self, record)
def test_setup_task_logging_customization(caplog):
# Setting the logging level via caplog.set_level is not sufficient. The Flask
# related settings from previous tests interfere with this.
workers_logger = logging.getLogger("cachito.workers")
workers_logger.disabled = False
workers_logger.setLevel(logging.INFO)
root_logger = logging.getLogger()
original_formatters = []
for handler in root_logger.handlers:
original_formatters.append(handler.formatter)
color_handler = logging.StreamHandler()
color_handler.setFormatter(ColorFormatter("%(message)s"))
root_logger.addHandler(color_handler)
task_id = mock.Mock()
task = mock.Mock()
def _dummy_task(msg, request_id):
return
task.__wrapped__ = _dummy_task
try:
celery_logging.setup_task_logging_customization(
task_id, task, args=["hello"], kwargs={"request_id": 3})
workers_logger.info("Test log message")
assert "#%(request_id)s" in color_handler.formatter._fmt
finally:
root_logger.removeHandler(color_handler)
for handler, formatter in zip(root_logger.handlers,
original_formatters):
handler.setFormatter(formatter)
# Verify that the request filter and formatter were properly set
expected = (
"#3 cachito.workers INFO test_celery_logging.test_setup_task_logging_customization]"
" Test log message")
assert expected in caplog.text
def cleanup_task_logging_customization(task_id, task, **kwargs):
"""
Clean up any logging customizations that were set in ``setup_task_logging_customization``.
:param str task_id: the task ID
:param class task: the class of the task being executed
"""
conf = get_worker_config()
root_logger = logging.getLogger()
for handler in root_logger.handlers:
if not isinstance(handler, logging.StreamHandler):
continue
if isinstance(handler.formatter, ColorFormatter):
formatter = ColorFormatter(conf.worker_log_format,
use_color=handler.formatter.use_color)
else:
formatter = logging.Formatter(conf.worker_log_format)
handler.setFormatter(formatter)
for log_filter in handler.filters:
if isinstance(log_filter, AddRequestIDFilter):
handler.removeFilter(log_filter)
def test_cleanup_task_logging_customization():
root_logger = logging.getLogger()
original_formatters = []
for handler in root_logger.handlers:
original_formatters.append(handler.formatter)
color_handler = logging.StreamHandler()
color_handler.setFormatter(ColorFormatter("%(message)s"))
log_filter = celery_logging.AddRequestIDFilter(5)
color_handler.addFilter(log_filter)
root_logger.addHandler(color_handler)
try:
celery_logging.cleanup_task_logging_customization(
mock.Mock(), mock.Mock())
finally:
root_logger.removeHandler(color_handler)
for handler, formatter in zip(root_logger.handlers,
original_formatters):
handler.setFormatter(formatter)
# Verify that the AddRequestIDFilter filter was removed from the color handler
assert len(color_handler.filters) == 0
def test_formatException_string(self, safe_str, fe, value="HELLO"):
x = ColorFormatter(value)
fe.return_value = value
self.assertTrue(x.formatException(value))
self.assertTrue(safe_str.called)
def setup_log_handler(
loglevel=None,
logfile=None,
format=None,
colorize=None,
log_console=None,
log_journal=None,
**kwargs,
):
"""Setup logging according to Software Heritage preferences.
We use the command-line loglevel for tasks only, as we never
really care about the debug messages from celery.
"""
if loglevel is None:
loglevel = logging.DEBUG
if isinstance(loglevel, str):
loglevel = logging._nameToLevel[loglevel]
formatter = logging.Formatter(format)
root_logger = logging.getLogger("")
root_logger.setLevel(logging.INFO)
log_target = os.environ.get("SWH_LOG_TARGET", "console")
if log_target == "console":
log_console = True
elif log_target == "journal":
log_journal = True
# this looks for log levels *higher* than DEBUG
if loglevel <= logging.DEBUG and log_console is None:
log_console = True
if log_console:
color_formatter = ColorFormatter(format) if colorize else formatter
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
console.setFormatter(color_formatter)
root_logger.addHandler(console)
if log_journal:
if not JournalHandler:
root_logger.warning("JournalHandler is not available, skipping. "
"Please install swh-core[logging].")
else:
systemd_journal = JournalHandler()
systemd_journal.setLevel(logging.DEBUG)
systemd_journal.setFormatter(formatter)
root_logger.addHandler(systemd_journal)
logging.getLogger("celery").setLevel(logging.INFO)
# Silence amqp heartbeat_tick messages
logger = logging.getLogger("amqp")
logger.addFilter(
lambda record: not record.msg.startswith("heartbeat_tick"))
logger.setLevel(loglevel)
# Silence useless "Starting new HTTP connection" messages
logging.getLogger("urllib3").setLevel(logging.WARNING)
# Completely disable azure logspam
azure_logger = logging.getLogger(
"azure.core.pipeline.policies.http_logging_policy")
azure_logger.setLevel(logging.WARNING)
logging.getLogger("swh").setLevel(loglevel)
# get_task_logger makes the swh tasks loggers children of celery.task
logging.getLogger("celery.task").setLevel(loglevel)
return loglevel
def test_formatException_string(self, safe_str, fe, value="HELLO"):
x = ColorFormatter(value)
fe.return_value = value
self.assertTrue(x.formatException(value))
self.assertTrue(safe_str.called)
def test_formatException_string(self, safe_str, fe, value='HELLO'):
x = ColorFormatter(value)
fe.return_value = value
self.assertTrue(x.formatException(value))
if sys.version_info[0] == 2:
self.assertTrue(safe_str.called)
def test_formatException_string(self, safe_str, fe, value='HELLO'):
x = ColorFormatter(value)
fe.return_value = value
self.assertTrue(x.formatException(value))
if sys.version_info[0] == 2:
self.assertTrue(safe_str.called)
