def test_format_raises(self, safe_str): x = ColorFormatter() def on_safe_str(s): try: raise ValueError('foo') finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str @python_2_unicode_compatible class Record(object): levelname = 'ERROR' msg = 'HELLO' exc_info = 1 exc_text = 'error text' stack_info = None def __str__(self): return on_safe_str('') def getMessage(self): return self.msg record = Record() safe_str.return_value = record msg = x.format(record) assert '<Unrepresentable' in msg assert safe_str.call_count == 1
def test_formatException_bytes(self, safe_str, fe): x = ColorFormatter() fe.return_value = b'HELLO' try: raise Exception() except Exception: assert x.formatException(sys.exc_info())
def test_format_raises(self, safe_str): x = ColorFormatter() def on_safe_str(s): try: raise ValueError("foo") finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str @python_2_unicode_compatible class Record(object): levelname = "ERROR" msg = "HELLO" exc_info = 1 exc_text = "error text" stack_info = None def __str__(self): return on_safe_str("") def getMessage(self): return self.msg record = Record() safe_str.return_value = record msg = x.format(record) self.assertIn("<Unrepresentable", msg) self.assertEqual(safe_str.call_count, 1)
def test_formatException_not_string(self, fe, safe_str): x = ColorFormatter() value = KeyError() fe.return_value = value assert x.formatException(value) is value fe.assert_called() safe_str.assert_not_called()
def test_formatException_not_string(self, fe, safe_str): x = ColorFormatter() value = KeyError() fe.return_value = value self.assertIs(x.formatException(value), value) fe.assert_called() safe_str.assert_not_called()
def test_format_object(self, _format): x = ColorFormatter() x.use_color = True record = Mock() record.levelname = 'ERROR' record.msg = object() self.assertTrue(x.format(record))
def test_format_raises(self, safe_str): x = ColorFormatter("HELLO") def on_safe_str(s): try: raise ValueError("foo") finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str class Record(object): levelname = "ERROR" msg = "HELLO" exc_text = "error text" stack_info = None def __str__(self): return on_safe_str("") def getMessage(self): return self.msg record = Record() safe_str.return_value = record x.format(record) self.assertIn("<Unrepresentable", record.msg) self.assertEqual(safe_str.call_count, 2)
def test_format_raises(self, safe_str): x = ColorFormatter() def on_safe_str(s): try: raise ValueError('foo') finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str class Record(object): levelname = 'ERROR' msg = 'HELLO' exc_info = 1 exc_text = 'error text' stack_info = None def __str__(self): return on_safe_str('') def getMessage(self): return self.msg record = Record() safe_str.return_value = record msg = x.format(record) self.assertIn('<Unrepresentable', msg) self.assertEqual(safe_str.call_count, 1)
def test_formatException_not_string(self, fe, safe_str): x = ColorFormatter() value = KeyError() fe.return_value = value self.assertIs(x.formatException(value), value) self.assertTrue(fe.called) self.assertFalse(safe_str.called)
def test_formatException_not_string(self, fe, safe_str): x = ColorFormatter('HELLO') value = KeyError() fe.return_value = value self.assertIs(x.formatException(value), value) self.assertTrue(fe.called) self.assertFalse(safe_str.called)
def test_format_raises(self, safe_str): x = ColorFormatter('HELLO') def on_safe_str(s): try: raise ValueError('foo') finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str class Record(object): levelname = 'ERROR' msg = 'HELLO' exc_text = 'error text' stack_info = None def __str__(self): return on_safe_str('') def getMessage(self): return self.msg record = Record() safe_str.return_value = record x.format(record) self.assertIn('<Unrepresentable', record.msg) self.assertEqual(safe_str.call_count, 2)
def test_format_raises_no_color(self, safe_str): x = ColorFormatter("HELLO", False) record = Mock() record.levelname = "ERROR" record.msg = "HELLO" record.exc_text = "error text" x.format(record) self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str): x = ColorFormatter(use_color=False) record = Mock() record.levelname = 'ERROR' record.msg = 'HELLO' record.exc_text = 'error text' x.format(record) self.assertEqual(safe_str.call_count, 1)
def test_formatException_bytes(self, safe_str, fe): x = ColorFormatter() fe.return_value = b'HELLO' try: raise Exception() except Exception: self.assertTrue(x.formatException(sys.exc_info())) if sys.version_info[0] == 2: self.assertTrue(safe_str.called)
def test_format_raises_no_color(self, safe_str): if sys.version_info[0] == 3: raise SkipTest("py3k") x = ColorFormatter("HELLO", False) record = Mock() record.levelname = "ERROR" record.msg = "HELLO" record.exc_text = "error text" x.format(record) self.assertEqual(safe_str.call_count, 1)
def test_format_raises_no_color(self, safe_str): if sys.version_info[0] == 3: raise SkipTest('py3k') x = ColorFormatter('HELLO', False) record = Mock() record.levelname = 'ERROR' record.msg = 'HELLO' record.exc_text = 'error text' x.format(record) self.assertEqual(safe_str.call_count, 1)
def setup_task_logging_customization(task_id, task, **kwargs): """ Customize the logging for the task. This adds a filter that sets ``request_id`` on the log record. If the request ID cannot be determined, "unknown" is set instead. This also sets the log format to the ``cachito_task_log_format`` config. :param str task_id: the task ID :param class task: the class of the task being executed """ conf = get_worker_config() request_id = _get_function_arg_value("request_id", task, kwargs["args"], kwargs["kwargs"]) log_filter = AddRequestIDFilter(str(request_id) or "unknown") root_logger = logging.getLogger() for handler in root_logger.handlers: if not isinstance(handler, logging.StreamHandler): continue handler.addFilter(log_filter) if isinstance(handler.formatter, ColorFormatter): formatter = ColorFormatter(conf.cachito_task_log_format, use_color=handler.formatter.use_color) else: formatter = logging.Formatter(conf.cachito_task_log_format) handler.setFormatter(formatter)
def format(self, record): task = get_current_task() if task and task.request: name = task.name.split('.')[-1] record.__dict__.update(task_id=task.request.id, task_name=name) else: record.__dict__.update(task_id='xxx', task_name='xxx') return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault("task_name", "???") record.__dict__.setdefault("task_id", "???") return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault("task_name", "???") record.__dict__.setdefault("task_id", "???") return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return ColorFormatter.format(self, record)
def test_format_raises(self, safe_str): x = ColorFormatter("HELLO") def on_safe_str(s): try: raise ValueError("foo") finally: safe_str.side_effect = None safe_str.side_effect = on_safe_str record = Mock() record.levelname = "ERROR" record.msg = "HELLO" record.exc_text = "error text" safe_str.return_value = record x.format(record) self.assertIn("<Unrepresentable", record.msg) self.assertEqual(safe_str.call_count, 2)
def format(self, record): self.use_color = False task = get_current_task() if task and task.request: record.__dict__.update(root_id=task.request.root_id, task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', None) record.__dict__.setdefault('task_id', None) record.__dict__.setdefault('root_id', None) return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task and task.request: # For gevent pool, task_id will be something like `7ab44cb4-aacf-444e-bc20-4cbaa2a7b082`. For logs # is better to get it short task_id = task.request.id[:8] # Task name usually has all the package, better cut the first part for logging task_name = task.name.split('.')[-1] record.__dict__.update(task_id=task_id, task_name=task_name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return ColorFormatter.format(self, record)
def setup_logging_handler(loglevel, logfile, format, colorize, **kwargs): if logfile: if os.name == 'nt': logfile = logfile.format(os.getpid()) handler = logging.handlers.RotatingFileHandler( logfile, maxBytes=LOGFILE_SIZE_BYTES, backupCount=LOGFILE_BACKUP_COUNT) handler.setFormatter(logging.Formatter(fmt=format)) else: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(ColorFormatter(fmt=format, use_color=colorize)) handler.setLevel(loglevel) logger = logging.getLogger() logger.handlers = [] logger.addHandler(handler) logger.setLevel(loglevel)
def format(self, record): """Append task-related values to the record upon format.""" task = get_current_task() if task and task.request: record.__dict__.update( task_id=task.request.id, task_name=task.name, task_root_id=task.request.root_id, task_parent_id=task.request.parent_id, ) else: record.__dict__.setdefault("task_name", "None") record.__dict__.setdefault("task_id", "None") record.__dict__.setdefault("task_root_id", "None") record.__dict__.setdefault("task_parent_id", "None") return ColorFormatter.format(self, record)
def test_setup_task_logging_customization(caplog): # Setting the logging level via caplog.set_level is not sufficient. The Flask # related settings from previous tests interfere with this. workers_logger = logging.getLogger("cachito.workers") workers_logger.disabled = False workers_logger.setLevel(logging.INFO) root_logger = logging.getLogger() original_formatters = [] for handler in root_logger.handlers: original_formatters.append(handler.formatter) color_handler = logging.StreamHandler() color_handler.setFormatter(ColorFormatter("%(message)s")) root_logger.addHandler(color_handler) task_id = mock.Mock() task = mock.Mock() def _dummy_task(msg, request_id): return task.__wrapped__ = _dummy_task try: celery_logging.setup_task_logging_customization( task_id, task, args=["hello"], kwargs={"request_id": 3}) workers_logger.info("Test log message") assert "#%(request_id)s" in color_handler.formatter._fmt finally: root_logger.removeHandler(color_handler) for handler, formatter in zip(root_logger.handlers, original_formatters): handler.setFormatter(formatter) # Verify that the request filter and formatter were properly set expected = ( "#3 cachito.workers INFO test_celery_logging.test_setup_task_logging_customization]" " Test log message") assert expected in caplog.text
def cleanup_task_logging_customization(task_id, task, **kwargs): """ Clean up any logging customizations that were set in ``setup_task_logging_customization``. :param str task_id: the task ID :param class task: the class of the task being executed """ conf = get_worker_config() root_logger = logging.getLogger() for handler in root_logger.handlers: if not isinstance(handler, logging.StreamHandler): continue if isinstance(handler.formatter, ColorFormatter): formatter = ColorFormatter(conf.worker_log_format, use_color=handler.formatter.use_color) else: formatter = logging.Formatter(conf.worker_log_format) handler.setFormatter(formatter) for log_filter in handler.filters: if isinstance(log_filter, AddRequestIDFilter): handler.removeFilter(log_filter)
def test_cleanup_task_logging_customization(): root_logger = logging.getLogger() original_formatters = [] for handler in root_logger.handlers: original_formatters.append(handler.formatter) color_handler = logging.StreamHandler() color_handler.setFormatter(ColorFormatter("%(message)s")) log_filter = celery_logging.AddRequestIDFilter(5) color_handler.addFilter(log_filter) root_logger.addHandler(color_handler) try: celery_logging.cleanup_task_logging_customization( mock.Mock(), mock.Mock()) finally: root_logger.removeHandler(color_handler) for handler, formatter in zip(root_logger.handlers, original_formatters): handler.setFormatter(formatter) # Verify that the AddRequestIDFilter filter was removed from the color handler assert len(color_handler.filters) == 0
def test_formatException_string(self, safe_str, fe, value="HELLO"): x = ColorFormatter(value) fe.return_value = value self.assertTrue(x.formatException(value)) self.assertTrue(safe_str.called)
def setup_log_handler( loglevel=None, logfile=None, format=None, colorize=None, log_console=None, log_journal=None, **kwargs, ): """Setup logging according to Software Heritage preferences. We use the command-line loglevel for tasks only, as we never really care about the debug messages from celery. """ if loglevel is None: loglevel = logging.DEBUG if isinstance(loglevel, str): loglevel = logging._nameToLevel[loglevel] formatter = logging.Formatter(format) root_logger = logging.getLogger("") root_logger.setLevel(logging.INFO) log_target = os.environ.get("SWH_LOG_TARGET", "console") if log_target == "console": log_console = True elif log_target == "journal": log_journal = True # this looks for log levels *higher* than DEBUG if loglevel <= logging.DEBUG and log_console is None: log_console = True if log_console: color_formatter = ColorFormatter(format) if colorize else formatter console = logging.StreamHandler() console.setLevel(logging.DEBUG) console.setFormatter(color_formatter) root_logger.addHandler(console) if log_journal: if not JournalHandler: root_logger.warning("JournalHandler is not available, skipping. " "Please install swh-core[logging].") else: systemd_journal = JournalHandler() systemd_journal.setLevel(logging.DEBUG) systemd_journal.setFormatter(formatter) root_logger.addHandler(systemd_journal) logging.getLogger("celery").setLevel(logging.INFO) # Silence amqp heartbeat_tick messages logger = logging.getLogger("amqp") logger.addFilter( lambda record: not record.msg.startswith("heartbeat_tick")) logger.setLevel(loglevel) # Silence useless "Starting new HTTP connection" messages logging.getLogger("urllib3").setLevel(logging.WARNING) # Completely disable azure logspam azure_logger = logging.getLogger( "azure.core.pipeline.policies.http_logging_policy") azure_logger.setLevel(logging.WARNING) logging.getLogger("swh").setLevel(loglevel) # get_task_logger makes the swh tasks loggers children of celery.task logging.getLogger("celery.task").setLevel(loglevel) return loglevel
def test_formatException_string(self, safe_str, fe, value='HELLO'): x = ColorFormatter(value) fe.return_value = value self.assertTrue(x.formatException(value)) if sys.version_info[0] == 2: self.assertTrue(safe_str.called)