def _setup_stdlib_logging(config, log_config, logBeginner: LogBeginner): """ Set up Python stdlib logging. """ if log_config is None: log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" " - %(message)s") logger = logging.getLogger("") logger.setLevel(logging.INFO) logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO) formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) else: logging.config.dictConfig(log_config) # Route Twisted's native logging through to the standard library logging # system. observer = STDLibLogObserver() def _log(event): if "log_text" in event: if event["log_text"].startswith( "DNSDatagramProtocol starting on "): return if event["log_text"].startswith("(UDP Port "): return if event["log_text"].startswith("Timing out client"): return return observer(event) logBeginner.beginLoggingTo([_log], redirectStandardIO=not config.no_redirect_stdio) if not config.no_redirect_stdio: print("Redirected stdout/stderr to logs") return observer
def _setup_stdlib_logging(config, log_config_path, logBeginner: LogBeginner) -> None: """ Set up Python standard library logging. """ if log_config_path is None: log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" " - %(message)s") logger = logging.getLogger("") logger.setLevel(logging.INFO) logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO) formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) else: # Load the logging configuration. _load_logging_config(log_config_path) # We add a log record factory that runs all messages through the # LoggingContextFilter so that we get the context *at the time we log* # rather than when we write to a handler. This can be done in config using # filter options, but care must when using e.g. MemoryHandler to buffer # writes. log_context_filter = LoggingContextFilter(request="") log_metadata_filter = MetadataFilter({"server_name": config.server_name}) old_factory = logging.getLogRecordFactory() def factory(*args, **kwargs): record = old_factory(*args, **kwargs) log_context_filter.filter(record) log_metadata_filter.filter(record) return record logging.setLogRecordFactory(factory) # Route Twisted's native logging through to the standard library logging # system. observer = STDLibLogObserver() threadlocal = threading.local() def _log(event): if "log_text" in event: if event["log_text"].startswith( "DNSDatagramProtocol starting on "): return if event["log_text"].startswith("(UDP Port "): return if event["log_text"].startswith("Timing out client"): return # this is a workaround to make sure we don't get stack overflows when the # logging system raises an error which is written to stderr which is redirected # to the logging system, etc. if getattr(threadlocal, "active", False): # write the text of the event, if any, to the *real* stderr (which may # be redirected to /dev/null, but there's not much we can do) try: event_text = eventAsText(event) print("logging during logging: %s" % event_text, file=sys.__stderr__) except Exception: # gah. pass return try: threadlocal.active = True return observer(event) finally: threadlocal.active = False logBeginner.beginLoggingTo([_log], redirectStandardIO=not config.no_redirect_stdio) if not config.no_redirect_stdio: print("Redirected stdout/stderr to logs")
def _setup_stdlib_logging(config, log_config, logBeginner: LogBeginner): """ Set up Python stdlib logging. """ if log_config is None: log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" " - %(message)s") logger = logging.getLogger("") logger.setLevel(logging.INFO) logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO) formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) else: logging.config.dictConfig(log_config) # We add a log record factory that runs all messages through the # LoggingContextFilter so that we get the context *at the time we log* # rather than when we write to a handler. This can be done in config using # filter options, but care must when using e.g. MemoryHandler to buffer # writes. log_filter = LoggingContextFilter(request="") old_factory = logging.getLogRecordFactory() def factory(*args, **kwargs): record = old_factory(*args, **kwargs) log_filter.filter(record) return record logging.setLogRecordFactory(factory) # Route Twisted's native logging through to the standard library logging # system. observer = STDLibLogObserver() def _log(event): if "log_text" in event: if event["log_text"].startswith( "DNSDatagramProtocol starting on "): return if event["log_text"].startswith("(UDP Port "): return if event["log_text"].startswith("Timing out client"): return return observer(event) logBeginner.beginLoggingTo([_log], redirectStandardIO=not config.no_redirect_stdio) if not config.no_redirect_stdio: print("Redirected stdout/stderr to logs") return observer
def setup_structured_logging( hs, config, log_config: dict, logBeginner: LogBeginner, redirect_stdlib_logging: bool = True, ) -> LogPublisher: """ Set up Twisted's structured logging system. Args: hs: The homeserver to use. config (HomeserverConfig): The configuration of the Synapse homeserver. log_config (dict): The log configuration to use. """ if config.no_redirect_stdio: raise ConfigError( "no_redirect_stdio cannot be defined using structured logging.") logger = Logger() if "drains" not in log_config: raise ConfigError( "The logging configuration requires a list of drains.") observers = [] # type: List[ILogObserver] for observer in parse_drain_configs(log_config["drains"]): # Pipe drains if observer.type == DrainType.CONSOLE: logger.debug("Starting up the {name} console logger drain", name=observer.name) observers.append(SynapseFileLogObserver(observer.location)) elif observer.type == DrainType.CONSOLE_JSON: logger.debug("Starting up the {name} JSON console logger drain", name=observer.name) observers.append(jsonFileLogObserver(observer.location)) elif observer.type == DrainType.CONSOLE_JSON_TERSE: logger.debug( "Starting up the {name} terse JSON console logger drain", name=observer.name, ) observers.append( TerseJSONToConsoleLogObserver(observer.location, metadata={})) # File drains elif observer.type == DrainType.FILE: logger.debug("Starting up the {name} file logger drain", name=observer.name) log_file = open(observer.location, "at", buffering=1, encoding="utf8") observers.append(SynapseFileLogObserver(log_file)) elif observer.type == DrainType.FILE_JSON: logger.debug("Starting up the {name} JSON file logger drain", name=observer.name) log_file = open(observer.location, "at", buffering=1, encoding="utf8") observers.append(jsonFileLogObserver(log_file)) elif observer.type == DrainType.NETWORK_JSON_TERSE: metadata = {"server_name": hs.config.server_name} log_observer = TerseJSONToTCPLogObserver( hs=hs, host=observer.location[0], port=observer.location[1], metadata=metadata, maximum_buffer=observer.options.maximum_buffer, ) log_observer.start() observers.append(log_observer) else: # We should never get here, but, just in case, throw an error. raise ConfigError("%s drain type cannot be configured" % (observer.type, )) publisher = StoppableLogPublisher(*observers) log_filter = LogLevelFilterPredicate() for namespace, namespace_config in log_config.get("loggers", DEFAULT_LOGGERS).items(): # Set the log level for twisted.logger.Logger namespaces log_filter.setLogLevelForNamespace( namespace, stdlib_log_level_to_twisted(namespace_config.get("level", "INFO")), ) # Also set the log levels for the stdlib logger namespaces, to prevent # them getting to PythonStdlibToTwistedLogger and having to be formatted if "level" in namespace_config: logging.getLogger(namespace).setLevel( namespace_config.get("level")) f = FilteringLogObserver(publisher, [log_filter]) lco = LogContextObserver(f) if redirect_stdlib_logging: stuff_into_twisted = PythonStdlibToTwistedLogger(lco) stdliblogger = logging.getLogger() stdliblogger.addHandler(stuff_into_twisted) # Always redirect standard I/O, otherwise other logging outputs might miss # it. logBeginner.beginLoggingTo([lco], redirectStandardIO=True) return publisher