def test_sd_not_sent(logger, handler_kwargs, logger_kwargs): sh = Rfc5424SysLogHandler(**handler_kwargs) logger.addHandler(sh) with patch.object(sh.transport, 'socket') as syslog_socket: logger.info(message, **logger_kwargs) syslog_socket.sendto.assert_not_called() logger.removeHandler(sh)
def logger_with_tcp_handler(logger): sh = Rfc5424SysLogHandler(address=address, socktype=socket.SOCK_STREAM) logger.addHandler(sh) with patch.object(sh.transport, 'socket', side_effect=connect_mock) as syslog_socket: yield logger, syslog_socket logger.removeHandler(sh)
def test_empty_msg(logger): sh = Rfc5424SysLogHandler(address=address, msg_as_utf8=False) logger.addHandler(sh) with patch.object(sh, 'socket') as syslog_socket: logger.info(None) expected = b'<14>1 2000-01-01T17:11:11.111111+06:00 testhostname root 111 - -' syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def init_logging(logging_configuration: dict = None) -> None: """Initialize Thoth's logging - respects all namespaces. This function allows you to control logging facilities in Thoth. Logging can be configured via env variables so that deployment can respect your configuration. The structure of environment variables is THOTH_LOG_<MODULE> and the value of env variable states verbosity level as in the logging module (DEBUG, INFO, WARNING, ERROR). >>> import os >>> os.environ['THOTH_LOG_SOLVER'] WARNING You can also specify more closely which sub-module logging you are configuring - submodules are separated with double dash: >>> os.environ['THOTH_LOG_SOLVER__PYTHON'] DEBUG You can also use arguments explicitly that override configuration in env variables (a shorthand for standard logging functionality): >>> init_logging({'thoth.solver': 'DEBUG'}) """ # TODO: JSON in deployments? # deployed_to_cluster = bool(int(os.getenv('THOTH_CLUSTER_DEPLOYMENT', '0'))) logging.basicConfig() root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) # Disable annoying unverified HTTPS request warnings. try: import urllib3 urllib3.disable_warnings() except ImportError: pass thoth_root_logger = logging.getLogger('thoth') thoth_root_logger.setLevel(logging.INFO) _init_log_levels(logging_configuration) if _RSYSLOG_HOST and _RSYSLOG_PORT: root_logger.info( f"Setting up logging to rsyslog endpoint {_RSYSLOG_HOST}:{_RSYSLOG_PORT}" ) syslog_handler = Rfc5424SysLogHandler(address=(_RSYSLOG_HOST, _RSYSLOG_PORT)) root_logger.addHandler(syslog_handler) elif int(bool(_RSYSLOG_PORT)) + int(bool(_RSYSLOG_HOST)) == 1: raise RuntimeError( f"Please provide both RSYSLOG_HOST and RSYSLOG_PORT configuration" f"in order to use rsyslog logging, host: {_RSYSLOG_HOST}, port: {_RSYSLOG_PORT}" ) else: root_logger.info("Logging to rsyslog endpoint is turned off")
def test_msg_any(logger): sh = Rfc5424SysLogHandler(address=address, msg_as_utf8=False) logger.addHandler(sh) with patch.object(sh.transport, 'socket') as syslog_socket: logger.info(message) expected = b'<14>1 2000-01-01T17:11:11.111111+06:00 testhostname root 111' \ b' - - This is an interesting message' syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def __init__(self, conf): # Keep syslog dependency optionnal by importing at init time from rfc5424logging import Rfc5424SysLogHandler sh = Rfc5424SysLogHandler( address=(conf['syslog_server'], conf['syslog_port']), socktype=getattr(socket, conf['syslog_stream']), # Use TCP or UDP appname='WPWatcher', **conf['syslog_kwargs']) self.syslog = logging.getLogger('wpwatcher-syslog') self.syslog.setLevel(logging.DEBUG) self.syslog.addHandler(sh)
def test_unicode_msg(logger): sh = Rfc5424SysLogHandler(address=address) logger.addHandler(sh) message = u"This is a ℛℯα∂α♭ℓℯ message" with patch.object(sh, 'socket') as syslog_socket: logger.info(message) expected = b'<14>1 2000-01-01T17:11:11.111111+06:00 testhostname root 111' \ b' - - \xef\xbb\xbfThis is a \xe2\x84\x9b\xe2\x84\xaf\xce\xb1\xe2\x88\x82' \ b'\xce\xb1\xe2\x99\xad\xe2\x84\x93\xe2\x84\xaf message' syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def __init__(self, conf: Dict[str, Any]): # Keep syslog dependency optionnal by importing at init time from rfc5424logging import Rfc5424SysLogHandler sh: Rfc5424SysLogHandler = Rfc5424SysLogHandler( address=(conf["syslog_server"], conf["syslog_port"]), socktype=getattr(socket, conf["syslog_stream"]), # Use TCP or UDP appname="WPWatcher", **conf["syslog_kwargs"], ) self.syslog = logging.getLogger("wpwatcher-syslog") self.syslog.setLevel(logging.DEBUG) self.syslog.addHandler(sh)
def test_adapter(logger, handler_kwargs, adapter_kwargs, logger_kwargs, expected): sh = Rfc5424SysLogHandler(**handler_kwargs) logger.addHandler(sh) adapter = Rfc5424SysLogAdapter(logger, **adapter_kwargs) with patch.object(sh, 'socket') as syslog_socket: adapter.info(message, **logger_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() adapter.log(logging.INFO, message, **logger_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def test_appname(logger, handler_kwargs, expected): sh = Rfc5424SysLogHandler(handler_kwargs["address"]) logger.addHandler(sh) with patch.object(sh, 'socket') as syslog_socket: logger.info(message, extra=handler_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.log(logging.INFO, message, extra=handler_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logging.info(message, extra=handler_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def test_msgid(logger, logger_kwargs, expected): sh = Rfc5424SysLogHandler(address=address) logger.addHandler(sh) with patch.object(sh.transport, 'socket') as syslog_socket: logger.info(message, **logger_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.log(logging.INFO, message, **logger_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logging.info(message, **logger_kwargs) syslog_socket.sendto.assert_called_once_with(expected, address) syslog_socket.sendto.reset_mock() logger.removeHandler(sh)
def test_procid_tcp(logger, handler_kwargs, expected): sh = Rfc5424SysLogHandler(**handler_kwargs) logger.addHandler(sh) with patch.object(sh, 'socket', side_effect=connect_mock) as syslog_socket: logger.info(message) syslog_socket.sendall.assert_called_once_with(expected) syslog_socket.sendall.reset_mock() logger.log(logging.INFO, message) syslog_socket.sendall.assert_called_once_with(expected) syslog_socket.sendall.reset_mock() logging.info(message) syslog_socket.sendall.assert_called_once_with(expected) syslog_socket.sendall.reset_mock() logger.removeHandler(sh)
def init_logging(logging_configuration: Optional[Dict[str, str]] = None, logging_env_var_start: Optional[str] = None) -> None: """Initialize Thoth's logging - respects all namespaces. This function allows you to control logging facilities in Thoth. Logging can be configured via env variables so that deployment can respect your configuration. The structure of environment variables is THOTH_LOG_(MODULE) and the value of env variable states verbosity level as in the logging module (DEBUG, INFO, WARNING, ERROR). >>> import os >>> os.environ['THOTH_LOG_SOLVER'] WARNING You can also specify more closely which sub-module logging you are configuring - submodules are separated with double dash: >>> os.environ['THOTH_LOG_SOLVER__PYTHON'] DEBUG You can also use arguments explicitly that override configuration in env variables (a shorthand for standard logging functionality): >>> init_logging({'thoth.solver': 'DEBUG'}) Optionally you can specify prefix of the logging environment variable determining logging configuration via env vars (defaults to THOTH_LOG_). """ # TODO: JSON in deployments? # deployed_to_cluster = bool(int(os.getenv('THOTH_CLUSTER_DEPLOYMENT', '0'))) formatter = daiquiri.formatter.ColorFormatter( fmt="%(asctime)s %(process)3d %(color)s%(levelname)-8.8s %(name)s:" "%(lineno)d: %(message)s%(color_stop)s") # Always log in UTC to be consistent with team members all over the world. formatter.converter = time.gmtime daiquiri.setup( level=logging.INFO, outputs=(daiquiri.output.Stream(formatter=formatter), ), ) root_logger = logging.getLogger("thoth.common") environment = os.getenv("SENTRY_ENVIRONMENT", os.getenv("THOTH_DEPLOYMENT_NAME")) # Disable annoying unverified HTTPS request warnings. try: import urllib3 urllib3.disable_warnings() except ImportError: pass _init_log_levels(logging_env_var_start or _DEFAULT_LOGGING_CONF_START, logging_configuration) _logging_adjust() ignored_loggers = os.getenv("THOTH_SENTRY_IGNORE_LOGGER") if ignored_loggers: for logger in ignored_loggers.split(","): ignore_logger(logger) ignored_exceptions = os.getenv("THOTH_SENTRY_IGNORE_EXCEPTION") if ignored_exceptions: exceptions_split = ignored_exceptions.split(',') for exception in exceptions_split: exception_parts = exception.rsplit('.', maxsplit=1) if len(exception_parts) == 2: exc_module, exc_name = exception_parts _IGNORED_EXCEPTIONS.append((exc_module, exc_name)) else: root_logger.error( "The following configuration for ignoring exception couldn't be parsed: %r ", exception) if _SENTRY_DSN: try: integrations = _get_sentry_integrations() root_logger.info( "Setting up logging to a Sentry instance %r, environment %r and integrations %r", _SENTRY_DSN.rsplit("@", maxsplit=1)[1], environment, [ integration.__class__.__name__ for integration in integrations ]) sentry_sdk_init(_SENTRY_DSN, environment=environment, integrations=integrations, before_send=before_send_handler) except Exception: root_logger.exception( "Failed to initialize logging to Sentry instance, check configuration" ) raise if environment is None: root_logger.warning( "No Sentry environment configured: it is recommended to provide Sentry environment " "to split reported exceptions based on different deployments when running in a cluster" ) else: root_logger.warning("Logging to a Sentry instance is turned off") if _RSYSLOG_HOST and _RSYSLOG_PORT: root_logger.info( f"Setting up logging to rsyslog endpoint {_RSYSLOG_HOST}:{_RSYSLOG_PORT}" ) try: syslog_handler = Rfc5424SysLogHandler(address=(_RSYSLOG_HOST, int(_RSYSLOG_PORT))) root_logger.addHandler(syslog_handler) except socket.gaierror as exc: root_logger.exception( f"RSYSLOG_HOST and RSYSLOG_PORT have been set but {_RSYSLOG_HOST}:{_RSYSLOG_PORT} cannot be reached" ) elif int(bool(_RSYSLOG_PORT)) + int(bool(_RSYSLOG_HOST)) == 1: raise RuntimeError( f"Please provide both RSYSLOG_HOST and RSYSLOG_PORT configuration" f"in order to use rsyslog logging, host: {_RSYSLOG_HOST}, port: {_RSYSLOG_PORT}" ) else: root_logger.info("Logging to rsyslog endpoint is turned off")
def init_logging( logging_configuration: Optional[Dict[str, str]] = None, logging_env_var_start: Optional[str] = None, ) -> None: """Initialize Thoth's logging - respects all namespaces. This function allows you to control logging facilities in Thoth. Logging can be configured via env variables so that deployment can respect your configuration. The structure of environment variables is THOTH_LOG_(MODULE) and the value of env variable states verbosity level as in the logging module (DEBUG, INFO, WARNING, ERROR). >>> import os >>> os.environ['THOTH_LOG_SOLVER'] > WARNING You can also specify more closely which sub-module logging you are configuring - submodules are separated with double dash: >>> os.environ['THOTH_LOG_SOLVER__PYTHON'] > DEBUG You can also use arguments explicitly that override configuration in env variables (a shorthand for standard logging functionality): >>> init_logging({'thoth.solver': 'DEBUG'}) Optionally you can specify prefix of the logging environment variable determining logging configuration via env vars (defaults to THOTH_LOG_). """ if not os.getenv("STI_SCRIPTS_PATH") or int( os.getenv("THOTH_LOGGING_NO_JSON", 0)): # Running outside the cluster or forced not to use structured logging. formatter = daiquiri.formatter.ColorFormatter( fmt="%(asctime)s %(process)3d %(color)s%(levelname)-8.8s %(name)s:" "%(lineno)d: %(message)s%(color_stop)s") # Always log in UTC to be consistent with team members all over the world. formatter.converter = time.gmtime daiquiri.setup(level=logging.INFO, outputs=(daiquiri.output.Stream(formatter=formatter), )) else: # The most straightforward way for setting up all the loggers (werkzeug, flask, gunicorn) is to # discard their default configuration and force them to use JSON logger configured for the root logger. logging._acquireLock() # type: ignore for logger in logging.Logger.manager.loggerDict.values( ): # type: ignore if not isinstance(logger, logging.Logger): continue if logger.name == "gunicorn.access": # We configure access log in gunicorn config file, see API deployment # configuration and gunicorn.conf.py file. continue logger.filters.clear() logger.handlers.clear() logger.propagate = True logging._releaseLock() # type: ignore handler = logging.StreamHandler() formatter = JsonFormatter(_JSON_LOGGING_FORMAT) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) logging.getLogger().setLevel(logging.WARNING) logging.getLogger().propagate = False root_logger = logging.getLogger("thoth.common") environment = os.getenv("SENTRY_ENVIRONMENT", os.getenv("THOTH_DEPLOYMENT_NAME")) # Disable annoying unverified HTTPS request warnings. try: import urllib3 urllib3.disable_warnings() except ImportError: pass _init_log_levels(logging_env_var_start or _DEFAULT_LOGGING_CONF_START, logging_configuration) _logging_adjust() ignored_loggers = os.getenv("THOTH_SENTRY_IGNORE_LOGGER") if ignored_loggers: for logger in ignored_loggers.split(","): ignore_logger(logger) ignored_exceptions = os.getenv("THOTH_SENTRY_IGNORE_EXCEPTION") if ignored_exceptions: exceptions_split = ignored_exceptions.split(",") for exception in exceptions_split: exception_parts = exception.rsplit(".", maxsplit=1) if len(exception_parts) == 2: exc_module, exc_name = exception_parts _IGNORED_EXCEPTIONS.append((exc_module, exc_name)) else: root_logger.error( "The following configuration for ignoring exception couldn't be parsed: %r ", exception, ) if _SENTRY_DSN: sentry_sdk_init_kwargs = {} try: import flask # noqa: F401 sentry_sdk_init_kwargs[ "traces_sample_rate"] = _SENTRY_TRACES_SAMPLE_RATE root_logger.info("Setting Sentry's traces sample rate to %f", _SENTRY_TRACES_SAMPLE_RATE) except ImportError: pass try: integrations = _get_sentry_integrations() root_logger.info( "Setting up logging to a Sentry instance %r, environment %r and integrations %r", _SENTRY_DSN.rsplit("@", maxsplit=1)[1], environment, [ integration.__class__.__name__ for integration in integrations ], ) sentry_sdk_init( _SENTRY_DSN, environment=environment, integrations=integrations, before_send=before_send_handler, **sentry_sdk_init_kwargs, ) except Exception: root_logger.exception( "Failed to initialize logging to Sentry instance, check configuration" ) raise if environment is None: root_logger.error( "No Sentry environment configured: it is recommended to provide Sentry environment " "to split reported exceptions based on different deployments when running in a cluster" ) else: root_logger.warning("Logging to a Sentry instance is turned off") if _RSYSLOG_HOST and _RSYSLOG_PORT: root_logger.info( f"Setting up logging to rsyslog endpoint {_RSYSLOG_HOST}:{_RSYSLOG_PORT}" ) try: syslog_handler = Rfc5424SysLogHandler(address=(_RSYSLOG_HOST, int(_RSYSLOG_PORT))) root_logger.addHandler(syslog_handler) except socket.gaierror: root_logger.exception( f"RSYSLOG_HOST and RSYSLOG_PORT have been set but {_RSYSLOG_HOST}:{_RSYSLOG_PORT} cannot be reached" ) elif int(bool(_RSYSLOG_PORT)) + int(bool(_RSYSLOG_HOST)) == 1: raise RuntimeError( f"Please provide both RSYSLOG_HOST and RSYSLOG_PORT configuration" f"in order to use rsyslog logging, host: {_RSYSLOG_HOST}, port: {_RSYSLOG_PORT}" ) else: root_logger.info("Logging to rsyslog endpoint is turned off")
#!/usr/bin/env python3 import logging from logging.handlers import SysLogHandler, QueueHandler, QueueListener from logging import Formatter from queue import SimpleQueue from syslog_rfc5424_formatter import RFC5424Formatter from rfc5424logging import Rfc5424SysLogHandler, NILVALUE import requests import sys logging.basicConfig(level=logging.DEBUG) LOG = logging.getLogger(__name__) #LOG.addHandler(QueueHandler(log_queue)) logs_hostname = sys.argv[1] syslog_handler = Rfc5424SysLogHandler(('127.0.0.1', 514), hostname=logs_hostname) #syslog_handler.setFormatter(RFC5424Formatter(msgid='36b1308f-e0e2-4d4a-ae98-284f51f39a8a')) LOG.addHandler(syslog_handler) LOG.info('this is an info message') LOG.debug('this is a debug message') LOG.info('this is another info message') LOG.debug('and this is another debug message') try: raise RuntimeError('foo') except Exception as ex: LOG.exception(ex) #queue_listener.stop()
def logger_with_udp_handler(logger): sh = Rfc5424SysLogHandler(address=address) logger.addHandler(sh) with patch.object(sh, 'socket') as syslog_socket: yield logger, syslog_socket logger.removeHandler(sh)
def init_logging(logging_configuration: dict = None, logging_env_var_start: str = None) -> None: """Initialize Thoth's logging - respects all namespaces. This function allows you to control logging facilities in Thoth. Logging can be configured via env variables so that deployment can respect your configuration. The structure of environment variables is THOTH_LOG_(MODULE) and the value of env variable states verbosity level as in the logging module (DEBUG, INFO, WARNING, ERROR). >>> import os >>> os.environ['THOTH_LOG_SOLVER'] WARNING You can also specify more closely which sub-module logging you are configuring - submodules are separated with double dash: >>> os.environ['THOTH_LOG_SOLVER__PYTHON'] DEBUG You can also use arguments explicitly that override configuration in env variables (a shorthand for standard logging functionality): >>> init_logging({'thoth.solver': 'DEBUG'}) Optionally you can specify prefix of the logging environment variable determining logging configuration via env vars (defaults to THOTH_LOG_). """ # TODO: JSON in deployments? # deployed_to_cluster = bool(int(os.getenv('THOTH_CLUSTER_DEPLOYMENT', '0'))) daiquiri.setup( level=logging.INFO, outputs= (daiquiri.output.Stream(formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [%(process)d] %(color)s%(levelname)-8.8s %(name)s:" "%(lineno)d: %(message)s%(color_stop)s")), ), ) root_logger = logging.getLogger() environment = os.getenv("SENTRY_ENVIRONMENT", os.getenv("THOTH_DEPLOYMENT_NAME")) # Disable annoying unverified HTTPS request warnings. try: import urllib3 urllib3.disable_warnings() except ImportError: pass _init_log_levels(logging_env_var_start or _DEFAULT_LOGGING_CONF_START, logging_configuration) if _SENTRY_DSN: try: root_logger.info( "Setting up logging to a Sentry instance %r, environment %r", _SENTRY_DSN.rsplit("@", maxsplit=1)[1], environment) sentry_sdk.init(_SENTRY_DSN, environment=environment) except Exception: root_logger.exception( "Failed to initialize logging to Sentry instance, check configuration" ) raise else: root_logger.info("Logging to a Sentry instance is turned off") if _RSYSLOG_HOST and _RSYSLOG_PORT: root_logger.info( f"Setting up logging to rsyslog endpoint {_RSYSLOG_HOST}:{_RSYSLOG_PORT}" ) try: syslog_handler = Rfc5424SysLogHandler(address=(_RSYSLOG_HOST, int(_RSYSLOG_PORT))) root_logger.addHandler(syslog_handler) except socket.gaierror as exc: root_logger.exception( f"RSYSLOG_HOST and RSYSLOG_PORT have been set but {_RSYSLOG_HOST}:{_RSYSLOG_PORT} cannot be reached" ) elif int(bool(_RSYSLOG_PORT)) + int(bool(_RSYSLOG_HOST)) == 1: raise RuntimeError( f"Please provide both RSYSLOG_HOST and RSYSLOG_PORT configuration" f"in order to use rsyslog logging, host: {_RSYSLOG_HOST}, port: {_RSYSLOG_PORT}" ) else: root_logger.info("Logging to rsyslog endpoint is turned off")
import logging import sys import time # We are using a 3rd syslog handler because the default python sysloghandler is not fully RFC5424 complient. # The compliance is a requirement of the in_syslog fluentd plugin. from rfc5424logging import Rfc5424SysLogHandler logger = logging.getLogger('to.syslog.example7') handler = Rfc5424SysLogHandler(address=('127.0.0.1', 5140)) logger.addHandler(handler) logger.setLevel(logging.DEBUG) while True: logger.debug('Logging a debug statement into syslog.') logger.error('Logging an error statement into syslog.') time.sleep(2)
log = logging.getLogger('mylog') log.setLevel(logging.DEBUG) # log_hdlr=SysLogHandler(facility=SysLogHandler.LOG_LOCAL5, address='/dev/log') # log_hdlr=SysLogHandler( # address=('log.u.rizhiyi.com', 5140), # facility=SysLogHandler.LOG_LOCAL5 # ) log_hdlr = SysLogHandler(address=('tbjohufa.u.rizhiyi.com', 5140), facility=SysLogHandler.LOG_LOCAL5) console = StreamHandler() rsyslog = Rfc5424SysLogHandler(address=('log.u.rizhiyi.com', 5140), socktype=socket.SOCK_STREAM, structured_data={rizhiyi_token: { 'tag': 'udp' }}, enterprise_id=32473) log_format = logging.Formatter( 'hhl-%(name)s-server[%(process)d]-%(levelname)s: %(message)s') log_hdlr.setFormatter(log_format) log_hdlr.setLevel(logging.ERROR) console.setFormatter(log_format) console.setLevel(logging.DEBUG) log.addHandler(log_hdlr) log.addHandler(console) log.addHandler(rsyslog)