def emit(self, level, format=None, **kwargs): if level not in LogLevel.iterconstants(): self.failure( "Got invalid log level {invalidLevel!r} in {logger}.emit().", Failure(InvalidLogLevelError(level)), invalidLevel=level, logger=self, ) return event = kwargs event.update( log_logger=self, log_level=level, log_namespace=self.namespace, log_source=self.source, log_format=format, log_time=time.time(), ) # ---------------------------------8<--------------------------------- # this is a workaround for the mess between twisted's legacy log system # and twistd's --syslog option. event["system"] = "%s#%s" % (self.namespace, level.name) # ---------------------------------8<--------------------------------- if "log_trace" in event: event["log_trace"].append((self, self.observer)) self.observer(event)
class TestObserveTwistedInternetTCP_Other(MAASTestCase): """Tests for `observe_tftp` with non-informational messages.""" scenarios = tuple((log_level.name, { "log_level": log_level }) for log_level in LogLevel.iterconstants() if log_level is not LogLevel.info) def test__propagates_other_events(self): event = make_event(log_level=self.log_level) with TwistedLoggerFixture() as logger: observe_tftp(event) self.assertThat(logger.events, Contains(event)) self.assertThat(event["log_level"], Is(self.log_level))
class TwistOptions(Options): """ Command line options for C{twist}. """ defaultReactorName = "default" defaultLogLevel = LogLevel.info def __init__(self): Options.__init__(self) self["reactorName"] = self.defaultReactorName self["logLevel"] = self.defaultLogLevel self["logFile"] = stdout def getSynopsis(self): return "{} plugin [plugin_options]".format(Options.getSynopsis(self)) def opt_version(self): """ Print version and exit. """ exit(ExitStatus.EX_OK, "{}".format(version)) def opt_reactor(self, name): """ The name of the reactor to use. (options: {options}) """ # Actually actually actually install the reactor right at this very # moment, before any other code (for example, a sub-command plugin) # runs and accidentally imports and installs the default reactor. try: self["reactor"] = self.installReactor(name) except NoSuchReactor: raise UsageError("Unknown reactor: {}".format(name)) else: self["reactorName"] = name opt_reactor.__doc__ = dedent(opt_reactor.__doc__ or "").format( options=", ".join('"{}"'.format(rt.shortName) for rt in getReactorTypes()), ) def installReactor(self, name): """ Install the reactor. """ if name == self.defaultReactorName: from twisted.internet import reactor return reactor else: return installReactor(name) def opt_log_level(self, levelName): """ Set default log level. (options: {options}; default: "{default}") """ try: self["logLevel"] = LogLevel.levelWithName(levelName) except InvalidLogLevelError: raise UsageError("Invalid log level: {}".format(levelName)) opt_log_level.__doc__ = dedent(opt_log_level.__doc__ or "").format( options=", ".join('"{}"'.format(constant.name) for constant in LogLevel.iterconstants()), default=defaultLogLevel.name, ) def opt_log_file(self, fileName): """ Log to file. ("-" for stdout, "+" for stderr; default: "-") """ if fileName == "-": self["logFile"] = stdout return if fileName == "+": self["logFile"] = stderr return try: self["logFile"] = openFile(fileName, "a") except EnvironmentError as e: exit( ExitStatus.EX_IOERR, "Unable to open log file {!r}: {}".format(fileName, e), ) def opt_log_format(self, format): """ Log file format. (options: "text", "json"; default: "text" if the log file is a tty, otherwise "json") """ format = format.lower() if format == "text": self["fileLogObserverFactory"] = textFileLogObserver elif format == "json": self["fileLogObserverFactory"] = jsonFileLogObserver else: raise UsageError("Invalid log format: {}".format(format)) self["logFormat"] = format opt_log_format.__doc__ = dedent(opt_log_format.__doc__ or "") def selectDefaultLogObserver(self): """ Set C{fileLogObserverFactory} to the default appropriate for the chosen C{logFile}. """ if "fileLogObserverFactory" not in self: logFile = self["logFile"] if hasattr(logFile, "isatty") and logFile.isatty(): self["fileLogObserverFactory"] = textFileLogObserver self["logFormat"] = "text" else: self["fileLogObserverFactory"] = jsonFileLogObserver self["logFormat"] = "json" def parseOptions(self, options=None): self.selectDefaultLogObserver() Options.parseOptions(self, options=options) if "reactor" not in self: self["reactor"] = self.installReactor(self["reactorName"]) @property def plugins(self): if "plugins" not in self: plugins = {} for plugin in getPlugins(IServiceMaker): plugins[plugin.tapname] = plugin self["plugins"] = plugins return self["plugins"] @property def subCommands(self): plugins = self.plugins for name in sorted(plugins): plugin = plugins[name] yield ( plugin.tapname, None, # Avoid resolving the options attribute right away, in case # it's a property with a non-trivial getter (eg, one which # imports modules). lambda plugin=plugin: plugin.options(), plugin.description, ) def postOptions(self): Options.postOptions(self) if self.subCommand is None: raise UsageError("No plugin specified.")
class IMSOptions(Options): """ Command line options for all IMS commands. """ log: ClassVar[Logger] = Logger() defaultLogLevel: ClassVar = LogLevel.info subCommands: ClassVar = [ ["server", None, ServerOptions, "Run the IMS server"], ["export", None, ExportOptions, "Export data"], ["import", None, ImportOptions, "Import data"], ["compare", None, CompareOptions, "Compare two export files"], ] # defaultSubCommand = "server" def getSynopsis(self) -> str: return f"{Options.getSynopsis(self)} command [command_options]" def opt_config(self, path: str) -> None: """ Location of configuration file. """ cast(MutableMapping, self)["configFile"] = Path(path) def opt_log_level(self, levelName: str) -> None: """ Set default log level. (options: {options}; default: "{default}") """ try: self["logLevel"] = LogLevel.levelWithName(levelName) except InvalidLogLevelError: raise UsageError(f"Invalid log level: {levelName}") opt_log_level.__doc__ = dedent(cast(str, opt_log_level.__doc__)).format( options=", ".join(f'"{l.name}"' for l in LogLevel.iterconstants()), default=defaultLogLevel.name, ) def opt_log_file(self, fileName: str) -> None: """ Log to file. ("-" for stdout, "+" for stderr; default: "-") """ self["logFileName"] = fileName def opt_log_format(self, logFormatName: str) -> None: """ Log file format. (options: "text", "json"; default: "text" if the log file is a tty, otherwise "json") """ try: logFormat = LogFormat[logFormatName.lower()] except KeyError: raise UsageError(f"Invalid log format: {logFormatName}") if logFormat is LogFormat.text: self["fileLogObserverFactory"] = textFileLogObserver elif logFormat is LogFormat.json: self["fileLogObserverFactory"] = jsonFileLogObserver else: raise AssertionError(f"Unhandled LogFormat: {logFormat}") self["logFormat"] = logFormat opt_log_format.__doc__ = dedent(cast(str, opt_log_format.__doc__)) def opt_option(self, arg: str) -> None: """ Set a configuration option. Format is "[section]name=value", eg: "[Core]Host=0.0.0.0". """ try: if arg.startswith("["): section, rest = arg[1:].split("]", 1) else: section = "Core" rest = arg name, value = rest.split("=", 1) except ValueError: raise UsageError(f"Invalid option specifier: {arg}") if "overrides" not in self: self["overrides"] = [] self["overrides"].append( Override(section=section, name=name, value=value)) def initConfig(self) -> None: try: configFile = cast(Optional[Path], cast(Mapping, self).get("configFile")) if configFile and not configFile.is_file(): self.log.info("Config file not found.") configFile = None configuration = Configuration.fromConfigFile(configFile) options = cast(MutableMapping, self) if "overrides" in options: for _override in options["overrides"]: raise NotImplementedError("Option overrides unimplemented") if "logFileName" in options: configuration = configuration.replace( logFilePath=Path(options["logFileName"])) self.opt_log_file(str(configuration.logFilePath)) if "logFormat" in options: configuration = configuration.replace( logFormat=options["logFormat"]) elif configuration.logFormat is not None: self.opt_log_format(configuration.logFormat.name) if "logLevel" in options: configuration = configuration.replace( logLevelName=options["logLevel"].name) elif configuration.logLevelName is not None: self.opt_log_level(configuration.logLevelName) options["configuration"] = configuration except Exception as e: exit(ExitStatus.EX_CONFIG, str(e)) def initLogFile(self) -> None: self["logFile"] = openFile(self["logFileName"], "a") def selectDefaultLogObserver(self) -> None: """ Set :func:`fileLogObserverFactory` to the default appropriate for the chosen log file. """ if "fileLogObserverFactory" not in self: logFile = self["logFile"] if hasattr(logFile, "isatty") and logFile.isatty(): self["fileLogObserverFactory"] = textFileLogObserver self["logFormat"] = "text" else: self["fileLogObserverFactory"] = jsonFileLogObserver self["logFormat"] = "json" def parseOptions(self, options: Optional[Sequence[str]] = None) -> None: Options.parseOptions(self, options=options) self.initLogFile() self.selectDefaultLogObserver() def postOptions(self) -> None: Options.postOptions(self) self.initConfig()
class TwistOptions(Options): """ Command line options for C{twist}. """ defaultReactorName = "default" defaultLogLevel = LogLevel.info def __init__(self) -> None: Options.__init__(self) self["reactorName"] = self.defaultReactorName self["logLevel"] = self.defaultLogLevel self["logFile"] = stdout # An empty long description is explicitly set here as otherwise # when executing from distributed trial twisted.python.usage will # pull the description from `__main__` which is another entry point. self.longdesc = "" def getSynopsis(self) -> str: return f"{Options.getSynopsis(self)} plugin [plugin_options]" def opt_version(self) -> "typing.NoReturn": """ Print version and exit. """ exit(ExitStatus.EX_OK, f"{version}") def opt_reactor(self, name: str) -> None: """ The name of the reactor to use. (options: {options}) """ # Actually actually actually install the reactor right at this very # moment, before any other code (for example, a sub-command plugin) # runs and accidentally imports and installs the default reactor. try: self["reactor"] = self.installReactor(name) except NoSuchReactor: raise UsageError(f"Unknown reactor: {name}") else: self["reactorName"] = name _update_doc( opt_reactor, options=", ".join(f'"{rt.shortName}"' for rt in getReactorTypes()), ) def installReactor(self, name: str) -> IReactorCore: """ Install the reactor. """ if name == self.defaultReactorName: from twisted.internet import reactor return cast(IReactorCore, reactor) else: return installReactor(name) def opt_log_level(self, levelName: str) -> None: """ Set default log level. (options: {options}; default: "{default}") """ try: self["logLevel"] = LogLevel.levelWithName(levelName) except InvalidLogLevelError: raise UsageError(f"Invalid log level: {levelName}") _update_doc( opt_log_level, options=", ".join( f'"{constant.name}"' for constant in LogLevel.iterconstants() ), default=defaultLogLevel.name, ) def opt_log_file(self, fileName: str) -> None: """ Log to file. ("-" for stdout, "+" for stderr; default: "-") """ if fileName == "-": self["logFile"] = stdout return if fileName == "+": self["logFile"] = stderr return try: self["logFile"] = openFile(fileName, "a") except OSError as e: exit( ExitStatus.EX_IOERR, f"Unable to open log file {fileName!r}: {e}", ) def opt_log_format(self, format: str) -> None: """ Log file format. (options: "text", "json"; default: "text" if the log file is a tty, otherwise "json") """ format = format.lower() if format == "text": self["fileLogObserverFactory"] = textFileLogObserver elif format == "json": self["fileLogObserverFactory"] = jsonFileLogObserver else: raise UsageError(f"Invalid log format: {format}") self["logFormat"] = format _update_doc(opt_log_format) def selectDefaultLogObserver(self) -> None: """ Set C{fileLogObserverFactory} to the default appropriate for the chosen C{logFile}. """ if "fileLogObserverFactory" not in self: logFile = self["logFile"] if hasattr(logFile, "isatty") and logFile.isatty(): self["fileLogObserverFactory"] = textFileLogObserver self["logFormat"] = "text" else: self["fileLogObserverFactory"] = jsonFileLogObserver self["logFormat"] = "json" def parseOptions(self, options: Optional[Sequence[str]] = None) -> None: self.selectDefaultLogObserver() Options.parseOptions(self, options=options) if "reactor" not in self: self["reactor"] = self.installReactor(self["reactorName"]) @property def plugins(self) -> Mapping[str, IServiceMaker]: if "plugins" not in self: plugins = {} for plugin in getPlugins(IServiceMaker): plugins[plugin.tapname] = plugin self["plugins"] = plugins return cast(Mapping[str, IServiceMaker], self["plugins"]) @property def subCommands( self, ) -> Iterable[Tuple[str, None, Callable[[IServiceMaker], Options], str]]: plugins = self.plugins for name in sorted(plugins): plugin = plugins[name] # Don't pass plugin.options along in order to avoid resolving the # options attribute right away, in case it's a property with a # non-trivial getter (eg, one which imports modules). def options(plugin: IServiceMaker = plugin) -> Options: return cast(Options, plugin.options()) yield (plugin.tapname, None, options, plugin.description) def postOptions(self) -> None: Options.postOptions(self) if self.subCommand is None: raise UsageError("No plugin specified.")
event = { "log_format": "{log_text}", "log_level": pick_log_level() if log_level is None else log_level, "log_text": make_log_text() if log_text is None else log_text, "log_time": pick_log_time() if log_time is None else log_time, } event.update(other) return event def make_log_text(): """Make some random log text.""" return factory.make_unicode_string(size=50, spaces=True) _log_levels = tuple(LogLevel.iterconstants()) def pick_log_level(): """Pick a random `LogLevel`.""" return random.choice(_log_levels) def pick_log_time(noise=float(60 * 60)): """Pick a random time based on now, but with some noise.""" return time.time() + (random.random() * noise) - (noise / 2) # Matches lines like: 2016-10-18 14:23:55 namespace: [level] message find_log_lines_re = re.compile( r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) (.*?): \[(.*)\] (.*)$",
class ServerOptions(Options): """ Command line options for the IMS server. """ defaultLogLevel = LogLevel.info def getSynopsis(self) -> str: return f"{Options.getSynopsis(self)} plugin [plugin_options]" def opt_version(self) -> None: """ Print version and exit. """ exit(ExitStatus.EX_OK, f"{version}") def opt_config(self, path: str) -> None: """ Location of configuration file. """ cast(MutableMapping, self)["configFile"] = Path(path) def initConfig(self) -> None: try: configFile = cast(Path, cast(Mapping, self).get("configFile")) if configFile is None: configuration = Configuration(None) else: if not configFile.is_file(): exit(ExitStatus.EX_CONFIG, "Config file not found.") configuration = Configuration(configFile) options = cast(MutableMapping, self) if "overrides" in options: for _override in options["overrides"]: raise NotImplementedError("Option overrides unimplemented") if "logFileName" in options: configuration.LogFilePath = Path(options["logFileName"]) else: self.opt_log_file(str(configuration.LogFilePath)) if "logFormat" in options: configuration.LogFormat = options["logFormat"] elif configuration.LogFormat is not None: self.opt_log_format(configuration.LogFormat) if "logLevel" in options: configuration.LogLevelName = options["logLevel"].name elif configuration.LogLevelName is not None: self.opt_log_level(configuration.LogLevelName) options["configuration"] = configuration except Exception as e: exit(ExitStatus.EX_CONFIG, str(e)) def opt_log_level(self, levelName: str) -> None: """ Set default log level. (options: {options}; default: "{default}") """ try: self["logLevel"] = LogLevel.levelWithName(levelName) except InvalidLogLevelError: raise UsageError(f"Invalid log level: {levelName}") opt_log_level.__doc__ = dedent(cast(str, opt_log_level.__doc__)).format( options=", ".join(f'"{l.name}"' for l in LogLevel.iterconstants()), default=defaultLogLevel.name, ) def opt_log_file(self, fileName: str) -> None: """ Log to file. ("-" for stdout, "+" for stderr; default: "-") """ self["logFileName"] = fileName if fileName == "-": self["logFile"] = stdout return if fileName == "+": self["logFile"] = stderr return try: self["logFile"] = openFile(fileName, "a") except EnvironmentError as e: exit(ExitStatus.EX_IOERR, f"Unable to open log file {fileName!r}: {e}") def opt_log_format(self, logFormat: str) -> None: """ Log file format. (options: "text", "json"; default: "text" if the log file is a tty, otherwise "json") """ logFormat = logFormat.lower() if logFormat == "text": self["fileLogObserverFactory"] = textFileLogObserver elif logFormat == "json": self["fileLogObserverFactory"] = jsonFileLogObserver else: raise UsageError(f"Invalid log format: {logFormat}") self["logFormat"] = logFormat opt_log_format.__doc__ = dedent(cast(str, opt_log_format.__doc__)) def opt_option(self, arg: str) -> None: """ Set a configuration option. Format is "[section]name=value", eg: "[Core]Host=0.0.0.0". """ try: if arg.startswith("["): section, rest = arg[1:].split("]", 1) else: section = "Core" rest = arg name, value = rest.split("=", 1) except ValueError: raise UsageError(f"Invalid option specifier: {arg}") if "overrides" not in self: self["overrides"] = [] self["overrides"].append( Override(section=section, name=name, value=value)) def selectDefaultLogObserver(self) -> None: """ Set :func:`fileLogObserverFactory` to the default appropriate for the chosen log file. """ if "fileLogObserverFactory" not in self: logFile = self["logFile"] if hasattr(logFile, "isatty") and logFile.isatty(): self["fileLogObserverFactory"] = textFileLogObserver self["logFormat"] = "text" else: self["fileLogObserverFactory"] = jsonFileLogObserver self["logFormat"] = "json" def parseOptions(self, options: Optional[Sequence[str]] = None) -> None: Options.parseOptions(self, options=options) self.selectDefaultLogObserver() def postOptions(self) -> None: Options.postOptions(self) self.initConfig()
class Options(TwistedOptions): """ Options. """ def opt_version(self): """ Print version and exit. """ from deps import __version__ as version exit(ExitStatus.EX_OK, "{}".format(version)) def opt_pid_file(self, name): """ File to store process ID in. """ self["pidFile"] = FilePath(name) def opt_kill(self): """ Exit running application. (Requires --pid-file.) """ self["kill"] = True def opt_log_file(self, fileName): """ Log to file. ("-" for stdout, "+" for stderr. default: "+") """ if fileName == "-": self["logFile"] = sys.stdout return if fileName == "+": self["logFile"] = sys.stderr return try: self["logFile"] = open(fileName, "a") self.setdefault("fileLogObserverFactory", jsonFileLogObserver) except EnvironmentError as e: exit( ExitStatus.EX_CANTCREAT, "Unable to open log file {!r}: {}".format(fileName, e) ) def opt_log_format(self, format): """ Set log file format to one of: (text, json). (default: text for stdout/stderr, otherwise json) """ format = format.lower() if format == "text": self["fileLogObserverFactory"] = textFileLogObserver elif format == "json": self["fileLogObserverFactory"] = jsonFileLogObserver else: exit( ExitStatus.EX_USAGE, "Invalid log format: {}".format(format) ) self["logFormat"] = format def opt_log_level(self, levelName): """ Set default log level to one of: {levelNames}. (default: info) """ try: self["logLevel"] = LogLevel.levelWithName(levelName) except InvalidLogLevelError: exit( ExitStatus.EX_USAGE, "Invalid log level: {}".format(levelName) ) # Format the docstring for opt_log_level. opt_log_level.__doc__ = opt_log_level.__doc__.format( levelNames=", ".join([l.name for l in LogLevel.iterconstants()]) )