def configLog(longlog, logLevels): """Configure logging system. Parameters ---------- longlog : bool If True then make log messages appear in "long format" logLevels : `list` of `tuple` per-component logging levels, each item in the list is a tuple (component, level), `component` is a logger name or `None` for root logger, `level` is a logging level name ('DEBUG', 'INFO', etc.) """ if longlog: message_fmt = "%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n" else: message_fmt = "%c %p: %m%n" # global logging config lsstLog.configure_prop(_logProp.format(message_fmt)) # configure individual loggers for component, level in logLevels: level = getattr(lsstLog.Log, level.upper(), None) if level is not None: logger = lsstLog.Log.getLogger(component or "") logger.setLevel(level)
def initLog(cls, longlog): """Initialize logging. This should only be called once per program execution. After the first call this will log a warning and return. If lsst.log is importable, will add its log handler to the python root logger's handlers. Parameters ---------- longlog : `bool` If True, make log messages appear in long format, by default False. """ if cls._initialized: # Unit tests that execute more than one command do end up # calling this function multiple times in one program execution, # so do log a debug but don't log an error or fail, just make the # re-initialization a no-op. log = logging.getLogger(__name__) log.debug( "Log is already initialized, returning without re-initializing." ) return cls._initialized = True if lsstLog is not None: # Initialize global logging config. Skip if the env var # LSST_LOG_CONFIG exists. The file it points to would already # configure lsst.log. if not os.path.isfile(os.environ.get("LSST_LOG_CONFIG", "")): lsstLog.configure_prop( _LOG_PROP.format(cls.lsstLog_longLogFmt if longlog else cls.lsstLog_normalLogFmt)) cls._recordComponentSetting(None) pythonLogger = logging.getLogger() pythonLogger.setLevel(logging.INFO) cls._lsstLogHandler = lsstLog.LogHandler() # Forward all Python logging to lsstLog pythonLogger.addHandler(cls._lsstLogHandler) else: cls._recordComponentSetting(None) if longlog: logging.basicConfig(level=logging.INFO, format=cls.pylog_longLogFmt, datefmt=cls.pylog_longLogDateFmt) else: logging.basicConfig(level=logging.INFO, format=cls.pylog_normalFmt) # also capture warnings and send them to logging logging.captureWarnings(True) # remember this call cls.configState.append((cls.initLog, longlog))
def initLog(cls, longlog): """Initialize logging. This should only be called once per program execution. After the first call this will log a warning and return. If lsst.log is importable, will add its log handler to the python root logger's handlers. Parameters ---------- longlog : `bool` If True, make log messages appear in long format, by default False. """ if cls._initialized: # Unit tests that execute more than one command do end up # calling this fucntion multiple times in one program execution, # so do log a debug but don't log an error or fail, just make the # re-initalization a no-op. log = logging.getLogger(__name__.partition(".")[2]) log.debug( "Log is already initialized, returning without re-initializing." ) return cls._initialized = True if lsstLog is not None: # global logging config lsstLog.configure_prop( _LOG_PROP.format( cls.longLogFmt if longlog else cls.normalLogFmt)) cls._recordComponentSetting(None) pythonLogger = logging.getLogger() pythonLogger.setLevel(logging.INFO) cls._lsstLogHandler = lsstLog.LogHandler() # Forward all Python logging to lsstLog pythonLogger.addHandler(cls._lsstLogHandler) else: cls._recordComponentSetting(None) logging.basicConfig(level=logging.INFO) # also capture warnings and send them to logging logging.captureWarnings(True)
def configure(self, configuration): """ Create a configuration file in the temporary directory and populate it with the provided string. """ log.configure_prop(configuration.format(self.outputFilename))
def parse_args(self, config, args=None, log=None, override=None): """Parse arguments for a command-line task. Parameters ---------- config : `lsst.pex.config.Config` Config for the task being run. args : `list`, optional Argument list; if `None` then ``sys.argv[1:]`` is used. log : `lsst.log.Log`, optional `~lsst.log.Log` instance; if `None` use the default log. override : callable, optional A config override function. It must take the root config object as its only argument and must modify the config in place. This function is called after camera-specific overrides files are applied, and before command-line config overrides are applied (thus allowing the user the final word). Returns ------- namespace : `argparse.Namespace` A `~argparse.Namespace` instance containing fields: - ``camera``: camera name. - ``config``: the supplied config with all overrides applied, validated and frozen. - ``butler``: a `lsst.daf.persistence.Butler` for the data. - An entry for each of the data ID arguments registered by `add_id_argument`, the value of which is a `~lsst.pipe.base.DataIdArgument` that includes public elements ``idList`` and ``refList``. - ``log``: a `lsst.log` Log. - An entry for each command-line argument, with the following exceptions: - config is the supplied config, suitably updated. - configfile, id and loglevel are all missing. - ``obsPkg``: name of the ``obs_`` package for this camera. """ if args is None: args = sys.argv[1:] if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"): self.print_help() if len(args) == 1 and args[0] in ("-h", "--help"): self.exit() else: self.exit("%s: error: Must specify input as first argument" % self.prog) # Note that --rerun may change namespace.input, but if it does we verify that the # new input has the same mapper class. namespace = argparse.Namespace() namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0]) if not os.path.isdir(namespace.input): self.error("Error: input=%r not found" % (namespace.input, )) namespace.config = config namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger( ) mapperClass = dafPersist.Butler.getMapperClass(namespace.input) namespace.camera = mapperClass.getCameraName() namespace.obsPkg = mapperClass.getPackageName() self.handleCamera(namespace) self._applyInitialOverrides(namespace) if override is not None: override(namespace.config) # Add data ID containers to namespace for dataIdArgument in self._dataIdArgDict.values(): setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level)) namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace) del namespace.configfile self._parseDirectories(namespace) namespace.template = _fixPath(DEFAULT_INPUT_NAME, namespace.rawTemplate) del namespace.rawTemplate if namespace.clobberOutput: if namespace.output is None: self.error( "--clobber-output is only valid with --output or --rerun") elif namespace.output == namespace.input: self.error( "--clobber-output is not valid when the output and input repos are the same" ) if os.path.exists(namespace.output): namespace.log.info( "Removing output repo %s for --clobber-output", namespace.output) shutil.rmtree(namespace.output) namespace.log.debug("input=%s", namespace.input) namespace.log.debug("calib=%s", namespace.calib) namespace.log.debug("output=%s", namespace.output) namespace.log.debug("template=%s", namespace.template) obeyShowArgument(namespace.show, namespace.config, exit=False) # No environment variable or --output or --rerun specified. if self.requireOutput and namespace.output is None and namespace.rerun is None: self.error( "no output directory specified.\n" "An output directory must be specified with the --output or --rerun\n" "command-line arguments.\n") self._makeButler(namespace) # convert data in each of the identifier lists to proper types # this is done after constructing the butler, hence after parsing the command line, # because it takes a long time to construct a butler self._processDataIds(namespace) if "data" in namespace.show: for dataIdName in self._dataIdArgDict.keys(): for dataRef in getattr(namespace, dataIdName).refList: print("%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId)) if namespace.show and "run" not in namespace.show: sys.exit(0) if namespace.debug: try: import debug assert debug # silence pyflakes except ImportError: sys.stderr.write("Warning: no 'debug' module found\n") namespace.debug = False del namespace.loglevel if namespace.longlog: lsstLog.configure_prop(""" log4j.rootLogger=INFO, A1 log4j.appender.A1=ConsoleAppender log4j.appender.A1.Target=System.out log4j.appender.A1.layout=PatternLayout log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n """) del namespace.longlog namespace.config.validate() namespace.config.freeze() return namespace
def configure(self, configuration): """ Create a configuration file in the temporary directory and populate it with the provided string. """ log.configure_prop(configuration.format(self.outputFilename))
import os import tempfile import lsst.log as log import lsst.ctrl.events as events if __name__ == "__main__": broker = "example.host.com" topic = "loggingtest" recv = events.EventReceiver(broker, topic) confStr = "log4j.rootLogger=TRACE, EA\n" confStr += "log4j.appender.EA=EventAppender\n" confStr += "log4j.appender.EA.BROKER=" + broker + "\n" confStr += "log4j.appender.EA.TOPIC=" + topic + "\n" tempDir = tempfile.mkdtemp() outputFileName = os.path.join(tempDir, "log.out") log.configure_prop(confStr.format(outputFileName)) # test a simple message #with log.LogContext("component"): log.trace("this is a trace message") ev = recv.receiveEvent() ps = ev.getPropertySet() print(ps.get("message"))
def parse_args(self, config, args=None, log=None, override=None): """!Parse arguments for a pipeline task @param[in,out] config config for the task being run @param[in] args argument list; if None use sys.argv[1:] @param[in] log log (instance lsst.log Log); if None use the default log @param[in] override a config override function; it must take the root config object as its only argument and must modify the config in place. This function is called after camera-specific overrides files are applied, and before command-line config overrides are applied (thus allowing the user the final word). @return namespace: an argparse.Namespace containing many useful fields including: - camera: camera name - config: the supplied config with all overrides applied, validated and frozen - butler: a butler for the data - an entry for each of the data ID arguments registered by add_id_argument(), the value of which is a DataIdArgument that includes public elements 'idList' and 'refList' - log: a lsst.log Log - an entry for each command-line argument, with the following exceptions: - config is the supplied config, suitably updated - configfile, id and loglevel are all missing - obsPkg: name of obs_ package for this camera """ if args is None: args = sys.argv[1:] if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"): self.print_help() if len(args) == 1 and args[0] in ("-h", "--help"): self.exit() else: self.exit("%s: error: Must specify input as first argument" % self.prog) # Note that --rerun may change namespace.input, but if it does we verify that the # new input has the same mapper class. namespace = argparse.Namespace() namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0]) if not os.path.isdir(namespace.input): self.error("Error: input=%r not found" % (namespace.input, )) namespace.config = config namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger( ) mapperClass = dafPersist.Butler.getMapperClass(namespace.input) namespace.camera = mapperClass.getCameraName() namespace.obsPkg = mapperClass.getPackageName() self.handleCamera(namespace) self._applyInitialOverrides(namespace) if override is not None: override(namespace.config) # Add data ID containers to namespace for dataIdArgument in self._dataIdArgDict.values(): setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level)) namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace) del namespace.configfile self._parseDirectories(namespace) if namespace.clobberOutput: if namespace.output is None: self.error( "--clobber-output is only valid with --output or --rerun") elif namespace.output == namespace.input: self.error( "--clobber-output is not valid when the output and input repos are the same" ) if os.path.exists(namespace.output): namespace.log.info( "Removing output repo %s for --clobber-output", namespace.output) shutil.rmtree(namespace.output) namespace.log.debug("input=%s", namespace.input) namespace.log.debug("calib=%s", namespace.calib) namespace.log.debug("output=%s", namespace.output) obeyShowArgument(namespace.show, namespace.config, exit=False) # No environment variable or --output or --rerun specified. if self.requireOutput and namespace.output is None and namespace.rerun is None: self.error( "no output directory specified.\n" "An output directory must be specified with the --output or --rerun\n" "command-line arguments.\n") butlerArgs = {} # common arguments for butler elements if namespace.calib: butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}} if namespace.output: outputs = {'root': namespace.output, 'mode': 'rw'} inputs = {'root': namespace.input} inputs.update(butlerArgs) outputs.update(butlerArgs) namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs) else: outputs = {'root': namespace.input, 'mode': 'rw'} outputs.update(butlerArgs) namespace.butler = dafPersist.Butler(outputs=outputs) # convert data in each of the identifier lists to proper types # this is done after constructing the butler, hence after parsing the command line, # because it takes a long time to construct a butler self._processDataIds(namespace) if "data" in namespace.show: for dataIdName in self._dataIdArgDict.keys(): for dataRef in getattr(namespace, dataIdName).refList: print("%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId)) if namespace.show and "run" not in namespace.show: sys.exit(0) if namespace.debug: try: import debug assert debug # silence pyflakes except ImportError: sys.stderr.write("Warning: no 'debug' module found\n") namespace.debug = False del namespace.loglevel if namespace.longlog: lsstLog.configure_prop(""" log4j.rootLogger=INFO, A1 log4j.appender.A1=ConsoleAppender log4j.appender.A1.Target=System.err log4j.appender.A1.layout=PatternLayout log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n """) del namespace.longlog namespace.config.validate() namespace.config.freeze() return namespace
def __init__(self, name, usage="%(prog)s input [options]", **kwargs): """!Construct an ArgumentParser @param[in] name name of top-level task; used to identify camera-specific override files @param[in] usage usage string @param[in] **kwargs additional keyword arguments for argparse.ArgumentParser """ self._name = name self._dataIdArgDict = { } # Dict of data identifier specifications, by argument name argparse.ArgumentParser.__init__( self, usage=usage, fromfile_prefix_chars='@', epilog=textwrap.dedent("""Notes: * --config, --configfile, --id, --loglevel and @file may appear multiple times; all values are used, in order left to right * @file reads command-line options from the specified file: * data may be distributed among multiple lines (e.g. one option per line) * data after # is treated as a comment and ignored * blank lines and lines starting with # are ignored * To specify multiple values for an option, do not use = after the option name: * right: --configfile foo bar * wrong: --configfile=foo bar """), formatter_class=argparse.RawDescriptionHelpFormatter, **kwargs) self.add_argument( metavar='input', dest="rawInput", help="path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME, )) self.add_argument( "--calib", dest="rawCalib", help="path to input calibration repository, relative to $%s" % (DEFAULT_CALIB_NAME, )) self.add_argument( "--output", dest="rawOutput", help= "path to output data repository (need not exist), relative to $%s" % (DEFAULT_OUTPUT_NAME, )) self.add_argument("--rerun", dest="rawRerun", metavar="[INPUT:]OUTPUT", help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; " "optionally sets ROOT to ROOT/rerun/INPUT") self.add_argument( "-c", "--config", nargs="*", action=ConfigValueAction, help="config override(s), e.g. -c foo=newfoo bar.baz=3", metavar="NAME=VALUE") self.add_argument("-C", "--configfile", dest="configfile", nargs="*", action=ConfigFileAction, help="config override file(s)") self.add_argument( "-L", "--loglevel", nargs="*", action=LogLevelAction, help= "logging level; supported levels are [trace|debug|info|warn|error|fatal]", metavar="LEVEL|COMPONENT=LEVEL") self.add_argument("--longlog", action="store_true", help="use a more verbose format for the logging") self.add_argument("--debug", action="store_true", help="enable debugging output?") self.add_argument( "--doraise", action="store_true", help= "raise an exception on error (else log a message and continue)?") self.add_argument( "--noExit", action="store_true", help= "Do not exit even upon failure (i.e. return a struct to the calling script)" ) self.add_argument("--profile", help="Dump cProfile statistics to filename") self.add_argument( "--show", nargs="+", default=(), help="display the specified information to stdout and quit " "(unless run is specified).") self.add_argument("-j", "--processes", type=int, default=1, help="Number of processes to use") self.add_argument( "-t", "--timeout", type=float, help="Timeout for multiprocessing; maximum wall time (sec)") self.add_argument( "--clobber-output", action="store_true", dest="clobberOutput", default=False, help= ("remove and re-create the output directory if it already exists " "(safe with -j, but not all other forms of parallel execution)")) self.add_argument( "--clobber-config", action="store_true", dest="clobberConfig", default=False, help= ("backup and then overwrite existing config files instead of checking them " "(safe with -j, but not all other forms of parallel execution)")) self.add_argument("--no-backup-config", action="store_true", dest="noBackupConfig", default=False, help="Don't copy config to file~N backup.") self.add_argument( "--clobber-versions", action="store_true", dest="clobberVersions", default=False, help= ("backup and then overwrite existing package versions instead of checking" "them (safe with -j, but not all other forms of parallel execution)" )) self.add_argument( "--no-versions", action="store_true", dest="noVersions", default=False, help="don't check package versions; useful for development") lsstLog.configure_prop(""" log4j.rootLogger=INFO, A1 log4j.appender.A1=ConsoleAppender log4j.appender.A1.Target=System.err log4j.appender.A1.layout=PatternLayout log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n """)
import os import tempfile import lsst.log as log import lsst.ctrl.events as events if __name__ == "__main__": broker = "example.host.com" topic = "loggingtest" recv = events.EventReceiver(broker, topic) confStr = "log4j.rootLogger=TRACE, EA\n" confStr += "log4j.appender.EA=EventAppender\n" confStr += "log4j.appender.EA.BROKER="+broker+"\n" confStr += "log4j.appender.EA.TOPIC="+topic+"\n" tempDir = tempfile.mkdtemp() outputFileName = os.path.join(tempDir, "log.out") log.configure_prop(confStr.format(outputFileName)) # test a simple message #with log.LogContext("component"): log.trace("this is a trace message") ev = recv.receiveEvent() ps = ev.getPropertySet() print(ps.get("message"))