def configLog(longlog, logLevels):
"""Configure logging system.
Parameters
----------
longlog : bool
If True then make log messages appear in "long format"
logLevels : `list` of `tuple`
per-component logging levels, each item in the list is a tuple
(component, level), `component` is a logger name or `None` for root
logger, `level` is a logging level name ('DEBUG', 'INFO', etc.)
"""
if longlog:
message_fmt = "%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n"
else:
message_fmt = "%c %p: %m%n"
# global logging config
lsstLog.configure_prop(_logProp.format(message_fmt))
# configure individual loggers
for component, level in logLevels:
level = getattr(lsstLog.Log, level.upper(), None)
if level is not None:
logger = lsstLog.Log.getLogger(component or "")
logger.setLevel(level)
def initLog(cls, longlog):
"""Initialize logging. This should only be called once per program
execution. After the first call this will log a warning and return.
If lsst.log is importable, will add its log handler to the python
root logger's handlers.
Parameters
----------
longlog : `bool`
If True, make log messages appear in long format, by default False.
"""
if cls._initialized:
# Unit tests that execute more than one command do end up
# calling this function multiple times in one program execution,
# so do log a debug but don't log an error or fail, just make the
# re-initialization a no-op.
log = logging.getLogger(__name__)
log.debug(
"Log is already initialized, returning without re-initializing."
)
return
cls._initialized = True
if lsstLog is not None:
# Initialize global logging config. Skip if the env var
# LSST_LOG_CONFIG exists. The file it points to would already
# configure lsst.log.
if not os.path.isfile(os.environ.get("LSST_LOG_CONFIG", "")):
lsstLog.configure_prop(
_LOG_PROP.format(cls.lsstLog_longLogFmt
if longlog else cls.lsstLog_normalLogFmt))
cls._recordComponentSetting(None)
pythonLogger = logging.getLogger()
pythonLogger.setLevel(logging.INFO)
cls._lsstLogHandler = lsstLog.LogHandler()
# Forward all Python logging to lsstLog
pythonLogger.addHandler(cls._lsstLogHandler)
else:
cls._recordComponentSetting(None)
if longlog:
logging.basicConfig(level=logging.INFO,
format=cls.pylog_longLogFmt,
datefmt=cls.pylog_longLogDateFmt)
else:
logging.basicConfig(level=logging.INFO,
format=cls.pylog_normalFmt)
# also capture warnings and send them to logging
logging.captureWarnings(True)
# remember this call
cls.configState.append((cls.initLog, longlog))
def initLog(cls, longlog):
"""Initialize logging. This should only be called once per program
execution. After the first call this will log a warning and return.
If lsst.log is importable, will add its log handler to the python
root logger's handlers.
Parameters
----------
longlog : `bool`
If True, make log messages appear in long format, by default False.
"""
if cls._initialized:
# Unit tests that execute more than one command do end up
# calling this fucntion multiple times in one program execution,
# so do log a debug but don't log an error or fail, just make the
# re-initalization a no-op.
log = logging.getLogger(__name__.partition(".")[2])
log.debug(
"Log is already initialized, returning without re-initializing."
)
return
cls._initialized = True
if lsstLog is not None:
# global logging config
lsstLog.configure_prop(
_LOG_PROP.format(
cls.longLogFmt if longlog else cls.normalLogFmt))
cls._recordComponentSetting(None)
pythonLogger = logging.getLogger()
pythonLogger.setLevel(logging.INFO)
cls._lsstLogHandler = lsstLog.LogHandler()
# Forward all Python logging to lsstLog
pythonLogger.addHandler(cls._lsstLogHandler)
else:
cls._recordComponentSetting(None)
logging.basicConfig(level=logging.INFO)
# also capture warnings and send them to logging
logging.captureWarnings(True)
def configure(self, configuration):
"""
Create a configuration file in the temporary directory and populate
it with the provided string.
"""
log.configure_prop(configuration.format(self.outputFilename))
def parse_args(self, config, args=None, log=None, override=None):
"""Parse arguments for a command-line task.
Parameters
----------
config : `lsst.pex.config.Config`
Config for the task being run.
args : `list`, optional
Argument list; if `None` then ``sys.argv[1:]`` is used.
log : `lsst.log.Log`, optional
`~lsst.log.Log` instance; if `None` use the default log.
override : callable, optional
A config override function. It must take the root config object as its only argument and must
modify the config in place. This function is called after camera-specific overrides files are
applied, and before command-line config overrides are applied (thus allowing the user the final
word).
Returns
-------
namespace : `argparse.Namespace`
A `~argparse.Namespace` instance containing fields:
- ``camera``: camera name.
- ``config``: the supplied config with all overrides applied, validated and frozen.
- ``butler``: a `lsst.daf.persistence.Butler` for the data.
- An entry for each of the data ID arguments registered by `add_id_argument`,
the value of which is a `~lsst.pipe.base.DataIdArgument` that includes public elements
``idList`` and ``refList``.
- ``log``: a `lsst.log` Log.
- An entry for each command-line argument, with the following exceptions:
- config is the supplied config, suitably updated.
- configfile, id and loglevel are all missing.
- ``obsPkg``: name of the ``obs_`` package for this camera.
"""
if args is None:
args = sys.argv[1:]
if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
self.print_help()
if len(args) == 1 and args[0] in ("-h", "--help"):
self.exit()
else:
self.exit("%s: error: Must specify input as first argument" %
self.prog)
# Note that --rerun may change namespace.input, but if it does we verify that the
# new input has the same mapper class.
namespace = argparse.Namespace()
namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
if not os.path.isdir(namespace.input):
self.error("Error: input=%r not found" % (namespace.input, ))
namespace.config = config
namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger(
)
mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
namespace.camera = mapperClass.getCameraName()
namespace.obsPkg = mapperClass.getPackageName()
self.handleCamera(namespace)
self._applyInitialOverrides(namespace)
if override is not None:
override(namespace.config)
# Add data ID containers to namespace
for dataIdArgument in self._dataIdArgDict.values():
setattr(namespace, dataIdArgument.name,
dataIdArgument.ContainerClass(level=dataIdArgument.level))
namespace = argparse.ArgumentParser.parse_args(self,
args=args,
namespace=namespace)
del namespace.configfile
self._parseDirectories(namespace)
namespace.template = _fixPath(DEFAULT_INPUT_NAME,
namespace.rawTemplate)
del namespace.rawTemplate
if namespace.clobberOutput:
if namespace.output is None:
self.error(
"--clobber-output is only valid with --output or --rerun")
elif namespace.output == namespace.input:
self.error(
"--clobber-output is not valid when the output and input repos are the same"
)
if os.path.exists(namespace.output):
namespace.log.info(
"Removing output repo %s for --clobber-output",
namespace.output)
shutil.rmtree(namespace.output)
namespace.log.debug("input=%s", namespace.input)
namespace.log.debug("calib=%s", namespace.calib)
namespace.log.debug("output=%s", namespace.output)
namespace.log.debug("template=%s", namespace.template)
obeyShowArgument(namespace.show, namespace.config, exit=False)
# No environment variable or --output or --rerun specified.
if self.requireOutput and namespace.output is None and namespace.rerun is None:
self.error(
"no output directory specified.\n"
"An output directory must be specified with the --output or --rerun\n"
"command-line arguments.\n")
self._makeButler(namespace)
# convert data in each of the identifier lists to proper types
# this is done after constructing the butler, hence after parsing the command line,
# because it takes a long time to construct a butler
self._processDataIds(namespace)
if "data" in namespace.show:
for dataIdName in self._dataIdArgDict.keys():
for dataRef in getattr(namespace, dataIdName).refList:
print("%s dataRef.dataId = %s" %
(dataIdName, dataRef.dataId))
if namespace.show and "run" not in namespace.show:
sys.exit(0)
if namespace.debug:
try:
import debug
assert debug # silence pyflakes
except ImportError:
sys.stderr.write("Warning: no 'debug' module found\n")
namespace.debug = False
del namespace.loglevel
if namespace.longlog:
lsstLog.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.out
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n
""")
del namespace.longlog
namespace.config.validate()
namespace.config.freeze()
return namespace
def configure(self, configuration):
"""
Create a configuration file in the temporary directory and populate
it with the provided string.
"""
log.configure_prop(configuration.format(self.outputFilename))
import os
import tempfile
import lsst.log as log
import lsst.ctrl.events as events
if __name__ == "__main__":
broker = "example.host.com"
topic = "loggingtest"
recv = events.EventReceiver(broker, topic)
confStr = "log4j.rootLogger=TRACE, EA\n"
confStr += "log4j.appender.EA=EventAppender\n"
confStr += "log4j.appender.EA.BROKER=" + broker + "\n"
confStr += "log4j.appender.EA.TOPIC=" + topic + "\n"
tempDir = tempfile.mkdtemp()
outputFileName = os.path.join(tempDir, "log.out")
log.configure_prop(confStr.format(outputFileName))
# test a simple message
#with log.LogContext("component"):
log.trace("this is a trace message")
ev = recv.receiveEvent()
ps = ev.getPropertySet()
print(ps.get("message"))
def parse_args(self, config, args=None, log=None, override=None):
"""!Parse arguments for a pipeline task
@param[in,out] config config for the task being run
@param[in] args argument list; if None use sys.argv[1:]
@param[in] log log (instance lsst.log Log); if None use the default log
@param[in] override a config override function; it must take the root config object
as its only argument and must modify the config in place.
This function is called after camera-specific overrides files are applied, and before
command-line config overrides are applied (thus allowing the user the final word).
@return namespace: an argparse.Namespace containing many useful fields including:
- camera: camera name
- config: the supplied config with all overrides applied, validated and frozen
- butler: a butler for the data
- an entry for each of the data ID arguments registered by add_id_argument(),
the value of which is a DataIdArgument that includes public elements 'idList' and 'refList'
- log: a lsst.log Log
- an entry for each command-line argument, with the following exceptions:
- config is the supplied config, suitably updated
- configfile, id and loglevel are all missing
- obsPkg: name of obs_ package for this camera
"""
if args is None:
args = sys.argv[1:]
if len(args) < 1 or args[0].startswith("-") or args[0].startswith("@"):
self.print_help()
if len(args) == 1 and args[0] in ("-h", "--help"):
self.exit()
else:
self.exit("%s: error: Must specify input as first argument" %
self.prog)
# Note that --rerun may change namespace.input, but if it does we verify that the
# new input has the same mapper class.
namespace = argparse.Namespace()
namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
if not os.path.isdir(namespace.input):
self.error("Error: input=%r not found" % (namespace.input, ))
namespace.config = config
namespace.log = log if log is not None else lsstLog.Log.getDefaultLogger(
)
mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
namespace.camera = mapperClass.getCameraName()
namespace.obsPkg = mapperClass.getPackageName()
self.handleCamera(namespace)
self._applyInitialOverrides(namespace)
if override is not None:
override(namespace.config)
# Add data ID containers to namespace
for dataIdArgument in self._dataIdArgDict.values():
setattr(namespace, dataIdArgument.name,
dataIdArgument.ContainerClass(level=dataIdArgument.level))
namespace = argparse.ArgumentParser.parse_args(self,
args=args,
namespace=namespace)
del namespace.configfile
self._parseDirectories(namespace)
if namespace.clobberOutput:
if namespace.output is None:
self.error(
"--clobber-output is only valid with --output or --rerun")
elif namespace.output == namespace.input:
self.error(
"--clobber-output is not valid when the output and input repos are the same"
)
if os.path.exists(namespace.output):
namespace.log.info(
"Removing output repo %s for --clobber-output",
namespace.output)
shutil.rmtree(namespace.output)
namespace.log.debug("input=%s", namespace.input)
namespace.log.debug("calib=%s", namespace.calib)
namespace.log.debug("output=%s", namespace.output)
obeyShowArgument(namespace.show, namespace.config, exit=False)
# No environment variable or --output or --rerun specified.
if self.requireOutput and namespace.output is None and namespace.rerun is None:
self.error(
"no output directory specified.\n"
"An output directory must be specified with the --output or --rerun\n"
"command-line arguments.\n")
butlerArgs = {} # common arguments for butler elements
if namespace.calib:
butlerArgs = {'mapperArgs': {'calibRoot': namespace.calib}}
if namespace.output:
outputs = {'root': namespace.output, 'mode': 'rw'}
inputs = {'root': namespace.input}
inputs.update(butlerArgs)
outputs.update(butlerArgs)
namespace.butler = dafPersist.Butler(inputs=inputs,
outputs=outputs)
else:
outputs = {'root': namespace.input, 'mode': 'rw'}
outputs.update(butlerArgs)
namespace.butler = dafPersist.Butler(outputs=outputs)
# convert data in each of the identifier lists to proper types
# this is done after constructing the butler, hence after parsing the command line,
# because it takes a long time to construct a butler
self._processDataIds(namespace)
if "data" in namespace.show:
for dataIdName in self._dataIdArgDict.keys():
for dataRef in getattr(namespace, dataIdName).refList:
print("%s dataRef.dataId = %s" %
(dataIdName, dataRef.dataId))
if namespace.show and "run" not in namespace.show:
sys.exit(0)
if namespace.debug:
try:
import debug
assert debug # silence pyflakes
except ImportError:
sys.stderr.write("Warning: no 'debug' module found\n")
namespace.debug = False
del namespace.loglevel
if namespace.longlog:
lsstLog.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.err
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddThh:mm:ss.sss} %c (%X{LABEL})(%F:%L)- %m%n
""")
del namespace.longlog
namespace.config.validate()
namespace.config.freeze()
return namespace
def __init__(self, name, usage="%(prog)s input [options]", **kwargs):
"""!Construct an ArgumentParser
@param[in] name name of top-level task; used to identify camera-specific override files
@param[in] usage usage string
@param[in] **kwargs additional keyword arguments for argparse.ArgumentParser
"""
self._name = name
self._dataIdArgDict = {
} # Dict of data identifier specifications, by argument name
argparse.ArgumentParser.__init__(
self,
usage=usage,
fromfile_prefix_chars='@',
epilog=textwrap.dedent("""Notes:
* --config, --configfile, --id, --loglevel and @file may appear multiple times;
all values are used, in order left to right
* @file reads command-line options from the specified file:
* data may be distributed among multiple lines (e.g. one option per line)
* data after # is treated as a comment and ignored
* blank lines and lines starting with # are ignored
* To specify multiple values for an option, do not use = after the option name:
* right: --configfile foo bar
* wrong: --configfile=foo bar
"""),
formatter_class=argparse.RawDescriptionHelpFormatter,
**kwargs)
self.add_argument(
metavar='input',
dest="rawInput",
help="path to input data repository, relative to $%s" %
(DEFAULT_INPUT_NAME, ))
self.add_argument(
"--calib",
dest="rawCalib",
help="path to input calibration repository, relative to $%s" %
(DEFAULT_CALIB_NAME, ))
self.add_argument(
"--output",
dest="rawOutput",
help=
"path to output data repository (need not exist), relative to $%s"
% (DEFAULT_OUTPUT_NAME, ))
self.add_argument("--rerun",
dest="rawRerun",
metavar="[INPUT:]OUTPUT",
help="rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
"optionally sets ROOT to ROOT/rerun/INPUT")
self.add_argument(
"-c",
"--config",
nargs="*",
action=ConfigValueAction,
help="config override(s), e.g. -c foo=newfoo bar.baz=3",
metavar="NAME=VALUE")
self.add_argument("-C",
"--configfile",
dest="configfile",
nargs="*",
action=ConfigFileAction,
help="config override file(s)")
self.add_argument(
"-L",
"--loglevel",
nargs="*",
action=LogLevelAction,
help=
"logging level; supported levels are [trace|debug|info|warn|error|fatal]",
metavar="LEVEL|COMPONENT=LEVEL")
self.add_argument("--longlog",
action="store_true",
help="use a more verbose format for the logging")
self.add_argument("--debug",
action="store_true",
help="enable debugging output?")
self.add_argument(
"--doraise",
action="store_true",
help=
"raise an exception on error (else log a message and continue)?")
self.add_argument(
"--noExit",
action="store_true",
help=
"Do not exit even upon failure (i.e. return a struct to the calling script)"
)
self.add_argument("--profile",
help="Dump cProfile statistics to filename")
self.add_argument(
"--show",
nargs="+",
default=(),
help="display the specified information to stdout and quit "
"(unless run is specified).")
self.add_argument("-j",
"--processes",
type=int,
default=1,
help="Number of processes to use")
self.add_argument(
"-t",
"--timeout",
type=float,
help="Timeout for multiprocessing; maximum wall time (sec)")
self.add_argument(
"--clobber-output",
action="store_true",
dest="clobberOutput",
default=False,
help=
("remove and re-create the output directory if it already exists "
"(safe with -j, but not all other forms of parallel execution)"))
self.add_argument(
"--clobber-config",
action="store_true",
dest="clobberConfig",
default=False,
help=
("backup and then overwrite existing config files instead of checking them "
"(safe with -j, but not all other forms of parallel execution)"))
self.add_argument("--no-backup-config",
action="store_true",
dest="noBackupConfig",
default=False,
help="Don't copy config to file~N backup.")
self.add_argument(
"--clobber-versions",
action="store_true",
dest="clobberVersions",
default=False,
help=
("backup and then overwrite existing package versions instead of checking"
"them (safe with -j, but not all other forms of parallel execution)"
))
self.add_argument(
"--no-versions",
action="store_true",
dest="noVersions",
default=False,
help="don't check package versions; useful for development")
lsstLog.configure_prop("""
log4j.rootLogger=INFO, A1
log4j.appender.A1=ConsoleAppender
log4j.appender.A1.Target=System.err
log4j.appender.A1.layout=PatternLayout
log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
""")
import os
import tempfile
import lsst.log as log
import lsst.ctrl.events as events
if __name__ == "__main__":
broker = "example.host.com"
topic = "loggingtest"
recv = events.EventReceiver(broker, topic)
confStr = "log4j.rootLogger=TRACE, EA\n"
confStr += "log4j.appender.EA=EventAppender\n"
confStr += "log4j.appender.EA.BROKER="+broker+"\n"
confStr += "log4j.appender.EA.TOPIC="+topic+"\n"
tempDir = tempfile.mkdtemp()
outputFileName = os.path.join(tempDir, "log.out")
log.configure_prop(confStr.format(outputFileName))
# test a simple message
#with log.LogContext("component"):
log.trace("this is a trace message")
ev = recv.receiveEvent()
ps = ev.getPropertySet()
print(ps.get("message"))