def showAllLogProperties(self, show):
"""control whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
@param show if true, show all the properties when a log message is
printed. If false, don't show them.
"""
Log.getDefaultLog().setShowAll(show)
def showAllLogProperties(self, show):
"""control whether log properties are displayed to the screen. These
include, for example, the DATE (and time) of the message.
@param show if true, show all the properties when a log message is
printed. If false, don't show them.
"""
Log.getDefaultLog().setShowAll(show)
def main():
"execute the launchMospipe script"
logger = Log(Log.getDefaultLog(), "launchMos")
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(-10 * cl.opts.verbosity)
if cl.opts.pipeverb is None:
cl.opts.pipeverb = "trace"
t = filter(lambda x: x.startswith(cl.opts.datatype.lower()),
datatypes.keys())
if len(t) > 1:
raise ValueError("Ambiguous data type name: " + cl.opts.datatype)
if len(t) == 0:
raise ValueError("Unrecognized data type name: "+ cl.opts.datatype)
cl.opts.datatype = datatypes[t[0]]
colls = []
# parse the collection names
if cl.opts.colls is not None:
colls = cl.opts.colls.split(',')
launchMos(cl.args[0], cl.args[1], cl.args[2:], colls, cl.opts, logger)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main():
"""execute the testEventLogger script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
props = {}
if cl.opts.stage:
props["stageId"] = cl.opts.stage
if cl.opts.pipeline:
props["pipeline"] = cl.opts.pipeline
broker = None
if len(cl.args) > 0:
broker = cl.args[0]
input = None
if cl.opts.stdin:
input = sys.stdin
testEventLogger(broker, cl.opts.runid, cl.opts.slice, props,
input, cl.opts.logname, cl.opts.logtopic)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main():
"""execute the watchLogs script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
if cl.opts.inclhosts:
cl.opts.inclhosts = cl.opts.inclhosts.split(',')
if cl.opts.exclhosts:
cl.opts.exclhosts = cl.opts.exclhosts.split(',')
hosts = cl.opts.inclhosts
if not hosts:
hosts = cl.opts.exclhosts
if cl.opts.minimport and cl.opts.maximport and \
cl.opts.minimport > cl.opts.maximport:
raise run.UsageError(
"-m value (%i) should be less than -M value (%i)" %
(cl.opts.minimport, cl.opts.maximport))
watchLogs(cl.args[0], cl.args[1:], cl.opts.sleep,
cl.opts.slice, hosts, not bool(cl.opts.inclhosts),
cl.opts.minimport, cl.opts.maximport)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def testEventLogger(broker, runid, sliceid, props, input=None, logname=None,
logtopic="LSSTLogging"):
"""
test logging through a broker. This will send a single message to a
the logger, and, if input is a list of string or and file input object,
it will also send all input message in order. If broker is None, the
log messages will not got to any event broker, only to the screen.
@param broker the host running the event broker
@param runid the runid to assume for the log (ignored if broker is None)
@param sliceid the sliceid to assume for the log (ignored if broker is None)
@param props a set of properties to attach to all messages
@param input if not None, messages to send, either in the form of
of a list of strings or a file object to read from
@param logname the log name to send messages to
@param logtopic the event topic to use (def: "LSSTLogging")
"""
if broker:
thresh = Log.getDefaultLog().getThreshold()
setEventLogger(broker, runid, sliceid, thresh <= Log.DEBUG, logtopic)
Log.getDefaultLog().setThreshold(thresh)
logger.log(VERB, "Created event logger")
else:
logger.log(VERB, "Messages only going to screen")
logger.setShowAll(Log.getDefaultLog().getThreshold() <= Log.DEBUG)
if logname is None:
logname = "showEvents"
uselog = Log(Log.getDefaultLog(), logname)
for key in props.keys():
uselog.addPreamblePropertyString(key, props[key])
testLogger(uselog, input)
def main():
"""execute the testEventLogger script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
props = {}
if cl.opts.stage:
props["stageId"] = cl.opts.stage
if cl.opts.pipeline:
props["pipeline"] = cl.opts.pipeline
broker = None
if len(cl.args) > 0:
broker = cl.args[0]
input = None
if cl.opts.stdin:
input = sys.stdin
testEventLogger(broker, cl.opts.runid, cl.opts.slice, props, input,
cl.opts.logname, cl.opts.logtopic)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main():
"execute the launchMospipe script"
logger = Log(Log.getDefaultLog(), "launchMos")
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(-10 * cl.opts.verbosity)
if cl.opts.pipeverb is None:
cl.opts.pipeverb = "trace"
t = filter(lambda x: x.startswith(cl.opts.datatype.lower()),
datatypes.keys())
if len(t) > 1:
raise ValueError("Ambiguous data type name: " + cl.opts.datatype)
if len(t) == 0:
raise ValueError("Unrecognized data type name: " +
cl.opts.datatype)
cl.opts.datatype = datatypes[t[0]]
colls = []
# parse the collection names
if cl.opts.colls is not None:
colls = cl.opts.colls.split(',')
launchMos(cl.args[0], cl.args[1], cl.args[2:], colls, cl.opts, logger)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def run():
Log.getDefaultLog()
memId0 = dafBase.Citizen_getNextMemId()
main()
# check for memory leaks
if dafBase.Citizen_census(0, memId0) != 0:
print dafBase.Citizen_census(0, memId0), 'Objects leaked:'
print dafBase.Citizen_census(dafBase.cout, memId0)
def run():
Log.getDefaultLog()
memId0 = dafBase.Citizen_getNextMemId()
main()
# check for memory leaks
if dafBase.Citizen_census(0, memId0) != 0:
print dafBase.Citizen_census(0, memId0), 'Objects leaked:'
print dafBase.Citizen_census(dafBase.cout, memId0)
def setLogThreshold(self, threshold):
"""set the importance threshold for the default log. This and
setDebugVerbosity are different APIs that affect the same underlying
limit that controls how many messages get logged. Normally one
uses one of the predefined values--Log.DEBUG, Log.INFO, Log.WARN, and
Log.FATAL--as input.
@param threshold the minimum importance of the message that will
get printed.
"""
Log.getDefaultLog().setThreshold(threshold)
def setDebugVerbosity(self, verbLimit):
"""set the verbosity of the default log. This and setLogThreshold()
are different APIs that affect the same underlying limit that controls
how many messages get logged.
@param verbLimit debug messages with a verbosity level larger
than this will not be printed. If positive
INFO, WARN, and FATAL messages will also
be printed.
"""
Log.getDefaultLog().setThreshold(-1 * verbLimit)
def setLogThreshold(self, threshold):
"""set the importance threshold for the default log. This and
setDebugVerbosity are different APIs that affect the same underlying
limit that controls how many messages get logged. Normally one
uses one of the predefined values--Log.DEBUG, Log.INFO, Log.WARN, and
Log.FATAL--as input.
@param threshold the minimum importance of the message that will
get printed.
"""
Log.getDefaultLog().setThreshold(threshold)
def setDebugVerbosity(self, verbLimit):
"""set the verbosity of the default log. This and setLogThreshold()
are different APIs that affect the same underlying limit that controls
how many messages get logged.
@param verbLimit debug messages with a verbosity level larger
than this will not be printed. If positive
INFO, WARN, and FATAL messages will also
be printed.
"""
Log.getDefaultLog().setThreshold(-1*verbLimit)
def setLogThreshold(self, level):
"""
set the default message importance threshold to be used for
recording messages. This will value be applied to the default
root (system-wide) logger (or what it will be after logging is
initialized) so that all software components are affected.
"""
if self.log is not None:
Log.getDefaultLog().setThreshold(level)
self.log.log(Log.INFO,
"Upating Root Log Message Threshold to %i" % level)
self.logthresh = level
def setLogThreshold(self, level):
"""
set the default message importance threshold to be used for
recording messages. This will value be applied to the default
root (system-wide) logger (or what it will be after logging is
initialized) so that all software components are affected.
"""
if self.log is not None:
Log.getDefaultLog().setThreshold(level)
self.log.log(Log.INFO,
"Upating Root Log Message Threshold to %i" % level)
self.logthresh = level
def main():
"""execute the showEvents script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
showEvents(cl.args[0], cl.args[1:], cl.opts.sleep)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def __init__(self, persistDir, logger=None, lock=SharedData()):
"""
create an empty blackboard
@param persistDir a path to a directory where the blackboard's
state can persisted.
@param logger the logger to gather error messages
@param lock a SharedData instance to be used to protect this
instance from simultaneous updates by different
threads. If not provided, a default is used.
"""
LockProtected.__init__(self, lock)
# the encompassing persistence directory
self._persistDir = persistDir
parent = os.path.dirname(self._persistDir)
if not os.path.isdir(parent) or not os.path.exists(parent):
raise BlackboardPersistError("Unable to create queue directory: %s: directory not found" % parent)
if not os.path.exists(self._persistDir):
os.mkdir(self._persistDir)
elif not os.path.isdir(self._persistDir):
raise BlackboardAccessError("Queue directory: %s: not a directory"
% self._persistDir)
# the logger to use
if not logger:
logger = Log.getDefaultLog()
self._log = Log(logger, "blackboard")
# prep the queues
self.queues = lock
with self.queues:
# a queue representing available datasets to be processed.
dir = os.path.join(self._persistDir, "dataAvailable")
self.queues.dataAvailable = DataQueue(dir, self._log, lock)
# a queue of datasets that have beend bundled into jobs and queued
# for processing.
dir = os.path.join(self._persistDir, "jobsPossible")
self.queues.jobsPossible = JobQueue(dir, self._log, lock)
# a queue of jobs that are ready to be processed
dir = os.path.join(self._persistDir, "jobsAvailable")
self.queues.jobsAvailable = JobQueue(dir, self._log, lock)
# a queue of jobs that have been taken up by a pipeline and are
# currently being processed.
dir = os.path.join(self._persistDir, "jobsInProgress")
self.queues.jobsInProgress = JobQueue(dir, self._log, lock)
# a queue for jobs that have been completed by a pipeline
dir = os.path.join(self._persistDir, "jobsDone")
self.queues.jobsDone = JobQueue(dir, self._log, lock)
# a queue of pipelines that are ready to accept a job
dir = os.path.join(self._persistDir, "pipelinesReady")
self.queues.pipelinesReady = JobQueue(dir, self._log, lock)
self._dbfail = 0
def __init__(self,
policy,
log=None,
stageId=-1,
eventBroker=None,
sysdata=None):
"""
initialize this stage with the policy that defines the stage and
some contextual system data. Applications normally do not directly
call this constructor. Instead they either construct a Stage subclass
or create a Stage instance using makeStage() or makeStageFromPolicy().
@param policy the policy that will configure the SerialProcessing
and ParallelProcessing
@param log the log object the stage instance should
should use. If not provided, a default will be
used.
@param eventBroker the name of the host where the event broker is
running. If not provided, an eventBroker will
not be available to the stage.
@param sysdata a dictionary of data describing the execution
context. The stage uses this information to
set some of its internal data. See
StageProcessing documentation for datails.
The name provided in the policy will override
the name in this dictionary
"""
if sysdata is None:
sysdata = {}
self.sysdata = sysdata
self.stagePolicy = policy
self.eventBroker = eventBroker
if log is None:
log = Log(Log.getDefaultLog(), "stage")
self.log = log
def __init__(self, policy, log=None, stageId=-1, eventBroker=None,
sysdata=None):
"""
initialize this stage with the policy that defines the stage and
some contextual system data. Applications normally do not directly
call this constructor. Instead they either construct a Stage subclass
or create a Stage instance using makeStage() or makeStageFromPolicy().
@param policy the policy that will configure the SerialProcessing
and ParallelProcessing
@param log the log object the stage instance should
should use. If not provided, a default will be
used.
@param eventBroker the name of the host where the event broker is
running. If not provided, an eventBroker will
not be available to the stage.
@param sysdata a dictionary of data describing the execution
context. The stage uses this information to
set some of its internal data. See
StageProcessing documentation for datails.
The name provided in the policy will override
the name in this dictionary
"""
if sysdata is None:
sysdata = {}
self.sysdata = sysdata
self.stagePolicy = policy
self.eventBroker = eventBroker
if log is None:
log = Log(Log.getDefaultLog(), "stage")
self.log = log
def __init__(self, persistDir, logger=None, runId=None,
fromSubclass=False):
"""
create the JobOffice
@param persistDir the directory where the blackboard should be
persisted
@param logger the logger to use. If not given, the default
will be used.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
"""
self._checkAbstract(fromSubclass, "JobOffice")
threading.Thread.__init__(self, name="JobOffice")
self.setDaemon(False)
self.bb = Blackboard(persistDir)
self.esys = EventSystem.getDefaultEventSystem()
self.runId = runId
self.brokerHost = None
self.brokerPort = None
self.originatorId = self.esys.createOriginatorId()
if logger is None:
logger = Log.getDefaultLog()
self.log = logger # override this in sub-class
self.halt = False
self.finalDatasetSent = False
self.jobOfficeCompletedSent = False
self.stopTopic = "JobOfficeStop"
self.stopThread = None
self.exc = None
def process(self, clipboard):
"""
Execute the needed processing code for this Stage
"""
print 'Python apps.SyncSetupStageParallel process : stageId %i' % self.stageId
print 'Python apps.SyncSetupStageParallel process : _rank %i' % self.rank
root = Log.getDefaultLog()
log = Log(root,
"lsst.pexhexamples.apps.SyncSetupStageParallel.process")
lr = LogRec(log, Log.INFO)
lr << "Retrieving Clipboard"
lr << LogRec.endr
propertySet = dafBase.PropertySet()
propertySet.setInt("sliceRank", self.rank)
propertySet.setString("Level", "Debug")
clipboard.put("rankKey", propertySet)
clipboard.setShared("rankKey", True)
lr = LogRec(log, Log.INFO)
lr << "Posted data to be Shared on Clipboard"
lr << LogRec.endr
def setup(self):
if not self.log:
self.log = Log.getDefaultLog()
self.mylog = Log(self.log, "output")
self.outputDatasetsKey = \
self.policy.getString("inputKeys.outputDatasets")
self.possibleDatasetsKey = \
self.policy.getString("inputKeys.possibleDatasets")
def setup(self):
if not self.log:
self.log = Log.getDefaultLog()
self.mylog = Log(self.log, "output")
self.outputDatasetsKey = \
self.policy.getString("inputKeys.outputDatasets")
self.possibleDatasetsKey = \
self.policy.getString("inputKeys.possibleDatasets")
def setup(self):
if not self.log:
self.log = Log.getDefaultLog()
self.mylog = Log(self.log, "fakeProcess")
self.jobIdentityItem = \
self.policy.getString("inputKeys.jobIdentity")
self.sleeptime = self.policy.getInt("sleep")
self.visitCount = 0
self.failOnVisitN = self.policy.getInt("failIteration")
def __init__(self, fullPath=None):
if fullPath is None:
pDir = os.environ["CAT_DIR"]
if pDir is None:
raise RuntimeError('CAT_DIR env var required')
fullPath = os.path.join(pDir, 'policy/defaultProdCatPolicy.paf')
self.policyObj = pexPolicy.Policy.createPolicy(fullPath)
log = Log(Log.getDefaultLog(), "cat")
log.log(Log.DEBUG, 'Reading policy from %s' % fullPath)
def __init__(self, stageId=-1, policy=None):
"""
Standard Stage initializer.
"""
lsst.pex.harness.Stage.Stage.__init__(self, stageId, policy)
self.mopsLog = Log(Log.getDefaultLog(), 'mops.stage')
if isinstance(self.mopsLog, log.ScreenLog):
self.mopsLog.setScreenVerbose(True)
return
def setup(self):
if not self.log:
self.log = Log.getDefaultLog()
self.mylog = Log(self.log, "fakeProcess")
self.jobIdentityItem = \
self.policy.getString("inputKeys.jobIdentity")
self.sleeptime = self.policy.getInt("sleep")
self.visitCount = 0
self.failOnVisitN = self.policy.getInt("failIteration")
def setEventLogger(broker, runid, sliceid, verbose=False, topic="LSSTLogging"):
topic = events.EventLog.getLoggingTopic()
logger.log(VERB, "using event topic=%s" % topic)
es = events.EventSystem.getDefaultEventSystem()
es.createTransmitter(broker, topic)
events.EventLog.createDefaultLog(runid, sliceid)
deflog = Log.getDefaultLog()
frmtr = logging.IndentedFormatter(verbose)
deflog.addDestination(logging.cout, Log.DEBUG, frmtr)
def setEventLogger(broker, runid, sliceid, verbose=False, topic="LSSTLogging"):
topic = events.EventLog.getLoggingTopic()
logger.log(VERB, "using event topic=%s" % topic)
es = events.EventSystem.getDefaultEventSystem()
es.createTransmitter(broker, topic)
events.EventLog.createDefaultLog(runid, sliceid)
deflog = Log.getDefaultLog()
frmtr = logging.IndentedFormatter(verbose)
deflog.addDestination(logging.cout, Log.DEBUG, frmtr)
def getLogThreshold(self):
"""
return the default message importance threshold being used for
recording messages. The returned value reflects the threshold
associated with the default root (system-wide) logger (or what it will
be after logging is initialized). Some underlying components may
override this threshold.
"""
if self.log is None:
return self.logthresh
else:
return Log.getDefaultLog().getThreshold()
def getLogThreshold(self):
"""
return the default message importance threshold being used for
recording messages. The returned value reflects the threshold
associated with the default root (system-wide) logger (or what it will
be after logging is initialized). Some underlying components may
override this threshold.
"""
if self.log is None:
return self.logthresh
else:
return Log.getDefaultLog().getThreshold()
def postprocess(self, clipboard):
"""
Execute the needed postprocessing code for SyncSetupStage
"""
print 'Python apps.SyncSetupStage postprocess : stageId %i' % self.stageId
print 'Python apps.SyncSetupStage postprocess : _rank %i' % self.rank
root = Log.getDefaultLog()
log = Log(root, "lsst.pexhexamples.apps.SyncSetupStage.postprocess")
lr = LogRec(log, Log.INFO)
lr << "In SyncSetupStage.postprocess"
lr << LogRec.endr
def testEventLogger(broker,
runid,
sliceid,
props,
input=None,
logname=None,
logtopic="LSSTLogging"):
"""
test logging through a broker. This will send a single message to a
the logger, and, if input is a list of string or and file input object,
it will also send all input message in order. If broker is None, the
log messages will not got to any event broker, only to the screen.
@param broker the host running the event broker
@param runid the runid to assume for the log (ignored if broker is None)
@param sliceid the sliceid to assume for the log (ignored if broker is None)
@param props a set of properties to attach to all messages
@param input if not None, messages to send, either in the form of
of a list of strings or a file object to read from
@param logname the log name to send messages to
@param logtopic the event topic to use (def: "LSSTLogging")
"""
if broker:
thresh = Log.getDefaultLog().getThreshold()
setEventLogger(broker, runid, sliceid, thresh <= Log.DEBUG, logtopic)
Log.getDefaultLog().setThreshold(thresh)
logger.log(VERB, "Created event logger")
else:
logger.log(VERB, "Messages only going to screen")
logger.setShowAll(Log.getDefaultLog().getThreshold() <= Log.DEBUG)
if logname is None:
logname = "showEvents"
uselog = Log(Log.getDefaultLog(), logname)
for key in props.keys():
uselog.addPreamblePropertyString(key, props[key])
testLogger(uselog, input)
def _inputSetup(stage):
"""Set up an InputStage."""
if not stage.log:
stage.log = BlockTimingLog(Log.getDefaultLog(),
"pex.harness.iostage.input")
else:
stage.log = BlockTimingLog(stage.log, "iostage.input")
defaultFile = pexPolicy.DefaultPolicyFile("pex_harness",
"InputStageDictionary.paf",
"policy")
defaults = pexPolicy.Policy.createPolicy(defaultFile,
defaultFile.getRepositoryPath())
stage.policy.mergeDefaults(defaults)
if stage.policy.exists('parameters.butler'):
stage.butler = _makeButler(stage.policy)
else:
stage.butler = None
def __init__(self, logger=None, fromSub=False):
"""
As this class is abstract, it should only be executed from a
subclass's constructor, in which case fromSub should be set to
True.
@param logger a logger to use for messages. This will be
passed to each recorder. If null, a
logger will be created as needed.
@param fromSub set this to True to indicate that it is being called
from a subclass constructor. If False (default),
an exception will be raised under the assumption
that one is trying instantiate it directly.
"""
# subclasses may wish to use a different logger name
if not logger:
logger = Log.getDefaultLog()
self._logger = Log(logger, "provenance")
if not fromSub:
raise RuntimeError("Attempt to instantiate abstract class, " +
"ProvenanceRecorder; see class docs")
def preprocess(self, clipboard):
"""
Execute the needed preprocessing code for this Stage
"""
print 'Python apps.SyncSetupStage preprocess : stageId %i' % self.stageId
print 'Python apps.SyncSetupStage preprocess : _rank %i' % self.rank
root = Log.getDefaultLog()
log = Log(root, "lsst.pexhexamples.apps.SyncSetupStage.preprocess")
lr = LogRec(log, Log.INFO)
lr << "Retrieving Clipboard"
lr << LogRec.endr
propertySet = dafBase.PropertySet()
propertySet.setInt("redHerring", self.rank)
clipboard.put("redKey", propertySet)
lr = LogRec(log, Log.INFO)
lr << "Posted data to Clipboard"
lr << LogRec.endr
def __init__(self, logger=None, fromSub=False):
"""
As this class is abstract, it should only be executed from a
subclass's constructor, in which case fromSub should be set to
True.
@param logger a logger to use for messages. This will be
passed to each recorder. If null, a
logger will be created as needed.
@param fromSub set this to True to indicate that it is being called
from a subclass constructor. If False (default),
an exception will be raised under the assumption
that one is trying instantiate it directly.
"""
# subclasses may wish to use a different logger name
if not logger:
logger = Log.getDefaultLog()
self._logger = Log(logger, "provenance")
if not fromSub:
raise RuntimeError("Attempt to instantiate abstract class, " +
"ProvenanceRecorder; see class docs")
def process(self, clipboard):
"""
Execute the needed processing code for this Stage
"""
print 'Python IPDPStage process : _rank %i stageId %d' % (self.rank,
self.stageId)
inputImage = clipboard.get("theProperty")
print "inputImage properties: "
nameList = inputImage.names()
root = Log.getDefaultLog()
log = Log(root, "lsst.pexhexamples.apps.IPDPStageParallel.process")
for name in nameList:
value = inputImage.getString(name)
print name, value
lr = LogRec(log, Log.INFO)
lr << name
lr << value
lr << LogRec.endr
def __init__(self, bbox, wcs, badMaskPlanes, logName="coadd.utils.Coadd"):
"""Create a coadd
@param[in] bbox: bounding box of coadd Exposure with respect to parent (lsst.afw.geom.Box2I):
coadd dimensions = bbox.getDimensions(); xy0 = bbox.getMin()
@param[in] wcs: WCS of coadd exposure (lsst.afw.math.Wcs)
@param[in] badMaskPlanes: mask planes to pay attention to when rejecting masked pixels.
Specify as a collection of names.
badMaskPlanes should always include "NO_DATA".
@param[in] logName: name by which messages are logged
"""
self._log = Log(Log.getDefaultLog(), logName)
self._bbox = bbox
self._wcs = wcs
self._badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)
self._coadd = afwImage.ExposureF(bbox, wcs)
self._weightMap = afwImage.ImageF(bbox)
self._filterDict = dict() # dict of filter name: filter object for all filters seen so far
self._statsControl = afwMath.StatisticsControl()
self._statsControl.setNumSigmaClip(3.0)
self._statsControl.setNumIter(2)
self._statsControl.setAndMask(self._badPixelMask)
def __init__(self,
persistDir,
logger=None,
runId=None,
fromSubclass=False):
"""
create the JobOffice
@param persistDir the directory where the blackboard should be
persisted
@param logger the logger to use. If not given, the default
will be used.
@param runId the run ID to restrict our attention to. If
not None, incoming events will be restricted
to the given runId.
"""
self._checkAbstract(fromSubclass, "JobOffice")
threading.Thread.__init__(self, name="JobOffice")
self.setDaemon(False)
self.bb = Blackboard(persistDir)
self.esys = EventSystem.getDefaultEventSystem()
self.runId = runId
self.brokerHost = None
self.brokerPort = None
self.originatorId = self.esys.createOriginatorId()
if logger is None:
logger = Log.getDefaultLog()
self.log = logger # override this in sub-class
self.halt = False
self.finalDatasetSent = False
self.jobOfficeCompletedSent = False
self.stopTopic = "JobOfficeStop"
self.stopThread = None
self.exc = None
def testBigXy0(self):
# test for ticket #2710
from lsst.pex.logging import Log
log = Log.getDefaultLog()
log.setThreshold(Log.DEBUG);
self.astrom.log = log
x0,y0 = 200000, 500000
cx = 500
a2 = 1e-5
cat = afwTable.SourceCatalog.readFits(self.filename)
print 'Catalog size', len(cat)
# Source x,y positions are ~ (500,1500) x (500,1500)
xKey = cat.table.getCentroidKey().getX()
yKey = cat.table.getCentroidKey().getY()
for src in cat:
x = src.get(xKey) - 500
dx = x - cx
x += a2 * (dx**2)
src.set(xKey, x + x0)
src.set(yKey, src.get(yKey) - 500 + y0)
bbox = afwGeom.Box2I(afwGeom.Point2I(x0, y0), afwGeom.Extent2I(1000, 1000))
res = self.astrom.determineWcs2(cat, bbox=bbox)
self.assertIsNotNone(res.sipWcs, "Failed to fit SIP terms")
print 'Got result', res
print 'SIP:', res.sipWcs.getFitsMetadata().toString()
wcs = res.wcs
for src in cat:
rd = wcs.pixelToSky(src.getCentroid())
xy = wcs.skyToPixel(rd)
#print 'src', src.getX(), src.getY()
#print 'rd', rd
#print 'xy', xy
#print 'dx,dy', xy[0] - src.getX(), xy[1] - src.getY()
self.assertLess(abs(xy[0] - src.getX()), 0.1)
self.assertLess(abs(xy[1] - src.getY()), 0.1)
pyPipeline = MpiPipeline(runId, policyFile, name)
if isinstance(logthresh, int):
pyPipeline.setLogThreshold(logthresh)
pyPipeline.initializeLogger()
pyPipeline.configurePipeline()
pyPipeline.initializeQueues()
pyPipeline.initializeStages()
pyPipeline.startShutdownThread()
pyPipeline.startSlices()
pyPipeline.startStagesLoop()
if (__name__ == '__main__'):
try:
main()
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
except Exception, e:
log = Log(Log.getDefaultLog(), "runPipeline")
log.log(Log.FATAL, str(e))
traceback.print_exc(file=sys.stderr)
sys.exit(2)
cl.add_option("-r", "--runid", action="store", default=None, dest="runid",
help="the runid the pipelines were launched under")
cl.add_option("-f", "--tell-fail", action="store_true", default=False,
dest="fail",
help="when applicable, set the success flag to False")
cl.add_option("-i", "--id-delim", action="store", dest="iddelim", default=" ",
help="for dataset events, the delimiters look for to separate the dataset IDs")
cl.add_option("-j", "--job-identity", action="store", dest="identity",
help="the identifiers and values that define the job being processed, in dataset format (for assign command)")
cl.add_option("-O", "--output-dataset", action="append", dest="outputs",
help="an output dataset, in dataset format; may appear multiple times (for assign command)")
cl.add_option("-o", "--orig-id", action="store", type="long", default=0L,
dest="origid",
help="for assign, the originator id to send to")
logger = Log(Log.getDefaultLog(), "sendevent")
def main():
(cl.opts, cl.args) = cl.parse_args()
if len(cl.args) < 1:
fail("missing event type to send (ready|assign|dataset|done)")
if len(cl.args) < 2:
fail("missing event topic to send on")
if not cl.opts.name:
cl.opts.name = 'pipeline'
warn("No pipeline name given; using '%s'", cl.opts.name)
if cl.opts.runid is None:
cl.opts.runid = "unkn_run"
def createLog():
log = Log(Log.getDefaultLog(), "dc2pipe.ensureMpdConf")
return log
def test1(self):
res = self.getAstrometrySolution()
matches = res.matches
print 'Test1'
logLevel = Log.DEBUG
log = Log(Log.getDefaultLog(),
'testPhotoCal',
logLevel)
schema = matches[0].second.schema
config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
config.fluxField = "base_PsfFlux_flux"
config.doWriteOutput = False # schema is fixed because we already loaded the data
task = PhotoCalTask(config=config, schema=schema)
pCal = task.run(exposure=self.exposure, matches=matches)
print "Ref flux fields list =", pCal.arrays.refFluxFieldList
refFluxField = pCal.arrays.refFluxFieldList[0]
# These are *all* the matches; we don't really expect to do that well.
diff=[]
for m in matches:
refFlux = m[0].get(refFluxField) # reference catalog flux
if refFlux <= 0:
continue
refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag
instFlux = m[1].getPsfFlux() #Instrumental Flux
if instFlux <= 0:
continue
instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag
diff.append(instMag - refMag)
diff = np.array(diff)
self.assertGreater(len(diff), 50)
log.info('%i magnitude differences; mean difference %g; mean abs diff %g' %
(len(diff), np.mean(diff), np.mean(np.abs(diff))))
self.assertLess(np.mean(diff), 0.6)
# Differences of matched objects that were used in the fit.
zp = pCal.calib.getMagnitude(1.)
log.logdebug('zeropoint: %g' % zp)
fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag
log.logdebug('number of sources used in fit: %i' % len(fitdiff))
log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5)
log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff)))
# zeropoint: 31.3145
# number of sources used in fit: 65
# median diff: -0.009681
# mean diff: 0.00331871
# median abs(diff): 0.0368904
# mean abs(diff): 0.0516589
self.assertLess(abs(zp - 31.3145), 0.05)
self.assertGreater(len(fitdiff), 50)
# Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and
# are not based on we'd expect to get given the data quality.
self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference
self.assertLess(np.median(np.abs(fitdiff)), 0.06) # median absolution difference
from Orbit import Orbit
import numpy
import auton
from lsst.daf.base import DateTime
import lsst.daf.persistence as dafPer
from lsst.pex.logging import endr, Log, Rec
import time
# Constants/globals.
RIDICOLOUSLY_VERBOSE = False
EXTRA_RIDICOLOUSLY_VERBOSE = False
MOPS_LOG = Log(Log.getDefaultLog(), 'mops.stage')
def logit(msg):
return(Rec(MOPS_LOG, Log.INFO) << msg << endr)
def selectOrbitsForFOV(dbLogicalLocation,
sliceId,
numSlices,
fovRA,
fovDec,
fovR,
mjd):
"""
def createLog():
log = Log(Log.getDefaultLog(), "harness.launchPipeline")
return log
# Import the rest of the packages.
from Orbit import Orbit
import numpy
import auton
from lsst.daf.base import DateTime
import lsst.daf.persistence as dafPer
from lsst.pex.logging import endr, Log, Rec
import time
# Constants/globals.
RIDICOLOUSLY_VERBOSE = False
EXTRA_RIDICOLOUSLY_VERBOSE = False
MOPS_LOG = Log(Log.getDefaultLog(), 'mops.stage')
def logit(msg):
return (Rec(MOPS_LOG, Log.INFO) << msg << endr)
def selectOrbitsForFOV(dbLogicalLocation, sliceId, numSlices, fovRA, fovDec,
fovR, mjd):
"""
Select from the orbit database those orbits that, at t=MJD, intersect the
FOV (field of view) specified by (fovRA, fovDec) and whose size is given by
fovR (which is the half width of the smallest circle enclosing the actual
FOV).
"""
if (RIDICOLOUSLY_VERBOSE):
def createLog():
log = Log(Log.getDefaultLog(), "harness.run.launchPipeline")
return log
def run():
Log.getDefaultLog()
main()
import unittest
import numpy as np
import lsst.meas.astrom as measAstrom
import lsst.afw.geom as afwGeom
import lsst.afw.table as afwTable
import lsst.afw.image as afwImage
import lsst.utils.tests as utilsTests
from lsst.pex.logging import Log
from lsst.pipe.tasks.photoCal import PhotoCalTask, PhotoCalConfig
import testFindAstrometryNetDataDir as helper
# Quiet down meas_astrom logging, so we can see PhotoCal logs better
Log(Log.getDefaultLog(), "meas.astrom.astrometry_net", Log.WARN)
Log(Log.getDefaultLog(), "meas.astrom.sip", Log.WARN)
Log(Log.getDefaultLog(), "astrometricSolver", Log.WARN)
class PhotoCalTest(unittest.TestCase):
def setUp(self):
self.conf = measAstrom.AstrometryConfig()
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(
os.path.join(testDir, "data", "v695833-e0-c000.xy.fits"))
self.srcCat["slot_ApFlux_fluxSigma"] = 1
self.srcCat["slot_PsfFlux_fluxSigma"] = 1
def setup(self):
if not self.log:
self.log = Log.getDefaultLog()
self.mylog = Log(self.log, "inputStage")
self.inputDatasetKey = \
self.policy.getString("inputKeys.inputDatasets")
def test1(self):
res = self.getAstrometrySolution()
matches = res.matches
print 'Test1'
logLevel = Log.DEBUG
log = Log(Log.getDefaultLog(), 'testPhotoCal', logLevel)
schema = matches[0].second.schema
config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
config.fluxField = "base_PsfFlux_flux"
config.doWriteOutput = False # schema is fixed because we already loaded the data
task = PhotoCalTask(config=config, schema=schema)
pCal = task.run(exposure=self.exposure, matches=matches)
print "Ref flux fields list =", pCal.arrays.refFluxFieldList
refFluxField = pCal.arrays.refFluxFieldList[0]
# These are *all* the matches; we don't really expect to do that well.
diff = []
for m in matches:
refFlux = m[0].get(refFluxField) # reference catalog flux
if refFlux <= 0:
continue
refMag = afwImage.abMagFromFlux(refFlux) # reference catalog mag
instFlux = m[1].getPsfFlux() #Instrumental Flux
if instFlux <= 0:
continue
instMag = pCal.calib.getMagnitude(instFlux) #Instrumental mag
diff.append(instMag - refMag)
diff = np.array(diff)
self.assertGreater(len(diff), 50)
log.info(
'%i magnitude differences; mean difference %g; mean abs diff %g' %
(len(diff), np.mean(diff), np.mean(np.abs(diff))))
self.assertLess(np.mean(diff), 0.6)
# Differences of matched objects that were used in the fit.
zp = pCal.calib.getMagnitude(1.)
log.logdebug('zeropoint: %g' % zp)
fitdiff = pCal.arrays.srcMag + zp - pCal.arrays.refMag
log.logdebug('number of sources used in fit: %i' % len(fitdiff))
log.logdebug('rms diff: %g' % np.mean(fitdiff**2)**0.5)
log.logdebug('median abs(diff): %g' % np.median(np.abs(fitdiff)))
# zeropoint: 31.3145
# number of sources used in fit: 65
# median diff: -0.009681
# mean diff: 0.00331871
# median abs(diff): 0.0368904
# mean abs(diff): 0.0516589
self.assertLess(abs(zp - 31.3145), 0.05)
self.assertGreater(len(fitdiff), 50)
# Tolerances are somewhat arbitrary; they're set simply to avoid regressions, and
# are not based on we'd expect to get given the data quality.
self.assertLess(np.mean(fitdiff**2)**0.5, 0.07) # rms difference
self.assertLess(np.median(np.abs(fitdiff)),
0.06) # median absolution difference
cl.add_option("-T",
"--log-topic",
action="store",
type="str",
default=events.EventLog.getLoggingTopic(),
dest="logtopic",
metavar="topic",
help="event topic name (def: 'LSSTLogging')")
cl.add_option("-i",
"--read-stdin",
action="store_true",
default=False,
dest="stdin",
help="read messages from standard input")
logger = Log(Log.getDefaultLog(), "showEvents")
VERB = logger.INFO - 2
def main():
"""execute the testEventLogger script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
props = {}
if cl.opts.stage:
props["stageId"] = cl.opts.stage
if cl.opts.pipeline:
import unittest
import numpy as np
import lsst.meas.astrom as measAstrom
import lsst.afw.geom as afwGeom
import lsst.afw.table as afwTable
import lsst.afw.image as afwImage
import lsst.utils.tests as utilsTests
from lsst.pex.logging import Log
from lsst.pipe.tasks.photoCal import PhotoCalTask, PhotoCalConfig
import testFindAstrometryNetDataDir as helper
# Quiet down meas_astrom logging, so we can see PhotoCal logs better
Log(Log.getDefaultLog(), "meas.astrom.astrometry_net", Log.WARN)
Log(Log.getDefaultLog(), "meas.astrom.sip", Log.WARN)
Log(Log.getDefaultLog(), "astrometricSolver", Log.WARN)
class PhotoCalTest(unittest.TestCase):
def setUp(self):
self.conf = measAstrom.AstrometryConfig()
# Load sample input from disk
testDir=os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(os.path.join(testDir, "data", "v695833-e0-c000.xy.fits"))
self.srcCat["slot_ApFlux_fluxSigma"] = 1
self.srcCat["slot_PsfFlux_fluxSigma"] = 1
# The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
