def testForwardToPython(self):
"""Test that `lsst.log` log messages can be forwarded to `logging`."""
log.configure()
# Without forwarding we only get python logger messages captured
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log warning message that will not be forwarded to Python")
logging.warning("Python logging message that will be captured")
self.assertEqual(len(cm.output), 1)
log.usePythonLogging()
# With forwarding we get 2 logging messages captured
with self.assertLogs(level="WARNING") as cm:
log.warn("This is a warning from lsst log meant for python logging")
logging.warning("Python warning log message to be captured")
self.assertEqual(len(cm.output), 2)
loggername = "newlogger"
log2 = log.Log.getLogger(loggername)
with self.assertLogs(level="INFO", logger=loggername):
log2.info("Info message to non-root lsst logger")
# Check that debug and info are working properly
# This test should return a single log message
with self.assertLogs(level="INFO", logger=loggername) as cm:
log2.info("Second INFO message to non-root lsst logger")
log.debug("Debug message to root lsst logger")
self.assertEqual(len(cm.output), 1, f"Got output: {cm.output}")
logging.shutdown()
def test_readInFits(self):
credFile = os.path.expanduser('~/.mysqlAuthLSST')
if not os.path.isfile(credFile):
log.warn("Required file with credentials '%s' not found.", credFile)
return
testFile = ("./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz")
self.assertTrue(isFitsExt('stuf.fits'))
self.assertFalse(isFitsExt('thing.txt'))
self.assertFalse(isFitsExt('item.tx.gz'))
self.assertTrue(isFitsExt(testFile))
self.assertTrue(isFits(testFile))
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
# test a specific file
self.assertFalse(metadataFits.isFileInDb(testFile))
metadataFits.insertFile(testFile)
log.info(metadataFits.showColumnsInTables())
self.assertTrue(metadataFits.isFileInDb(testFile))
# test crawler
rootDir = '~/test_md'
rootDir = os.path.expanduser(rootDir)
if not os.path.exists(rootDir):
log.error("Data directory {} is required".format(rootDir))
return
directoryCrawl(rootDir, metadataFits)
def test_readInFits(self):
credFile = os.path.expanduser('~/.mysqlAuthLSST')
if not os.path.isfile(credFile):
log.warn("Required file with credentials '%s' not found.",
credFile)
return
testFile = (
"./tests/testData/imsim_886258731_R33_S21_C12_E000.fits.gz")
self.assertTrue(isFitsExt('stuf.fits'))
self.assertFalse(isFitsExt('thing.txt'))
self.assertFalse(isFitsExt('item.tx.gz'))
self.assertTrue(isFitsExt(testFile))
self.assertTrue(isFits(testFile))
# Destroy existing tables and re-create them
dbDestroyCreate(credFile, "DELETE")
# Open a connection to the database.
metadataFits = MetadataFitsDb(credFile)
# test a specific file
self.assertFalse(metadataFits.isFileInDb(testFile))
metadataFits.insertFile(testFile)
log.info(metadataFits.showColumnsInTables())
self.assertTrue(metadataFits.isFileInDb(testFile))
# test crawler
rootDir = '~/test_md'
rootDir = os.path.expanduser(rootDir)
if not os.path.exists(rootDir):
log.error("Data directory {} is required".format(rootDir))
return
directoryCrawl(rootDir, metadataFits)
def testRedir(self):
"""
Test redirection to stream.
"""
with TestRedir.StdoutCapture(self.outputFilename):
log.configure()
dest = io.StringIO()
log_utils.enable_notebook_logging(dest)
log.log(log.getDefaultLogger().getName(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log_utils.disable_notebook_logging()
log.warn("Format %d %g %s", 3, 2.71828, "foo")
self.assertEqual(
dest.getvalue(),
"""root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
""",
)
self.check(
"""
root WARN: Format 3 2.71828 foo
"""
)
def testBasic(self):
"""
Test basic log output with default configuration.
Since the default threshold is INFO, the DEBUG or TRACE
message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.log(log.getDefaultLogger(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.critical("This is CRITICAL")
log.warning("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
root FATAL: This is CRITICAL
root WARN: Format 3 2.71828 foo
""")
def testForwardToPython(self):
"""Test that `lsst.log` log messages can be forwarded to `logging`."""
log.configure()
# Without forwarding we only get python logger messages captured
with self.assertLogs(level="WARNING") as cm:
log.warn(
"lsst.log warning message that will not be forwarded to Python"
)
logging.warning("Python logging message that will be captured")
self.assertEqual(len(cm.output), 1)
log.usePythonLogging()
# With forwarding we get 2 logging messages captured
with self.assertLogs(level="WARNING") as cm:
log.warn(
"This is a warning from lsst log meant for python logging")
logging.warning("Python warning log message to be captured")
self.assertEqual(len(cm.output), 2)
loggername = "newlogger"
log2 = log.Log.getLogger(loggername)
with self.assertLogs(level="INFO", logger=loggername):
log2.info("Info message to non-root lsst logger")
# Check that debug and info are working properly
# This test should return a single log message
with self.assertLogs(level="INFO", logger=loggername) as cm:
log2.info("Second INFO message to non-root lsst logger")
log.debug("Debug message to root lsst logger")
self.assertEqual(len(cm.output), 1, f"Got output: {cm.output}")
logging.shutdown()
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False):
"""Return an image of the specified ccd, and also the (possibly updated) ccd"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage")
if self.isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
im = None
if self.butler is not None:
err = None
for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]:
try:
im = self.butler.get(self.type, dataId, **self.kwargs)
except FitsError as e: # no point trying another dataId
err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error
break
except Exception as e: # try a different dataId
if err is None:
err = e
continue
else:
ccd = im.getDetector() # possibly modified by assembleCcdTask
break
if im:
if asMaskedImage:
im = im.getMaskedImage()
else:
im = im.getMaskedImage().getImage()
else:
if self.verbose:
# Lost by jupyterlab.
print(f"Reading {ccd.getId()}: {err}")
log.warn(f"Reading {ccd.getId()}: {err}")
if im is None:
return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd
if self.type == "raw":
if hasattr(im, 'convertF'):
im = im.convertF()
if False and self.callback is None: # we need to trim the raw image
self.callback = rawCallback
allowRotate = True
if self.callback:
try:
im = self.callback(im, ccd, imageSource=self)
except Exception as e:
if self.verbose:
log.error(f"callback failed: {e}")
im = imageFactory(*bbox.getDimensions())
else:
allowRotate = False # the callback was responsible for any rotations
return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def main():
args = _Parser().parse_args()
log = lsst.log.Log.getLogger("add_gen3_repo")
# To convert consistently, don't use any previous output
dataset = ap_verify.dataset.Dataset(args.dataset)
gen3_repo = os.path.join(dataset.datasetRoot, "preloaded")
if os.path.exists(gen3_repo):
log.warn("Clearing out %s and making it from scratch...", gen3_repo)
shutil.rmtree(gen3_repo)
os.makedirs(gen3_repo)
mode = "copy" if args.drop_gen2 else "relsymlink"
log.info("Converting templates...")
gen2_templates = dataset.templateLocation
_migrate_gen2_to_gen3(dataset, gen2_templates, None, gen3_repo, mode,
config_file="convertRepo_templates.py")
log.info("Converting calibs...")
with tempfile.TemporaryDirectory() as tmp:
workspace = ap_verify.workspace.Workspace(tmp)
ap_verify.ingestion.ingestDataset(dataset, workspace)
gen2_repo = workspace.dataRepo
gen2_calibs = workspace.calibRepo
# Files stored in the Gen 2 part of the dataset, can be safely linked
_migrate_gen2_to_gen3(dataset, gen2_repo, gen2_calibs, gen3_repo, mode,
config_file="convertRepo_calibs.py")
# Our refcats and defects are temporary files, and must not be linked
_migrate_gen2_to_gen3(dataset, gen2_repo, gen2_calibs, gen3_repo, mode="copy",
config_file="convertRepo_copied.py")
log.info("Exporting Gen 3 registry to configure new repos...")
_export_for_copy(dataset, gen3_repo)
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1, asMaskedImage=False):
"""Return an image of the specified ccd, and also the (possibly updated) ccd"""
log = lsst.log.Log.getLogger("afw.cameraGeom.utils.ButlerImage")
if self.isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
im = None
if self.butler is not None:
err = None
for dataId in [dict(detector=ccd.getId()), dict(ccd=ccd.getId()), dict(ccd=ccd.getName())]:
try:
im = self.butler.get(self.type, dataId, **self.kwargs)
except FitsError as e: # no point trying another dataId
err = IOError(e.args[0].split('\n')[0]) # It's a very chatty error
break
except Exception as e: # try a different dataId
if err is None:
err = e
continue
else:
ccd = im.getDetector() # possibly modified by assembleCcdTask
break
if im:
if asMaskedImage:
im = im.getMaskedImage()
else:
im = im.getMaskedImage().getImage()
else:
if self.verbose:
print("Reading %s: %s" % (ccd.getId(), err)) # lost by jupyterLab
log.warn("Reading %s: %s", ccd.getId(), err)
if im is None:
return self._prepareImage(ccd, imageFactory(*bbox.getDimensions()), binSize), ccd
if self.type == "raw":
if hasattr(im, 'convertF'):
im = im.convertF()
if False and self.callback is None: # we need to trim the raw image
self.callback = rawCallback
allowRotate = True
if self.callback:
try:
im = self.callback(im, ccd, imageSource=self)
except Exception as e:
if self.verbose:
log.error("callback failed: %s" % e)
im = imageFactory(*bbox.getDimensions())
else:
allowRotate = False # the callback was responsible for any rotations
return self._prepareImage(ccd, im, binSize, allowRotate=allowRotate), ccd
def validate(self):
CoaddBaseTask.ConfigClass.validate(self)
if not self.makePsfMatched and not self.makeDirect:
raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
if self.doPsfMatch:
# Backwards compatibility.
log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
self.makePsfMatched = True
self.makeDirect = False
def validate(self):
CoaddBaseTask.ConfigClass.validate(self)
if not self.makePsfMatched and not self.makeDirect:
raise RuntimeError("At least one of config.makePsfMatched and config.makeDirect must be True")
if self.doPsfMatch:
# Backwards compatibility.
log.warn("Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False")
self.makePsfMatched = True
self.makeDirect = False
def runPipeline(self, graph, butler, args):
"""
Parameters
----------
graph : `QuantumGraph`
Execution graph.
butler : `Butler`
data butler instance
args : `argparse.Namespace`
Parsed command line
"""
# how many processes do we want
numProc = args.processes
# pre-flight check
for taskNodes in graph:
taskDef, quanta = taskNodes.taskDef, taskNodes.quanta
task = self.taskFactory.makeTask(taskDef.taskClass, taskDef.config,
None, butler)
if not self.precall(task, butler, args):
# non-zero means failure
return 1
if numProc > 1 and not taskDef.taskClass.canMultiprocess:
lsstLog.warn(
"Task %s does not support multiprocessing; using one process",
taskDef.taskName)
numProc = 1
# chose map function being simple sequential map or multi-process map
if numProc > 1:
timeout = getattr(args, 'timeout', None)
if timeout is None or timeout <= 0:
timeout = self.MP_TIMEOUT
mapFunc = _MPMap(numProc, timeout)
else:
def _mapFunc(func, iterable):
"""Call function for all items sequentially"""
return [func(item) for item in iterable]
mapFunc = _mapFunc
# tasks are executed sequentially but quanta can run in parallel
for taskNodes in graph:
taskDef, quanta = taskNodes.taskDef, taskNodes.quanta
# targets for map function
target_list = [(taskDef.taskClass, taskDef.config, quantum, butler)
for quantum in quanta]
# call task on each argument in a list
profile_name = getattr(args, "profile", None)
with util.profile(profile_name, lsstLog):
mapFunc(self._executeSuperTask, target_list)
def validate(self):
pexConfig.Config.validate(self)
if not self.makePsfMatched and not self.makeDirect:
raise RuntimeError(
"At least one of config.makePsfMatched and config.makeDirect must be True"
)
if self.doPsfMatch:
# Courtesy backwards compatibility.
# Configs do not have loggers
log.warn(
"Config doPsfMatch deprecated. Setting makePsfMatched=True and makeDirect=False"
)
self.makePsfMatched = True
self.makeDirect = False
def testForwardToPythonAppenderWithMDC(self):
"""Test that `log4cxx` appender forwards it all to logging and modifies
message with MDC info"""
self.configure("""
log4j.rootLogger=DEBUG, PyLog
log4j.appender.PyLog = PyLogAppender
log4j.appender.PyLog.MessagePattern = %m (LABEL=%X{{LABEL}})
""")
log.MDC("LABEL", "some.task")
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded")
log.MDCRemove("LABEL")
self.assertEqual(len(cm.records), 1)
self.assertEqual(cm.records[0].MDC, {"LABEL": "some.task"})
self.assertEqual(cm.records[0].msg,
"lsst.log: forwarded (LABEL=some.task)")
def makeWcs(self, visitInfo, detector):
"""Create a SkyWcs from information about the exposure.
If VisitInfo is not None, use it and the detector to create a SkyWcs,
otherwise return the metadata-based SkyWcs (always created, so that
the relevant metadata keywords are stripped).
Parameters
----------
visitInfo : `~lsst.afw.image.VisitInfo`
The information about the telescope boresight and camera
orientation angle for this exposure.
detector : `~lsst.afw.cameraGeom.Detector`
The detector used to acquire this exposure.
Returns
-------
skyWcs : `~lsst.afw.geom.SkyWcs`
Reversible mapping from pixel coordinates to sky coordinates.
Raises
------
InitialSkyWcsError
Raised if there is an error generating the SkyWcs, chained from the
lower-level exception if available.
"""
if not self.isOnSky():
# This is not an on-sky observation
return None
skyWcs = self._createSkyWcsFromMetadata()
log = lsst.log.Log.getLogger("fitsRawFormatter")
if visitInfo is None:
msg = "No VisitInfo; cannot access boresight information. Defaulting to metadata-based SkyWcs."
log.warn(msg)
if skyWcs is None:
raise InitialSkyWcsError(
"Failed to create both metadata and boresight-based SkyWcs."
"See warnings in log messages for details.")
return skyWcs
return self.makeRawSkyWcsFromBoresight(
visitInfo.getBoresightRaDec(), visitInfo.getBoresightRotAngle(),
detector)
def testForwardToPythonAppender(self):
"""Test that `log4cxx` appender forwards it all to logging"""
self.configure("""
log4j.rootLogger=DEBUG, PyLog
log4j.appender.PyLog = PyLogAppender
""")
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded")
logging.warning("Python logging: also captured")
self.assertEqual(len(cm.output), 2)
# check that MDC is stored in LogRecord
log.MDC("LABEL", "some.task")
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded")
log.MDCRemove("LABEL")
self.assertEqual(len(cm.records), 1)
self.assertEqual(cm.records[0].MDC, {"LABEL": "some.task"})
self.assertEqual(cm.records[0].msg, "lsst.log: forwarded")
def _createSkyWcsFromMetadata(self):
"""Create a SkyWcs from the FITS header metadata in an Exposure.
Returns
-------
skyWcs: `lsst.afw.geom.SkyWcs`, or None
The WCS that was created from ``self.metadata``, or None if that
creation fails due to invalid metadata.
"""
if not self.isOnSky():
# This is not an on-sky observation
return None
try:
return lsst.afw.geom.makeSkyWcs(self.metadata, strip=True)
except TypeError as e:
log = lsst.log.Log.getLogger("fitsRawFormatter")
log.warn("Cannot create a valid WCS from metadata: %s", e.args[0])
return None
def testForwardToPythonContextManager(self):
"""Test that `lsst.log` log messages can be forwarded to `logging`
using context manager"""
log.configure()
# Without forwarding we only get python logger messages captured
with self.assertLogs(level="WARNING") as cm:
log.warning("lsst.log: not forwarded")
logging.warning("Python logging: captured")
self.assertEqual(len(cm.output), 1)
# Temporarily turn on forwarding
with log.UsePythonLogging():
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded")
logging.warning("Python logging: also captured")
self.assertEqual(len(cm.output), 2)
# Verify that forwarding is disabled
self.assertFalse(log.Log.UsePythonLogging)
def testForwardToPythonContextManager(self):
"""Test that `lsst.log` log messages can be forwarded to `logging`
using context manager"""
log.configure()
# Without forwarding we only get python logger messages captured
with self.assertLogs(level="WARNING") as cm:
log.warning("lsst.log: not forwarded")
logging.warning("Python logging: captured")
self.assertEqual(len(cm.output), 1)
# Temporarily turn on forwarding
with log.UsePythonLogging():
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded")
logging.warning("Python logging: also captured")
self.assertEqual(len(cm.output), 2)
# Verify that forwarding is disabled
self.assertFalse(log.Log.UsePythonLogging)
def testBasic(self):
"""
Test basic log output. Since the default threshold is INFO, the
TRACE message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.trace("This is TRACE")
log.info("This is INFO")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.info("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
INFO root null - This is INFO
DEBUG root null - This is DEBUG
WARN root null - This is WARN
ERROR root null - This is ERROR
FATAL root null - This is FATAL
INFO root null - Format 3 2.71828 foo
""")
def main():
args = _Parser().parse_args()
log = lsst.log.Log.getLogger("add_gen3_repo")
# To convert consistently, don't use any previous output
dataset = ap_verify.dataset.Dataset("test")
gen3_repo = os.path.join(dataset.datasetRoot, "preloaded")
if os.path.exists(gen3_repo):
log.warn("Clearing out %s and making it from scratch...", gen3_repo)
shutil.rmtree(gen3_repo)
os.makedirs(gen3_repo)
mode = "copy" if args.drop_gen2 else "relsymlink"
log.info("Converting calibs...")
with tempfile.TemporaryDirectory() as tmp:
workspace = ap_verify.workspace.WorkspaceGen2(tmp)
ap_verify.ingestion.ingestDataset(dataset, workspace)
gen2_repo = workspace.dataRepo
gen2_calibs = workspace.calibRepo
# Files stored in the Gen 2 part of the dataset, can be safely linked
_migrate_gen2_to_gen3(dataset, gen2_repo, gen2_calibs, gen3_repo, mode,
curated=True,
config_file="convertRepo_calibs.py")
# Our refcats and defects are temporary files, and must not be linked
_migrate_gen2_to_gen3(dataset, gen2_repo, gen2_calibs, gen3_repo, mode="copy",
curated=False,
config_file="convertRepo_copied.py")
# ap_verify assumes specific collections are present
log.info("Adding unpopulated collections...")
butler = daf_butler.Butler(gen3_repo, writeable=True)
butler.registry.registerCollection("skymaps", daf_butler.CollectionType.RUN)
butler.registry.registerCollection("templates/deep", daf_butler.CollectionType.RUN)
log.info("Exporting Gen 3 registry to configure new repos...")
_export_for_copy(dataset, gen3_repo)
def testBasic(self):
"""
Test basic log output with default configuration.
Since the default threshold is INFO, the DEBUG or TRACE
message is not emitted.
"""
with TestLog.StdoutCapture(self.outputFilename):
log.configure()
log.log(log.getDefaultLoggerName(), log.INFO, "This is INFO")
log.info(u"This is unicode INFO")
log.trace("This is TRACE")
log.debug("This is DEBUG")
log.warn("This is WARN")
log.error("This is ERROR")
log.fatal("This is FATAL")
log.warn("Format %d %g %s", 3, 2.71828, "foo")
self.check("""
root INFO: This is INFO
root INFO: This is unicode INFO
root WARN: This is WARN
root ERROR: This is ERROR
root FATAL: This is FATAL
root WARN: Format 3 2.71828 foo
""")
def testForwardToPythonAppenderFormatMDC(self):
"""Test that we can format `log4cxx` MDC on Python side"""
# remember old record factory
old_factory = logging.getLogRecordFactory()
# configure things using convenience method
log.configure_pylog_MDC("INFO")
with self.assertLogs(level="WARNING") as cm:
log.warn("lsst.log: forwarded 1")
log.MDC("LABEL", "task1")
log.warn("lsst.log: forwarded 2")
log.MDC("LABEL-X", "task2")
log.warn("lsst.log: forwarded 3")
logging.warning("Python logging: also captured")
log.MDCRemove("LABEL")
log.MDCRemove("LABEL-X")
self.assertEqual(len(cm.records), 4)
# restore factory
logging.setLogRecordFactory(old_factory)
# %-style formatting, only works on whole MDC
formatter = logging.Formatter(fmt="%(levelname)s:%(name)s:%(message)s",
style="%")
self.assertEqual(formatter.format(cm.records[0]),
"WARNING:root:lsst.log: forwarded 1")
formatter = logging.Formatter(
fmt="%(levelname)s:%(name)s:%(message)s:%(MDC)s", style="%")
self.assertEqual(formatter.format(cm.records[0]),
"WARNING:root:lsst.log: forwarded 1:{}")
self.assertEqual(formatter.format(cm.records[1]),
"WARNING:root:lsst.log: forwarded 2:{LABEL=task1}")
self.assertEqual(
formatter.format(cm.records[2]),
"WARNING:root:lsst.log: forwarded 3:{LABEL=task1, LABEL-X=task2}")
self.assertEqual(formatter.format(cm.records[3]),
"WARNING:root:Python logging: also captured:{}")
# format-style formatting, without MDC first
formatter = logging.Formatter(fmt="{levelname}:{name}:{message}",
style="{")
self.assertEqual(formatter.format(cm.records[0]),
"WARNING:root:lsst.log: forwarded 1")
# format-style formatting, with full MDC
formatter = logging.Formatter(fmt="{levelname}:{name}:{message}:{MDC}",
style="{")
self.assertEqual(formatter.format(cm.records[0]),
"WARNING:root:lsst.log: forwarded 1:{}")
self.assertEqual(formatter.format(cm.records[1]),
"WARNING:root:lsst.log: forwarded 2:{LABEL=task1}")
self.assertEqual(
formatter.format(cm.records[2]),
"WARNING:root:lsst.log: forwarded 3:{LABEL=task1, LABEL-X=task2}")
self.assertEqual(formatter.format(cm.records[3]),
"WARNING:root:Python logging: also captured:{}")
# format-style, using index access to MDC items, works for almost any
# item names
formatter = logging.Formatter(
fmt="{levelname}:{name}:{message}:{MDC[LABEL-X]}", style="{")
self.assertEqual(formatter.format(cm.records[0]),
"WARNING:root:lsst.log: forwarded 1:")
self.assertEqual(formatter.format(cm.records[1]),
"WARNING:root:lsst.log: forwarded 2:")
self.assertEqual(formatter.format(cm.records[2]),
"WARNING:root:lsst.log: forwarded 3:task2")
self.assertEqual(formatter.format(cm.records[3]),
"WARNING:root:Python logging: also captured:")
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
matchKernelAmplitudes=False, keepPlots=True):
"""Plot the PSF spatial model."""
log = lsst.log.Log.getLogger("utils.plotPsfSpatialModel")
try:
import matplotlib.pyplot as plt
import matplotlib as mpl
except ImportError as e:
log.warn("Unable to import matplotlib: %s", e)
return
noSpatialKernel = psf.getKernel()
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
if not showBadCandidates and cand.isBad():
continue
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception:
continue
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * k.getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
zGood = numpy.array(candFits)
xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
zBad = numpy.array(badFits)
numBad = len(badPos)
xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)
kernel = psf.getKernel()
nKernelComponents = kernel.getNKernelParameters()
#
# Figure out how many panels we'll need
#
nPanelX = int(math.sqrt(nKernelComponents))
nPanelY = nKernelComponents//nPanelX
while nPanelY*nPanelX < nKernelComponents:
nPanelX += 1
fig = plt.figure(1)
fig.clf()
try:
fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
except Exception: # protect against API changes
pass
#
# Generator for axes arranged in panels
#
mpl.rcParams["figure.titlesize"] = "x-small"
subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)
for k in range(nKernelComponents):
func = kernel.getSpatialFunction(k)
dfGood = zGood[:, k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
yMin = dfGood.min()
yMax = dfGood.max()
if numBad > 0:
dfBad = zBad[:, k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
yMin = min([yMin, dfBad.min()])
yMax = max([yMax, dfBad.max()])
yMin -= 0.05 * (yMax - yMin)
yMax += 0.05 * (yMax - yMin)
yMin = -0.01
yMax = 0.01
fRange = numpy.ndarray((len(xRange), len(yRange)))
for j, yVal in enumerate(yRange):
for i, xVal in enumerate(xRange):
fRange[j][i] = func(xVal, yVal)
ax = next(subplots)
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getHeight())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(yGood, dfGood, 'b+')
if numBad > 0:
ax.plot(yBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('Residuals(y)')
ax = next(subplots)
if matchKernelAmplitudes and k == 0:
vmin = 0.0
vmax = 1.1
else:
vmin = fRange.min()
vmax = fRange.max()
norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
ax.set_title('Spatial poly')
plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
ax = next(subplots)
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(xGood, dfGood, 'b+')
if numBad > 0:
ax.plot(xBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('K%d Residuals(x)' % k)
ax = next(subplots)
photoCalib = exposure.getPhotoCalib()
# If there is no calibration factor, use 1.0.
if photoCalib.getCalibrationMean() <= 0:
photoCalib = afwImage.PhotoCalib(1.0)
ampMag = [photoCalib.instFluxToMagnitude(candAmp) for candAmp in candAmps]
ax.plot(ampMag, zGood[:, k], 'b+')
if numBad > 0:
badAmpMag = [photoCalib.instFluxToMagnitude(badAmp) for badAmp in badAmps]
ax.plot(badAmpMag, zBad[:, k], 'r+')
ax.set_title('Flux variation')
fig.show()
global keptPlots
if keepPlots and not keptPlots:
# Keep plots open when done
def show():
print("%s: Please close plots when done." % __name__)
try:
plt.show()
except Exception:
pass
print("Plots closed, exiting...")
import atexit
atexit.register(show)
keptPlots = True
def makeSubplots(fig, nx=2, ny=2, Nx=1, Ny=1, plottingArea=(0.1, 0.1, 0.85, 0.80),
pxgutter=0.05, pygutter=0.05, xgutter=0.04, ygutter=0.04,
headroom=0.0, panelBorderWeight=0, panelColor='black'):
"""Return a generator of a set of subplots, a set of Nx*Ny panels of nx*ny plots. Each panel is fully
filled by row (starting in the bottom left) before the next panel is started. If panelBorderWidth is
greater than zero a border is drawn around each panel, adjusted to enclose the axis labels.
E.g.
subplots = makeSubplots(fig, 2, 2, Nx=1, Ny=1, panelColor='k')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (0,0)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (1,0)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (0,1)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (1,1)')
fig.show()
@param fig The matplotlib figure to draw
@param nx The number of plots in each row of each panel
@param ny The number of plots in each column of each panel
@param Nx The number of panels in each row of the figure
@param Ny The number of panels in each column of the figure
@param plottingArea (x0, y0, x1, y1) for the part of the figure containing all the panels
@param pxgutter Spacing between columns of panels in units of (x1 - x0)
@param pygutter Spacing between rows of panels in units of (y1 - y0)
@param xgutter Spacing between columns of plots within a panel in units of (x1 - x0)
@param ygutter Spacing between rows of plots within a panel in units of (y1 - y0)
@param headroom Extra spacing above each plot for e.g. a title
@param panelBorderWeight Width of border drawn around panels
@param panelColor Colour of border around panels
"""
log = lsst.log.Log.getLogger("utils.makeSubplots")
try:
import matplotlib.pyplot as plt
except ImportError as e:
log.warn("Unable to import matplotlib: %s", e)
return
# Make show() call canvas.draw() too so that we know how large the axis labels are. Sigh
try:
fig.__show
except AttributeError:
fig.__show = fig.show
def myShow(fig):
fig.__show()
fig.canvas.draw()
import types
fig.show = types.MethodType(myShow, fig)
#
# We can't get the axis sizes until after draw()'s been called, so use a callback Sigh^2
#
axes = {} # all axes in all the panels we're drawing: axes[panel][0] etc.
#
def on_draw(event):
"""
Callback to draw the panel borders when the plots are drawn to the canvas
"""
if panelBorderWeight <= 0:
return False
for p in axes.keys():
bboxes = []
for ax in axes[p]:
bboxes.append(ax.bbox.union([label.get_window_extent() for label in
ax.get_xticklabels() + ax.get_yticklabels()]))
ax = axes[p][0]
# this is the bbox that bounds all the bboxes, again in relative
# figure coords
bbox = ax.bbox.union(bboxes)
xy0, xy1 = ax.transData.inverted().transform(bbox)
x0, y0 = xy0
x1, y1 = xy1
w, h = x1 - x0, y1 - y0
# allow a little space around BBox
x0 -= 0.02*w
w += 0.04*w
y0 -= 0.02*h
h += 0.04*h
h += h*headroom
# draw BBox
ax.patches = [] # remove old ones
rec = ax.add_patch(plt.Rectangle((x0, y0), w, h, fill=False,
lw=panelBorderWeight, edgecolor=panelColor))
rec.set_clip_on(False)
return False
fig.canvas.mpl_connect('draw_event', on_draw)
#
# Choose the plotting areas for each subplot
#
x0, y0 = plottingArea[0:2]
W, H = plottingArea[2:4]
w = (W - (Nx - 1)*pxgutter - (nx*Nx - 1)*xgutter)/float(nx*Nx)
h = (H - (Ny - 1)*pygutter - (ny*Ny - 1)*ygutter)/float(ny*Ny)
#
# OK! Time to create the subplots
#
for panel in range(Nx*Ny):
axes[panel] = []
px = panel%Nx
py = panel//Nx
for window in range(nx*ny):
x = nx*px + window%nx
y = ny*py + window//nx
ax = fig.add_axes((x0 + xgutter + pxgutter + x*w + (px - 1)*pxgutter + (x - 1)*xgutter,
y0 + ygutter + pygutter + y*h + (py - 1)*pygutter + (y - 1)*ygutter,
w, h), frame_on=True, facecolor='w')
axes[panel].append(ax)
yield ax
def doSuperTask(self, args, extra_args):
"""Implementation of run/show for SuperTask.
Parameters
----------
args : `argparse.Namespace`
Parsed command line
extra_args : `list` of `str`
extra arguments for sub-command which were not parsed by parser
"""
if args.do_help:
# before displaying help populate sub-parser with task-specific options
self._copyParserOptions(self.task_class.makeArgumentParser(),
args.subparser)
args.subparser.print_help()
return
if args.subcommand not in ("run", "show"):
print("unexpected command {}".format(args.subcommand),
file=sys.stderr)
return 2
# parse remaining extra options
task_args = self._reParseArgs(args, extra_args)
# do all --show first
# currently task parser handles that, we have to implement something
# similar when we get rid of that
if args.subcommand == "show":
# stop here
return
# make task instance
task = self._makeSuperTask(task_args.butler)
# how many processed do we want
numProc = task_args.processes
if numProc > 1 and not self.task_class.canMultiprocess:
lsstLog.warn(
"This task does not support multiprocessing; using one process"
)
numProc = 1
# execute it
if self.precall(task, task_args):
# chose map function being simple sequential map or multi-process map
if numProc > 1:
timeout = getattr(task_args, 'timeout', None)
if timeout is None or timeout <= 0:
timeout = self.MP_TIMEOUT
mapFunc = _MPMap(numProc, timeout)
else:
# map in Py3 returns iterable and we want a complete result
mapFunc = lambda func, iterable: list(map(func, iterable))
log = task_args.log
target_list = self._makeTargetList(task_args)
if target_list:
# call task on each argument in a list
profile_name = getattr(task_args, "profile", None)
with profile(profile_name, log):
mapFunc(self._executeSuperTask, target_list)
else:
log.warn(
"Not running the task because there is no data to process; "
"you may preview data using \"--show-data\"")
def run(self):
credFileName = "~/.lsst/dbAuth-dbServ.ini"
engine = getEngineFromFile(credFileName)
dbName = "{}_fitsTest".format(engine.url.username)
metaDb = MetadataFitsDb(credFileName)
resp = None
try:
resp = self.client.search(WATCH_FOLDER, version="current", site="all",
query="scanStatus = 'UNSCANNED'", max_num=1000)
except DcException as error:
if hasattr(error, "message"):
log.warn("Error occurred:\nMessage: %s", error.message)
if hasattr(error, "type"):
log.warn("Type: %s", error.type)
if hasattr(error, "cause"):
log.warn("Cause: %s", error.cause)
else:
# Should have content
log.warn(error.content)
sys.exit(1)
results = unpack(resp.content)
for dataset in results:
locations = dataset.locations
check_location = None
for location in locations:
if location.site == WATCH_SITE:
check_location = location
break
file_path = check_location.resource
dataset_path = dataset.path
stat = os.stat(file_path)
cksum = self.get_cksum(file_path)
# Note: While there may only be one version of a dataset,
# we tie the metadata to versionMetadata
scan_result = {}
scan_result["size"] = stat.st_size
scan_result["checksum"] = str(cksum)
# UTC datetime in ISO format (Note: We need Z to denote UTC Time Zone)
scan_result["locationScanned"] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
scan_result["scanStatus"] = "OK"
md = self.get_metadata(file_path)
if md:
scan_result["versionMetadata"] = md
try:
log.debug("patch_resp %s", str(file_path))
patch_resp = self.client.patch_dataset(dataset_path, scan_result,
versionId=dataset.versionId, site=WATCH_SITE)
log.debug("Inserting %s", str(file_path))
fileId = metaDb.insertFile(file_path)
metadata = {"fileId":fileId}
md_patch = {}
md_patch["versionMetadata"] = metadata
md_patch_resp = self.client.patch_dataset(dataset_path, md_patch,
versionId=dataset.versionId)
log.info("Inserted %d %s", fileId, str(file_path))
except DcException as err:
log.warn("Encountered error while updating dataset %s", str(file_path), err)
def makeSubplots(fig, nx=2, ny=2, Nx=1, Ny=1, plottingArea=(0.1, 0.1, 0.85, 0.80),
pxgutter=0.05, pygutter=0.05, xgutter=0.04, ygutter=0.04,
headroom=0.0, panelBorderWeight=0, panelColor='black'):
"""Return a generator of a set of subplots, a set of Nx*Ny panels of nx*ny plots. Each panel is fully
filled by row (starting in the bottom left) before the next panel is started. If panelBorderWidth is
greater than zero a border is drawn around each panel, adjusted to enclose the axis labels.
E.g.
subplots = makeSubplots(fig, 2, 2, Nx=1, Ny=1, panelColor='k')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (0,0)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (1,0)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (0,1)')
ax = subplots.next(); ax.text(0.3, 0.5, '[0, 0] (1,1)')
fig.show()
@param fig The matplotlib figure to draw
@param nx The number of plots in each row of each panel
@param ny The number of plots in each column of each panel
@param Nx The number of panels in each row of the figure
@param Ny The number of panels in each column of the figure
@param plottingArea (x0, y0, x1, y1) for the part of the figure containing all the panels
@param pxgutter Spacing between columns of panels in units of (x1 - x0)
@param pygutter Spacing between rows of panels in units of (y1 - y0)
@param xgutter Spacing between columns of plots within a panel in units of (x1 - x0)
@param ygutter Spacing between rows of plots within a panel in units of (y1 - y0)
@param headroom Extra spacing above each plot for e.g. a title
@param panelBorderWeight Width of border drawn around panels
@param panelColor Colour of border around panels
"""
log = lsst.log.Log.getLogger("utils.makeSubplots")
try:
import matplotlib.pyplot as plt
except ImportError as e:
log.warn("Unable to import matplotlib: %s", e)
return
# Make show() call canvas.draw() too so that we know how large the axis labels are. Sigh
try:
fig.__show
except AttributeError:
fig.__show = fig.show
def myShow(fig):
fig.__show()
fig.canvas.draw()
import types
fig.show = types.MethodType(myShow, fig)
#
# We can't get the axis sizes until after draw()'s been called, so use a callback Sigh^2
#
axes = {} # all axes in all the panels we're drawing: axes[panel][0] etc.
#
def on_draw(event):
"""
Callback to draw the panel borders when the plots are drawn to the canvas
"""
if panelBorderWeight <= 0:
return False
for p in axes.keys():
bboxes = []
for ax in axes[p]:
bboxes.append(ax.bbox.union([label.get_window_extent() for label in
ax.get_xticklabels() + ax.get_yticklabels()]))
ax = axes[p][0]
# this is the bbox that bounds all the bboxes, again in relative
# figure coords
bbox = ax.bbox.union(bboxes)
xy0, xy1 = ax.transData.inverted().transform(bbox)
x0, y0 = xy0
x1, y1 = xy1
w, h = x1 - x0, y1 - y0
# allow a little space around BBox
x0 -= 0.02*w
w += 0.04*w
y0 -= 0.02*h
h += 0.04*h
h += h*headroom
# draw BBox
ax.patches = [] # remove old ones
rec = ax.add_patch(plt.Rectangle((x0, y0), w, h, fill=False,
lw=panelBorderWeight, edgecolor=panelColor))
rec.set_clip_on(False)
return False
fig.canvas.mpl_connect('draw_event', on_draw)
#
# Choose the plotting areas for each subplot
#
x0, y0 = plottingArea[0:2]
W, H = plottingArea[2:4]
w = (W - (Nx - 1)*pxgutter - (nx*Nx - 1)*xgutter)/float(nx*Nx)
h = (H - (Ny - 1)*pygutter - (ny*Ny - 1)*ygutter)/float(ny*Ny)
#
# OK! Time to create the subplots
#
for panel in range(Nx*Ny):
axes[panel] = []
px = panel%Nx
py = panel//Nx
for window in range(nx*ny):
x = nx*px + window%nx
y = ny*py + window//nx
ax = fig.add_axes((x0 + xgutter + pxgutter + x*w + (px - 1)*pxgutter + (x - 1)*xgutter,
y0 + ygutter + pygutter + y*h + (py - 1)*pygutter + (y - 1)*ygutter,
w, h), frame_on=True, facecolor='w')
axes[panel].append(ax)
yield ax
def runOneFilter(repo,
visitDataIds,
brightSnrMin=None,
brightSnrMax=None,
makeJson=True,
filterName=None,
outputPrefix='',
doApplyExternalPhotoCalib=False,
externalPhotoCalibName=None,
doApplyExternalSkyWcs=False,
externalSkyWcsName=None,
skipTEx=False,
verbose=False,
metrics_package='verify_metrics',
instrument='Unknown',
dataset_repo_url='./',
skipNonSrd=False,
**kwargs):
r"""Main executable for the case where there is just one filter.
Plot files and JSON files are generated in the local directory
prefixed with the repository name (where '_' replace path separators),
unless overriden by specifying `outputPrefix`.
E.g., Analyzing a repository ``CFHT/output``
will result in filenames that start with ``CFHT_output_``.
Parameters
----------
repo : string or Butler
A Butler or a repository URL that can be used to construct one.
dataIds : list of dict
List of `butler` data IDs of Image catalogs to compare to reference.
The `calexp` pixel image is needed for the photometric calibration
unless doApplyExternalPhotoCalib is True such
that the appropriate `photoCalib` dataset is used. Note that these
have data IDs that include the tract number.
brightSnrMin : float, optional
Minimum median SNR for a source to be considered bright; passed to
`lsst.validate.drp.matchreduce.build_matched_dataset`.
brightSnrMax : float, optional
Maximum median SNR for a source to be considered bright; passed to
`lsst.validate.drp.matchreduce.build_matched_dataset`.
makeJson : bool, optional
Create JSON output file for metrics. Saved to current working directory.
outputPrefix : str, optional
Specify the beginning filename for output files.
filterName : str, optional
Name of the filter (bandpass).
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True.
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
externalSkyWcsName : str, optional
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if "doApplyExternalSkyWcs" is True.
skipTEx : bool, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
verbose : bool, optional
Output additional information on the analysis steps.
skipNonSrd : bool, optional
Skip any metrics not defined in the LSST SRD.
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""
if kwargs:
log.warn(
f"Extra kwargs - {kwargs}, will be ignored. Did you add extra things to your config file?"
)
if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError(
"Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True."
)
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError(
"Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")
# collect just the common key, value pairs to omit the keys that are aggregated over
job_metadata = dict(
set.intersection(*[set(vid.items()) for vid in visitDataIds]))
# update with metadata passed into the method
job_metadata.update({
'instrument': instrument,
'filter_name': filterName,
'dataset_repo_url': dataset_repo_url
})
job = Job.load_metrics_package(meta=job_metadata,
subset='validate_drp',
package_name_or_path=metrics_package)
matchedDataset = build_matched_dataset(
repo,
visitDataIds,
doApplyExternalPhotoCalib=doApplyExternalPhotoCalib,
externalPhotoCalibName=externalPhotoCalibName,
doApplyExternalSkyWcs=doApplyExternalSkyWcs,
externalSkyWcsName=externalSkyWcsName,
skipTEx=skipTEx,
skipNonSrd=skipNonSrd,
brightSnrMin=brightSnrMin,
brightSnrMax=brightSnrMax)
snr = matchedDataset['snr'].quantity
bright = (matchedDataset['brightSnrMin'].quantity <
snr) & (snr < matchedDataset['brightSnrMax'].quantity)
photomModel = build_photometric_error_model(matchedDataset, bright)
astromModel = build_astrometric_error_model(matchedDataset, bright)
linkedBlobs = [matchedDataset, photomModel, astromModel]
metrics = job.metrics
specs = job.specs
def add_measurement(measurement):
for blob in linkedBlobs:
measurement.link_blob(blob)
job.measurements.insert(measurement)
for x, D in zip((1, 2, 3), (5., 20., 200.)):
amxName = 'AM{0:d}'.format(x)
afxName = 'AF{0:d}'.format(x)
adxName = 'AD{0:d}'.format(x)
amx = measureAMx(metrics['validate_drp.' + amxName],
matchedDataset,
D * u.arcmin,
verbose=verbose)
add_measurement(amx)
afx_spec_set = specs.subset(required_meta={'instrument': 'HSC'},
spec_tags=[
afxName,
])
adx_spec_set = specs.subset(required_meta={'instrument': 'HSC'},
spec_tags=[
adxName,
])
for afx_spec_key, adx_spec_key in zip(afx_spec_set, adx_spec_set):
afx_spec = afx_spec_set[afx_spec_key]
adx_spec = adx_spec_set[adx_spec_key]
adx = measureADx(metrics[adx_spec.metric_name], amx, afx_spec)
add_measurement(adx)
afx = measureAFx(metrics[afx_spec.metric_name], amx, adx, adx_spec)
add_measurement(afx)
pa1 = measurePA1(metrics['validate_drp.PA1'], filterName,
matchedDataset.matchesBright, matchedDataset.magKey)
add_measurement(pa1)
pf1_spec_set = specs.subset(required_meta={
'instrument': instrument,
'filter_name': filterName
},
spec_tags=[
'PF1',
])
pa2_spec_set = specs.subset(required_meta={
'instrument': instrument,
'filter_name': filterName
},
spec_tags=[
'PA2',
])
# I worry these might not always be in the right order. Sorting...
pf1_spec_keys = list(pf1_spec_set.keys())
pa2_spec_keys = list(pa2_spec_set.keys())
pf1_spec_keys.sort()
pa2_spec_keys.sort()
for pf1_spec_key, pa2_spec_key in zip(pf1_spec_keys, pa2_spec_keys):
pf1_spec = pf1_spec_set[pf1_spec_key]
pa2_spec = pa2_spec_set[pa2_spec_key]
pa2 = measurePA2(metrics[pa2_spec.metric_name], pa1,
pf1_spec.threshold)
add_measurement(pa2)
pf1 = measurePF1(metrics[pf1_spec.metric_name], pa1, pa2_spec)
add_measurement(pf1)
if not skipTEx:
for x, D, bin_range_operator in zip((1, 2), (1.0, 5.0), ("<=", ">=")):
texName = 'TE{0:d}'.format(x)
tex = measureTEx(metrics['validate_drp.' + texName],
matchedDataset,
D * u.arcmin,
bin_range_operator,
verbose=verbose)
add_measurement(tex)
if not skipNonSrd:
model_phot_reps = measure_model_phot_rep(metrics, filterName,
matchedDataset)
for measurement in model_phot_reps:
add_measurement(measurement)
if makeJson:
job.write(outputPrefix + '.json')
return job
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
matchKernelAmplitudes=False, keepPlots=True):
"""Plot the PSF spatial model."""
log = lsst.log.Log.getLogger("utils.plotPsfSpatialModel")
try:
import matplotlib.pyplot as plt
import matplotlib as mpl
except ImportError as e:
log.warn("Unable to import matplotlib: %s", e)
return
noSpatialKernel = psf.getKernel()
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
if not showBadCandidates and cand.isBad():
continue
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception:
continue
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * k.getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
zGood = numpy.array(candFits)
xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
zBad = numpy.array(badFits)
numBad = len(badPos)
xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)
kernel = psf.getKernel()
nKernelComponents = kernel.getNKernelParameters()
#
# Figure out how many panels we'll need
#
nPanelX = int(math.sqrt(nKernelComponents))
nPanelY = nKernelComponents//nPanelX
while nPanelY*nPanelX < nKernelComponents:
nPanelX += 1
fig = plt.figure(1)
fig.clf()
try:
fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
except Exception: # protect against API changes
pass
#
# Generator for axes arranged in panels
#
mpl.rcParams["figure.titlesize"] = "x-small"
subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)
for k in range(nKernelComponents):
func = kernel.getSpatialFunction(k)
dfGood = zGood[:, k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
yMin = dfGood.min()
yMax = dfGood.max()
if numBad > 0:
dfBad = zBad[:, k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
yMin = min([yMin, dfBad.min()])
yMax = max([yMax, dfBad.max()])
yMin -= 0.05 * (yMax - yMin)
yMax += 0.05 * (yMax - yMin)
yMin = -0.01
yMax = 0.01
fRange = numpy.ndarray((len(xRange), len(yRange)))
for j, yVal in enumerate(yRange):
for i, xVal in enumerate(xRange):
fRange[j][i] = func(xVal, yVal)
ax = next(subplots)
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getHeight())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(yGood, dfGood, 'b+')
if numBad > 0:
ax.plot(yBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('Residuals(y)')
ax = next(subplots)
if matchKernelAmplitudes and k == 0:
vmin = 0.0
vmax = 1.1
else:
vmin = fRange.min()
vmax = fRange.max()
norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
ax.set_title('Spatial poly')
plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
ax = next(subplots)
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(xGood, dfGood, 'b+')
if numBad > 0:
ax.plot(xBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('K%d Residuals(x)' % k)
ax = next(subplots)
photoCalib = exposure.getPhotoCalib()
# If there is no calibration factor, use 1.0.
if photoCalib.getCalibrationMean() <= 0:
photoCalib = afwImage.PhotoCalib(1.0)
ampMag = [photoCalib.isntFluxToMagnitude(candAmp) for candAmp in candAmps]
ax.plot(ampMag, zGood[:, k], 'b+')
if numBad > 0:
badAmpMag = [photoCalib.isntFluxToMagnitude(badAmp) for badAmp in badAmps]
ax.plot(badAmpMag, zBad[:, k], 'r+')
ax.set_title('Flux variation')
fig.show()
global keptPlots
if keepPlots and not keptPlots:
# Keep plots open when done
def show():
print("%s: Please close plots when done." % __name__)
try:
plt.show()
except Exception:
pass
print("Plots closed, exiting...")
import atexit
atexit.register(show)
keptPlots = True
