def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection task
#
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
#
# And the measurement Task
#
config = SingleFrameMeasurementTask.ConfigClass()
config.algorithms.names = ["base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux"]
config.algorithms["base_CircularApertureFlux"].radii = [1, 2, 4, 8, 12, 16] # pixels
config.slots.gaussianFlux = None
config.slots.modelFlux = None
config.slots.psfFlux = None
algMetadata = dafBase.PropertyList()
measureTask = SingleFrameMeasurementTask(schema, algMetadata=algMetadata, config=config)
radii = algMetadata.getArray("base_CircularApertureFlux_radii")
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
result = detectionTask.run(tab, exposure)
sources = result.sources
print("Found %d sources (%d +ve, %d -ve)" % (len(sources), result.fpSets.numPos, result.fpSets.numNeg))
measureTask.run(sources, exposure)
if display: # display image (see also --debug argparse option)
afwDisplay.setDefaultMaskTransparency(75)
frame = 1
disp = afwDisplay.Display(frame=frame)
disp.mtv(exposure)
with disp.Buffering():
for s in sources:
xy = s.getCentroid()
disp.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
disp.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
for radius in radii:
disp.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
def display(self, exposure, results, convolvedImage=None):
"""Display detections if so configured
Displays the ``exposure`` in frame 0, overlays the detection peaks.
Requires that ``lsstDebug`` has been set up correctly, so that
``lsstDebug.Info("lsst.meas.algorithms.detection")`` evaluates `True`.
If the ``convolvedImage`` is non-`None` and
``lsstDebug.Info("lsst.meas.algorithms.detection") > 1``, the
``convolvedImage`` will be displayed in frame 1.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure to display, on which will be plotted the detections.
results : `lsst.pipe.base.Struct`
Results of the 'detectFootprints' method, containing positive and
negative footprints (which contain the peak positions that we will
plot). This is a `Struct` with ``positive`` and ``negative``
elements that are of type `lsst.afw.detection.FootprintSet`.
convolvedImage : `lsst.afw.image.Image`, optional
Convolved image used for thresholding.
"""
try:
import lsstDebug
display = lsstDebug.Info(__name__).display
except ImportError:
try:
display
except NameError:
display = False
if not display:
return
afwDisplay.setDefaultMaskTransparency(75)
disp0 = afwDisplay.Display(frame=0)
disp0.mtv(exposure, title="detection")
def plotPeaks(fps, ctype):
if fps is None:
return
with disp0.Buffering():
for fp in fps.getFootprints():
for pp in fp.getPeaks():
disp0.dot("+", pp.getFx(), pp.getFy(), ctype=ctype)
plotPeaks(results.positive, "yellow")
plotPeaks(results.negative, "red")
if convolvedImage and display > 1:
disp1 = afwDisplay.Display(frame=1)
disp1.mtv(convolvedImage, title="PSF smoothed")
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
"""Provide visualization of the inputs and ouputs to the Psf-matching code
Parameters
----------
kernelCellSet : `lsst.afw.math.SpatialCellSet`
The SpatialCellSet used in determining the matching kernel and background
spatialKernel : `lsst.afw.math.LinearCombinationKernel`
Spatially varying Psf-matching kernel
spatialBackground : `lsst.afw.math.Function2D`
Spatially varying background-matching function
"""
import lsstDebug
displayCandidates = lsstDebug.Info(__name__).displayCandidates
displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if displayCandidates:
diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
frame=lsstDebug.frame,
showBadCandidates=showBadCandidates)
lsstDebug.frame += 1
diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
frame=lsstDebug.frame,
showBadCandidates=showBadCandidates,
kernels=True)
lsstDebug.frame += 1
diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
frame=lsstDebug.frame,
showBadCandidates=showBadCandidates,
resids=True)
lsstDebug.frame += 1
if displayKernelBasis:
diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
lsstDebug.frame += 1
if displayKernelMosaic:
diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
lsstDebug.frame += 1
if plotKernelSpatialModel:
diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
def displayImages(root):
"""Display coadd images in different frames, with the bounding boxes of the
observations that went into them overlayed.
"""
import lsst.afw.display as afwDisplay
afwDisplay.setDefaultMaskTransparency(75)
butler = lsst.daf.persistence.Butler(root=root)
skyMap = butler.get("deepCoadd_skyMap")
tractInfo = skyMap[0]
task = lsst.pipe.tasks.mocks.MockCoaddTask()
coadds = [patchRef.get("deepCoadd", immediate=True)
for patchRef in task.iterPatchRefs(butler, tractInfo)]
for n, coadd in enumerate(coadds):
afwDisplay.Display(frame=n + 1).mtv(coadd, title="displayImages: coadd")
for n, coadd in enumerate(coadds):
afwDisplay.utils.drawCoaddInputs(coadd, frame=n + 1)
return butler
def subtractExposures(self, templateExposure, scienceExposure,
templateFwhmPix=None, scienceFwhmPix=None,
candidateList=None, doWarping=True, convolveTemplate=True):
"""Register, Psf-match and subtract two Exposures.
Do the following, in order:
- Warp templateExposure to match scienceExposure, if their WCSs do not already match
- Determine a PSF matching kernel and differential background model
that matches templateExposure to scienceExposure
- PSF-match templateExposure to scienceExposure
- Compute subtracted exposure (see return values for equation).
Parameters
----------
templateExposure : `lsst.afw.image.Exposure`
Exposure to PSF-match to scienceExposure
scienceExposure : `lsst.afw.image.Exposure`
Reference Exposure
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
doWarping : `bool`
What to do if ``templateExposure``` and ``scienceExposure`` WCSs do
not match:
- if `True` then warp ``templateExposure`` to match ``scienceExposure``
- if `False` then raise an Exception
convolveTemplate : `bool`
Convolve the template image or the science image
- if `True`, ``templateExposure`` is warped if doWarping,
``templateExposure`` is convolved
- if `False`, ``templateExposure`` is warped if doWarping,
``scienceExposure is`` convolved
Returns
-------
result : `lsst.pipe.base.Struct`
An `lsst.pipe.base.Struct` containing these fields:
- ``subtractedExposure`` : subtracted Exposure
scienceExposure - (matchedImage + backgroundModel)
- ``matchedImage`` : ``templateExposure`` after warping to match
``templateExposure`` (if doWarping true),
and convolving with psfMatchingKernel
- ``psfMatchingKernel`` : PSF matching kernel
- ``backgroundModel`` : differential background model
- ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel
"""
results = self.matchExposures(
templateExposure=templateExposure,
scienceExposure=scienceExposure,
templateFwhmPix=templateFwhmPix,
scienceFwhmPix=scienceFwhmPix,
candidateList=candidateList,
doWarping=doWarping,
convolveTemplate=convolveTemplate
)
subtractedExposure = afwImage.ExposureF(scienceExposure, True)
if convolveTemplate:
subtractedMaskedImage = subtractedExposure.getMaskedImage()
subtractedMaskedImage -= results.matchedExposure.getMaskedImage()
subtractedMaskedImage -= results.backgroundModel
else:
subtractedExposure.setMaskedImage(results.warpedExposure.getMaskedImage())
subtractedMaskedImage = subtractedExposure.getMaskedImage()
subtractedMaskedImage -= results.matchedExposure.getMaskedImage()
subtractedMaskedImage -= results.backgroundModel
# Preserve polarity of differences
subtractedMaskedImage *= -1
# Place back on native photometric scale
subtractedMaskedImage /= results.psfMatchingKernel.computeImage(
afwImage.ImageD(results.psfMatchingKernel.getDimensions()), False)
import lsstDebug
display = lsstDebug.Info(__name__).display
displayDiffIm = lsstDebug.Info(__name__).displayDiffIm
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if display and displayDiffIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(templateExposure, title="Template")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(results.matchedExposure, title="Matched template")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(scienceExposure, title="Science Image")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(subtractedExposure, title="Difference Image")
lsstDebug.frame += 1
results.subtractedExposure = subtractedExposure
return results
def matchMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList,
templateFwhmPix=None, scienceFwhmPix=None):
"""PSF-match a MaskedImage (templateMaskedImage) to a reference MaskedImage (scienceMaskedImage).
Do the following, in order:
- Determine a PSF matching kernel and differential background model
that matches templateMaskedImage to scienceMaskedImage
- Convolve templateMaskedImage by the PSF matching kernel
Parameters
----------
templateMaskedImage : `lsst.afw.image.MaskedImage`
masked image to PSF-match to the reference masked image;
must be warped to match the reference masked image
scienceMaskedImage : `lsst.afw.image.MaskedImage`
maskedImage whose PSF is to be matched to
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
Returns
-------
result : `callable`
An `lsst.pipe.base.Struct` containing these fields:
- psfMatchedMaskedImage: the PSF-matched masked image =
``templateMaskedImage`` convolved with psfMatchingKernel.
This has the same xy0, dimensions and wcs as ``scienceMaskedImage``.
- psfMatchingKernel: the PSF matching kernel
- backgroundModel: differential background model
- kernelCellSet: SpatialCellSet used to solve for the PSF matching kernel
Raises
------
RuntimeError
Raised if input images have different dimensions
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayTemplate = lsstDebug.Info(__name__).displayTemplate
displaySciIm = lsstDebug.Info(__name__).displaySciIm
displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if not candidateList:
raise RuntimeError("Candidate list must be populated by makeCandidateList")
if not self._validateSize(templateMaskedImage, scienceMaskedImage):
self.log.error("ERROR: Input images different size")
raise RuntimeError("Input images different size")
if display and displayTemplate:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(templateMaskedImage, title="Image to convolve")
lsstDebug.frame += 1
if display and displaySciIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(scienceMaskedImage, title="Image to not convolve")
lsstDebug.frame += 1
kernelCellSet = self._buildCellSet(templateMaskedImage,
scienceMaskedImage,
candidateList)
if display and displaySpatialCells:
diffimUtils.showKernelSpatialCells(scienceMaskedImage, kernelCellSet,
symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame,
title="Image to not convolve")
lsstDebug.frame += 1
if templateFwhmPix and scienceFwhmPix:
self.log.info("Matching Psf FWHM %.2f -> %.2f pix", templateFwhmPix, scienceFwhmPix)
if self.kConfig.useBicForKernelBasis:
tmpKernelCellSet = self._buildCellSet(templateMaskedImage,
scienceMaskedImage,
candidateList)
nbe = diffimTools.NbasisEvaluator(self.kConfig, templateFwhmPix, scienceFwhmPix)
bicDegrees = nbe(tmpKernelCellSet, self.log)
basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix,
alardDegGauss=bicDegrees[0], metadata=self.metadata)
del tmpKernelCellSet
else:
basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix,
metadata=self.metadata)
spatialSolution, psfMatchingKernel, backgroundModel = self._solve(kernelCellSet, basisList)
psfMatchedMaskedImage = afwImage.MaskedImageF(templateMaskedImage.getBBox())
doNormalize = False
afwMath.convolve(psfMatchedMaskedImage, templateMaskedImage, psfMatchingKernel, doNormalize)
return pipeBase.Struct(
matchedImage=psfMatchedMaskedImage,
psfMatchingKernel=psfMatchingKernel,
backgroundModel=backgroundModel,
kernelCellSet=kernelCellSet,
)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
"""Determine a PCA PSF model for an exposure given a list of PSF candidates.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure containing the psf candidates.
psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate`
A sequence of PSF candidates typically obtained by detecting sources
and then running them through a star selector.
metadata : `lsst.daf.base import PropertyList` or `None`, optional
A home for interesting tidbits of information.
flagKey : `str`, optional
Schema key used to mark sources actually used in PSF determination.
Returns
-------
psf : `lsst.meas.algorithms.PcaPsf`
The measured PSF.
psfCellSet : `lsst.afw.math.SpatialCellSet`
The PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates
displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration
displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components
displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals
displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
# match Kernel amplitudes for spatial plots
matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
# Keep matplotlib alive post mortem
keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model?
showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates
# Normalize residuals by object amplitude
normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration?
if display:
afwDisplay.setDefaultMaskTransparency(75)
if display > 1:
pause = True
mi = exposure.getMaskedImage()
if len(psfCandidateList) == 0:
raise RuntimeError("No PSF candidates supplied.")
# construct and populate a spatial cell set
bbox = mi.getBBox()
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
sizes = []
for i, psfCandidate in enumerate(psfCandidateList):
if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement
continue
try:
psfCellSet.insertCandidate(psfCandidate)
except Exception as e:
self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
continue
source = psfCandidate.getSource()
quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
axes = afwEll.Axes(quad)
sizes.append(axes.getA())
if len(sizes) == 0:
raise RuntimeError("No usable PSF candidates supplied")
nEigenComponents = self.config.nEigenComponents # initial version
if self.config.kernelSize >= 15:
self.log.warn("WARNING: NOT scaling kernelSize by stellar quadrupole moment "
"because config.kernelSize=%s >= 15; "
"using config.kernelSize as as the width, instead",
self.config.kernelSize)
actualKernelSize = int(self.config.kernelSize)
else:
medSize = numpy.median(sizes)
actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
print("Median size=%s" % (medSize,))
self.log.trace("Kernel size=%s", actualKernelSize)
# Set size of image returned around candidate
psfCandidateList[0].setHeight(actualKernelSize)
psfCandidateList[0].setWidth(actualKernelSize)
if self.config.doRejectBlends:
# Remove blended candidates completely
blendedCandidates = [] # Candidates to remove; can't do it while iterating
for cell, cand in candidatesIter(psfCellSet, False):
if len(cand.getSource().getFootprint().getPeaks()) > 1:
blendedCandidates.append((cell, cand))
continue
if display:
print("Removing %d blended Psf candidates" % len(blendedCandidates))
for cell, cand in blendedCandidates:
cell.removeCandidate(cand)
if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
raise RuntimeError("All PSF candidates removed as blends")
if display:
if displayExposure:
disp = afwDisplay.Display(frame=0)
disp.mtv(exposure, title="psf determination")
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o",
ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
size=4, display=disp)
#
# Do a PCA decomposition of those PSF candidates
#
reply = "y" # used in interactive mode
for iterNum in range(self.config.nIterForPsf):
if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates
stamps = []
for cell in psfCellSet.getCellList():
for cand in cell.begin(not showBadCandidates): # maybe include bad candidates
try:
im = cand.getMaskedImage()
chi2 = cand.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
stamps.append((im, "%d%s" %
(utils.splitId(cand.getSource().getId(), True)["objId"], chi2),
cand.getStatus()))
except Exception:
continue
if len(stamps) == 0:
print("WARNING: No PSF candidates to show; try setting showBadCandidates=True")
else:
mos = afwDisplay.utils.Mosaic()
for im, label, status in stamps:
im = type(im)(im, True)
try:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
except NotImplementedError:
pass
mos.append(im, label,
(afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else
afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else
afwDisplay.RED))
disp8 = afwDisplay.Display(frame=8)
mos.makeMosaic(display=disp8, title="Psf Candidates")
# Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
cleanChi2 = False # Any naughty (negative/NAN) chi^2 values?
while not cleanChi2:
cleanChi2 = True
#
# First, estimate the PSF
#
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# In clipping, allow all candidates to be innocent until proven guilty on this iteration.
# Throw out any prima facie guilty candidates (naughty chi^2 values)
#
for cell in psfCellSet.getCellList():
awfulCandidates = []
for cand in cell.begin(False): # include bad candidates
cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty
rchi2 = cand.getChi2()
if not numpy.isfinite(rchi2) or rchi2 <= 0:
# Guilty prima facie
awfulCandidates.append(cand)
cleanChi2 = False
self.log.debug("chi^2=%s; id=%s",
cand.getChi2(), cand.getSource().getId())
for cand in awfulCandidates:
if display:
print("Removing bad candidate: id=%d, chi^2=%f" %
(cand.getSource().getId(), cand.getChi2()))
cell.removeCandidate(cand)
#
# Clip out bad fits based on reduced chi^2
#
badCandidates = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate
assert rchi2 > 0
if rchi2 > self.config.reducedChi2ForPsfCandidates:
badCandidates.append(cand)
badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(numBad), badCandidates):
if display:
chi2 = c.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2))
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = psf.getKernel()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
except Exception:
continue
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p*k.getSum()
predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for
k in range(kernel.getNKernelParameters())]
residuals.append([a/amp - p for a, p in zip(params, predict)])
candidates.append(cand)
residuals = numpy.array(residuals)
for k in range(kernel.getNKernelParameters()):
if False:
# Straight standard deviation
mean = residuals[:, k].mean()
rms = residuals[:, k].std()
elif False:
# Using interquartile range
sr = numpy.sort(residuals[:, k])
mean = (sr[int(0.5*len(sr))] if len(sr)%2 else
0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1]))
rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))])
else:
stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
mean = stats.getValue(afwMath.MEANCLIP)
rms = stats.getValue(afwMath.STDEVCLIP)
rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues
if display:
print("Mean for component %d is %f" % (k, mean))
print("RMS for component %d is %f" % (k, rms))
badCandidates = list()
for i, cand in enumerate(candidates):
if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms:
badCandidates.append(i)
badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates):
cand = candidates[c]
if display:
print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" %
(cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
residuals[badCandidates[i], k], self.config.spatialReject*rms))
cand.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Display results
#
if display and displayIterations:
if displayExposure:
if iterNum > 0:
disp.erase()
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", size=10, display=disp,
ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED)
if displayResiduals:
while True:
try:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
disp5 = afwDisplay.Display(frame=5)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates,
variance=True)
except Exception:
if not showBadCandidates:
showBadCandidates = True
continue
break
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale('linear', 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
if pause:
while True:
try:
reply = input("Next iteration? [ynchpqQs] ").strip()
except EOFError:
reply = "n"
reply = reply.split()
if reply:
reply, args = reply[0], reply[1:]
else:
reply = ""
if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"):
if reply == "c":
pause = False
elif reply == "h":
print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] "
"s[ave fileName] y[es]")
continue
elif reply == "p":
import pdb
pdb.set_trace()
elif reply == "q":
display = False
elif reply == "Q":
sys.exit(1)
elif reply == "s":
fileName = args.pop(0)
if not fileName:
print("Please provide a filename")
continue
print("Saving to %s" % fileName)
utils.saveSpatialCellSet(psfCellSet, fileName=fileName)
continue
break
else:
print("Unrecognised response: %s" % reply, file=sys.stderr)
if reply == "n":
break
# One last time, to take advantage of the last iteration
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# Display code for debugging
#
if display and reply != "n":
disp = afwDisplay.Display(frame=0)
if displayExposure:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=8, display=disp)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=10, display=disp)
if displayResiduals:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale("linear", 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
#
# Generate some QA information
#
# Count PSF stars
#
numGoodStars = 0
numAvailStars = 0
avgX = 0.0
avgY = 0.0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # don't ignore BAD stars
numAvailStars += 1
for cand in cell.begin(True): # do ignore BAD stars
src = cand.getSource()
if flagKey is not None:
src.set(flagKey, True)
avgX += src.getX()
avgY += src.getY()
numGoodStars += 1
avgX /= numGoodStars
avgY /= numGoodStars
if metadata is not None:
metadata.set("spatialFitChi2", fitChi2)
metadata.set("numGoodStars", numGoodStars)
metadata.set("numAvailStars", numAvailStars)
metadata.set("avgX", avgX)
metadata.set("avgY", avgY)
psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY))
return psf, psfCellSet
import lsst.utils
import lsst.utils.tests
import lsst.geom
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
import lsst.afw.math.detail as mathDetail
import lsst.pex.exceptions as pexExcept
from test_kernel import makeDeltaFunctionKernelList, makeGaussianKernelList
from lsst.log import Log
import lsst.afw.display as afwDisplay
Log.getLogger("afw.image.Mask").setLevel(Log.INFO)
afwDisplay.setDefaultMaskTransparency(75)
try:
display
except NameError:
display = False
try:
dataDir = os.path.join(lsst.utils.getPackageDir("afwdata"), "data")
except pexExcept.NotFoundError:
dataDir = None
else:
InputMaskedImagePath = os.path.join(dataDir, "medexp.fits")
FullMaskedImage = afwImage.MaskedImageF(InputMaskedImagePath)
# input image contains a saturated star, a bad column, and a faint star
def matchMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList,
templateFwhmPix=None, scienceFwhmPix=None):
"""PSF-match a MaskedImage (templateMaskedImage) to a reference MaskedImage (scienceMaskedImage).
Do the following, in order:
- Determine a PSF matching kernel and differential background model
that matches templateMaskedImage to scienceMaskedImage
- Convolve templateMaskedImage by the PSF matching kernel
Parameters
----------
templateMaskedImage : `lsst.afw.image.MaskedImage`
masked image to PSF-match to the reference masked image;
must be warped to match the reference masked image
scienceMaskedImage : `lsst.afw.image.MaskedImage`
maskedImage whose PSF is to be matched to
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
Returns
-------
result : `callable`
An `lsst.pipe.base.Struct` containing these fields:
- psfMatchedMaskedImage: the PSF-matched masked image =
``templateMaskedImage`` convolved with psfMatchingKernel.
This has the same xy0, dimensions and wcs as ``scienceMaskedImage``.
- psfMatchingKernel: the PSF matching kernel
- backgroundModel: differential background model
- kernelCellSet: SpatialCellSet used to solve for the PSF matching kernel
Raises
------
RuntimeError
Raised if input images have different dimensions
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayTemplate = lsstDebug.Info(__name__).displayTemplate
displaySciIm = lsstDebug.Info(__name__).displaySciIm
displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if not candidateList:
raise RuntimeError("Candidate list must be populated by makeCandidateList")
if not self._validateSize(templateMaskedImage, scienceMaskedImage):
self.log.error("ERROR: Input images different size")
raise RuntimeError("Input images different size")
if display and displayTemplate:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(templateMaskedImage, title="Image to convolve")
lsstDebug.frame += 1
if display and displaySciIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(scienceMaskedImage, title="Image to not convolve")
lsstDebug.frame += 1
kernelCellSet = self._buildCellSet(templateMaskedImage,
scienceMaskedImage,
candidateList)
if display and displaySpatialCells:
diffimUtils.showKernelSpatialCells(scienceMaskedImage, kernelCellSet,
symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame,
title="Image to not convolve")
lsstDebug.frame += 1
if templateFwhmPix and scienceFwhmPix:
self.log.info("Matching Psf FWHM %.2f -> %.2f pix", templateFwhmPix, scienceFwhmPix)
if self.kConfig.useBicForKernelBasis:
tmpKernelCellSet = self._buildCellSet(templateMaskedImage,
scienceMaskedImage,
candidateList)
nbe = diffimTools.NbasisEvaluator(self.kConfig, templateFwhmPix, scienceFwhmPix)
bicDegrees = nbe(tmpKernelCellSet, self.log)
basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix,
alardDegGauss=bicDegrees[0], metadata=self.metadata)
del tmpKernelCellSet
else:
basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix,
metadata=self.metadata)
spatialSolution, psfMatchingKernel, backgroundModel = self._solve(kernelCellSet, basisList)
psfMatchedMaskedImage = afwImage.MaskedImageF(templateMaskedImage.getBBox())
convolutionControl = afwMath.ConvolutionControl()
convolutionControl.setDoNormalize(False)
afwMath.convolve(psfMatchedMaskedImage, templateMaskedImage, psfMatchingKernel, convolutionControl)
return pipeBase.Struct(
matchedImage=psfMatchedMaskedImage,
psfMatchingKernel=psfMatchingKernel,
backgroundModel=backgroundModel,
kernelCellSet=kernelCellSet,
)
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of psf-like objects.
Parameters
----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch` or None
Ignored by this source selector.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used for debug display.
Return
------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `numpy.ndarray` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = display and \
lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
plotFwhmHistogram = display and plt and \
lsstDebug.Info(__name__).plotFwhmHistogram # Plot histogram of FWHM
plotFlags = display and plt and \
lsstDebug.Info(__name__).plotFlags # Plot the sources coloured by their flags
plotRejection = display and plt and \
lsstDebug.Info(__name__).plotRejection # Plot why sources are rejected
afwDisplay.setDefaultMaskTransparency(75)
fluxName = self.config.fluxName
fluxErrName = self.config.fluxErrName
minFwhm = self.config.minFwhm
maxFwhm = self.config.maxFwhm
maxFwhmVariability = self.config.maxFwhmVariability
maxbad = self.config.maxbad
maxbadflag = self.config.maxbadflag
maxellip = self.config.maxellip
minsn = self.config.minsn
maxelong = (maxellip + 1.0) / (1.0 -
maxellip) if maxellip < 1.0 else 100
# Unpack the catalogue
shape = sourceCat.getShapeDefinition()
ixx = sourceCat.get("%s.xx" % shape)
iyy = sourceCat.get("%s.yy" % shape)
fwhm = 2 * np.sqrt(2 * np.log(2)) * np.sqrt(0.5 * (ixx + iyy))
elong = 0.5 * (ixx - iyy) / (ixx + iyy)
flux = sourceCat.get(fluxName)
fluxErr = sourceCat.get(fluxErrName)
sn = flux / np.where(fluxErr > 0, fluxErr, 1)
sn[fluxErr <= 0] = -psfexLib.BIG
flags = 0x0
for i, f in enumerate(self.config.badFlags):
flags = np.bitwise_or(flags, np.where(sourceCat.get(f), 1 << i, 0))
#
# Estimate the acceptable range of source widths
#
good = np.logical_and(sn > minsn, np.logical_not(flags))
good = np.logical_and(good, elong < maxelong)
good = np.logical_and(good, fwhm >= minFwhm)
good = np.logical_and(good, fwhm < maxFwhm)
fwhmMode, fwhmMin, fwhmMax = compute_fwhmrange(
fwhm[good],
maxFwhmVariability,
minFwhm,
maxFwhm,
plot=dict(fwhmHistogram=plotFwhmHistogram))
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Here's select_candidates
#
# ---- Apply some selection over flags, fluxes...
bad = (flags != 0)
# set.setBadFlags(int(sum(bad)))
if plotRejection:
selectionVectors = []
selectionVectors.append((bad, "flags %d" % sum(bad)))
dbad = sn < minsn
# set.setBadSN(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
dbad = fwhm < fwhmMin
# set.setBadFrmin(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMin %d" % sum(dbad)))
dbad = fwhm > fwhmMax
# set.setBadFrmax(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMax %d" % sum(dbad)))
dbad = elong > maxelong
# set.setBadElong(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "elong %d" % sum(dbad)))
# -- ... and check the integrity of the sample
if maxbadflag:
nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
dbad = nbad > maxbad
# set.setBadPix(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
good = np.logical_not(bad)
#
# We know enough to plot, if so requested
#
frame = 0
if displayExposure:
mi = exposure.getMaskedImage()
disp = afwDisplay.Display(frame=frame)
disp.mtv(mi, title="PSF candidates")
with disp.Buffering():
for i, source in enumerate(sourceCat):
if good[i]:
ctype = afwDisplay.GREEN # star candidate
else:
ctype = afwDisplay.RED # not star
disp.dot("+",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
ctype=ctype)
if plotFlags or plotRejection:
imag = -2.5 * np.log10(flux)
plt.clf()
alpha = 0.5
if plotFlags:
isSet = np.where(flags == 0x0)[0]
plt.plot(imag[isSet],
fwhm[isSet],
'o',
alpha=alpha,
label="good")
for i, f in enumerate(self.config.badFlags):
mask = 1 << i
isSet = np.where(np.bitwise_and(flags, mask))[0]
if isSet.any():
if np.isfinite(imag[isSet] + fwhm[isSet]).any():
label = re.sub(
r"\_flag", "",
re.sub(
r"^base\_", "",
re.sub(r"^.*base\_PixelFlags\_flag\_", "",
f)))
plt.plot(imag[isSet],
fwhm[isSet],
'o',
alpha=alpha,
label=label)
else:
for bad, label in selectionVectors:
plt.plot(imag[bad],
fwhm[bad],
'o',
alpha=alpha,
label=label)
plt.plot(imag[good],
fwhm[good],
'o',
color="black",
label="selected")
[plt.axhline(_, color='red') for _ in [fwhmMin, fwhmMax]]
plt.xlim(np.median(imag[good]) + 5 * np.array([-1, 1]))
plt.ylim(fwhm[np.where(np.isfinite(fwhm + imag))].min(),
2 * fwhmMax)
plt.legend(loc=2)
plt.xlabel("Instrumental %s Magnitude" %
fluxName.split(".")[-1].title())
plt.ylabel("fwhm")
title = "PSFEX Star Selection"
plt.title("%s %d selected" % (title, sum(good)))
if displayExposure:
global eventHandler
eventHandler = EventHandler(plt.axes(),
imag,
fwhm,
sourceCat.getX(),
sourceCat.getY(),
frames=[frame])
if plotFlags or plotRejection:
while True:
try:
reply = input(
"continue? [y[es] h(elp) p(db) q(uit)] ").strip()
except EOFError:
reply = "y"
if not reply:
reply = "y"
if reply[0] == "h":
print("""\
At this prompt, you can continue with almost any key; 'p' enters pdb,
'q' returns to the shell, and
'h' prints this text
""",
end=' ')
if displayExposure:
print("""
If you put the cursor on a point in the matplotlib scatter plot and hit 'p' you'll see it in ds9."""
)
elif reply[0] == "p":
import pdb
pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
return Struct(selected=good)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
"""Determine a PCA PSF model for an exposure given a list of PSF candidates.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure containing the psf candidates.
psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate`
A sequence of PSF candidates typically obtained by detecting sources
and then running them through a star selector.
metadata : `lsst.daf.base import PropertyList` or `None`, optional
A home for interesting tidbits of information.
flagKey : `str`, optional
Schema key used to mark sources actually used in PSF determination.
Returns
-------
psf : `lsst.meas.algorithms.PcaPsf`
The measured PSF.
psfCellSet : `lsst.afw.math.SpatialCellSet`
The PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates
displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration
displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components
displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals
displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
# match Kernel amplitudes for spatial plots
matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
# Keep matplotlib alive post mortem
keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model?
showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates
# Normalize residuals by object amplitude
normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration?
if display:
afwDisplay.setDefaultMaskTransparency(75)
if display > 1:
pause = True
mi = exposure.getMaskedImage()
if len(psfCandidateList) == 0:
raise RuntimeError("No PSF candidates supplied.")
# construct and populate a spatial cell set
bbox = mi.getBBox()
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
sizes = []
for i, psfCandidate in enumerate(psfCandidateList):
if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement
continue
try:
psfCellSet.insertCandidate(psfCandidate)
except Exception as e:
self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
continue
source = psfCandidate.getSource()
quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
axes = afwEll.Axes(quad)
sizes.append(axes.getA())
if len(sizes) == 0:
raise RuntimeError("No usable PSF candidates supplied")
nEigenComponents = self.config.nEigenComponents # initial version
if self.config.kernelSize >= 15:
self.log.warning("WARNING: NOT scaling kernelSize by stellar quadrupole moment "
"because config.kernelSize=%s >= 15; "
"using config.kernelSize as the width, instead",
self.config.kernelSize)
actualKernelSize = int(self.config.kernelSize)
else:
medSize = numpy.median(sizes)
actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
print("Median size=%s" % (medSize,))
self.log.trace("Kernel size=%s", actualKernelSize)
# Set size of image returned around candidate
psfCandidateList[0].setHeight(actualKernelSize)
psfCandidateList[0].setWidth(actualKernelSize)
if self.config.doRejectBlends:
# Remove blended candidates completely
blendedCandidates = [] # Candidates to remove; can't do it while iterating
for cell, cand in candidatesIter(psfCellSet, False):
if len(cand.getSource().getFootprint().getPeaks()) > 1:
blendedCandidates.append((cell, cand))
continue
if display:
print("Removing %d blended Psf candidates" % len(blendedCandidates))
for cell, cand in blendedCandidates:
cell.removeCandidate(cand)
if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
raise RuntimeError("All PSF candidates removed as blends")
if display:
if displayExposure:
disp = afwDisplay.Display(frame=0)
disp.mtv(exposure, title="psf determination")
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o",
ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
size=4, display=disp)
#
# Do a PCA decomposition of those PSF candidates
#
reply = "y" # used in interactive mode
for iterNum in range(self.config.nIterForPsf):
if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates
stamps = []
for cell in psfCellSet.getCellList():
for cand in cell.begin(not showBadCandidates): # maybe include bad candidates
try:
im = cand.getMaskedImage()
chi2 = cand.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
stamps.append((im, "%d%s" %
(utils.splitId(cand.getSource().getId(), True)["objId"], chi2),
cand.getStatus()))
except Exception:
continue
if len(stamps) == 0:
print("WARNING: No PSF candidates to show; try setting showBadCandidates=True")
else:
mos = afwDisplay.utils.Mosaic()
for im, label, status in stamps:
im = type(im)(im, True)
try:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
except NotImplementedError:
pass
mos.append(im, label,
(afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else
afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else
afwDisplay.RED))
disp8 = afwDisplay.Display(frame=8)
mos.makeMosaic(display=disp8, title="Psf Candidates")
# Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
cleanChi2 = False # Any naughty (negative/NAN) chi^2 values?
while not cleanChi2:
cleanChi2 = True
#
# First, estimate the PSF
#
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# In clipping, allow all candidates to be innocent until proven guilty on this iteration.
# Throw out any prima facie guilty candidates (naughty chi^2 values)
#
for cell in psfCellSet.getCellList():
awfulCandidates = []
for cand in cell.begin(False): # include bad candidates
cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty
rchi2 = cand.getChi2()
if not numpy.isfinite(rchi2) or rchi2 <= 0:
# Guilty prima facie
awfulCandidates.append(cand)
cleanChi2 = False
self.log.debug("chi^2=%s; id=%s",
cand.getChi2(), cand.getSource().getId())
for cand in awfulCandidates:
if display:
print("Removing bad candidate: id=%d, chi^2=%f" %
(cand.getSource().getId(), cand.getChi2()))
cell.removeCandidate(cand)
#
# Clip out bad fits based on reduced chi^2
#
badCandidates = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate
assert rchi2 > 0
if rchi2 > self.config.reducedChi2ForPsfCandidates:
badCandidates.append(cand)
badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(numBad), badCandidates):
if display:
chi2 = c.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2))
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = psf.getKernel()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
except Exception:
continue
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p*k.getSum()
predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for
k in range(kernel.getNKernelParameters())]
residuals.append([a/amp - p for a, p in zip(params, predict)])
candidates.append(cand)
residuals = numpy.array(residuals)
for k in range(kernel.getNKernelParameters()):
if False:
# Straight standard deviation
mean = residuals[:, k].mean()
rms = residuals[:, k].std()
elif False:
# Using interquartile range
sr = numpy.sort(residuals[:, k])
mean = (sr[int(0.5*len(sr))] if len(sr)%2 else
0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1]))
rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))])
else:
stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
mean = stats.getValue(afwMath.MEANCLIP)
rms = stats.getValue(afwMath.STDEVCLIP)
rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues
if display:
print("Mean for component %d is %f" % (k, mean))
print("RMS for component %d is %f" % (k, rms))
badCandidates = list()
for i, cand in enumerate(candidates):
if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms:
badCandidates.append(i)
badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates):
cand = candidates[c]
if display:
print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" %
(cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
residuals[badCandidates[i], k], self.config.spatialReject*rms))
cand.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Display results
#
if display and displayIterations:
if displayExposure:
if iterNum > 0:
disp.erase()
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", size=10, display=disp,
ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED)
if displayResiduals:
while True:
try:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
disp5 = afwDisplay.Display(frame=5)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates,
variance=True)
except Exception:
if not showBadCandidates:
showBadCandidates = True
continue
break
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale('linear', 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
if pause:
while True:
try:
reply = input("Next iteration? [ynchpqQs] ").strip()
except EOFError:
reply = "n"
reply = reply.split()
if reply:
reply, args = reply[0], reply[1:]
else:
reply = ""
if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"):
if reply == "c":
pause = False
elif reply == "h":
print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] "
"s[ave fileName] y[es]")
continue
elif reply == "p":
import pdb
pdb.set_trace()
elif reply == "q":
display = False
elif reply == "Q":
sys.exit(1)
elif reply == "s":
fileName = args.pop(0)
if not fileName:
print("Please provide a filename")
continue
print("Saving to %s" % fileName)
utils.saveSpatialCellSet(psfCellSet, fileName=fileName)
continue
break
else:
print("Unrecognised response: %s" % reply, file=sys.stderr)
if reply == "n":
break
# One last time, to take advantage of the last iteration
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# Display code for debugging
#
if display and reply != "n":
disp = afwDisplay.Display(frame=0)
if displayExposure:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=8, display=disp)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=10, display=disp)
if displayResiduals:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale("linear", 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
#
# Generate some QA information
#
# Count PSF stars
#
numGoodStars = 0
numAvailStars = 0
avgX = 0.0
avgY = 0.0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # don't ignore BAD stars
numAvailStars += 1
for cand in cell.begin(True): # do ignore BAD stars
src = cand.getSource()
if flagKey is not None:
src.set(flagKey, True)
avgX += src.getX()
avgY += src.getY()
numGoodStars += 1
avgX /= numGoodStars
avgY /= numGoodStars
if metadata is not None:
metadata["spatialFitChi2"] = fitChi2
metadata["numGoodStars"] = numGoodStars
metadata["numAvailStars"] = numAvailStars
metadata["avgX"] = avgX
metadata["avgY"] = avgY
psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY))
return psf, psfCellSet
def subtractMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList,
templateFwhmPix=None, scienceFwhmPix=None):
"""Psf-match and subtract two MaskedImages.
Do the following, in order:
- PSF-match templateMaskedImage to scienceMaskedImage
- Determine the differential background
- Return the difference: scienceMaskedImage
((warped templateMaskedImage convolved with psfMatchingKernel) + backgroundModel)
Parameters
----------
templateMaskedImage : `lsst.afw.image.MaskedImage`
MaskedImage to PSF-match to ``scienceMaskedImage``
scienceMaskedImage : `lsst.afw.image.MaskedImage`
Reference MaskedImage
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
Returns
-------
results : `lsst.pipe.base.Struct`
An `lsst.pipe.base.Struct` containing these fields:
- ``subtractedMaskedImage`` : ``scienceMaskedImage`` - (matchedImage + backgroundModel)
- ``matchedImage`` : templateMaskedImage convolved with psfMatchingKernel
- `psfMatchingKernel`` : PSF matching kernel
- ``backgroundModel`` : differential background model
- ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel
"""
if not candidateList:
raise RuntimeError("Candidate list must be populated by makeCandidateList")
results = self.matchMaskedImages(
templateMaskedImage=templateMaskedImage,
scienceMaskedImage=scienceMaskedImage,
candidateList=candidateList,
templateFwhmPix=templateFwhmPix,
scienceFwhmPix=scienceFwhmPix,
)
subtractedMaskedImage = afwImage.MaskedImageF(scienceMaskedImage, True)
subtractedMaskedImage -= results.matchedImage
subtractedMaskedImage -= results.backgroundModel
results.subtractedMaskedImage = subtractedMaskedImage
import lsstDebug
display = lsstDebug.Info(__name__).display
displayDiffIm = lsstDebug.Info(__name__).displayDiffIm
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if display and displayDiffIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(subtractedMaskedImage, title="Subtracted masked image")
lsstDebug.frame += 1
return results
def subtractExposures(self, templateExposure, scienceExposure,
templateFwhmPix=None, scienceFwhmPix=None,
candidateList=None, doWarping=True, convolveTemplate=True):
"""Register, Psf-match and subtract two Exposures.
Do the following, in order:
- Warp templateExposure to match scienceExposure, if their WCSs do not already match
- Determine a PSF matching kernel and differential background model
that matches templateExposure to scienceExposure
- PSF-match templateExposure to scienceExposure
- Compute subtracted exposure (see return values for equation).
Parameters
----------
templateExposure : `lsst.afw.image.Exposure`
Exposure to PSF-match to scienceExposure
scienceExposure : `lsst.afw.image.Exposure`
Reference Exposure
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
doWarping : `bool`
What to do if ``templateExposure``` and ``scienceExposure`` WCSs do
not match:
- if `True` then warp ``templateExposure`` to match ``scienceExposure``
- if `False` then raise an Exception
convolveTemplate : `bool`
Convolve the template image or the science image
- if `True`, ``templateExposure`` is warped if doWarping,
``templateExposure`` is convolved
- if `False`, ``templateExposure`` is warped if doWarping,
``scienceExposure is`` convolved
Returns
-------
result : `lsst.pipe.base.Struct`
An `lsst.pipe.base.Struct` containing these fields:
- ``subtractedExposure`` : subtracted Exposure
scienceExposure - (matchedImage + backgroundModel)
- ``matchedImage`` : ``templateExposure`` after warping to match
``templateExposure`` (if doWarping true),
and convolving with psfMatchingKernel
- ``psfMatchingKernel`` : PSF matching kernel
- ``backgroundModel`` : differential background model
- ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel
"""
results = self.matchExposures(
templateExposure=templateExposure,
scienceExposure=scienceExposure,
templateFwhmPix=templateFwhmPix,
scienceFwhmPix=scienceFwhmPix,
candidateList=candidateList,
doWarping=doWarping,
convolveTemplate=convolveTemplate
)
subtractedExposure = afwImage.ExposureF(scienceExposure, True)
if convolveTemplate:
subtractedMaskedImage = subtractedExposure.getMaskedImage()
subtractedMaskedImage -= results.matchedExposure.getMaskedImage()
subtractedMaskedImage -= results.backgroundModel
else:
subtractedExposure.setMaskedImage(results.warpedExposure.getMaskedImage())
subtractedMaskedImage = subtractedExposure.getMaskedImage()
subtractedMaskedImage -= results.matchedExposure.getMaskedImage()
subtractedMaskedImage -= results.backgroundModel
# Preserve polarity of differences
subtractedMaskedImage *= -1
# Place back on native photometric scale
subtractedMaskedImage /= results.psfMatchingKernel.computeImage(
afwImage.ImageD(results.psfMatchingKernel.getDimensions()), False)
import lsstDebug
display = lsstDebug.Info(__name__).display
displayDiffIm = lsstDebug.Info(__name__).displayDiffIm
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if display and displayDiffIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(templateExposure, title="Template")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(results.matchedExposure, title="Matched template")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(scienceExposure, title="Science Image")
lsstDebug.frame += 1
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(subtractedExposure, title="Difference Image")
lsstDebug.frame += 1
results.subtractedExposure = subtractedExposure
return results
def subtractMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList,
templateFwhmPix=None, scienceFwhmPix=None):
"""Psf-match and subtract two MaskedImages.
Do the following, in order:
- PSF-match templateMaskedImage to scienceMaskedImage
- Determine the differential background
- Return the difference: scienceMaskedImage
((warped templateMaskedImage convolved with psfMatchingKernel) + backgroundModel)
Parameters
----------
templateMaskedImage : `lsst.afw.image.MaskedImage`
MaskedImage to PSF-match to ``scienceMaskedImage``
scienceMaskedImage : `lsst.afw.image.MaskedImage`
Reference MaskedImage
templateFwhmPix : `float`
FWHM (in pixels) of the Psf in the template image (image to convolve)
scienceFwhmPix : `float`
FWHM (in pixels) of the Psf in the science image
candidateList : `list`, optional
A list of footprints/maskedImages for kernel candidates;
if `None` then source detection is run.
- Currently supported: list of Footprints or measAlg.PsfCandidateF
Returns
-------
results : `lsst.pipe.base.Struct`
An `lsst.pipe.base.Struct` containing these fields:
- ``subtractedMaskedImage`` : ``scienceMaskedImage`` - (matchedImage + backgroundModel)
- ``matchedImage`` : templateMaskedImage convolved with psfMatchingKernel
- `psfMatchingKernel`` : PSF matching kernel
- ``backgroundModel`` : differential background model
- ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel
"""
if not candidateList:
raise RuntimeError("Candidate list must be populated by makeCandidateList")
results = self.matchMaskedImages(
templateMaskedImage=templateMaskedImage,
scienceMaskedImage=scienceMaskedImage,
candidateList=candidateList,
templateFwhmPix=templateFwhmPix,
scienceFwhmPix=scienceFwhmPix,
)
subtractedMaskedImage = afwImage.MaskedImageF(scienceMaskedImage, True)
subtractedMaskedImage -= results.matchedImage
subtractedMaskedImage -= results.backgroundModel
results.subtractedMaskedImage = subtractedMaskedImage
import lsstDebug
display = lsstDebug.Info(__name__).display
displayDiffIm = lsstDebug.Info(__name__).displayDiffIm
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if display and displayDiffIm:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(subtractedMaskedImage, title="Subtracted masked image")
lsstDebug.frame += 1
return results
def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection task
#
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
#
# And the measurement Task
#
config = SingleFrameMeasurementTask.ConfigClass()
config.algorithms.names = [
"base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux"
]
config.algorithms["base_CircularApertureFlux"].radii = [
1, 2, 4, 8, 12, 16
] # pixels
config.slots.gaussianFlux = None
config.slots.modelFlux = None
config.slots.psfFlux = None
algMetadata = dafBase.PropertyList()
measureTask = SingleFrameMeasurementTask(schema,
algMetadata=algMetadata,
config=config)
radii = algMetadata.getArray("base_CircularApertureFlux_radii")
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
result = detectionTask.run(tab, exposure)
sources = result.sources
print("Found %d sources (%d +ve, %d -ve)" %
(len(sources), result.fpSets.numPos, result.fpSets.numNeg))
measureTask.run(sources, exposure)
if display: # display image (see also --debug argparse option)
afwDisplay.setDefaultMaskTransparency(75)
frame = 1
disp = afwDisplay.Display(frame=frame)
disp.mtv(exposure)
with disp.Buffering():
for s in sources:
xy = s.getCentroid()
disp.dot('+',
*xy,
ctype=afwDisplay.CYAN
if s.get("flags_negative") else afwDisplay.GREEN)
disp.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
for radius in radii:
disp.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of psf-like objects.
Parameters
----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch` or None
Ignored by this source selector.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used for debug display.
Return
------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `numpy.ndarray` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = display and \
lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
plotFwhmHistogram = display and plt and \
lsstDebug.Info(__name__).plotFwhmHistogram # Plot histogram of FWHM
plotFlags = display and plt and \
lsstDebug.Info(__name__).plotFlags # Plot the sources coloured by their flags
plotRejection = display and plt and \
lsstDebug.Info(__name__).plotRejection # Plot why sources are rejected
afwDisplay.setDefaultMaskTransparency(75)
fluxName = self.config.fluxName
fluxErrName = self.config.fluxErrName
minFwhm = self.config.minFwhm
maxFwhm = self.config.maxFwhm
maxFwhmVariability = self.config.maxFwhmVariability
maxbad = self.config.maxbad
maxbadflag = self.config.maxbadflag
maxellip = self.config.maxellip
minsn = self.config.minsn
maxelong = (maxellip + 1.0)/(1.0 - maxellip) if maxellip < 1.0 else 100
# Unpack the catalogue
shape = sourceCat.getShapeDefinition()
ixx = sourceCat.get("%s.xx" % shape)
iyy = sourceCat.get("%s.yy" % shape)
fwhm = 2*np.sqrt(2*np.log(2))*np.sqrt(0.5*(ixx + iyy))
elong = 0.5*(ixx - iyy)/(ixx + iyy)
flux = sourceCat.get(fluxName)
fluxErr = sourceCat.get(fluxErrName)
sn = flux/np.where(fluxErr > 0, fluxErr, 1)
sn[fluxErr <= 0] = -psfexLib.BIG
flags = 0x0
for i, f in enumerate(self.config.badFlags):
flags = np.bitwise_or(flags, np.where(sourceCat.get(f), 1 << i, 0))
#
# Estimate the acceptable range of source widths
#
good = np.logical_and(sn > minsn, np.logical_not(flags))
good = np.logical_and(good, elong < maxelong)
good = np.logical_and(good, fwhm >= minFwhm)
good = np.logical_and(good, fwhm < maxFwhm)
fwhmMode, fwhmMin, fwhmMax = compute_fwhmrange(fwhm[good], maxFwhmVariability, minFwhm, maxFwhm,
plot=dict(fwhmHistogram=plotFwhmHistogram))
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Here's select_candidates
#
# ---- Apply some selection over flags, fluxes...
bad = (flags != 0)
# set.setBadFlags(int(sum(bad)))
if plotRejection:
selectionVectors = []
selectionVectors.append((bad, "flags %d" % sum(bad)))
dbad = sn < minsn
# set.setBadSN(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
dbad = fwhm < fwhmMin
# set.setBadFrmin(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMin %d" % sum(dbad)))
dbad = fwhm > fwhmMax
# set.setBadFrmax(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMax %d" % sum(dbad)))
dbad = elong > maxelong
# set.setBadElong(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "elong %d" % sum(dbad)))
# -- ... and check the integrity of the sample
if maxbadflag:
nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet])
dbad = nbad > maxbad
# set.setBadPix(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
good = np.logical_not(bad)
#
# We know enough to plot, if so requested
#
frame = 0
if displayExposure:
mi = exposure.getMaskedImage()
disp = afwDisplay.Display(frame=frame)
disp.mtv(mi, title="PSF candidates")
with disp.Buffering():
for i, source in enumerate(sourceCat):
if good[i]:
ctype = afwDisplay.GREEN # star candidate
else:
ctype = afwDisplay.RED # not star
disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
ctype=ctype)
if plotFlags or plotRejection:
imag = -2.5*np.log10(flux)
plt.clf()
alpha = 0.5
if plotFlags:
isSet = np.where(flags == 0x0)[0]
plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label="good")
for i, f in enumerate(self.config.badFlags):
mask = 1 << i
isSet = np.where(np.bitwise_and(flags, mask))[0]
if isSet.any():
if np.isfinite(imag[isSet] + fwhm[isSet]).any():
label = re.sub(r"\_flag", "",
re.sub(r"^base\_", "",
re.sub(r"^.*base\_PixelFlags\_flag\_", "", f)))
plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label=label)
else:
for bad, label in selectionVectors:
plt.plot(imag[bad], fwhm[bad], 'o', alpha=alpha, label=label)
plt.plot(imag[good], fwhm[good], 'o', color="black", label="selected")
[plt.axhline(_, color='red') for _ in [fwhmMin, fwhmMax]]
plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
plt.ylim(fwhm[np.where(np.isfinite(fwhm + imag))].min(), 2*fwhmMax)
plt.legend(loc=2)
plt.xlabel("Instrumental %s Magnitude" % fluxName.split(".")[-1].title())
plt.ylabel("fwhm")
title = "PSFEX Star Selection"
plt.title("%s %d selected" % (title, sum(good)))
if displayExposure:
global eventHandler
eventHandler = EventHandler(plt.axes(), imag, fwhm, sourceCat.getX(), sourceCat.getY(),
frames=[frame])
if plotFlags or plotRejection:
while True:
try:
reply = input("continue? [y[es] h(elp) p(db) q(uit)] ").strip()
except EOFError:
reply = "y"
if not reply:
reply = "y"
if reply[0] == "h":
print("""\
At this prompt, you can continue with almost any key; 'p' enters pdb,
'q' returns to the shell, and
'h' prints this text
""", end=' ')
if displayExposure:
print("""
If you put the cursor on a point in the matplotlib scatter plot and hit 'p' you'll see it in ds9.""")
elif reply[0] == "p":
import pdb
pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
return Struct(selected=good)
def _buildCellSet(self, exposure, referencePsfModel):
"""Build a SpatialCellSet for use with the solve method
Parameters
----------
exposure : `lsst.afw.image.Exposure`
The science exposure that will be convolved; must contain a Psf
referencePsfModel : `lsst.afw.detection.Psf`
Psf model to match to
Returns
-------
result : `struct`
- ``kernelCellSet`` : a SpatialCellSet to be used by self._solve
- ``referencePsfModel`` : Validated and/or modified
reference model used to populate the SpatialCellSet
Notes
-----
If the reference Psf model and science Psf model have different dimensions,
adjust the referencePsfModel (the model to which the exposure PSF will be matched)
to match that of the science Psf. If the science Psf dimensions vary across the image,
as is common with a WarpedPsf, either pad or clip (depending on config.padPsf)
the dimensions to be constant.
"""
sizeCellX = self.kConfig.sizeCellX
sizeCellY = self.kConfig.sizeCellY
scienceBBox = exposure.getBBox()
# Extend for proper spatial matching kernel all the way to edge, especially for narrow strips
scienceBBox.grow(geom.Extent2I(sizeCellX, sizeCellY))
sciencePsfModel = exposure.getPsf()
dimenR = referencePsfModel.getLocalKernel().getDimensions()
psfWidth, psfHeight = dimenR
regionSizeX, regionSizeY = scienceBBox.getDimensions()
scienceX0, scienceY0 = scienceBBox.getMin()
kernelCellSet = afwMath.SpatialCellSet(geom.Box2I(scienceBBox), sizeCellX, sizeCellY)
nCellX = regionSizeX//sizeCellX
nCellY = regionSizeY//sizeCellY
if nCellX == 0 or nCellY == 0:
raise ValueError("Exposure dimensions=%s and sizeCell=(%s, %s). Insufficient area to match" %
(scienceBBox.getDimensions(), sizeCellX, sizeCellY))
# Survey the PSF dimensions of the Spatial Cell Set
# to identify the minimum enclosed or maximum bounding square BBox.
widthList = []
heightList = []
for row in range(nCellY):
posY = sizeCellY*row + sizeCellY//2 + scienceY0
for col in range(nCellX):
posX = sizeCellX*col + sizeCellX//2 + scienceX0
widthS, heightS = sciencePsfModel.computeBBox(geom.Point2D(posX, posY)).getDimensions()
widthList.append(widthS)
heightList.append(heightS)
psfSize = max(max(heightList), max(widthList))
if self.config.doAutoPadPsf:
minPsfSize = nextOddInteger(self.kConfig.kernelSize*self.config.autoPadPsfTo)
paddingPix = max(0, minPsfSize - psfSize)
else:
if self.config.padPsfBy % 2 != 0:
raise ValueError("Config padPsfBy (%i pixels) must be even number." %
self.config.padPsfBy)
paddingPix = self.config.padPsfBy
if paddingPix > 0:
self.log.info("Padding Science PSF from (%s, %s) to (%s, %s) pixels" %
(psfSize, psfSize, paddingPix + psfSize, paddingPix + psfSize))
psfSize += paddingPix
# Check that PSF is larger than the matching kernel
maxKernelSize = psfSize - 1
if maxKernelSize % 2 == 0:
maxKernelSize -= 1
if self.kConfig.kernelSize > maxKernelSize:
message = """
Kernel size (%d) too big to match Psfs of size %d.
Please reconfigure by setting one of the following:
1) kernel size to <= %d
2) doAutoPadPsf=True
3) padPsfBy to >= %s
""" % (self.kConfig.kernelSize, psfSize,
maxKernelSize, self.kConfig.kernelSize - maxKernelSize)
raise ValueError(message)
dimenS = geom.Extent2I(psfSize, psfSize)
if (dimenR != dimenS):
try:
referencePsfModel = referencePsfModel.resized(psfSize, psfSize)
self.log.info("Adjusted dimensions of reference PSF model from %s to %s" % (dimenR, dimenS))
except Exception as e:
self.log.warn("Zero padding or clipping the reference PSF model of type %s and dimensions %s"
" to the science Psf dimensions %s because: %s",
referencePsfModel.__class__.__name__, dimenR, dimenS, e)
dimenR = dimenS
ps = pexConfig.makePropertySet(self.kConfig)
for row in range(nCellY):
# place at center of cell
posY = sizeCellY*row + sizeCellY//2 + scienceY0
for col in range(nCellX):
# place at center of cell
posX = sizeCellX*col + sizeCellX//2 + scienceX0
log.log("TRACE4." + self.log.getName(), log.DEBUG,
"Creating Psf candidate at %.1f %.1f", posX, posY)
# reference kernel image, at location of science subimage
referenceMI = self._makePsfMaskedImage(referencePsfModel, posX, posY, dimensions=dimenR)
# kernel image we are going to convolve
scienceMI = self._makePsfMaskedImage(sciencePsfModel, posX, posY, dimensions=dimenR)
# The image to convolve is the science image, to the reference Psf.
kc = diffimLib.makeKernelCandidate(posX, posY, scienceMI, referenceMI, ps)
kernelCellSet.insertCandidate(kc)
import lsstDebug
display = lsstDebug.Info(__name__).display
displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if not maskTransparency:
maskTransparency = 0
if display:
afwDisplay.setDefaultMaskTransparency(maskTransparency)
if display and displaySpatialCells:
dituils.showKernelSpatialCells(exposure.getMaskedImage(), kernelCellSet,
symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame,
title="Image to be convolved")
lsstDebug.frame += 1
return pipeBase.Struct(kernelCellSet=kernelCellSet,
referencePsfModel=referencePsfModel,
)
import numpy as np
import lsst.utils.tests
import lsst.afw.math as afwMath
import lsst.afw.image as afwImage
import lsst.afw.image.utils as afwImageUtils
from lsst.ip.isr.fringe import FringeTask
try:
display
except NameError:
display = False
else:
import lsst.afw.display as afwDisplay
afwDisplay.setDefaultMaskTransparency(75)
class FringeDataRef(object):
"""Quacks like a ButlerDataRef, so we can provide an in-memory fringe frame.
"""
def __init__(self, fringe):
self.fringe = fringe
self.dataId = {'test': True}
def get(self, name="fringe", immediate=False):
if name == "fringe":
return self.fringe
if name == "ccdExposureId":
return 1000
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
"""Determine a PSFEX PSF model for an exposure given a list of PSF candidates.
Parameters
----------
exposure: `lsst.afw.image.Exposure`
Exposure containing the PSF candidates.
psfCandidateList: iterable of `lsst.meas.algorithms.PsfCandidate`
Sequence of PSF candidates typically obtained by detecting sources and then running them through a
star selector.
metadata: metadata, optional
A home for interesting tidbits of information.
flagKey: `lsst.afw.table.Key`, optional
Schema key used to mark sources actually used in PSF determination.
Returns
-------
psf: `lsst.meas.extensions.psfex.PsfexPsf`
The determined PSF.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = display and \
lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
displayPsfComponents = display and \
lsstDebug.Info(__name__).displayPsfComponents # show the basis functions
showBadCandidates = display and \
lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks)
displayResiduals = display and \
lsstDebug.Info(__name__).displayResiduals # show residuals
displayPsfMosaic = display and \
lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
afwDisplay.setDefaultMaskTransparency(75)
# Normalise residuals by object amplitude
mi = exposure.getMaskedImage()
nCand = len(psfCandidateList)
if nCand == 0:
raise RuntimeError("No PSF candidates supplied.")
#
# How big should our PSF models be?
#
if display: # only needed for debug plots
# construct and populate a spatial cell set
bbox = mi.getBBox(afwImage.PARENT)
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
else:
psfCellSet = None
sizes = np.empty(nCand)
for i, psfCandidate in enumerate(psfCandidateList):
try:
if psfCellSet:
psfCellSet.insertCandidate(psfCandidate)
except Exception as e:
self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
continue
source = psfCandidate.getSource()
quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
rmsSize = quad.getTraceRadius()
sizes[i] = rmsSize
if self.config.kernelSize >= 15:
self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value")
actualKernelSize = int(self.config.kernelSize)
else:
actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
rms = np.median(sizes)
print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms))
# If we manually set the resolution then we need the size in pixel units
pixKernelSize = actualKernelSize
if self.config.samplingSize > 0:
pixKernelSize = int(actualKernelSize*self.config.samplingSize)
if pixKernelSize % 2 == 0:
pixKernelSize += 1
self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize)
psfCandidateList[0].setHeight(pixKernelSize)
psfCandidateList[0].setWidth(pixKernelSize)
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
#
# Insert the good candidates into the set
#
defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex")
args_md = dafBase.PropertySet()
args_md.set("BASIS_TYPE", str(self.config.psfexBasis))
args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
args_md.set("PSF_SIZE", str(actualKernelSize))
args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
prefs = psfex.Prefs(defaultsFile, args_md)
prefs.setCommandLine([])
prefs.addCatalog("psfexPsfDeterminer")
prefs.use()
principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN)
if False else psfex.Context.KEEPHIDDEN)
context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(),
prefs.getGroupDeg(), principalComponentExclusionFlag)
set = psfex.Set(context)
set.setVigSize(pixKernelSize, pixKernelSize)
set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes))
set.setRecentroid(self.config.recentroid)
catindex, ext = 0, 0
backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
ccd = exposure.getDetector()
if ccd:
gain = np.mean(np.array([a.getGain() for a in ccd]))
else:
gain = 1.0
self.log.warn("Setting gain to %g" % (gain,))
contextvalp = []
for i, key in enumerate(context.getName()):
if context.getPcflag(i):
contextvalp.append(pcval[pc])
pc += 1
elif key[0] == ':':
try:
contextvalp.append(exposure.getMetadata().getScalar(key[1:]))
except KeyError:
raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
(key[1:], prefs.getContextName()))
else:
try:
contextvalp.append(np.array([psfCandidateList[_].getSource().get(key)
for _ in range(nCand)]))
except KeyError:
raise RuntimeError("*Error*: %s parameter not found" % (key,))
set.setContextname(i, key)
if display:
frame = 0
if displayExposure:
disp = afwDisplay.Display(frame=frame)
disp.mtv(exposure, title="psf determination")
badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
fluxName = prefs.getPhotfluxRkey()
fluxFlagName = "base_" + fluxName + "_flag"
xpos, ypos = [], []
for i, psfCandidate in enumerate(psfCandidateList):
source = psfCandidate.getSource()
xc, yc = source.getX(), source.getY()
try:
int(xc), int(yc)
except ValueError:
continue
try:
pstamp = psfCandidate.getMaskedImage().clone()
except Exception:
continue
if fluxFlagName in source.schema and source.get(fluxFlagName):
continue
flux = source.get(fluxName)
if flux < 0 or np.isnan(flux):
continue
# From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate).
# Having created the sample, we must proceed to configure it, and then fini (finalize),
# or it will be malformed.
try:
sample = set.newSample()
sample.setCatindex(catindex)
sample.setExtindex(ext)
sample.setObjindex(i)
imArray = pstamp.getImage().getArray()
imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
-2*psfex.BIG
sample.setVig(imArray)
sample.setNorm(flux)
sample.setBacknoise2(backnoise2)
sample.setGain(gain)
sample.setX(xc)
sample.setY(yc)
sample.setFluxrad(sizes[i])
for j in range(set.getNcontext()):
sample.setContext(j, float(contextvalp[j][i]))
except Exception as e:
self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e)
continue
else:
set.finiSample(sample)
xpos.append(xc) # for QA
ypos.append(yc)
if displayExposure:
with disp.Buffering():
disp.dot("o", xc, yc, ctype=afwDisplay.CYAN, size=4)
if set.getNsample() == 0:
raise RuntimeError("No good PSF candidates to pass to PSFEx")
# ---- Update min and max and then the scaling
for i in range(set.getNcontext()):
cmin = contextvalp[i].min()
cmax = contextvalp[i].max()
set.setContextScale(i, cmax - cmin)
set.setContextOffset(i, (cmin + cmax)/2.0)
# Don't waste memory!
set.trimMemory()
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
#
# Do a PSFEX decomposition of those PSF candidates
#
fields = []
field = psfex.Field("Unknown")
field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample())
field.finalize()
fields.append(field)
sets = []
sets.append(set)
psfex.makeit(fields, sets)
psfs = field.getPsfs()
# Flag which objects were actually used in psfex by
good_indices = []
for i in range(sets[0].getNsample()):
index = sets[0].getSample(i).getObjindex()
if index > -1:
good_indices.append(index)
if flagKey is not None:
for i, psfCandidate in enumerate(psfCandidateList):
source = psfCandidate.getSource()
if i in good_indices:
source.set(flagKey, True)
xpos = np.array(xpos)
ypos = np.array(ypos)
numGoodStars = len(good_indices)
avgX, avgY = np.mean(xpos), np.mean(ypos)
psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY))
if False and (displayResiduals or displayPsfMosaic):
ext = 0
frame = 1
diagnostics = True
catDir = "."
title = "psfexPsfDeterminer"
psfex.psfex.showPsf(psfs, set, ext,
[(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())],
nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir,
title=title)
#
# Display code for debugging
#
if display:
assert psfCellSet is not None
if displayExposure:
maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=8, display=disp)
if displayResiduals:
disp4 = afwDisplay.Display(frame=4)
maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
maUtils.showPsf(psf, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
maUtils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp.scale('linear', 0, 1)
#
# Generate some QA information
#
# Count PSF stars
#
if metadata is not None:
metadata.set("spatialFitChi2", np.nan)
metadata.set("numAvailStars", nCand)
metadata.set("numGoodStars", numGoodStars)
metadata.set("avgX", avgX)
metadata.set("avgY", avgY)
return psf, psfCellSet
