def setUp(self):
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(-20, -30), lsst.geom.Extent2I(240, 160))
self.dataset = measBaseTests.TestDataset(self.bbox)
# first two sources are points
self.pointCentroid1 = lsst.geom.Point2D(50.1, 49.8)
self.pointCentroid2 = lsst.geom.Point2D(-11.6, -1.7)
self.dataset.addSource(instFlux=1E5, centroid=self.pointCentroid1)
self.dataset.addSource(instFlux=2E5, centroid=self.pointCentroid2)
# third source is extended
self.extendedCentroid = lsst.geom.Point2D(149.9, 50.3)
self.dataset.addSource(instFlux=1E5, centroid=self.extendedCentroid,
shape=afwGeom.Quadrupole(8, 9, 3))
self.config = self.makeSingleFrameMeasurementConfig("base_SdssShape")
def dot(self,
symb,
c,
r,
size=2,
ctype=None,
origin=afwImage.PARENT,
*args,
**kwargs):
"""!Draw a symbol onto the specified DISPLAY frame at (col,row) = (c,r)
[0-based coordinates]
Possible values are:
+ Draw a +
x Draw an x
* Draw a *
o Draw a circle
@:Mxx,Mxy,Myy Draw an ellipse with moments (Mxx, Mxy, Myy) (argument size is ignored)
An object derived from afwGeom.ellipses.BaseCore Draw the ellipse (argument size is ignored)
Any other value is interpreted as a string to be drawn. Strings obey the fontFamily (which may be
extended with other characteristics, e.g. "times bold italic". Text will be drawn rotated by
textAngle (textAngle is ignored otherwise).
N.b. objects derived from BaseCore include Axes and Quadrupole.
"""
if isinstance(symb, int):
symb = "%d" % (symb)
if origin == afwImage.PARENT and self._xy0 is not None:
x0, y0 = self._xy0
r -= y0
c -= x0
if isinstance(symb, afwGeom.ellipses.BaseCore) or re.search(
r"^@:", symb):
try:
mat = re.search(r"^@:([^,]+),([^,]+),([^,]+)", symb)
except TypeError:
pass
else:
if mat:
mxx, mxy, myy = [float(_) for _ in mat.groups()]
symb = afwGeom.Quadrupole(mxx, myy, mxy)
symb = afwGeom.ellipses.Axes(symb)
self._impl._dot(symb, c, r, size, ctype, **kwargs)
def testRejectBlends(self):
"""Test the PcaPsfDeterminerTask blend removal."""
"""
We give it a single blended source, asking it to remove blends,
and check that it barfs in the expected way.
"""
psfDeterminerClass = measAlg.psfDeterminerRegistry["pca"]
config = psfDeterminerClass.ConfigClass()
config.doRejectBlends = True
psfDeterminer = psfDeterminerClass(config=config)
schema = afwTable.SourceTable.makeMinimalSchema()
# Use The single frame measurement task to populate the schema with standard keys
measBase.SingleFrameMeasurementTask(schema)
catalog = afwTable.SourceCatalog(schema)
source = catalog.addNew()
# Make the source blended, with necessary information to calculate pca
spanShift = afwGeom.Point2I(54, 123)
spans = afwGeom.SpanSet.fromShape(6, offset=spanShift)
foot = afwDetection.Footprint(spans, self.exposure.getBBox())
foot.addPeak(45, 123, 6)
foot.addPeak(47, 126, 5)
source.setFootprint(foot)
centerKey = afwTable.Point2DKey(source.schema['slot_Centroid'])
shapeKey = afwTable.QuadrupoleKey(schema['slot_Shape'])
source.set(centerKey, afwGeom.Point2D(46, 124))
source.set(shapeKey, afwGeom.Quadrupole(1.1, 2.2, 1))
candidates = [measAlg.makePsfCandidate(source, self.exposure)]
metadata = dafBase.PropertyList()
with self.assertRaises(RuntimeError) as cm:
psfDeterminer.determinePsf(self.exposure, candidates, metadata)
self.assertEqual(str(cm.exception),
"All PSF candidates removed as blends")
def dot(self,
symb,
c,
r,
size=2,
ctype=None,
origin=afwImage.PARENT,
*args,
**kwargs):
"""Draw a symbol onto the specified display frame
Parameters
----------
symb
Possible values are:
``"+"``
Draw a +
``"x"``
Draw an x
``"*"``
Draw a *
``"o"``
Draw a circle
``"@:Mxx,Mxy,Myy"``
Draw an ellipse with moments (Mxx, Mxy, Myy) (argument size is ignored)
`lsst.afw.geom.ellipses.BaseCore`
Draw the ellipse (argument size is ignored). N.b. objects
derived from `~lsst.afw.geom.ellipses.BaseCore` include
`~lsst.afw.geom.ellipses.Axes` and `~lsst.afw.geom.ellipses.Quadrupole`.
Any other value
Interpreted as a string to be drawn.
c, r
The column and row where the symbol is drawn [0-based coordinates]
size : `int`
Size of symbol, in pixels
ctype : `str`
The desired color, either e.g. `lsst.afw.display.RED` or a color name known to X11
origin : `lsst.afw.image.ImageOrigin`
Coordinate system for the given positions.
*args
Extra arguments to backend
**kwargs
Extra keyword arguments to backend
"""
if isinstance(symb, int):
symb = f"{symb:d}"
if origin == afwImage.PARENT and self._xy0 is not None:
x0, y0 = self._xy0
r -= y0
c -= x0
if isinstance(symb, afwGeom.ellipses.BaseCore) or re.search(
r"^@:", symb):
try:
mat = re.search(r"^@:([^,]+),([^,]+),([^,]+)", symb)
except TypeError:
pass
else:
if mat:
mxx, mxy, myy = [float(_) for _ in mat.groups()]
symb = afwGeom.Quadrupole(mxx, myy, mxy)
symb = afwGeom.ellipses.Axes(symb)
self._impl._dot(symb, c, r, size, ctype, **kwargs)
def plantSources(x0, y0, nx, ny, sky, nObj, wid, detector, useRandom=False):
pixToTanPix = detector.getTransform(cameraGeom.PIXELS,
cameraGeom.TAN_PIXELS)
img0 = afwImage.ImageF(afwGeom.ExtentI(nx, ny))
img = afwImage.ImageF(afwGeom.ExtentI(nx, ny))
ixx0, iyy0, ixy0 = wid * wid, wid * wid, 0.0
edgeBuffer = 40.0 * wid
flux = 1.0e4
nkx, nky = int(10 * wid) + 1, int(10 * wid) + 1
xhwid, yhwid = nkx // 2, nky // 2
nRow = int(math.sqrt(nObj))
xstep = (nx - 1 - 0.0 * edgeBuffer) // (nRow + 1)
ystep = (ny - 1 - 0.0 * edgeBuffer) // (nRow + 1)
if useRandom:
nObj = nRow * nRow
goodAdded0 = []
goodAdded = []
for i in range(nObj):
# get our position
if useRandom:
xcen0, ycen0 = np.random.uniform(nx), np.random.uniform(ny)
else:
xcen0, ycen0 = xstep * (
(i % nRow) + 1), ystep * (int(i / nRow) + 1)
ixcen0, iycen0 = int(xcen0), int(ycen0)
# distort position and shape
pTan = afwGeom.Point2D(xcen0, ycen0)
p = pixToTanPix.applyInverse(pTan)
linTransform = afwGeom.linearizeTransform(pixToTanPix,
p).invert().getLinear()
m = afwGeom.Quadrupole(ixx0, iyy0, ixy0)
m.transform(linTransform)
xcen, ycen = xcen0, ycen0 # p.getX(), p.getY()
if (xcen < 1.0 * edgeBuffer or (nx - xcen) < 1.0 * edgeBuffer
or ycen < 1.0 * edgeBuffer or (ny - ycen) < 1.0 * edgeBuffer):
continue
ixcen, iycen = int(xcen), int(ycen)
ixx, iyy, ixy = m.getIxx(), m.getIyy(), m.getIxy()
# plant the object
tmp = 0.25 * (ixx - iyy)**2 + ixy**2
a2 = 0.5 * (ixx + iyy) + np.sqrt(tmp)
b2 = 0.5 * (ixx + iyy) - np.sqrt(tmp)
theta = 0.5 * np.arctan2(2.0 * ixy, ixx - iyy)
a = np.sqrt(a2)
b = np.sqrt(b2)
c, s = math.cos(theta), math.sin(theta)
good0, good = True, True
for y in range(nky):
iy = iycen + y - yhwid
iy0 = iycen0 + y - yhwid
for x in range(nkx):
ix = ixcen + x - xhwid
ix0 = ixcen0 + x - xhwid
if ix >= 0 and ix < nx and iy >= 0 and iy < ny:
dx, dy = ix - xcen, iy - ycen
u = c * dx + s * dy
v = -s * dx + c * dy
I0 = flux / (2 * math.pi * a * b)
val = I0 * math.exp(-0.5 * ((u / a)**2 + (v / b)**2))
if val < 0:
val = 0
prevVal = img.get(ix, iy)
img.set(ix, iy, val + prevVal)
else:
good = False
if ix0 >= 0 and ix0 < nx and iy0 >= 0 and iy0 < ny:
dx, dy = ix - xcen, iy - ycen
I0 = flux / (2 * math.pi * wid * wid)
val = I0 * math.exp(-0.5 * ((dx / wid)**2 + (dy / wid)**2))
if val < 0:
val = 0
prevVal = img0.get(ix0, iy0)
img0.set(ix0, iy0, val + prevVal)
else:
good0 = False
if good0:
goodAdded0.append([xcen, ycen])
if good:
goodAdded.append([xcen, ycen])
# add sky and noise
img += sky
img0 += sky
noise = afwImage.ImageF(afwGeom.ExtentI(nx, ny))
noise0 = afwImage.ImageF(afwGeom.ExtentI(nx, ny))
for i in range(nx):
for j in range(ny):
noise.set(i, j, np.random.poisson(img.get(i, j)))
noise0.set(i, j, np.random.poisson(img0.get(i, j)))
edgeWidth = int(0.5 * edgeBuffer)
mask = afwImage.Mask(afwGeom.ExtentI(nx, ny))
left = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.ExtentI(edgeWidth, ny))
right = afwGeom.Box2I(afwGeom.Point2I(nx - edgeWidth, 0),
afwGeom.ExtentI(edgeWidth, ny))
top = afwGeom.Box2I(afwGeom.Point2I(0, ny - edgeWidth),
afwGeom.ExtentI(nx, edgeWidth))
bottom = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.ExtentI(nx, edgeWidth))
for pos in [left, right, top, bottom]:
msk = afwImage.Mask(mask, pos, deep=False)
msk.set(msk.getPlaneBitMask('EDGE'))
expos = afwImage.makeExposure(
afwImage.makeMaskedImage(noise, mask, afwImage.ImageF(noise, True)))
expos0 = afwImage.makeExposure(
afwImage.makeMaskedImage(noise0, mask, afwImage.ImageF(noise0, True)))
im = expos.getMaskedImage().getImage()
im0 = expos0.getMaskedImage().getImage()
im -= sky
im0 -= sky
return expos, goodAdded, expos0, goodAdded0
def main(dataDir, visit, title="", outputTxtFileName=None,
showFwhm=False, minFwhm=None, maxFwhm=None,
correctDistortion=False,
showEllipticity=False, ellipticityDirection=False,
showNdataFwhm=False, showNdataEll=False,
minNdata=None, maxNdata=None,
gridPoints=30, verbose=False):
butler = dafPersist.ButlerFactory(mapper=hscSim.HscSimMapper(root=dataDir)).create()
camera = butler.get("camera")
if not (showFwhm or showEllipticity or showNdataFwhm or showNdataEll or outputTxtFileName):
showFwhm = True
#
# Get a dict of cameraGeom::Ccd indexed by serial number
#
ccds = {}
for raft in camera:
for ccd in raft:
ccd.setTrimmed(True)
ccds[ccd.getId().getSerial()] = ccd
#
# Read all the tableSeeingMap files, converting their (x, y) to focal
# plane coordinates
#
xArr = []
yArr = []
ellArr = []
fwhmArr = []
paArr = []
aArr = []
bArr = []
e1Arr = []
e2Arr = []
elle1e2Arr = []
for tab in butler.subset("tableSeeingMap", visit=visit):
# we could use tab.datasetExists() but it prints a rude message
fileName = butler.get("tableSeeingMap_filename", **tab.dataId)[0]
if not os.path.exists(fileName):
continue
with open(fileName) as fd:
ccd = None
for line in fd.readlines():
if re.search(r"^\s*#", line):
continue
fields = [float(_) for _ in line.split()]
if ccd is None:
ccd = ccds[int(fields[0])]
x, y, fwhm, ell, pa, a, b = fields[1:8]
x, y = ccd.getPositionFromPixel(geom.PointD(x, y)).getMm()
xArr.append(x)
yArr.append(y)
ellArr.append(ell)
fwhmArr.append(fwhm)
paArr.append(pa)
aArr.append(a)
bArr.append(b)
if len(fields) == 11:
e1 = fields[8]
e2 = fields[9]
elle1e2 = fields[10]
else:
e1 = -9999.
e2 = -9999.
elle1e2 = -9999.
e1Arr.append(e1)
e2Arr.append(e2)
elle1e2Arr.append(elle1e2)
xArr = np.array(xArr)
yArr = np.array(yArr)
ellArr = np.array(ellArr)
fwhmArr = np.array(fwhmArr)*0.168 # arcseconds
paArr = np.radians(np.array(paArr))
aArr = np.array(aArr)
bArr = np.array(bArr)
e1Arr = np.array(e1Arr)
e2Arr = np.array(e2Arr)
elle1e2Arr = np.array(elle1e2Arr)
if correctDistortion:
import lsst.afw.geom.ellipses as afwEllipses
dist = camera.getDistortion()
for i in range(len(aArr)):
axes = afwEllipses.Axes(aArr[i], bArr[i], paArr[i])
if False: # testing only!
axes = afwEllipses.Axes(1.0, 1.0, np.arctan2(yArr[i], xArr[i]))
quad = afwGeom.Quadrupole(axes)
quad = quad.transform(dist.computeQuadrupoleTransform(geom.PointD(xArr[i], yArr[i]), False))
axes = afwEllipses.Axes(quad)
aArr[i], bArr[i], paArr[i] = axes.getA(), axes.getB(), axes.getTheta()
ellArr = 1 - bArr/aArr
if len(xArr) == 0:
gridPoints = 0
xs, ys = [], []
else:
N = gridPoints*1j
extent = [min(xArr), max(xArr), min(yArr), max(yArr)]
xs, ys = np.mgrid[extent[0]:extent[1]:N, extent[2]:extent[3]:N]
title = [title, ]
title.append("\n#")
if outputTxtFileName:
f = open(outputTxtFileName, 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
for x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2 \
in zip(xArr, yArr, ellArr, fwhmArr, paArr, aArr, bArr, e1Arr, e2Arr, elle1e2Arr):
f.write('%f %f %f %f %f %f %f %f %f %f\n' % (x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2))
if showFwhm:
title.append("FWHM (arcsec)")
if len(xs) > 0:
fwhmResampled = griddata(xArr, yArr, fwhmArr, xs, ys)
plt.imshow(fwhmResampled.T, extent=extent, vmin=minFwhm, vmax=maxFwhm, origin='lower')
plt.colorbar()
if outputTxtFileName:
ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys)
f = open(outputTxtFileName+'-fwhm-grid.txt', 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
for xline, yline, fwhmline, ndataline \
in zip(xs.tolist(), ys.tolist(), fwhmResampled.tolist(), ndataGrids):
for xx, yy, fwhm, ndata in zip(xline, yline, fwhmline, ndataline):
if fwhm is None:
fwhm = -9999
f.write('%f %f %f %d\n' % (xx, yy, fwhm, ndata))
elif showEllipticity:
title.append("Ellipticity")
scale = 4
if ellipticityDirection: # we don't care about the magnitude
ellArr = 0.1
u = -ellArr*np.cos(paArr)
v = -ellArr*np.sin(paArr)
if gridPoints > 0:
u = griddata(xArr, yArr, u, xs, ys)
v = griddata(xArr, yArr, v, xs, ys)
x, y = xs, ys
else:
x, y = xArr, yArr
Q = plt.quiver(x, y, u, v, scale=scale,
pivot="middle",
headwidth=0,
headlength=0,
headaxislength=0,
)
keyLen = 0.10
if not ellipticityDirection: # we care about the magnitude
plt.quiverkey(Q, 0.20, 0.95, keyLen, "e=%g" % keyLen, labelpos='W')
if outputTxtFileName:
ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys)
f = open(outputTxtFileName+'-ell-grid.txt', 'w')
f.write("# %s visit %s\n" % (" ".join(title), visit))
# f.write('# %f %f %f %f %f %f %f\n' % (x, y, ell, fwhm, pa, a, b))
for xline, yline, uline, vline, ndataline \
in zip(x.tolist(), y.tolist(), u.tolist(), v.tolist(), ndataGrids):
for xx, yy, uu, vv, ndata in zip(xline, yline, uline, vline, ndataline):
if uu is None:
uu = -9999
if vv is None:
vv = -9999
f.write('%f %f %f %f %d\n' % (xx, yy, uu, vv, ndata))
elif showNdataFwhm:
title.append("N per fwhm grid")
if len(xs) > 0:
ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys)
plt.imshow(ndataGrids, interpolation='nearest', extent=extent,
vmin=minNdata, vmax=maxNdata, origin='lower')
plt.colorbar()
else:
pass
elif showNdataEll:
title.append("N per ell grid")
if len(xs) > 0:
ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys)
plt.imshow(ndataGrids, interpolation='nearest', extent=extent,
vmin=minNdata, vmax=maxNdata, origin='lower')
plt.colorbar()
else:
pass
# plt.plot(xArr, yArr, "r.")
# plt.plot(xs, ys, "b.")
plt.axes().set_aspect('equal')
plt.axis([-20000, 20000, -20000, 20000])
def frameInfoFrom(filepath):
with fits.open(filepath) as hdul:
h = hdul[0].header
# 'object=ABELL2163 filter=HSC-I exptime=360.0 alt=62.11143274 '
# ' azm=202.32265181 hst=(23:40:08.363-23:40:48.546)'
return 'object=%s filter=%s exptime=%.1f azm=%.2f hst=%s' % \
(h['OBJECT'], h['FILTER01'], h['EXPTIME'], h['AZIMUTH'], h['HST'])
title.insert(0, frameInfoFrom(butler.get('raw_filename', {'visit': visit, 'ccd': 0})[0]))
title.append(r'$\langle$FWHM$\rangle %4.2f$"' % np.median(fwhmArr))
plt.title("%s visit=%s" % (" ".join(title), visit), fontsize=9)
return plt
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
"""Determine a PCA PSF model for an exposure given a list of PSF candidates.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure containing the psf candidates.
psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate`
A sequence of PSF candidates typically obtained by detecting sources
and then running them through a star selector.
metadata : `lsst.daf.base import PropertyList` or `None`, optional
A home for interesting tidbits of information.
flagKey : `str`, optional
Schema key used to mark sources actually used in PSF determination.
Returns
-------
psf : `lsst.meas.algorithms.PcaPsf`
The measured PSF.
psfCellSet : `lsst.afw.math.SpatialCellSet`
The PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates
displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration
displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components
displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals
displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
# match Kernel amplitudes for spatial plots
matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
# Keep matplotlib alive post mortem
keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model?
showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates
# Normalize residuals by object amplitude
normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration?
if display:
afwDisplay.setDefaultMaskTransparency(75)
if display > 1:
pause = True
mi = exposure.getMaskedImage()
if len(psfCandidateList) == 0:
raise RuntimeError("No PSF candidates supplied.")
# construct and populate a spatial cell set
bbox = mi.getBBox()
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
sizes = []
for i, psfCandidate in enumerate(psfCandidateList):
if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement
continue
try:
psfCellSet.insertCandidate(psfCandidate)
except Exception as e:
self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
continue
source = psfCandidate.getSource()
quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
axes = afwEll.Axes(quad)
sizes.append(axes.getA())
if len(sizes) == 0:
raise RuntimeError("No usable PSF candidates supplied")
nEigenComponents = self.config.nEigenComponents # initial version
if self.config.kernelSize >= 15:
self.log.warn("WARNING: NOT scaling kernelSize by stellar quadrupole moment "
"because config.kernelSize=%s >= 15; "
"using config.kernelSize as as the width, instead",
self.config.kernelSize)
actualKernelSize = int(self.config.kernelSize)
else:
medSize = numpy.median(sizes)
actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
print("Median size=%s" % (medSize,))
self.log.trace("Kernel size=%s", actualKernelSize)
# Set size of image returned around candidate
psfCandidateList[0].setHeight(actualKernelSize)
psfCandidateList[0].setWidth(actualKernelSize)
if self.config.doRejectBlends:
# Remove blended candidates completely
blendedCandidates = [] # Candidates to remove; can't do it while iterating
for cell, cand in candidatesIter(psfCellSet, False):
if len(cand.getSource().getFootprint().getPeaks()) > 1:
blendedCandidates.append((cell, cand))
continue
if display:
print("Removing %d blended Psf candidates" % len(blendedCandidates))
for cell, cand in blendedCandidates:
cell.removeCandidate(cand)
if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
raise RuntimeError("All PSF candidates removed as blends")
if display:
if displayExposure:
disp = afwDisplay.Display(frame=0)
disp.mtv(exposure, title="psf determination")
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o",
ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW,
size=4, display=disp)
#
# Do a PCA decomposition of those PSF candidates
#
reply = "y" # used in interactive mode
for iterNum in range(self.config.nIterForPsf):
if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates
stamps = []
for cell in psfCellSet.getCellList():
for cand in cell.begin(not showBadCandidates): # maybe include bad candidates
try:
im = cand.getMaskedImage()
chi2 = cand.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
stamps.append((im, "%d%s" %
(utils.splitId(cand.getSource().getId(), True)["objId"], chi2),
cand.getStatus()))
except Exception:
continue
if len(stamps) == 0:
print("WARNING: No PSF candidates to show; try setting showBadCandidates=True")
else:
mos = afwDisplay.utils.Mosaic()
for im, label, status in stamps:
im = type(im)(im, True)
try:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
except NotImplementedError:
pass
mos.append(im, label,
(afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else
afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else
afwDisplay.RED))
disp8 = afwDisplay.Display(frame=8)
mos.makeMosaic(display=disp8, title="Psf Candidates")
# Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
cleanChi2 = False # Any naughty (negative/NAN) chi^2 values?
while not cleanChi2:
cleanChi2 = True
#
# First, estimate the PSF
#
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# In clipping, allow all candidates to be innocent until proven guilty on this iteration.
# Throw out any prima facie guilty candidates (naughty chi^2 values)
#
for cell in psfCellSet.getCellList():
awfulCandidates = []
for cand in cell.begin(False): # include bad candidates
cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty
rchi2 = cand.getChi2()
if not numpy.isfinite(rchi2) or rchi2 <= 0:
# Guilty prima facie
awfulCandidates.append(cand)
cleanChi2 = False
self.log.debug("chi^2=%s; id=%s",
cand.getChi2(), cand.getSource().getId())
for cand in awfulCandidates:
if display:
print("Removing bad candidate: id=%d, chi^2=%f" %
(cand.getSource().getId(), cand.getChi2()))
cell.removeCandidate(cand)
#
# Clip out bad fits based on reduced chi^2
#
badCandidates = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate
assert rchi2 > 0
if rchi2 > self.config.reducedChi2ForPsfCandidates:
badCandidates.append(cand)
badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(numBad), badCandidates):
if display:
chi2 = c.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2))
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = psf.getKernel()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
except Exception:
continue
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p*k.getSum()
predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for
k in range(kernel.getNKernelParameters())]
residuals.append([a/amp - p for a, p in zip(params, predict)])
candidates.append(cand)
residuals = numpy.array(residuals)
for k in range(kernel.getNKernelParameters()):
if False:
# Straight standard deviation
mean = residuals[:, k].mean()
rms = residuals[:, k].std()
elif False:
# Using interquartile range
sr = numpy.sort(residuals[:, k])
mean = (sr[int(0.5*len(sr))] if len(sr)%2 else
0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1]))
rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))])
else:
stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
mean = stats.getValue(afwMath.MEANCLIP)
rms = stats.getValue(afwMath.STDEVCLIP)
rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues
if display:
print("Mean for component %d is %f" % (k, mean))
print("RMS for component %d is %f" % (k, rms))
badCandidates = list()
for i, cand in enumerate(candidates):
if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms:
badCandidates.append(i)
badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates):
cand = candidates[c]
if display:
print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" %
(cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
residuals[badCandidates[i], k], self.config.spatialReject*rms))
cand.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Display results
#
if display and displayIterations:
if displayExposure:
if iterNum > 0:
disp.erase()
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW,
ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", size=10, display=disp,
ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED)
if displayResiduals:
while True:
try:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
disp5 = afwDisplay.Display(frame=5)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates,
variance=True)
except Exception:
if not showBadCandidates:
showBadCandidates = True
continue
break
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale('linear', 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
if pause:
while True:
try:
reply = input("Next iteration? [ynchpqQs] ").strip()
except EOFError:
reply = "n"
reply = reply.split()
if reply:
reply, args = reply[0], reply[1:]
else:
reply = ""
if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"):
if reply == "c":
pause = False
elif reply == "h":
print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] "
"s[ave fileName] y[es]")
continue
elif reply == "p":
import pdb
pdb.set_trace()
elif reply == "q":
display = False
elif reply == "Q":
sys.exit(1)
elif reply == "s":
fileName = args.pop(0)
if not fileName:
print("Please provide a filename")
continue
print("Saving to %s" % fileName)
utils.saveSpatialCellSet(psfCellSet, fileName=fileName)
continue
break
else:
print("Unrecognised response: %s" % reply, file=sys.stderr)
if reply == "n":
break
# One last time, to take advantage of the last iteration
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# Display code for debugging
#
if display and reply != "n":
disp = afwDisplay.Display(frame=0)
if displayExposure:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=8, display=disp)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED,
size=10, display=disp)
if displayResiduals:
disp4 = afwDisplay.Display(frame=4)
utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
if displayPsfComponents:
disp6 = afwDisplay.Display(frame=6)
utils.showPsf(psf, eigenValues, display=disp6)
if displayPsfMosaic:
disp7 = afwDisplay.Display(frame=7)
utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True)
disp7.scale("linear", 0, 1)
if displayPsfSpatialModel:
utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
#
# Generate some QA information
#
# Count PSF stars
#
numGoodStars = 0
numAvailStars = 0
avgX = 0.0
avgY = 0.0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # don't ignore BAD stars
numAvailStars += 1
for cand in cell.begin(True): # do ignore BAD stars
src = cand.getSource()
if flagKey is not None:
src.set(flagKey, True)
avgX += src.getX()
avgY += src.getY()
numGoodStars += 1
avgX /= numGoodStars
avgY /= numGoodStars
if metadata is not None:
metadata.set("spatialFitChi2", fitChi2)
metadata.set("numGoodStars", numGoodStars)
metadata.set("numAvailStars", numAvailStars)
metadata.set("avgX", avgX)
metadata.set("avgY", avgY)
psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY))
return psf, psfCellSet
def setUp(self):
ixx, iyy, ixy = 1.0, 1.0, 0.0
self.data = afwGeom.Quadrupole(ixx, iyy, ixy)
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of PSF candidates that represent likely stars.
A list of PSF candidates may be used by a PSF fitter to construct a PSF.
Parameters:
-----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch` or None
Ignored in this SourceSelector.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used to get the detector
to transform to TanPix, and for debug display.
Return
------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `array` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(
__name__).displayExposure # display the Exposure + spatialCells
plotMagSize = lsstDebug.Info(
__name__).plotMagSize # display the magnitude-size relation
dumpData = lsstDebug.Info(
__name__).dumpData # dump data to pickle file?
detector = None
pixToTanPix = None
if exposure:
detector = exposure.getDetector()
if detector:
pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS)
#
# Look at the distribution of stars in the magnitude-size plane
#
flux = sourceCat.get(self.config.sourceFluxField)
fluxErr = sourceCat.get(self.config.sourceFluxField + "Err")
xx = numpy.empty(len(sourceCat))
xy = numpy.empty_like(xx)
yy = numpy.empty_like(xx)
for i, source in enumerate(sourceCat):
Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
if pixToTanPix:
p = lsst.geom.Point2D(source.getX(), source.getY())
linTransform = afwGeom.linearizeTransform(pixToTanPix,
p).getLinear()
m = afwGeom.Quadrupole(Ixx, Iyy, Ixy)
m.transform(linTransform)
Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy()
xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
width = numpy.sqrt(0.5 * (xx + yy))
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)),
self.config.badFlags, False)
bad = numpy.logical_or(bad,
numpy.logical_not(numpy.isfinite(width)))
bad = numpy.logical_or(bad,
numpy.logical_not(numpy.isfinite(flux)))
if self.config.doFluxLimit:
bad = numpy.logical_or(bad, flux < self.config.fluxMin)
if self.config.fluxMax > 0:
bad = numpy.logical_or(bad, flux > self.config.fluxMax)
if self.config.doSignalToNoiseLimit:
bad = numpy.logical_or(
bad, flux / fluxErr < self.config.signalToNoiseMin)
if self.config.signalToNoiseMax > 0:
bad = numpy.logical_or(
bad, flux / fluxErr > self.config.signalToNoiseMax)
bad = numpy.logical_or(bad, width < self.config.widthMin)
bad = numpy.logical_or(bad, width > self.config.widthMax)
good = numpy.logical_not(bad)
if not numpy.any(good):
raise RuntimeError(
"No objects passed our cuts for consideration as psf stars")
mag = -2.5 * numpy.log10(flux[good])
width = width[good]
#
# Look for the maximum in the size histogram, then search upwards for the minimum that separates
# the initial peak (of, we presume, stars) from the galaxies
#
if dumpData:
import os
import pickle as pickle
_ii = 0
while True:
pickleFile = os.path.expanduser(
os.path.join("~", "widths-%d.pkl" % _ii))
if not os.path.exists(pickleFile):
break
_ii += 1
with open(pickleFile, "wb") as fd:
pickle.dump(mag, fd, -1)
pickle.dump(width, fd, -1)
centers, clusterId = _kcenters(
width,
nCluster=4,
useMedian=True,
widthStdAllowed=self.config.widthStdAllowed)
if display and plotMagSize:
fig = plot(
mag,
width,
centers,
clusterId,
magType=self.config.sourceFluxField.split(".")[-1].title(),
marker="+",
markersize=3,
markeredgewidth=None,
ltype=':',
clear=True)
else:
fig = None
clusterId = _improveCluster(
width,
centers,
clusterId,
nsigma=self.config.nSigmaClip,
widthStdAllowed=self.config.widthStdAllowed)
if display and plotMagSize:
plot(mag,
width,
centers,
clusterId,
marker="x",
markersize=3,
markeredgewidth=None,
clear=False)
stellar = (clusterId == 0)
#
# We know enough to plot, if so requested
#
frame = 0
if fig:
if display and displayExposure:
disp = afwDisplay.Display(frame=frame)
disp.mtv(exposure.getMaskedImage(), title="PSF candidates")
global eventHandler
eventHandler = EventHandler(fig.get_axes()[0],
mag,
width,
sourceCat.getX()[good],
sourceCat.getY()[good],
frames=[frame])
fig.show()
while True:
try:
reply = input("continue? [c h(elp) q(uit) p(db)] ").strip()
except EOFError:
reply = None
if not reply:
reply = "c"
if reply:
if reply[0] == "h":
print("""\
We cluster the points; red are the stellar candidates and the other colours are other clusters.
Points labelled + are rejects from the cluster (only for cluster 0).
At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the
image display.
""")
elif reply[0] == "p":
import pdb
pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
if display and displayExposure:
mi = exposure.getMaskedImage()
with disp.Buffering():
for i, source in enumerate(sourceCat):
if good[i]:
ctype = afwDisplay.GREEN # star candidate
else:
ctype = afwDisplay.RED # not star
disp.dot("+",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
ctype=ctype)
# stellar only applies to good==True objects
mask = good == True # noqa (numpy bool comparison): E712
good[mask] = stellar
return Struct(selected=good)
