def showPsf(psf, eigenValues=None, XY=None, normalize=True, frame=None):
"""Display a PSF's eigen images
If normalize is True, set the largest absolute value of each eigenimage to 1.0 (n.b. sum == 0.0 for i > 0)
"""
if eigenValues:
coeffs = eigenValues
elif XY is not None:
coeffs = psf.getLocalKernel(afwGeom.PointD(XY[0], XY[1])).getKernelParameters()
else:
coeffs = None
mos = displayUtils.Mosaic(gutter=2, background=-0.1)
for i, k in enumerate(afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList()):
im = afwImage.ImageD(k.getDimensions())
k.computeImage(im, False)
if normalize:
im /= numpy.max(numpy.abs(im.getArray()))
if coeffs:
mos.append(im, "%g" % (coeffs[i]/coeffs[0]))
else:
mos.append(im)
mos.makeMosaic(frame=frame, title="Kernel Basis Functions")
return mos
def showPsf(psf, eigenValues=None, XY=None, normalize=True, frame=None):
"""Display a PSF's eigen images
If normalize is True, set the largest absolute value of each eigenimage to 1.0 (n.b. sum == 0.0 for i > 0)
"""
if eigenValues:
coeffs = eigenValues
elif XY is not None:
coeffs = psf.getLocalKernel(afwGeom.PointD(XY[0], XY[1])).getKernelParameters()
else:
coeffs = None
mos = displayUtils.Mosaic(gutter=2, background=-0.1)
for i, k in enumerate(afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList()):
im = afwImage.ImageD(k.getDimensions())
k.computeImage(im, False)
if normalize:
im /= numpy.max(numpy.abs(im.getArray()))
if coeffs:
mos.append(im, "%g" % (coeffs[i]/coeffs[0]))
else:
mos.append(im)
mos.makeMosaic(frame=frame, title="Kernel Basis Functions")
return mos
def plotPsfSpatialModel(exposure,
psf,
psfCellSet,
showBadCandidates=True,
numSample=128,
matchKernelAmplitudes=False,
keepPlots=True):
"""Plot the PSF spatial model."""
if plt is None:
print >> sys.stderr, "Unable to import matplotlib"
return
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
if not showBadCandidates and cand.isBad():
continue
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception, e:
continue
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im,
candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
matchKernelAmplitudes=False, keepPlots=True):
"""Plot the PSF spatial model."""
if plt is None:
print >> sys.stderr, "Unable to import matplotlib"
return
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
if not showBadCandidates and cand.isBad():
continue
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception, e:
continue
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
print "Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
except Exception, e:
continue
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
print templateIdList
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
False)
else:
raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
"but template sources not available. Cannot match science " +
"sources with template sources. Run process* on data from " +
"which templates are built.")
kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory
)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
exposure.getWcs(), exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId() : x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
+ 2.5*numpy.log10(srcToMatch[x].get("r"))
for x in sids if x in srcToMatch.keys()])
dlong = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
dlat = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
idx1 = numpy.where(colors<self.sourceSelector.config.grMin)
idx2 = numpy.where((colors>=self.sourceSelector.config.grMin)&
(colors<=self.sourceSelector.config.grMax))
idx3 = numpy.where(colors>self.sourceSelector.config.grMax)
rms1Long = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25))
rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25))
rms2Long = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25))
rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25))
rms3Long = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25))
rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
rms1Long,
numpy.median(dlat[idx1]),
rms1Lat))
self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
rms2Long,
numpy.median(dlat[idx2]),
rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
rms3Long,
numpy.median(dlat[idx3]),
rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
self.measurement.run(diaSources, subtractedExposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = measAstrom.AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
fitBasisComponents=False, variance=None, chi=None):
"""Display the PSF candidates.
If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals)
If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
If fitBasisComponents is true, also find the best linear combination of the PSF's components (if they exist)
"""
if chi is None:
if variance is not None: # old name for chi
chi = variance
#
# Show us the ccandidates
#
mos = displayUtils.Mosaic()
#
candidateCenters = []
candidateCentersBad = []
candidateIndex = 0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
rchi2 = cand.getChi2()
if rchi2 > 1e100:
rchi2 = numpy.nan
if not showBadCandidates and cand.isBad():
continue
if psf:
im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")
try:
im = cand.getMaskedImage() # copy of this object's image
xc, yc = cand.getXCenter(), cand.getYCenter()
margin = 0 if True else 5
w, h = im.getDimensions()
bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
var = bim.getVariance(); var.set(stdev**2); del var
sbim = im.Factory(bim, bbox)
sbim <<= im
del sbim
im = bim
xc += margin; yc += margin
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
except:
continue
if not variance:
im_resid.append(im.Factory(im, True))
if True: # tweak up centroids
mi = im
psfIm = mi.getImage()
config = measAlg.SourceMeasurementConfig()
config.centroider.name = "centroid.sdss"
config.slots.centroid = config.centroider.name
schema = afwTable.SourceTable.makeMinimalSchema()
measureSources = config.makeMeasureSources(schema)
catalog = afwTable.SourceCatalog(schema)
config.slots.setupTable(catalog.table)
extra = 10 # enough margin to run the sdss centroider
miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra)
miBig[extra:-extra, extra:-extra] = mi
miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra)
mi = miBig; del miBig
exp = afwImage.makeExposure(mi)
exp.setPsf(psf)
footprintSet = afwDet.FootprintSet(mi,
afwDet.Threshold(0.5*numpy.max(psfIm.getArray())),
"DETECTED")
footprintSet.makeSources(catalog)
if len(catalog) == 0:
raise RuntimeError("Failed to detect any objects")
elif len(catalog) == 1:
source = catalog[0]
else: # more than one source; find the once closest to (xc, yc)
for i, s in enumerate(catalog):
d = numpy.hypot(xc - s.getX(), yc - s.getY())
if i == 0 or d < dmin:
source, dmin = s, d
measureSources.applyWithPeak(source, exp)
xc, yc = source.getCentroid()
# residuals using spatial model
try:
chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
except:
chi2 = numpy.nan
continue
resid = im
if variance:
resid = resid.getImage()
var = im.getVariance()
var = var.Factory(var, True)
numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
resid /= var
im_resid.append(resid)
# Fit the PSF components directly to the data (i.e. ignoring the spatial model)
if fitBasisComponents:
im = cand.getMaskedImage()
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = afwMath.KernelList(fit[1])
outputKernel = afwMath.LinearCombinationKernel(kernels, params)
outImage = afwImage.ImageD(outputKernel.getDimensions())
outputKernel.computeImage(outImage, False)
im -= outImage.convertF()
resid = im
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
sbim = im.Factory(bim, bbox)
sbim <<= resid
del sbim
resid = bim
if variance:
resid = resid.getImage()
resid /= var
im_resid.append(resid)
im = im_resid.makeMosaic()
else:
im = cand.getMaskedImage()
if normalize:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
objId = splitId(cand.getSource().getId(), True)["objId"]
if psf:
lab = "%d chi^2 %.1f" % (objId, rchi2)
ctype = ds9.RED if cand.isBad() else ds9.GREEN
else:
lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
ctype = ds9.GREEN
mos.append(im, lab, ctype)
if False and numpy.isnan(rchi2):
ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
print "amp", cand.getAmplitude()
im = cand.getMaskedImage()
center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
candidateIndex += 1
if cand.isBad():
candidateCentersBad.append(center)
else:
candidateCenters.append(center)
if variance:
title = "chi(Psf fit)"
else:
title = "Stars & residuals"
mosaicImage = mos.makeMosaic(frame=frame, title=title)
with ds9.Buffering():
for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
for cen in centers:
bbox = mos.getBBox(cen[0])
ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)
return mosaicImage
def testGetPcaKernel(self):
"""Convert our cellSet to a LinearCombinationKernel"""
nEigenComponents = 2
spatialOrder = 1
kernelSize = 21
nStarPerCell = 2
nStarPerCellSpatialFit = 2
tolerance = 1e-5
if display:
ds9.mtv(self.mi, frame=0)
#
# Show the candidates we're using
#
for cell in self.cellSet.getCellList():
i = 0
for cand in cell:
i += 1
source = algorithms.cast_PsfCandidateF(cand).getSource()
xc, yc = source.getXAstrom() - self.mi.getX0(), source.getYAstrom() - self.mi.getY0()
if i <= nStarPerCell:
ds9.dot("o", xc, yc, ctype=ds9.GREEN)
else:
ds9.dot("o", xc, yc, ctype=ds9.YELLOW)
pair = algorithms.createKernelFromPsfCandidates(self.cellSet, self.exposure.getDimensions(),
self.exposure.getXY0(), nEigenComponents, spatialOrder,
kernelSize, nStarPerCell)
kernel, eigenValues = pair[0], pair[1]; del pair
print "lambda", " ".join(["%g" % l for l in eigenValues])
pair = algorithms.fitSpatialKernelFromPsfCandidates(kernel, self.cellSet, nStarPerCellSpatialFit, tolerance)
status, chi2 = pair[0], pair[1]; del pair
print "Spatial fit: %s chi^2 = %.2g" % (status, chi2)
psf = algorithms.PcaPsf.swigConvert(roundTripPsf(5, algorithms.PcaPsf(kernel))) # Hurrah!
self.assertTrue(afwMath.cast_AnalyticKernel(psf.getKernel()) is None)
self.assertTrue(afwMath.cast_LinearCombinationKernel(psf.getKernel()) is not None)
self.checkTablePersistence(psf)
if display:
#print psf.getKernel().toString()
eImages = []
for k in afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList():
im = afwImage.ImageD(k.getDimensions())
k.computeImage(im, False)
eImages.append(im)
mos = displayUtils.Mosaic()
frame = 3
ds9.mtv(mos.makeMosaic(eImages), frame=frame)
ds9.dot("Eigen Images", 0, 0, frame=frame)
#
# Make a mosaic of PSF candidates
#
stamps = []
stampInfo = []
for cell in self.cellSet.getCellList():
for cand in cell:
#
# Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate; all
# it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates
# don't know about getMaskedImage; so cast the pointer to PsfCandidate
#
cand = algorithms.cast_PsfCandidateF(cand)
s = cand.getSource()
im = cand.getMaskedImage()
stamps.append(im)
stampInfo.append("[%d 0x%x]" % (s.getId(), s.getFlagForDetection()))
mos = displayUtils.Mosaic()
frame = 1
ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True)
for i in range(len(stampInfo)):
ds9.dot(stampInfo[i], mos.getBBox(i).getX0(), mos.getBBox(i).getY0(), frame=frame, ctype=ds9.RED)
psfImages = []
labels = []
if False:
nx, ny = 3, 4
for iy in range(ny):
for ix in range(nx):
x = int((ix + 0.5)*self.mi.getWidth()/nx)
y = int((iy + 0.5)*self.mi.getHeight()/ny)
im = psf.getImage(x, y)
psfImages.append(im.Factory(im, True))
labels.append("PSF(%d,%d)" % (int(x), int(y)))
if True:
print x, y, "PSF parameters:", psf.getKernel().getKernelParameters()
else:
nx, ny = 2, 2
for x, y in [(20, 20), (60, 20),
(60, 210), (20, 210)]:
im = psf.computeImage(afwGeom.PointD(x, y))
psfImages.append(im.Factory(im, True))
labels.append("PSF(%d,%d)" % (int(x), int(y)))
if True:
print x, y, "PSF parameters:", psf.getKernel().getKernelParameters()
frame = 2
mos.makeMosaic(psfImages, frame=frame, mode=nx)
mos.drawLabels(labels, frame=frame)
if display:
ds9.mtv(self.mi, frame=0)
psfImages = []
labels = []
if False:
nx, ny = 3, 4
for iy in range(ny):
for ix in range(nx):
x = int((ix + 0.5)*self.mi.getWidth()/nx)
y = int((iy + 0.5)*self.mi.getHeight()/ny)
algorithms.subtractPsf(psf, self.mi, x, y)
else:
nx, ny = 2, 2
for x, y in [(20, 20), (60, 20),
(60, 210), (20, 210)]:
if False: # Test subtraction with non-centered psfs
x += 0.5; y -= 0.5
#algorithms.subtractPsf(psf, self.mi, x, y)
ds9.mtv(self.mi, frame=1)
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
matchKernelAmplitudes=False, keepPlots=True):
"""Plot the PSF spatial model."""
if not plt:
print >> sys.stderr, "Unable to import matplotlib"
return
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
if not showBadCandidates and cand.isBad():
continue
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception:
continue
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
numCandidates = len(candFits)
numBasisFuncs = noSpatialKernel.getNBasisKernels()
xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
zGood = numpy.array(candFits)
ampGood = numpy.array(candAmps)
xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
zBad = numpy.array(badFits)
ampBad = numpy.array(badAmps)
numBad = len(badPos)
xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)
kernel = psf.getKernel()
nKernelComponents = kernel.getNKernelParameters()
#
# Figure out how many panels we'll need
#
nPanelX = int(math.sqrt(nKernelComponents))
nPanelY = nKernelComponents//nPanelX
while nPanelY*nPanelX < nKernelComponents:
nPanelX += 1
fig = plt.figure(1)
fig.clf()
try:
fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
except: # protect against API changes
pass
#
# Generator for axes arranged in panels
#
subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)
for k in range(nKernelComponents):
func = kernel.getSpatialFunction(k)
dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
yMin = dfGood.min()
yMax = dfGood.max()
if numBad > 0:
dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
yMin = min([yMin, dfBad.min()])
yMax = max([yMax, dfBad.max()])
yMin -= 0.05 * (yMax - yMin)
yMax += 0.05 * (yMax - yMin)
yMin = -0.01
yMax = 0.01
fRange = numpy.ndarray((len(xRange), len(yRange)))
for j, yVal in enumerate(yRange):
for i, xVal in enumerate(xRange):
fRange[j][i] = func(xVal, yVal)
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getHeight())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(yGood, dfGood, 'b+')
if numBad > 0:
ax.plot(yBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('Residuals(y)')
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
if matchKernelAmplitudes and k == 0:
vmin = 0.0
vmax = 1.1
else:
vmin = fRange.min()
vmax = fRange.max()
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
ax.set_title('Spatial poly')
plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(xGood, dfGood, 'b+')
if numBad > 0:
ax.plot(xBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('K%d Residuals(x)' % k)
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
if False:
ax.scatter(xGood, yGood, c=dfGood, marker='o')
ax.scatter(xBad, yBad, c=dfBad, marker='x')
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=0, upper=exposure.getHeight())
ax.set_title('Spatial residuals')
plt.colorbar(im, orientation='horizontal')
else:
calib = exposure.getCalib()
if calib.getFluxMag0()[0] <= 0:
calib = type(calib)()
calib.setFluxMag0(1.0)
with CalibNoThrow():
ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+')
if numBad > 0:
ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+')
ax.set_title('Flux variation')
fig.show()
global keptPlots
if keepPlots and not keptPlots:
# Keep plots open when done
def show():
print "%s: Please close plots when done." % __name__
try:
plt.show()
except:
pass
print "Plots closed, exiting..."
import atexit
atexit.register(show)
keptPlots = True
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
fitBasisComponents=False, variance=None, chi=None):
"""Display the PSF candidates.
If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs
(and residuals)
If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
If fitBasisComponents is true, also find the best linear combination of the PSF's components
(if they exist)
"""
if chi is None:
if variance is not None: # old name for chi
chi = variance
#
# Show us the ccandidates
#
mos = displayUtils.Mosaic()
#
candidateCenters = []
candidateCentersBad = []
candidateIndex = 0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
rchi2 = cand.getChi2()
if rchi2 > 1e100:
rchi2 = numpy.nan
if not showBadCandidates and cand.isBad():
continue
if psf:
im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")
try:
im = cand.getMaskedImage() # copy of this object's image
xc, yc = cand.getXCenter(), cand.getYCenter()
margin = 0 if True else 5
w, h = im.getDimensions()
bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim.getVariance().set(stdev**2)
bim.assign(im, bbox)
im = bim
xc += margin
yc += margin
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
except:
continue
if not variance:
im_resid.append(im.Factory(im, True))
if True: # tweak up centroids
mi = im
psfIm = mi.getImage()
config = measBase.SingleFrameMeasurementTask.ConfigClass()
config.slots.centroid = "base_SdssCentroid"
schema = afwTable.SourceTable.makeMinimalSchema()
measureSources = measBase.SingleFrameMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
extra = 10 # enough margin to run the sdss centroider
miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra)
miBig[extra:-extra, extra:-extra] = mi
miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra)
mi = miBig
del miBig
exp = afwImage.makeExposure(mi)
exp.setPsf(psf)
footprintSet = afwDet.FootprintSet(mi,
afwDet.Threshold(0.5*numpy.max(psfIm.getArray())),
"DETECTED")
footprintSet.makeSources(catalog)
if len(catalog) == 0:
raise RuntimeError("Failed to detect any objects")
measureSources.run(catalog, exp)
if len(catalog) == 1:
source = catalog[0]
else: # more than one source; find the once closest to (xc, yc)
dmin = None # an invalid value to catch logic errors
for i, s in enumerate(catalog):
d = numpy.hypot(xc - s.getX(), yc - s.getY())
if i == 0 or d < dmin:
source, dmin = s, d
xc, yc = source.getCentroid()
# residuals using spatial model
try:
chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
except:
chi2 = numpy.nan
continue
resid = im
if variance:
resid = resid.getImage()
var = im.getVariance()
var = var.Factory(var, True)
numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
resid /= var
im_resid.append(resid)
# Fit the PSF components directly to the data (i.e. ignoring the spatial model)
if fitBasisComponents:
im = cand.getMaskedImage()
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
try:
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
except:
noSpatialKernel = None
if noSpatialKernel:
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = afwMath.KernelList(fit[1])
outputKernel = afwMath.LinearCombinationKernel(kernels, params)
outImage = afwImage.ImageD(outputKernel.getDimensions())
outputKernel.computeImage(outImage, False)
im -= outImage.convertF()
resid = im
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
bim.assign(resid, bbox)
resid = bim
if variance:
resid = resid.getImage()
resid /= var
im_resid.append(resid)
im = im_resid.makeMosaic()
else:
im = cand.getMaskedImage()
if normalize:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
objId = splitId(cand.getSource().getId(), True)["objId"]
if psf:
lab = "%d chi^2 %.1f" % (objId, rchi2)
ctype = ds9.RED if cand.isBad() else ds9.GREEN
else:
lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
ctype = ds9.GREEN
mos.append(im, lab, ctype)
if False and numpy.isnan(rchi2):
ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
print "amp", cand.getAmplitude()
im = cand.getMaskedImage()
center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
candidateIndex += 1
if cand.isBad():
candidateCentersBad.append(center)
else:
candidateCenters.append(center)
if variance:
title = "chi(Psf fit)"
else:
title = "Stars & residuals"
mosaicImage = mos.makeMosaic(frame=frame, title=title)
with ds9.Buffering():
for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
for cen in centers:
bbox = mos.getBBox(cen[0])
ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)
return mosaicImage
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure,
sensorRef,
templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(
psfAttr.ADAPTIVE_MOMENT)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(),
afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(
psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel(
).getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight,
scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn(
"Src product does not exist; running detection, measurement, selection"
)
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(
makeKernelBasisList(
self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(
exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
matches = afwTable.matchRaDec(templateSources,
selectSources,
1.0 * afwGeom.arcseconds,
False)
else:
raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
+
"but template sources not available. Cannot match science "
+
"sources with template sources. Run process* on data from "
+ "which templates are built.")
kernelSources = self.sourceSelector.selectStars(
exposure, selectSources, matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [
k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize
]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources),
len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources,
templateExposure,
selectSources)
warpedExp = self.register.warpExposure(templateExposure,
wcsResults.wcs,
exposure.getWcs(),
exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable(
).getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable(
).getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [
m.first.get(refCoordKey) for m in wcsResults.matches
]
residuals = [
m.first.get(refCoordKey).getOffsetFrom(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [
m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
colors = numpy.array([
-2.5 * numpy.log10(srcToMatch[x].get("g")) +
2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
if x in srcToMatch.keys()
])
dlong = numpy.array([
r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
dlat = numpy.array([
r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
idx1 = numpy.where(
colors < self.sourceSelector.config.grMin)
idx2 = numpy.where(
(colors >= self.sourceSelector.config.grMin)
& (colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(
colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma * (numpy.percentile(
dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25))
rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma * (numpy.percentile(
dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25))
rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma * (numpy.percentile(
dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25))
rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian",
numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian",
numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian",
numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian",
numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian",
numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian",
numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure,
self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(
exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
# already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel)
subtractedExposure = decorrResult.correctedExposure
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" %
(len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if not self.config.doDipoleFitting:
self.measurement.run(diaSources, subtractedExposure)
else:
if self.config.doSubtract:
self.measurement.run(diaSources, subtractedExposure,
exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure,
exposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs(
).pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"),
diaSources, matchRadPixel,
True)
srcMatchDict = dict([(srcMatch.second.getId(),
srcMatch.first.getId())
for srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" %
(len(srcMatchDict), len(diaSources)))
else:
self.log.warn(
"Src product does not exist; cannot match with diaSources"
)
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = measAstrom.AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure,
sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn(
"No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info(
"Matched %d / %d diaSources to reference catalog" %
(len(refMatches), len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources,
self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(
KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids,
diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources,
self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources,
diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
class PcaPsfDeterminer(object):
"""!
A measurePsfTask psf estimator
"""
ConfigClass = PcaPsfDeterminerConfig
def __init__(self, config):
"""!Construct a PCA PSF Fitter
\param[in] config instance of PcaPsfDeterminerConfig
"""
self.config = config
# N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging
# independent of which determiner is active
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(),
"meas.algorithms.psfDeterminer")
def _fitPsf(self, exposure, psfCellSet, kernelSize, nEigenComponents):
algorithmsLib.PsfCandidateF.setPixelThreshold(
self.config.pixelThreshold)
algorithmsLib.PsfCandidateF.setMaskBlends(self.config.doMaskBlends)
#
# Loop trying to use nEigenComponents, but allowing smaller numbers if necessary
#
for nEigen in range(nEigenComponents, 0, -1):
# Determine KL components
try:
kernel, eigenValues = algorithmsLib.createKernelFromPsfCandidates(
psfCellSet, exposure.getDimensions(), exposure.getXY0(),
nEigen, self.config.spatialOrder, kernelSize,
self.config.nStarPerCell, bool(self.config.constantWeight))
break # OK, we can get nEigen components
except pexExceptions.LengthError as e:
if nEigen == 1: # can't go any lower
raise IndexError("No viable PSF candidates survive")
self.warnLog.log(
pexLog.Log.WARN,
"%s: reducing number of eigen components" % e.what())
#
# We got our eigen decomposition so let's use it
#
# Express eigenValues in units of reduced chi^2 per star
size = kernelSize + 2 * self.config.borderWidth
nu = size * size - 1 # number of degrees of freedom/star for chi^2
eigenValues = [
l / float(
algorithmsLib.countPsfCandidates(
psfCellSet, self.config.nStarPerCell) * nu)
for l in eigenValues
]
# Fit spatial model
status, chi2 = algorithmsLib.fitSpatialKernelFromPsfCandidates(
kernel, psfCellSet, bool(self.config.nonLinearSpatialFit),
self.config.nStarPerCellSpatialFit, self.config.tolerance,
self.config.lam)
psf = algorithmsLib.PcaPsf(kernel)
return psf, eigenValues, nEigen, chi2
def determinePsf(self,
exposure,
psfCandidateList,
metadata=None,
flagKey=None):
"""!Determine a PCA PSF model for an exposure given a list of PSF candidates
\param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
\param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
typically obtained by detecting sources and then running them through a star selector
\param[in,out] metadata a home for interesting tidbits of information
\param[in] flagKey schema key used to mark sources actually used in PSF determination
\return a list of
- psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
- cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(
__name__).displayExposure # display the Exposure + spatialCells
displayPsfCandidates = lsstDebug.Info(
__name__).displayPsfCandidates # show the viable candidates
displayIterations = lsstDebug.Info(
__name__).displayIterations # display on each PSF iteration
displayPsfComponents = lsstDebug.Info(
__name__).displayPsfComponents # show the PCA components
displayResiduals = lsstDebug.Info(
__name__).displayResiduals # show residuals
displayPsfMosaic = lsstDebug.Info(
__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
matchKernelAmplitudes = lsstDebug.Info(
__name__).matchKernelAmplitudes # match Kernel amplitudes
# for spatial plots
keepMatplotlibPlots = lsstDebug.Info(
__name__).keepMatplotlibPlots # Keep matplotlib alive
# post mortem
displayPsfSpatialModel = lsstDebug.Info(
__name__).displayPsfSpatialModel # Plot spatial model?
showBadCandidates = lsstDebug.Info(
__name__).showBadCandidates # Include bad candidates
normalizeResiduals = lsstDebug.Info(
__name__).normalizeResiduals # Normalise residuals by
# object amplitude
pause = lsstDebug.Info(
__name__).pause # Prompt user after each iteration?
if display > 1:
pause = True
mi = exposure.getMaskedImage()
if len(psfCandidateList) == 0:
raise RuntimeError("No PSF candidates supplied.")
# construct and populate a spatial cell set
bbox = mi.getBBox()
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
self.config.sizeCellY)
sizes = []
for i, psfCandidate in enumerate(psfCandidateList):
if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement
continue
try:
psfCellSet.insertCandidate(psfCandidate)
except Exception, e:
self.debugLog.debug(
2, "Skipping PSF candidate %d of %d: %s" %
(i, len(psfCandidateList), e))
continue
source = psfCandidate.getSource()
quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
source.getIxy())
axes = afwEll.Axes(quad)
sizes.append(axes.getA())
if len(sizes) == 0:
raise RuntimeError("No usable PSF candidates supplied")
nEigenComponents = self.config.nEigenComponents # initial version
if self.config.kernelSize >= 15:
self.debugLog.debug(1, \
"WARNING: NOT scaling kernelSize by stellar quadrupole moment " +
"because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead" \
% (self.config.kernelSize,)
)
actualKernelSize = int(self.config.kernelSize)
else:
medSize = numpy.median(sizes)
actualKernelSize = 2 * int(self.config.kernelSize *
math.sqrt(medSize) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
print "Median size=%s" % (medSize, )
self.debugLog.debug(3, "Kernel size=%s" % (actualKernelSize, ))
# Set size of image returned around candidate
psfCandidateList[0].setHeight(actualKernelSize)
psfCandidateList[0].setWidth(actualKernelSize)
if self.config.doRejectBlends:
# Remove blended candidates completely
blendedCandidates = [
] # Candidates to remove; can't do it while iterating
for cell, cand in candidatesIter(psfCellSet, False):
if len(cand.getSource().getFootprint().getPeaks()) > 1:
blendedCandidates.append((cell, cand))
continue
if display:
print "Removing %d blended Psf candidates" % len(
blendedCandidates)
for cell, cand in blendedCandidates:
cell.removeCandidate(cand)
if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
raise RuntimeError("All PSF candidates removed as blends")
if display:
frame = 0
if displayExposure:
ds9.mtv(exposure, frame=frame, title="psf determination")
maUtils.showPsfSpatialCells(exposure,
psfCellSet,
self.config.nStarPerCell,
symb="o",
ctype=ds9.CYAN,
ctypeUnused=ds9.YELLOW,
size=4,
frame=frame)
#
# Do a PCA decomposition of those PSF candidates
#
reply = "y" # used in interactive mode
for iter in range(self.config.nIterForPsf):
if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates
#
import lsst.afw.display.utils as displayUtils
stamps = []
for cell in psfCellSet.getCellList():
for cand in cell.begin(not showBadCandidates
): # maybe include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
try:
im = cand.getMaskedImage()
chi2 = cand.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
stamps.append(
(im, "%d%s" %
(maUtils.splitId(cand.getSource().getId(),
True)["objId"], chi2),
cand.getStatus()))
except Exception, e:
continue
if len(stamps) == 0:
print "WARNING: No PSF candidates to show; try setting showBadCandidates=True"
else:
mos = displayUtils.Mosaic()
for im, label, status in stamps:
im = type(im)(im, True)
try:
im /= afwMath.makeStatistics(
im, afwMath.MAX).getValue()
except NotImplementedError:
pass
mos.append(
im, label, ds9.GREEN
if status == afwMath.SpatialCellCandidate.GOOD else
ds9.YELLOW if status ==
afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED)
mos.makeMosaic(frame=8, title="Psf Candidates")
# Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
cleanChi2 = False # Any naughty (negative/NAN) chi^2 values?
while not cleanChi2:
cleanChi2 = True
#
# First, estimate the PSF
#
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# In clipping, allow all candidates to be innocent until proven guilty on this iteration.
# Throw out any prima facie guilty candidates (naughty chi^2 values)
#
for cell in psfCellSet.getCellList():
awfulCandidates = []
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN
) # until proven guilty
rchi2 = cand.getChi2()
if not numpy.isfinite(rchi2) or rchi2 <= 0:
# Guilty prima facie
awfulCandidates.append(cand)
cleanChi2 = False
self.debugLog.debug(
2, "chi^2=%s; id=%s" %
(cand.getChi2(), cand.getSource().getId()))
for cand in awfulCandidates:
if display:
print "Removing bad candidate: id=%d, chi^2=%f" % \
(cand.getSource().getId(), cand.getChi2())
cell.removeCandidate(cand)
#
# Clip out bad fits based on reduced chi^2
#
badCandidates = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
rchi2 = cand.getChi2(
) # reduced chi^2 when fitting PSF to candidate
assert rchi2 > 0
if rchi2 > self.config.reducedChi2ForPsfCandidates:
badCandidates.append(cand)
badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
numBad = int(
len(badCandidates) * (iter + 1) / self.config.nIterForPsf +
0.5)
for i, c in zip(range(numBad), badCandidates):
if display:
chi2 = c.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
print "Chi^2 clipping %-4d %.2g" % (c.getSource().getId(),
chi2)
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = afwMath.cast_LinearCombinationKernel(
psf.getKernel())
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
candCenter = afwGeom.PointD(cand.getXCenter(),
cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(),
kernel.getHeight())
except Exception, e:
continue
fit = algorithmsLib.fitKernelParamsToImage(
noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
predict = [
kernel.getSpatialFunction(k)(candCenter.getX(),
candCenter.getY())
for k in range(kernel.getNKernelParameters())
]
#print cand.getSource().getId(), [a / amp for a in params], predict
residuals.append(
[a / amp - p for a, p in zip(params, predict)])
candidates.append(cand)
def run(self, sensorRef):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0:
self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS")
bbox = exposure.getBBox(afwImage.PARENT)
bbox.grow(-self.config.winter2013borderMask)
self.setEdgeBits(exposure.getMaskedImage(), bbox,
exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA"))
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = psf
afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix = scienceSigmaPost * FwhmPerSigma,
targetFwhmPix = templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
self.kcQa = diUtils.KernelCandidateQa(nparam, self.log)
selectSources = self.kcQa.addToSchema(selectSources)
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig())
astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure)
matches = astromRet.matches
kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches)
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
[kernelSources.remove(x) for x in controlSources]
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# First step: we need to subtract the background out
# for detection and measurement. Use large binsize
# for the background estimation.
binsize = self.config.templateBgBinSize
# Second step: we need to run detection on the
# background-subtracted template
#
# Estimate FWHM for detection
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma = templateSigma,
doSmooth = True,
idFactory = idFactory,
binsize = binsize,
)
# Third step: we need to fit the relative astrometry.
#
# One problem is that the SIP fits are w.r.t. CRPIX,
# and these coadd patches have the CRPIX of the entire
# tract, i.e. off the image. This causes
# register.fitWcs to fail. A workaround for now is to
# re-fit the Wcs which returns with a CRPIX that is on
# the image, and *then* to fit for the relative Wcs.
#
# Requires low Sip order to avoid overfitting
# useWinter2013Hacks includes us using the deep calexp
# as the template. In this case we don't need to
# refit the Wcs.
if not self.config.useWinter2013Hacks:
sipOrder = self.config.templateSipOrder
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder))
newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs()
results = self.register.run(templateSources, newWcs,
templateExposure.getBBox(afwImage.PARENT), selectSources)
else:
if self.config.winter2013WcsShift > 0.0:
offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
self.config.winter2013WcsShift)
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
elif self.config.winter2013WcsRms > 0.0:
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
self.config.winter2013WcsRms*numpy.random.normal())
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
results = self.register.run(templateSources, templateExposure.getWcs(),
templateExposure.getBBox(afwImage.PARENT), selectSources)
warpedExp = self.register.warpExposure(templateExposure, results.wcs,
exposure.getWcs(), exposure.getBBox(afwImage.PARENT))
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
refCoordKey = results.matches[0].first.getTable().getCoordKey()
inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in results.matches]
positions = [m.first.get(refCoordKey) for m in results.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(
results.wcs.pixelToSky(m.second.get(inCentroidKey))) for
m in results.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure = templateExposure,
scienceExposure = exposure,
scienceFwhmPix = scienceSigmaPost * FwhmPerSigma,
templateFwhmPix = templateSigma * FwhmPerSigma,
candidateList = kernelSources,
convolveTemplate = self.config.convolveTemplate,
doWarping = not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
subtractedExposure.setPsf(templateExposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table = table,
exposure = subtractedExposure,
doSmooth = not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if len(diaSources) < self.config.maxDiaSourcesToMeasure:
self.dipolemeasurement.run(subtractedExposure, diaSources)
else:
self.measurement.run(subtractedExposure, diaSources)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
# This does not do what I expect so I cobbled together a brute force method in python
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec))
astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources
else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags)
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, dobuild=True, basisList=basisList)
self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
self.kcQa.aggregate(selectSources, self.metadata, allresids)
#Persist using butler
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure = subtractedExposure,
subtractRes = subtractRes,
sources = diaSources,
)
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
variance=None, chi=None):
"""Display the PSF candidates.
If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals)
If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
"""
if chi is None:
if variance is not None: # old name for chi
chi = variance
#
# Show us the ccandidates
#
mos = displayUtils.Mosaic()
#
candidateCenters = []
candidateCentersBad = []
candidateIndex = 0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.cast_PsfCandidateF(cand)
rchi2 = cand.getChi2()
if rchi2 > 1e100:
rchi2 = numpy.nan
if not showBadCandidates and cand.isBad():
continue
if psf:
im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")
try:
im = cand.getMaskedImage() # copy of this object's image
xc, yc = cand.getXCenter(), cand.getYCenter()
margin = 0 if True else 5
w, h = im.getDimensions()
bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
var = bim.getVariance(); var.set(stdev**2); del var
sbim = im.Factory(bim, bbox)
sbim <<= im
del sbim
im = bim
xc += margin; yc += margin
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
except:
continue
if not variance:
im_resid.append(im.Factory(im, True))
# residuals using spatial model
chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
resid = im
if variance:
resid = resid.getImage()
var = im.getVariance()
var = var.Factory(var, True)
numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
resid /= var
im_resid.append(resid)
# Fit the PSF components directly to the data (i.e. ignoring the spatial model)
im = cand.getMaskedImage()
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = afwMath.KernelList(fit[1])
outputKernel = afwMath.LinearCombinationKernel(kernels, params)
outImage = afwImage.ImageD(outputKernel.getDimensions())
outputKernel.computeImage(outImage, False)
im -= outImage.convertF()
resid = im
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
sbim = im.Factory(bim, bbox)
sbim <<= resid
del sbim
resid = bim
if variance:
resid = resid.getImage()
resid /= var
im_resid.append(resid)
im = im_resid.makeMosaic()
else:
im = cand.getMaskedImage()
if normalize:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
objId = splitId(cand.getSource().getId(), True)["objId"]
if psf:
lab = "%d chi^2 %.1f" % (objId, rchi2)
ctype = ds9.RED if cand.isBad() else ds9.GREEN
else:
lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
ctype = ds9.GREEN
mos.append(im, lab, ctype)
if False and numpy.isnan(rchi2):
ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
print "amp", cand.getAmplitude()
im = cand.getMaskedImage()
center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
candidateIndex += 1
if cand.isBad():
candidateCentersBad.append(center)
else:
candidateCenters.append(center)
if variance:
title = "chi(Psf fit)"
else:
title = "Stars & residuals"
mosaicImage = mos.makeMosaic(frame=frame, title=title)
with ds9.Buffering():
for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
for cen in centers:
bbox = mos.getBBox(cen[0])
ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)
return mosaicImage
chi2)
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = afwMath.cast_LinearCombinationKernel(
psf.getKernel())
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
candCenter = afwGeom.PointD(cand.getXCenter(),
cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(),
kernel.getHeight())
except Exception, e:
continue
fit = algorithmsLib.fitKernelParamsToImage(
noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
def testGetPcaKernel(self):
"""Convert our cellSet to a LinearCombinationKernel"""
nEigenComponents = 2
spatialOrder = 1
kernelSize = 21
nStarPerCell = 2
nStarPerCellSpatialFit = 2
tolerance = 1e-5
if display:
ds9.mtv(self.mi, frame=0)
#
# Show the candidates we're using
#
for cell in self.cellSet.getCellList():
i = 0
for cand in cell:
i += 1
source = algorithms.PsfCandidateF.cast(cand).getSource()
xc, yc = source.getXAstrom() - self.mi.getX0(
), source.getYAstrom() - self.mi.getY0()
if i <= nStarPerCell:
ds9.dot("o", xc, yc, ctype=ds9.GREEN)
else:
ds9.dot("o", xc, yc, ctype=ds9.YELLOW)
pair = algorithms.createKernelFromPsfCandidates(
self.cellSet, self.exposure.getDimensions(),
self.exposure.getXY0(), nEigenComponents, spatialOrder, kernelSize,
nStarPerCell)
kernel, eigenValues = pair[0], pair[1]
del pair
print("lambda", " ".join(["%g" % l for l in eigenValues]))
pair = algorithms.fitSpatialKernelFromPsfCandidates(
kernel, self.cellSet, nStarPerCellSpatialFit, tolerance)
status, chi2 = pair[0], pair[1]
del pair
print("Spatial fit: %s chi^2 = %.2g" % (status, chi2))
psf = algorithms.PcaPsf.swigConvert(
roundTripPsf(5, algorithms.PcaPsf(kernel))) # Hurrah!
self.assertIsNone(afwMath.cast_AnalyticKernel(psf.getKernel()))
self.assertIsNotNone(
afwMath.cast_LinearCombinationKernel(psf.getKernel()))
self.checkTablePersistence(psf)
if display:
# print psf.getKernel().toString()
eImages = []
for k in afwMath.cast_LinearCombinationKernel(
psf.getKernel()).getKernelList():
im = afwImage.ImageD(k.getDimensions())
k.computeImage(im, False)
eImages.append(im)
mos = displayUtils.Mosaic()
frame = 3
ds9.mtv(mos.makeMosaic(eImages), frame=frame)
ds9.dot("Eigen Images", 0, 0, frame=frame)
#
# Make a mosaic of PSF candidates
#
stamps = []
stampInfo = []
for cell in self.cellSet.getCellList():
for cand in cell:
#
# Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate; all
# it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates
# don't know about getMaskedImage; so cast the pointer to PsfCandidate
#
cand = algorithms.PsfCandidateF.cast(cand)
s = cand.getSource()
im = cand.getMaskedImage()
stamps.append(im)
stampInfo.append("[%d 0x%x]" %
(s.getId(), s.getFlagForDetection()))
mos = displayUtils.Mosaic()
frame = 1
ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True)
for i in range(len(stampInfo)):
ds9.dot(stampInfo[i],
mos.getBBox(i).getX0(),
mos.getBBox(i).getY0(),
frame=frame,
ctype=ds9.RED)
psfImages = []
labels = []
if False:
nx, ny = 3, 4
for iy in range(ny):
for ix in range(nx):
x = int((ix + 0.5) * self.mi.getWidth() / nx)
y = int((iy + 0.5) * self.mi.getHeight() / ny)
im = psf.getImage(x, y)
psfImages.append(im.Factory(im, True))
labels.append("PSF(%d,%d)" % (int(x), int(y)))
if True:
print((x, y, "PSF parameters:",
psf.getKernel().getKernelParameters()))
else:
nx, ny = 2, 2
for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]:
im = psf.computeImage(afwGeom.PointD(x, y))
psfImages.append(im.Factory(im, True))
labels.append("PSF(%d,%d)" % (int(x), int(y)))
if True:
print(x, y, "PSF parameters:",
psf.getKernel().getKernelParameters())
frame = 2
mos.makeMosaic(psfImages, frame=frame, mode=nx)
mos.drawLabels(labels, frame=frame)
if display:
ds9.mtv(self.mi, frame=0)
psfImages = []
labels = []
if False:
nx, ny = 3, 4
for iy in range(ny):
for ix in range(nx):
x = int((ix + 0.5) * self.mi.getWidth() / nx)
y = int((iy + 0.5) * self.mi.getHeight() / ny)
algorithms.subtractPsf(psf, self.mi, x, y)
else:
nx, ny = 2, 2
for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]:
if False: # Test subtraction with non-centered psfs
x += 0.5
y -= 0.5
#algorithms.subtractPsf(psf, self.mi, x, y)
ds9.mtv(self.mi, frame=1)
def determinePsf(self,
exposure,
psfCandidateList,
metadata=None,
flagKey=None):
"""!Determine a PCA PSF model for an exposure given a list of PSF candidates
\param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
\param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
typically obtained by detecting sources and then running them through a star selector
\param[in,out] metadata a home for interesting tidbits of information
\param[in] flagKey schema key used to mark sources actually used in PSF determination
\return a list of
- psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
- cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(
__name__).displayExposure # display the Exposure + spatialCells
displayPsfCandidates = lsstDebug.Info(
__name__).displayPsfCandidates # show the viable candidates
displayIterations = lsstDebug.Info(
__name__).displayIterations # display on each PSF iteration
displayPsfComponents = lsstDebug.Info(
__name__).displayPsfComponents # show the PCA components
displayResiduals = lsstDebug.Info(
__name__).displayResiduals # show residuals
displayPsfMosaic = lsstDebug.Info(
__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
# match Kernel amplitudes for spatial plots
matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
# Keep matplotlib alive post mortem
keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
displayPsfSpatialModel = lsstDebug.Info(
__name__).displayPsfSpatialModel # Plot spatial model?
showBadCandidates = lsstDebug.Info(
__name__).showBadCandidates # Include bad candidates
# Normalize residuals by object amplitude
normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
pause = lsstDebug.Info(
__name__).pause # Prompt user after each iteration?
if display > 1:
pause = True
mi = exposure.getMaskedImage()
if len(psfCandidateList) == 0:
raise RuntimeError("No PSF candidates supplied.")
# construct and populate a spatial cell set
bbox = mi.getBBox()
psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
self.config.sizeCellY)
sizes = []
for i, psfCandidate in enumerate(psfCandidateList):
if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement
continue
try:
psfCellSet.insertCandidate(psfCandidate)
except Exception as e:
self.log.debug("Skipping PSF candidate %d of %d: %s", i,
len(psfCandidateList), e)
continue
source = psfCandidate.getSource()
quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
source.getIxy())
axes = afwEll.Axes(quad)
sizes.append(axes.getA())
if len(sizes) == 0:
raise RuntimeError("No usable PSF candidates supplied")
nEigenComponents = self.config.nEigenComponents # initial version
if self.config.kernelSize >= 15:
self.log.warn(
"WARNING: NOT scaling kernelSize by stellar quadrupole moment "
+
"because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead",
self.config.kernelSize)
actualKernelSize = int(self.config.kernelSize)
else:
medSize = numpy.median(sizes)
actualKernelSize = 2 * int(self.config.kernelSize *
math.sqrt(medSize) + 0.5) + 1
if actualKernelSize < self.config.kernelSizeMin:
actualKernelSize = self.config.kernelSizeMin
if actualKernelSize > self.config.kernelSizeMax:
actualKernelSize = self.config.kernelSizeMax
if display:
print("Median size=%s" % (medSize, ))
self.log.trace("Kernel size=%s", actualKernelSize)
# Set size of image returned around candidate
psfCandidateList[0].setHeight(actualKernelSize)
psfCandidateList[0].setWidth(actualKernelSize)
if self.config.doRejectBlends:
# Remove blended candidates completely
blendedCandidates = [
] # Candidates to remove; can't do it while iterating
for cell, cand in candidatesIter(psfCellSet, False):
if len(cand.getSource().getFootprint().getPeaks()) > 1:
blendedCandidates.append((cell, cand))
continue
if display:
print("Removing %d blended Psf candidates" %
len(blendedCandidates))
for cell, cand in blendedCandidates:
cell.removeCandidate(cand)
if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
raise RuntimeError("All PSF candidates removed as blends")
if display:
frame = 0
if displayExposure:
ds9.mtv(exposure, frame=frame, title="psf determination")
maUtils.showPsfSpatialCells(exposure,
psfCellSet,
self.config.nStarPerCell,
symb="o",
ctype=ds9.CYAN,
ctypeUnused=ds9.YELLOW,
size=4,
frame=frame)
#
# Do a PCA decomposition of those PSF candidates
#
reply = "y" # used in interactive mode
for iterNum in range(self.config.nIterForPsf):
if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates
#
import lsst.afw.display.utils as displayUtils
stamps = []
for cell in psfCellSet.getCellList():
for cand in cell.begin(not showBadCandidates
): # maybe include bad candidates
cand = algorithmsLib.PsfCandidateF.cast(cand)
try:
im = cand.getMaskedImage()
chi2 = cand.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
stamps.append(
(im, "%d%s" %
(maUtils.splitId(cand.getSource().getId(),
True)["objId"], chi2),
cand.getStatus()))
except Exception as e:
continue
if len(stamps) == 0:
print(
"WARNING: No PSF candidates to show; try setting showBadCandidates=True"
)
else:
mos = displayUtils.Mosaic()
for im, label, status in stamps:
im = type(im)(im, True)
try:
im /= afwMath.makeStatistics(
im, afwMath.MAX).getValue()
except NotImplementedError:
pass
mos.append(
im, label, ds9.GREEN
if status == afwMath.SpatialCellCandidate.GOOD else
ds9.YELLOW if status ==
afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED)
mos.makeMosaic(frame=8, title="Psf Candidates")
# Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
cleanChi2 = False # Any naughty (negative/NAN) chi^2 values?
while not cleanChi2:
cleanChi2 = True
#
# First, estimate the PSF
#
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# In clipping, allow all candidates to be innocent until proven guilty on this iteration.
# Throw out any prima facie guilty candidates (naughty chi^2 values)
#
for cell in psfCellSet.getCellList():
awfulCandidates = []
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.PsfCandidateF.cast(cand)
cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN
) # until proven guilty
rchi2 = cand.getChi2()
if not numpy.isfinite(rchi2) or rchi2 <= 0:
# Guilty prima facie
awfulCandidates.append(cand)
cleanChi2 = False
self.log.debug("chi^2=%s; id=%s", cand.getChi2(),
cand.getSource().getId())
for cand in awfulCandidates:
if display:
print("Removing bad candidate: id=%d, chi^2=%f" % \
(cand.getSource().getId(), cand.getChi2()))
cell.removeCandidate(cand)
#
# Clip out bad fits based on reduced chi^2
#
badCandidates = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
cand = algorithmsLib.PsfCandidateF.cast(cand)
rchi2 = cand.getChi2(
) # reduced chi^2 when fitting PSF to candidate
assert rchi2 > 0
if rchi2 > self.config.reducedChi2ForPsfCandidates:
badCandidates.append(cand)
badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(numBad), badCandidates):
if display:
chi2 = c.getChi2()
if chi2 > 1e100:
chi2 = numpy.nan
print("Chi^2 clipping %-4d %.2g" %
(c.getSource().getId(), chi2))
c.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Clip out bad fits based on spatial fitting.
#
# This appears to be better at getting rid of sources that have a single dominant kernel component
# (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
# set the spatial model) don't contain that kernel component, and so the spatial modeling
# downweights the component.
#
residuals = list()
candidates = list()
kernel = psf.getKernel()
noSpatialKernel = afwMath.cast_LinearCombinationKernel(
psf.getKernel())
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.PsfCandidateF.cast(cand)
candCenter = afwGeom.PointD(cand.getXCenter(),
cand.getYCenter())
try:
im = cand.getMaskedImage(kernel.getWidth(),
kernel.getHeight())
except Exception as e:
continue
fit = algorithmsLib.fitKernelParamsToImage(
noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
predict = [
kernel.getSpatialFunction(k)(candCenter.getX(),
candCenter.getY())
for k in range(kernel.getNKernelParameters())
]
#print cand.getSource().getId(), [a / amp for a in params], predict
residuals.append(
[a / amp - p for a, p in zip(params, predict)])
candidates.append(cand)
residuals = numpy.array(residuals)
for k in range(kernel.getNKernelParameters()):
if False:
# Straight standard deviation
mean = residuals[:, k].mean()
rms = residuals[:, k].std()
elif False:
# Using interquartile range
sr = numpy.sort(residuals[:, k])
mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \
0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1])
rms = 0.74 * (sr[int(0.75 * len(sr))] -
sr[int(0.25 * len(sr))])
else:
stats = afwMath.makeStatistics(
residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
mean = stats.getValue(afwMath.MEANCLIP)
rms = stats.getValue(afwMath.STDEVCLIP)
rms = max(
1.0e-4,
rms) # Don't trust RMS below this due to numerical issues
if display:
print("Mean for component %d is %f" % (k, mean))
print("RMS for component %d is %f" % (k, rms))
badCandidates = list()
for i, cand in enumerate(candidates):
if numpy.fabs(residuals[i, k] -
mean) > self.config.spatialReject * rms:
badCandidates.append(i)
badCandidates.sort(
key=lambda x: numpy.fabs(residuals[x, k] - mean),
reverse=True)
numBad = numCandidatesToReject(len(badCandidates), iterNum,
self.config.nIterForPsf)
for i, c in zip(range(min(len(badCandidates), numBad)),
badCandidates):
cand = candidates[c]
if display:
print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \
(cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
residuals[badCandidates[i], k], self.config.spatialReject * rms))
cand.setStatus(afwMath.SpatialCellCandidate.BAD)
#
# Display results
#
if display and displayIterations:
if displayExposure:
if iterNum > 0:
ds9.erase(frame=frame)
maUtils.showPsfSpatialCells(exposure,
psfCellSet,
self.config.nStarPerCell,
showChi2=True,
symb="o",
size=8,
frame=frame,
ctype=ds9.YELLOW,
ctypeBad=ds9.RED,
ctypeUnused=ds9.MAGENTA)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
maUtils.showPsfSpatialCells(
exposure,
psfCellSet,
self.config.nStarPerCellSpatialFit,
symb="o",
size=10,
frame=frame,
ctype=ds9.YELLOW,
ctypeBad=ds9.RED)
if displayResiduals:
while True:
try:
maUtils.showPsfCandidates(
exposure,
psfCellSet,
psf=psf,
frame=4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
maUtils.showPsfCandidates(
exposure,
psfCellSet,
psf=psf,
frame=5,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates,
variance=True)
except:
if not showBadCandidates:
showBadCandidates = True
continue
break
if displayPsfComponents:
maUtils.showPsf(psf, eigenValues, frame=6)
if displayPsfMosaic:
maUtils.showPsfMosaic(exposure,
psf,
frame=7,
showFwhm=True)
ds9.scale('linear', 0, 1, frame=7)
if displayPsfSpatialModel:
maUtils.plotPsfSpatialModel(
exposure,
psf,
psfCellSet,
showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
if pause:
while True:
try:
reply = input(
"Next iteration? [ynchpqQs] ").strip()
except EOFError:
reply = "n"
reply = reply.split()
if reply:
reply, args = reply[0], reply[1:]
else:
reply = ""
if reply in ("", "c", "h", "n", "p", "q", "Q", "s",
"y"):
if reply == "c":
pause = False
elif reply == "h":
print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \
"s[ave fileName] y[es]")
continue
elif reply == "p":
import pdb
pdb.set_trace()
elif reply == "q":
display = False
elif reply == "Q":
sys.exit(1)
elif reply == "s":
fileName = args.pop(0)
if not fileName:
print("Please provide a filename")
continue
print("Saving to %s" % fileName)
maUtils.saveSpatialCellSet(psfCellSet,
fileName=fileName)
continue
break
else:
print("Unrecognised response: %s" % reply,
file=sys.stderr)
if reply == "n":
break
# One last time, to take advantage of the last iteration
psf, eigenValues, nEigenComponents, fitChi2 = \
self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
#
# Display code for debugging
#
if display and reply != "n":
if displayExposure:
maUtils.showPsfSpatialCells(exposure,
psfCellSet,
self.config.nStarPerCell,
showChi2=True,
symb="o",
ctype=ds9.YELLOW,
ctypeBad=ds9.RED,
size=8,
frame=frame)
if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
maUtils.showPsfSpatialCells(
exposure,
psfCellSet,
self.config.nStarPerCellSpatialFit,
symb="o",
ctype=ds9.YELLOW,
ctypeBad=ds9.RED,
size=10,
frame=frame)
if displayResiduals:
maUtils.showPsfCandidates(
exposure,
psfCellSet,
psf=psf,
frame=4,
normalize=normalizeResiduals,
showBadCandidates=showBadCandidates)
if displayPsfComponents:
maUtils.showPsf(psf, eigenValues, frame=6)
if displayPsfMosaic:
maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
ds9.scale("linear", 0, 1, frame=7)
if displayPsfSpatialModel:
maUtils.plotPsfSpatialModel(
exposure,
psf,
psfCellSet,
showBadCandidates=True,
matchKernelAmplitudes=matchKernelAmplitudes,
keepPlots=keepMatplotlibPlots)
#
# Generate some QA information
#
# Count PSF stars
#
numGoodStars = 0
numAvailStars = 0
avgX = 0.0
avgY = 0.0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # don't ignore BAD stars
numAvailStars += 1
for cand in cell.begin(True): # do ignore BAD stars
cand = algorithmsLib.PsfCandidateF.cast(cand)
src = cand.getSource()
if flagKey is not None:
src.set(flagKey, True)
avgX += src.getX()
avgY += src.getY()
numGoodStars += 1
avgX /= numGoodStars
avgY /= numGoodStars
if metadata is not None:
metadata.set("spatialFitChi2", fitChi2)
metadata.set("numGoodStars", numGoodStars)
metadata.set("numAvailStars", numAvailStars)
metadata.set("avgX", avgX)
metadata.set("avgY", avgY)
psf = algorithmsLib.PcaPsf(psf.getKernel(),
afwGeom.Point2D(avgX, avgY))
return psf, psfCellSet
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
matchKernelAmplitudes=False, keepPlots=True):
"""Plot the PSF spatial model."""
if not plt:
print >> sys.stderr, "Unable to import matplotlib"
return
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
candPos = list()
candFits = list()
badPos = list()
badFits = list()
candAmps = list()
badAmps = list()
for cell in psfCellSet.getCellList():
for cand in cell.begin(False):
cand = algorithmsLib.cast_PsfCandidateF(cand)
if not showBadCandidates and cand.isBad():
continue
candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
try:
im = cand.getMaskedImage()
except Exception:
continue
fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = fit[1]
amp = 0.0
for p, k in zip(params, kernels):
amp += p * afwMath.cast_FixedKernel(k).getSum()
targetFits = badFits if cand.isBad() else candFits
targetPos = badPos if cand.isBad() else candPos
targetAmps = badAmps if cand.isBad() else candAmps
targetFits.append([x / amp for x in params])
targetPos.append(candCenter)
targetAmps.append(amp)
numCandidates = len(candFits)
numBasisFuncs = noSpatialKernel.getNBasisKernels()
xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
zGood = numpy.array(candFits)
ampGood = numpy.array(candAmps)
xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
zBad = numpy.array(badFits)
ampBad = numpy.array(badAmps)
numBad = len(badPos)
xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)
kernel = psf.getKernel()
nKernelComponents = kernel.getNKernelParameters()
#
# Figure out how many panels we'll need
#
nPanelX = int(math.sqrt(nKernelComponents))
nPanelY = nKernelComponents//nPanelX
while nPanelY*nPanelX < nKernelComponents:
nPanelX += 1
fig = plt.figure(1)
fig.clf()
try:
fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
except: # protect against API changes
pass
#
# Generator for axes arranged in panels
#
subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)
for k in range(nKernelComponents):
func = kernel.getSpatialFunction(k)
dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
yMin = dfGood.min()
yMax = dfGood.max()
if numBad > 0:
dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
yMin = min([yMin, dfBad.min()])
yMax = max([yMax, dfBad.max()])
yMin -= 0.05 * (yMax - yMin)
yMax += 0.05 * (yMax - yMin)
yMin = -0.01
yMax = 0.01
fRange = numpy.ndarray((len(xRange), len(yRange)))
for j, yVal in enumerate(yRange):
for i, xVal in enumerate(xRange):
fRange[j][i] = func(xVal, yVal)
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getHeight())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(yGood, dfGood, 'b+')
if numBad > 0:
ax.plot(yBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('Residuals(y)')
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
if matchKernelAmplitudes and k == 0:
vmin = 0.0
vmax = 1.1
else:
vmin = fRange.min()
vmax = fRange.max()
norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
ax.set_title('Spatial poly')
plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
ax.set_autoscale_on(False)
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=yMin, upper=yMax)
ax.plot(xGood, dfGood, 'b+')
if numBad > 0:
ax.plot(xBad, dfBad, 'r+')
ax.axhline(0.0)
ax.set_title('K%d Residuals(x)' % k)
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
ax = subplots.next()
if False:
ax.scatter(xGood, yGood, c=dfGood, marker='o')
ax.scatter(xBad, yBad, c=dfBad, marker='x')
ax.set_xbound(lower=0, upper=exposure.getWidth())
ax.set_ybound(lower=0, upper=exposure.getHeight())
ax.set_title('Spatial residuals')
plt.colorbar(im, orientation='horizontal')
else:
calib = exposure.getCalib()
if calib.getFluxMag0()[0] <= 0:
calib = type(calib)()
calib.setFluxMag0(1.0)
with CalibNoThrow():
ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+')
if numBad > 0:
ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+')
ax.set_title('Flux variation')
fig.show()
global keptPlots
if keepPlots and not keptPlots:
# Keep plots open when done
def show():
print "%s: Please close plots when done." % __name__
try:
plt.show()
except:
pass
print "Plots closed, exiting..."
import atexit
atexit.register(show)
keptPlots = True