Example #1
0
def showPsf(psf, eigenValues=None, XY=None, normalize=True, frame=None):
    """Display a PSF's eigen images

    If normalize is True, set the largest absolute value of each eigenimage to 1.0 (n.b. sum == 0.0 for i > 0)
    """

    if eigenValues:
        coeffs = eigenValues
    elif XY is not None:
        coeffs = psf.getLocalKernel(afwGeom.PointD(XY[0], XY[1])).getKernelParameters()
    else:
        coeffs = None

    mos = displayUtils.Mosaic(gutter=2, background=-0.1)
    for i, k in enumerate(afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList()):
        im = afwImage.ImageD(k.getDimensions())
        k.computeImage(im, False)
        if normalize:
            im /= numpy.max(numpy.abs(im.getArray()))
            
        if coeffs:
            mos.append(im, "%g" % (coeffs[i]/coeffs[0]))
        else:
            mos.append(im)

    mos.makeMosaic(frame=frame, title="Kernel Basis Functions")

    return mos
Example #2
0
def showPsf(psf, eigenValues=None, XY=None, normalize=True, frame=None):
    """Display a PSF's eigen images

    If normalize is True, set the largest absolute value of each eigenimage to 1.0 (n.b. sum == 0.0 for i > 0)
    """

    if eigenValues:
        coeffs = eigenValues
    elif XY is not None:
        coeffs = psf.getLocalKernel(afwGeom.PointD(XY[0], XY[1])).getKernelParameters()
    else:
        coeffs = None

    mos = displayUtils.Mosaic(gutter=2, background=-0.1)
    for i, k in enumerate(afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList()):
        im = afwImage.ImageD(k.getDimensions())
        k.computeImage(im, False)
        if normalize:
            im /= numpy.max(numpy.abs(im.getArray()))

        if coeffs:
            mos.append(im, "%g" % (coeffs[i]/coeffs[0]))
        else:
            mos.append(im)

    mos.makeMosaic(frame=frame, title="Kernel Basis Functions")

    return mos
Example #3
0
def plotPsfSpatialModel(exposure,
                        psf,
                        psfCellSet,
                        showBadCandidates=True,
                        numSample=128,
                        matchKernelAmplitudes=False,
                        keepPlots=True):
    """Plot the PSF spatial model."""

    if plt is None:
        print >> sys.stderr, "Unable to import matplotlib"
        return

    noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
    candPos = list()
    candFits = list()
    badPos = list()
    badFits = list()
    candAmps = list()
    badAmps = list()
    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False):
            cand = algorithmsLib.cast_PsfCandidateF(cand)
            if not showBadCandidates and cand.isBad():
                continue
            candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
            try:
                im = cand.getMaskedImage()
            except Exception, e:
                continue

            fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im,
                                                       candCenter)
            params = fit[0]
            kernels = fit[1]
            amp = 0.0
            for p, k in zip(params, kernels):
                amp += p * afwMath.cast_FixedKernel(k).getSum()

            targetFits = badFits if cand.isBad() else candFits
            targetPos = badPos if cand.isBad() else candPos
            targetAmps = badAmps if cand.isBad() else candAmps

            targetFits.append([x / amp for x in params])
            targetPos.append(candCenter)
            targetAmps.append(amp)
Example #4
0
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
                        matchKernelAmplitudes=False, keepPlots=True):
    """Plot the PSF spatial model."""

    if plt is None:
        print >> sys.stderr, "Unable to import matplotlib"
        return
    
    noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
    candPos = list()
    candFits = list()
    badPos = list()
    badFits = list()
    candAmps = list()
    badAmps = list()
    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False):
            cand = algorithmsLib.cast_PsfCandidateF(cand)
            if not showBadCandidates and cand.isBad():
                continue
            candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
            try:
                im = cand.getMaskedImage()
            except Exception, e:
                continue

            fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
            params = fit[0]
            kernels = fit[1]
            amp = 0.0
            for p, k in zip(params, kernels):
                amp += p * afwMath.cast_FixedKernel(k).getSum()

            targetFits = badFits if cand.isBad() else candFits
            targetPos = badPos if cand.isBad() else candPos
            targetAmps = badAmps if cand.isBad() else candAmps

            targetFits.append([x / amp for x in params])
            targetPos.append(candCenter)
            targetAmps.append(amp)
                    print "Chi^2 clipping %-4d  %.2g" % (c.getSource().getId(), chi2)
                c.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Clip out bad fits based on spatial fitting.
            #
            # This appears to be better at getting rid of sources that have a single dominant kernel component
            # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
            # set the spatial model) don't contain that kernel component, and so the spatial modeling
            # downweights the component.
            #

            residuals = list()
            candidates = list()
            kernel = psf.getKernel()
            noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):
                    cand = algorithmsLib.cast_PsfCandidateF(cand)
                    candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
                    try:
                        im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
                    except Exception, e:
                        continue

                    fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
                    params = fit[0]
                    kernels = fit[1]
                    amp = 0.0
                    for p, k in zip(params, kernels):
                        amp += p * afwMath.cast_FixedKernel(k).getSum()
Example #6
0
    def run(self, sensorRef, templateIdList=None):
        """Subtract an image from a template coadd and measure the result

        Steps include:
        - warp template coadd to match WCS of image
        - PSF match image to warped template
        - subtract image from PSF-matched, warped template
        - persist difference image
        - detect sources
        - measure sources

        @param sensorRef: sensor-level butler data reference, used for the following data products:
        Input only:
        - calexp
        - psf
        - ccdExposureId
        - ccdExposureId_bits
        - self.config.coaddName + "Coadd_skyMap"
        - self.config.coaddName + "Coadd"
        Input or output, depending on config:
        - self.config.coaddName + "Diff_subtractedExp"
        Output, depending on config:
        - self.config.coaddName + "Diff_matchedExp"
        - self.config.coaddName + "Diff_src"

        @return pipe_base Struct containing these fields:
        - subtractedExposure: exposure after subtracting template;
            the unpersisted version if subtraction not run but detection run
            None if neither subtraction nor detection run (i.e. nothing useful done)
        - subtractRes: results of subtraction task; None if subtraction not run
        - sources: detected and possibly measured sources; None if detection not run
        """
        self.log.info("Processing %s" % (sensorRef.dataId))

        # initialize outputs and some intermediate products
        subtractedExposure = None
        subtractRes = None
        selectSources = None
        kernelSources = None
        controlSources = None
        diaSources = None

        # We make one IdFactory that will be used by both icSrc and src datasets;
        # I don't know if this is the way we ultimately want to do things, but at least
        # this ensures the source IDs are fully unique.
        expBits = sensorRef.get("ccdExposureId_bits")
        expId = long(sensorRef.get("ccdExposureId"))
        idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)

        # Retrieve the science image we wish to analyze
        exposure = sensorRef.get("calexp", immediate=True)
        if self.config.doAddCalexpBackground:
            mi = exposure.getMaskedImage()
            mi += sensorRef.get("calexpBackground").getImage()
        if not exposure.hasPsf():
            raise pipeBase.TaskError("Exposure has no psf")
        sciencePsf = exposure.getPsf()

        subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
        templateExposure = None  # Stitched coadd exposure
        templateSources = None   # Sources on the template image
        if self.config.doSubtract:
            print templateIdList
            template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
            templateExposure = template.exposure
            templateSources = template.sources

            # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
            ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
            psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
            scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)

            # sigma of PSF of template image before warping
            ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
            psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
            templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)

            # if requested, convolve the science exposure with its PSF
            # (properly, this should be a cross-correlation, but our code does not yet support that)
            # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
            # else sigma of original science exposure
            if self.config.doPreConvolve:
                convControl = afwMath.ConvolutionControl()
                # cannot convolve in place, so make a new MI to receive convolved image
                srcMI = exposure.getMaskedImage()
                destMI = srcMI.Factory(srcMI.getDimensions())
                srcPsf = sciencePsf
                if self.config.useGaussianForPreConvolution:
                    # convolve with a simplified PSF model: a double Gaussian
                    kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
                    preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
                else:
                    # convolve with science exposure's PSF model
                    preConvPsf = srcPsf
                afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
                exposure.setMaskedImage(destMI)
                scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
            else:
                scienceSigmaPost = scienceSigmaOrig

            # If requested, find sources in the image
            if self.config.doSelectSources:
                if not sensorRef.datasetExists("src"):
                    self.log.warn("Src product does not exist; running detection, measurement, selection")
                    # Run own detection and measurement; necessary in nightly processing
                    selectSources = self.subtract.getSelectSources(
                        exposure,
                        sigma = scienceSigmaPost,
                        doSmooth = not self.doPreConvolve,
                        idFactory = idFactory,
                    )
                else:
                    self.log.info("Source selection via src product")
                    # Sources already exist; for data release processing
                    selectSources = sensorRef.get("src")

                # Number of basis functions
                nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
                                                 referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
                                                 targetFwhmPix=templateSigma * FwhmPerSigma))

                if self.config.doAddMetrics:
                    # Modify the schema of all Sources
                    kcQa = KernelCandidateQa(nparam)
                    selectSources = kcQa.addToSchema(selectSources)

                if self.config.kernelSourcesFromRef:
                    # match exposure sources to reference catalog
                    astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
                    matches = astromRet.matches
                elif templateSources:
                    # match exposure sources to template sources
                    matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
                                                  False)
                else:
                    raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
                                       "but template sources not available. Cannot match science " +
                                       "sources with template sources. Run process* on data from " +
                                       "which templates are built.")

                kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
                    matches=matches).starCat

                random.shuffle(kernelSources, random.random)
                controlSources = kernelSources[::self.config.controlStepSize]
                kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize]

                if self.config.doSelectDcrCatalog:
                    redSelector  = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
                    redSources   = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
                    controlSources.extend(redSources)

                    blueSelector = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
                    blueSources  = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
                    controlSources.extend(blueSources)

                if self.config.doSelectVariableCatalog:
                    varSelector = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(includeVariable=True))
                    varSources  = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
                    controlSources.extend(varSources)

                self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" 
                              % (len(kernelSources), len(selectSources), len(controlSources)))
            allresids = {}
            if self.config.doUseRegister:
                self.log.info("Registering images")

                if templateSources is None:
                    # Run detection on the template, which is
                    # temporarily background-subtracted
                    templateSources = self.subtract.getSelectSources(
                        templateExposure,
                        sigma=templateSigma,
                        doSmooth=True,
                        idFactory=idFactory
                    )

                # Third step: we need to fit the relative astrometry.
                #
                wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
                warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
                                            exposure.getWcs(), exposure.getBBox())
                templateExposure = warpedExp

                # Create debugging outputs on the astrometric
                # residuals as a function of position.  Persistence
                # not yet implemented; expected on (I believe) #2636.
                if self.config.doDebugRegister:
                    # Grab matches to reference catalog
                    srcToMatch = {x.second.getId() : x.first for x in matches}

                    refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
                    inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
                    sids      = [m.first.getId() for m in wcsResults.matches]
                    positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
                    residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
                                m.second.get(inCentroidKey))) for m in wcsResults.matches]
                    allresids = dict(zip(sids, zip(positions, residuals)))

                    cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
                            wcsResults.wcs.pixelToSky(
                                m.second.get(inCentroidKey))) for m in wcsResults.matches]
                    colors    = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
                                              + 2.5*numpy.log10(srcToMatch[x].get("r")) 
                                              for x in sids if x in srcToMatch.keys()])
                    dlong     = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals) 
                                             if s in srcToMatch.keys()])
                    dlat      = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals) 
                                             if s in srcToMatch.keys()])
                    idx1      = numpy.where(colors<self.sourceSelector.config.grMin)
                    idx2      = numpy.where((colors>=self.sourceSelector.config.grMin)&
                                            (colors<=self.sourceSelector.config.grMax))
                    idx3      = numpy.where(colors>self.sourceSelector.config.grMax)
                    rms1Long  = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25))
                    rms1Lat   = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25))
                    rms2Long  = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25))
                    rms2Lat   = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25))
                    rms3Long  = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25))
                    rms3Lat   = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25))
                    self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f"  % (numpy.median(dlong[idx1]), 
                                                                                  rms1Long,
                                                                                  numpy.median(dlat[idx1]), 
                                                                                  rms1Lat))
                    self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f"  % (numpy.median(dlong[idx2]), 
                                                                                   rms2Long,
                                                                                   numpy.median(dlat[idx2]), 
                                                                                   rms2Lat))
                    self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f"  % (numpy.median(dlong[idx3]), 
                                                                                 rms3Long,
                                                                                 numpy.median(dlat[idx3]), 
                                                                                 rms3Lat))

                    self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
                    self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
                    self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
                    self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
                    self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
                    self.metadata.add("RegisterRedLongOffsetStd", rms3Long)

                    self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
                    self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
                    self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
                    self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
                    self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
                    self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)

            # warp template exposure to match exposure,
            # PSF match template exposure to exposure,
            # then return the difference

            #Return warped template...  Construct sourceKernelCand list after subtract
            self.log.info("Subtracting images")
            subtractRes = self.subtract.subtractExposures(
                templateExposure=templateExposure,
                scienceExposure=exposure,
                candidateList=kernelSources,
                convolveTemplate=self.config.convolveTemplate,
                doWarping=not self.config.doUseRegister
            )
            subtractedExposure = subtractRes.subtractedExposure

            if self.config.doWriteMatchedExp:
                sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")

        if self.config.doDetection:
            self.log.info("Running diaSource detection")
            if subtractedExposure is None:
                subtractedExposure = sensorRef.get(subtractedExposureName)

            # Get Psf from the appropriate input image if it doesn't exist
            if not subtractedExposure.hasPsf():
                if self.config.convolveTemplate:
                    subtractedExposure.setPsf(exposure.getPsf())
                else:
                    if templateExposure is None:
                        template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
                    subtractedExposure.setPsf(template.exposure.getPsf())

            # Erase existing detection mask planes
            mask  = subtractedExposure.getMaskedImage().getMask()
            mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))

            table = afwTable.SourceTable.make(self.schema, idFactory)
            table.setMetadata(self.algMetadata)
            results = self.detection.makeSourceCatalog(
                table=table,
                exposure=subtractedExposure,
                doSmooth=not self.config.doPreConvolve
                )

            if self.config.doMerge:
                fpSet = results.fpSets.positive
                fpSet.merge(results.fpSets.negative, self.config.growFootprint,
                            self.config.growFootprint, False)
                diaSources = afwTable.SourceCatalog(table)
                fpSet.makeSources(diaSources)
                self.log.info("Merging detections into %d sources" % (len(diaSources)))
            else:
                diaSources = results.sources

            if self.config.doMeasurement:
                self.log.info("Running diaSource measurement")
                self.measurement.run(diaSources, subtractedExposure)

            # Match with the calexp sources if possible
            if self.config.doMatchSources:
                if sensorRef.datasetExists("src"):
                    # Create key,val pair where key=diaSourceId and val=sourceId
                    matchRadAsec = self.config.diaSourceMatchRadius
                    matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()

                    srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
                    srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for 
                                         srcMatch in srcMatches])
                    self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
                                                                             len(diaSources)))
                else:
                    self.log.warn("Src product does not exist; cannot match with diaSources")
                    srcMatchDict = {}

                # Create key,val pair where key=diaSourceId and val=refId
                refAstromConfig = measAstrom.AstrometryConfig()
                refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
                refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
                astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
                refMatches = astromRet.matches
                if refMatches is None:
                    self.log.warn("No diaSource matches with reference catalog")
                    refMatchDict = {}
                else:
                    self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
                                                                                       len(diaSources)))
                    refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
                                             refMatch in refMatches])

                # Assign source Ids
                for diaSource in diaSources:
                    sid = diaSource.getId()
                    if srcMatchDict.has_key(sid):
                        diaSource.set("srcMatchId", srcMatchDict[sid])
                    if refMatchDict.has_key(sid):
                        diaSource.set("refMatchId", refMatchDict[sid])

            if diaSources is not None and self.config.doWriteSources:
                sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")

            if self.config.doAddMetrics and self.config.doSelectSources:
                self.log.info("Evaluating metrics and control sample")

                kernelCandList = []
                for cell in subtractRes.kernelCellSet.getCellList():
                    for cand in cell.begin(False): # include bad candidates
                        kernelCandList.append(cast_KernelCandidateF(cand))

                # Get basis list to build control sample kernels
                basisList = afwMath.cast_LinearCombinationKernel(
                    kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()

                controlCandList = \
                    diffimTools.sourceTableToCandidateList(controlSources, 
                                                           subtractRes.warpedExposure, exposure,
                                                           self.config.subtract.kernel.active,
                                                           self.config.subtract.kernel.active.detectionConfig,
                                                           self.log, doBuild=True, basisList=basisList)

                kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
                                dof=nparam)
                kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)

                if self.config.doDetection:
                    kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
                else:
                    kcQa.aggregate(selectSources, self.metadata, allresids)

                sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")

        if self.config.doWriteSubtractedExp:
            sensorRef.put(subtractedExposure, subtractedExposureName)

        self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
        return pipeBase.Struct(
            subtractedExposure=subtractedExposure,
            subtractRes=subtractRes,
            sources=diaSources,
        )
Example #7
0
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
                      fitBasisComponents=False, variance=None, chi=None):
    """Display the PSF candidates.
If psf is provided include PSF model and residuals;  if normalize is true normalize the PSFs (and residuals)

If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi

If fitBasisComponents is true, also find the best linear combination of the PSF's components (if they exist)
"""
    if chi is None:
        if variance is not None:        # old name for chi
            chi = variance
    #
    # Show us the ccandidates
    #
    mos = displayUtils.Mosaic()
    #
    candidateCenters = []
    candidateCentersBad = []
    candidateIndex = 0

    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False): # include bad candidates
            cand = algorithmsLib.cast_PsfCandidateF(cand)

            rchi2 = cand.getChi2()
            if rchi2 > 1e100:
                rchi2 = numpy.nan

            if not showBadCandidates and cand.isBad():
                continue

            if psf:
                im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")

                try:
                    im = cand.getMaskedImage() # copy of this object's image
                    xc, yc = cand.getXCenter(), cand.getYCenter()

                    margin = 0 if True else 5
                    w, h = im.getDimensions()
                    bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())

                    if margin > 0:
                        bim = im.Factory(w + 2*margin, h + 2*margin)

                        stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
                        afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                        bim *= stdev
                        var = bim.getVariance(); var.set(stdev**2); del var

                        sbim = im.Factory(bim, bbox)
                        sbim <<= im
                        del sbim
                        im = bim
                        xc += margin; yc += margin

                    im = im.Factory(im, True)
                    im.setXY0(cand.getMaskedImage().getXY0())
                except:
                    continue

                if not variance:
                    im_resid.append(im.Factory(im, True))

                if True:                # tweak up centroids
                    mi = im
                    psfIm = mi.getImage()

                    config = measAlg.SourceMeasurementConfig()
                    config.centroider.name = "centroid.sdss"
                    config.slots.centroid = config.centroider.name

                    schema = afwTable.SourceTable.makeMinimalSchema()
                    measureSources = config.makeMeasureSources(schema)
                    catalog = afwTable.SourceCatalog(schema)
                    config.slots.setupTable(catalog.table)

                    extra = 10          # enough margin to run the sdss centroider
                    miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra)
                    miBig[extra:-extra, extra:-extra] = mi
                    miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra)
                    mi = miBig; del miBig
                    
                    exp = afwImage.makeExposure(mi)
                    exp.setPsf(psf)

                    footprintSet = afwDet.FootprintSet(mi,
                                                       afwDet.Threshold(0.5*numpy.max(psfIm.getArray())),
                                                       "DETECTED")
                    footprintSet.makeSources(catalog)
                    if len(catalog) == 0:
                        raise RuntimeError("Failed to detect any objects")
                    elif len(catalog) == 1:
                        source = catalog[0]
                    else:               # more than one source; find the once closest to (xc, yc)
                        for i, s in enumerate(catalog):
                            d = numpy.hypot(xc - s.getX(), yc - s.getY())
                            if i == 0 or d < dmin:
                                source, dmin = s, d
                                                    
                    measureSources.applyWithPeak(source, exp)
                    xc, yc = source.getCentroid()

                # residuals using spatial model
                try:
                    chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
                except:
                    chi2 = numpy.nan
                    continue
                
                resid = im
                if variance:
                    resid = resid.getImage()
                    var = im.getVariance()
                    var = var.Factory(var, True)
                    numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
                    resid /= var
                    
                im_resid.append(resid)

                # Fit the PSF components directly to the data (i.e. ignoring the spatial model)
                if fitBasisComponents:
                    im = cand.getMaskedImage()
                    
                    im = im.Factory(im, True)
                    im.setXY0(cand.getMaskedImage().getXY0())

                    noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
                    candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
                    fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
                    params = fit[0]
                    kernels = afwMath.KernelList(fit[1])
                    outputKernel = afwMath.LinearCombinationKernel(kernels, params)

                    outImage = afwImage.ImageD(outputKernel.getDimensions())
                    outputKernel.computeImage(outImage, False)

                    im -= outImage.convertF()
                    resid = im

                    if margin > 0:
                        bim = im.Factory(w + 2*margin, h + 2*margin)
                        afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                        bim *= stdev

                        sbim = im.Factory(bim, bbox)
                        sbim <<= resid
                        del sbim
                        resid = bim

                    if variance:
                        resid = resid.getImage()
                        resid /= var

                    im_resid.append(resid)

                im = im_resid.makeMosaic()
            else:
                im = cand.getMaskedImage()

            if normalize:
                im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()

            objId = splitId(cand.getSource().getId(), True)["objId"]
            if psf:
                lab = "%d chi^2 %.1f" % (objId, rchi2)
                ctype = ds9.RED if cand.isBad() else ds9.GREEN
            else:
                lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
                ctype = ds9.GREEN

            mos.append(im, lab, ctype)

            if False and numpy.isnan(rchi2):
                ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
                print "amp",  cand.getAmplitude()

            im = cand.getMaskedImage()
            center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
            candidateIndex += 1
            if cand.isBad():
                candidateCentersBad.append(center)
            else:
                candidateCenters.append(center)

    if variance:
        title = "chi(Psf fit)"
    else:
        title = "Stars & residuals"
    mosaicImage = mos.makeMosaic(frame=frame, title=title)

    with ds9.Buffering():
        for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
            for cen in centers:
                bbox = mos.getBBox(cen[0])
                ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)

    return mosaicImage
Example #8
0
    def testGetPcaKernel(self):
        """Convert our cellSet to a LinearCombinationKernel"""

        nEigenComponents = 2
        spatialOrder  =    1
        kernelSize =      21
        nStarPerCell =     2
        nStarPerCellSpatialFit = 2
        tolerance =     1e-5

        if display:
            ds9.mtv(self.mi, frame=0)
            #
            # Show the candidates we're using
            #
            for cell in self.cellSet.getCellList():
                i = 0
                for cand in cell:
                    i += 1
                    source = algorithms.cast_PsfCandidateF(cand).getSource()
                    
                    xc, yc = source.getXAstrom() - self.mi.getX0(), source.getYAstrom() - self.mi.getY0()
                    if i <= nStarPerCell:
                        ds9.dot("o", xc, yc, ctype=ds9.GREEN)
                    else:
                        ds9.dot("o", xc, yc, ctype=ds9.YELLOW)

        pair = algorithms.createKernelFromPsfCandidates(self.cellSet, self.exposure.getDimensions(),
                                                        self.exposure.getXY0(), nEigenComponents, spatialOrder,
                                                        kernelSize, nStarPerCell)

        kernel, eigenValues = pair[0], pair[1]; del pair

        print "lambda", " ".join(["%g" % l for l in eigenValues])

        pair = algorithms.fitSpatialKernelFromPsfCandidates(kernel, self.cellSet, nStarPerCellSpatialFit, tolerance)
        status, chi2 = pair[0], pair[1]; del pair
        print "Spatial fit: %s chi^2 = %.2g" % (status, chi2)

        psf = algorithms.PcaPsf.swigConvert(roundTripPsf(5, algorithms.PcaPsf(kernel))) # Hurrah!

        self.assertTrue(afwMath.cast_AnalyticKernel(psf.getKernel()) is None)
        self.assertTrue(afwMath.cast_LinearCombinationKernel(psf.getKernel()) is not None)

        self.checkTablePersistence(psf)

        if display:
            #print psf.getKernel().toString()

            eImages = []
            for k in afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList():
                im = afwImage.ImageD(k.getDimensions())
                k.computeImage(im, False)
                eImages.append(im)

            mos = displayUtils.Mosaic()
            frame = 3
            ds9.mtv(mos.makeMosaic(eImages), frame=frame)
            ds9.dot("Eigen Images", 0, 0, frame=frame)
            #
            # Make a mosaic of PSF candidates
            #
            stamps = []
            stampInfo = []

            for cell in self.cellSet.getCellList():
                for cand in cell:
                    #
                    # Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate;  all
                    # it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates
                    # don't know about getMaskedImage;  so cast the pointer to PsfCandidate
                    #
                    cand = algorithms.cast_PsfCandidateF(cand)
                    s = cand.getSource()

                    im = cand.getMaskedImage()

                    stamps.append(im)
                    stampInfo.append("[%d 0x%x]" % (s.getId(), s.getFlagForDetection()))
        
                    mos = displayUtils.Mosaic()
            frame = 1
            ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True)
            for i in range(len(stampInfo)):
                ds9.dot(stampInfo[i], mos.getBBox(i).getX0(), mos.getBBox(i).getY0(), frame=frame, ctype=ds9.RED)

            psfImages = []
            labels = []
            if False:
                nx, ny = 3, 4
                for iy in range(ny):
                    for ix in range(nx):
                        x = int((ix + 0.5)*self.mi.getWidth()/nx)
                        y = int((iy + 0.5)*self.mi.getHeight()/ny)

                        im = psf.getImage(x, y)
                        psfImages.append(im.Factory(im, True))
                        labels.append("PSF(%d,%d)" % (int(x), int(y)))

                        if True:
                            print x, y, "PSF parameters:", psf.getKernel().getKernelParameters()
            else:
                nx, ny = 2, 2
                for x, y in [(20, 20), (60, 20), 
                             (60, 210), (20, 210)]:

                    im = psf.computeImage(afwGeom.PointD(x, y))
                    psfImages.append(im.Factory(im, True))
                    labels.append("PSF(%d,%d)" % (int(x), int(y)))
                    
                    if True:
                        print x, y, "PSF parameters:", psf.getKernel().getKernelParameters()
                    
            frame = 2
            mos.makeMosaic(psfImages, frame=frame, mode=nx)
            mos.drawLabels(labels, frame=frame)

        if display:
            
            ds9.mtv(self.mi, frame=0)

            psfImages = []
            labels = []
            if False:
                nx, ny = 3, 4
                for iy in range(ny):
                    for ix in range(nx):
                        x = int((ix + 0.5)*self.mi.getWidth()/nx)
                        y = int((iy + 0.5)*self.mi.getHeight()/ny)

                        algorithms.subtractPsf(psf, self.mi, x, y)
            else:
                nx, ny = 2, 2
                for x, y in [(20, 20), (60, 20), 
                             (60, 210), (20, 210)]:
                        
                    if False:               # Test subtraction with non-centered psfs
                        x += 0.5; y -= 0.5

                    #algorithms.subtractPsf(psf, self.mi, x, y)

            ds9.mtv(self.mi, frame=1)
Example #9
0
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
                        matchKernelAmplitudes=False, keepPlots=True):
    """Plot the PSF spatial model."""

    if not plt:
        print >> sys.stderr, "Unable to import matplotlib"
        return

    noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
    candPos = list()
    candFits = list()
    badPos = list()
    badFits = list()
    candAmps = list()
    badAmps = list()
    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False):
            cand = algorithmsLib.cast_PsfCandidateF(cand)
            if not showBadCandidates and cand.isBad():
                continue
            candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
            try:
                im = cand.getMaskedImage()
            except Exception:
                continue

            fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
            params = fit[0]
            kernels = fit[1]
            amp = 0.0
            for p, k in zip(params, kernels):
                amp += p * afwMath.cast_FixedKernel(k).getSum()

            targetFits = badFits if cand.isBad() else candFits
            targetPos = badPos if cand.isBad() else candPos
            targetAmps = badAmps if cand.isBad() else candAmps

            targetFits.append([x / amp for x in params])
            targetPos.append(candCenter)
            targetAmps.append(amp)

    numCandidates = len(candFits)
    numBasisFuncs = noSpatialKernel.getNBasisKernels()

    xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
    yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
    zGood = numpy.array(candFits)
    ampGood = numpy.array(candAmps)

    xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
    yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
    zBad = numpy.array(badFits)
    ampBad = numpy.array(badAmps)
    numBad = len(badPos)

    xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
    yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)

    kernel = psf.getKernel()
    nKernelComponents = kernel.getNKernelParameters()
    #
    # Figure out how many panels we'll need
    #
    nPanelX = int(math.sqrt(nKernelComponents))
    nPanelY = nKernelComponents//nPanelX
    while nPanelY*nPanelX < nKernelComponents:
        nPanelX += 1

    fig = plt.figure(1)
    fig.clf()
    try:
        fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
    except:                                 # protect against API changes
        pass
    #
    # Generator for axes arranged in panels
    #
    subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)

    for k in range(nKernelComponents):
        func = kernel.getSpatialFunction(k)
        dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
        yMin = dfGood.min()
        yMax = dfGood.max()
        if numBad > 0:
            dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
            yMin = min([yMin, dfBad.min()])
            yMax = max([yMax, dfBad.max()])
        yMin -= 0.05 * (yMax - yMin)
        yMax += 0.05 * (yMax - yMin)

        yMin = -0.01
        yMax = 0.01

        fRange = numpy.ndarray((len(xRange), len(yRange)))
        for j, yVal in enumerate(yRange):
            for i, xVal in enumerate(xRange):
                fRange[j][i] = func(xVal, yVal)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        ax.set_autoscale_on(False)
        ax.set_xbound(lower=0, upper=exposure.getHeight())
        ax.set_ybound(lower=yMin, upper=yMax)
        ax.plot(yGood, dfGood, 'b+')
        if numBad > 0:
            ax.plot(yBad, dfBad, 'r+')
        ax.axhline(0.0)
        ax.set_title('Residuals(y)')

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        if matchKernelAmplitudes and k == 0:
            vmin = 0.0
            vmax = 1.1
        else:
            vmin = fRange.min()
            vmax = fRange.max()

        norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
        im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
                       extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
        ax.set_title('Spatial poly')
        plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()
        ax.set_autoscale_on(False)
        ax.set_xbound(lower=0, upper=exposure.getWidth())
        ax.set_ybound(lower=yMin, upper=yMax)
        ax.plot(xGood, dfGood, 'b+')
        if numBad > 0:
            ax.plot(xBad, dfBad, 'r+')
        ax.axhline(0.0)
        ax.set_title('K%d Residuals(x)' % k)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        if False:
            ax.scatter(xGood, yGood, c=dfGood, marker='o')
            ax.scatter(xBad, yBad, c=dfBad, marker='x')
            ax.set_xbound(lower=0, upper=exposure.getWidth())
            ax.set_ybound(lower=0, upper=exposure.getHeight())
            ax.set_title('Spatial residuals')
            plt.colorbar(im, orientation='horizontal')
        else:
            calib = exposure.getCalib()
            if calib.getFluxMag0()[0] <= 0:
                calib = type(calib)()
                calib.setFluxMag0(1.0)

            with CalibNoThrow():
                ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+')
                if numBad > 0:
                    ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+')

            ax.set_title('Flux variation')

    fig.show()

    global keptPlots
    if keepPlots and not keptPlots:
        # Keep plots open when done
        def show():
            print "%s: Please close plots when done." % __name__
            try:
                plt.show()
            except:
                pass
            print "Plots closed, exiting..."
        import atexit
        atexit.register(show)
        keptPlots = True
Example #10
0
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
                      fitBasisComponents=False, variance=None, chi=None):
    """Display the PSF candidates.

    If psf is provided include PSF model and residuals;  if normalize is true normalize the PSFs
    (and residuals)

    If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi

    If fitBasisComponents is true, also find the best linear combination of the PSF's components
    (if they exist)
    """
    if chi is None:
        if variance is not None:        # old name for chi
            chi = variance
    #
    # Show us the ccandidates
    #
    mos = displayUtils.Mosaic()
    #
    candidateCenters = []
    candidateCentersBad = []
    candidateIndex = 0

    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False): # include bad candidates
            cand = algorithmsLib.cast_PsfCandidateF(cand)

            rchi2 = cand.getChi2()
            if rchi2 > 1e100:
                rchi2 = numpy.nan

            if not showBadCandidates and cand.isBad():
                continue

            if psf:
                im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")

                try:
                    im = cand.getMaskedImage() # copy of this object's image
                    xc, yc = cand.getXCenter(), cand.getYCenter()

                    margin = 0 if True else 5
                    w, h = im.getDimensions()
                    bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())

                    if margin > 0:
                        bim = im.Factory(w + 2*margin, h + 2*margin)

                        stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
                        afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                        bim.getVariance().set(stdev**2)

                        bim.assign(im, bbox)
                        im = bim
                        xc += margin
                        yc += margin

                    im = im.Factory(im, True)
                    im.setXY0(cand.getMaskedImage().getXY0())
                except:
                    continue

                if not variance:
                    im_resid.append(im.Factory(im, True))

                if True:                # tweak up centroids
                    mi = im
                    psfIm = mi.getImage()
                    config = measBase.SingleFrameMeasurementTask.ConfigClass()
                    config.slots.centroid = "base_SdssCentroid"

                    schema = afwTable.SourceTable.makeMinimalSchema()
                    measureSources =  measBase.SingleFrameMeasurementTask(schema, config=config)
                    catalog = afwTable.SourceCatalog(schema)

                    extra = 10          # enough margin to run the sdss centroider
                    miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra)
                    miBig[extra:-extra, extra:-extra] = mi
                    miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra)
                    mi = miBig
                    del miBig

                    exp = afwImage.makeExposure(mi)
                    exp.setPsf(psf)

                    footprintSet = afwDet.FootprintSet(mi,
                                                       afwDet.Threshold(0.5*numpy.max(psfIm.getArray())),
                                                       "DETECTED")
                    footprintSet.makeSources(catalog)

                    if len(catalog) == 0:
                        raise RuntimeError("Failed to detect any objects")

                    measureSources.run(catalog, exp)
                    if len(catalog) == 1:
                        source = catalog[0]
                    else:               # more than one source; find the once closest to (xc, yc)
                        dmin = None # an invalid value to catch logic errors
                        for i, s in enumerate(catalog):
                            d = numpy.hypot(xc - s.getX(), yc - s.getY())
                            if i == 0 or d < dmin:
                                source, dmin = s, d
                    xc, yc = source.getCentroid()

                # residuals using spatial model
                try:
                    chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
                except:
                    chi2 = numpy.nan
                    continue

                resid = im
                if variance:
                    resid = resid.getImage()
                    var = im.getVariance()
                    var = var.Factory(var, True)
                    numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
                    resid /= var

                im_resid.append(resid)

                # Fit the PSF components directly to the data (i.e. ignoring the spatial model)
                if fitBasisComponents:
                    im = cand.getMaskedImage()

                    im = im.Factory(im, True)
                    im.setXY0(cand.getMaskedImage().getXY0())

                    try:
                        noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
                    except:
                        noSpatialKernel = None

                    if noSpatialKernel:
                        candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
                        fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
                        params = fit[0]
                        kernels = afwMath.KernelList(fit[1])
                        outputKernel = afwMath.LinearCombinationKernel(kernels, params)

                        outImage = afwImage.ImageD(outputKernel.getDimensions())
                        outputKernel.computeImage(outImage, False)

                        im -= outImage.convertF()
                        resid = im

                        if margin > 0:
                            bim = im.Factory(w + 2*margin, h + 2*margin)
                            afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                            bim *= stdev

                            bim.assign(resid, bbox)
                            resid = bim

                        if variance:
                            resid = resid.getImage()
                            resid /= var

                        im_resid.append(resid)

                im = im_resid.makeMosaic()
            else:
                im = cand.getMaskedImage()

            if normalize:
                im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()

            objId = splitId(cand.getSource().getId(), True)["objId"]
            if psf:
                lab = "%d chi^2 %.1f" % (objId, rchi2)
                ctype = ds9.RED if cand.isBad() else ds9.GREEN
            else:
                lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
                ctype = ds9.GREEN

            mos.append(im, lab, ctype)

            if False and numpy.isnan(rchi2):
                ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
                print "amp",  cand.getAmplitude()

            im = cand.getMaskedImage()
            center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
            candidateIndex += 1
            if cand.isBad():
                candidateCentersBad.append(center)
            else:
                candidateCenters.append(center)

    if variance:
        title = "chi(Psf fit)"
    else:
        title = "Stars & residuals"
    mosaicImage = mos.makeMosaic(frame=frame, title=title)

    with ds9.Buffering():
        for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
            for cen in centers:
                bbox = mos.getBBox(cen[0])
                ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)

    return mosaicImage
Example #11
0
    def run(self, sensorRef, templateIdList=None):
        """Subtract an image from a template coadd and measure the result

        Steps include:
        - warp template coadd to match WCS of image
        - PSF match image to warped template
        - subtract image from PSF-matched, warped template
        - persist difference image
        - detect sources
        - measure sources

        @param sensorRef: sensor-level butler data reference, used for the following data products:
        Input only:
        - calexp
        - psf
        - ccdExposureId
        - ccdExposureId_bits
        - self.config.coaddName + "Coadd_skyMap"
        - self.config.coaddName + "Coadd"
        Input or output, depending on config:
        - self.config.coaddName + "Diff_subtractedExp"
        Output, depending on config:
        - self.config.coaddName + "Diff_matchedExp"
        - self.config.coaddName + "Diff_src"

        @return pipe_base Struct containing these fields:
        - subtractedExposure: exposure after subtracting template;
            the unpersisted version if subtraction not run but detection run
            None if neither subtraction nor detection run (i.e. nothing useful done)
        - subtractRes: results of subtraction task; None if subtraction not run
        - sources: detected and possibly measured sources; None if detection not run
        """
        self.log.info("Processing %s" % (sensorRef.dataId))

        # initialize outputs and some intermediate products
        subtractedExposure = None
        subtractRes = None
        selectSources = None
        kernelSources = None
        controlSources = None
        diaSources = None

        # We make one IdFactory that will be used by both icSrc and src datasets;
        # I don't know if this is the way we ultimately want to do things, but at least
        # this ensures the source IDs are fully unique.
        expBits = sensorRef.get("ccdExposureId_bits")
        expId = long(sensorRef.get("ccdExposureId"))
        idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)

        # Retrieve the science image we wish to analyze
        exposure = sensorRef.get("calexp", immediate=True)
        if self.config.doAddCalexpBackground:
            mi = exposure.getMaskedImage()
            mi += sensorRef.get("calexpBackground").getImage()
        if not exposure.hasPsf():
            raise pipeBase.TaskError("Exposure has no psf")
        sciencePsf = exposure.getPsf()

        subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
        templateExposure = None  # Stitched coadd exposure
        templateSources = None  # Sources on the template image
        if self.config.doSubtract:
            template = self.getTemplate.run(exposure,
                                            sensorRef,
                                            templateIdList=templateIdList)
            templateExposure = template.exposure
            templateSources = template.sources

            # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
            ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
            psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
            scienceSigmaOrig = psfAttr.computeGaussianWidth(
                psfAttr.ADAPTIVE_MOMENT)

            # sigma of PSF of template image before warping
            ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
            psfAttr = PsfAttributes(templateExposure.getPsf(),
                                    afwGeom.Point2I(ctr))
            templateSigma = psfAttr.computeGaussianWidth(
                psfAttr.ADAPTIVE_MOMENT)

            # if requested, convolve the science exposure with its PSF
            # (properly, this should be a cross-correlation, but our code does not yet support that)
            # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
            # else sigma of original science exposure
            if self.config.doPreConvolve:
                convControl = afwMath.ConvolutionControl()
                # cannot convolve in place, so make a new MI to receive convolved image
                srcMI = exposure.getMaskedImage()
                destMI = srcMI.Factory(srcMI.getDimensions())
                srcPsf = sciencePsf
                if self.config.useGaussianForPreConvolution:
                    # convolve with a simplified PSF model: a double Gaussian
                    kWidth, kHeight = sciencePsf.getLocalKernel(
                    ).getDimensions()
                    preConvPsf = SingleGaussianPsf(kWidth, kHeight,
                                                   scienceSigmaOrig)
                else:
                    # convolve with science exposure's PSF model
                    preConvPsf = srcPsf
                afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
                                 convControl)
                exposure.setMaskedImage(destMI)
                scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
            else:
                scienceSigmaPost = scienceSigmaOrig

            # If requested, find sources in the image
            if self.config.doSelectSources:
                if not sensorRef.datasetExists("src"):
                    self.log.warn(
                        "Src product does not exist; running detection, measurement, selection"
                    )
                    # Run own detection and measurement; necessary in nightly processing
                    selectSources = self.subtract.getSelectSources(
                        exposure,
                        sigma=scienceSigmaPost,
                        doSmooth=not self.doPreConvolve,
                        idFactory=idFactory,
                    )
                else:
                    self.log.info("Source selection via src product")
                    # Sources already exist; for data release processing
                    selectSources = sensorRef.get("src")

                # Number of basis functions
                nparam = len(
                    makeKernelBasisList(
                        self.subtract.config.kernel.active,
                        referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
                        targetFwhmPix=templateSigma * FwhmPerSigma))

                if self.config.doAddMetrics:
                    # Modify the schema of all Sources
                    kcQa = KernelCandidateQa(nparam)
                    selectSources = kcQa.addToSchema(selectSources)

                if self.config.kernelSourcesFromRef:
                    # match exposure sources to reference catalog
                    astromRet = self.astrometer.loadAndMatch(
                        exposure=exposure, sourceCat=selectSources)
                    matches = astromRet.matches
                elif templateSources:
                    # match exposure sources to template sources
                    matches = afwTable.matchRaDec(templateSources,
                                                  selectSources,
                                                  1.0 * afwGeom.arcseconds,
                                                  False)
                else:
                    raise RuntimeError(
                        "doSelectSources=True and kernelSourcesFromRef=False,"
                        +
                        "but template sources not available. Cannot match science "
                        +
                        "sources with template sources. Run process* on data from "
                        + "which templates are built.")

                kernelSources = self.sourceSelector.selectStars(
                    exposure, selectSources, matches=matches).starCat

                random.shuffle(kernelSources, random.random)
                controlSources = kernelSources[::self.config.controlStepSize]
                kernelSources = [
                    k for i, k in enumerate(kernelSources)
                    if i % self.config.controlStepSize
                ]

                if self.config.doSelectDcrCatalog:
                    redSelector = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(
                            grMin=self.sourceSelector.config.grMax,
                            grMax=99.999))
                    redSources = redSelector.selectStars(
                        exposure, selectSources, matches=matches).starCat
                    controlSources.extend(redSources)

                    blueSelector = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(
                            grMin=-99.999,
                            grMax=self.sourceSelector.config.grMin))
                    blueSources = blueSelector.selectStars(
                        exposure, selectSources, matches=matches).starCat
                    controlSources.extend(blueSources)

                if self.config.doSelectVariableCatalog:
                    varSelector = DiaCatalogSourceSelectorTask(
                        DiaCatalogSourceSelectorConfig(includeVariable=True))
                    varSources = varSelector.selectStars(
                        exposure, selectSources, matches=matches).starCat
                    controlSources.extend(varSources)

                self.log.info(
                    "Selected %d / %d sources for Psf matching (%d for control sample)"
                    % (len(kernelSources), len(selectSources),
                       len(controlSources)))
            allresids = {}
            if self.config.doUseRegister:
                self.log.info("Registering images")

                if templateSources is None:
                    # Run detection on the template, which is
                    # temporarily background-subtracted
                    templateSources = self.subtract.getSelectSources(
                        templateExposure,
                        sigma=templateSigma,
                        doSmooth=True,
                        idFactory=idFactory)

                # Third step: we need to fit the relative astrometry.
                #
                wcsResults = self.fitAstrometry(templateSources,
                                                templateExposure,
                                                selectSources)
                warpedExp = self.register.warpExposure(templateExposure,
                                                       wcsResults.wcs,
                                                       exposure.getWcs(),
                                                       exposure.getBBox())
                templateExposure = warpedExp

                # Create debugging outputs on the astrometric
                # residuals as a function of position.  Persistence
                # not yet implemented; expected on (I believe) #2636.
                if self.config.doDebugRegister:
                    # Grab matches to reference catalog
                    srcToMatch = {x.second.getId(): x.first for x in matches}

                    refCoordKey = wcsResults.matches[0].first.getTable(
                    ).getCoordKey()
                    inCentroidKey = wcsResults.matches[0].second.getTable(
                    ).getCentroidKey()
                    sids = [m.first.getId() for m in wcsResults.matches]
                    positions = [
                        m.first.get(refCoordKey) for m in wcsResults.matches
                    ]
                    residuals = [
                        m.first.get(refCoordKey).getOffsetFrom(
                            wcsResults.wcs.pixelToSky(
                                m.second.get(inCentroidKey)))
                        for m in wcsResults.matches
                    ]
                    allresids = dict(zip(sids, zip(positions, residuals)))

                    cresiduals = [
                        m.first.get(refCoordKey).getTangentPlaneOffset(
                            wcsResults.wcs.pixelToSky(
                                m.second.get(inCentroidKey)))
                        for m in wcsResults.matches
                    ]
                    colors = numpy.array([
                        -2.5 * numpy.log10(srcToMatch[x].get("g")) +
                        2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
                        if x in srcToMatch.keys()
                    ])
                    dlong = numpy.array([
                        r[0].asArcseconds() for s, r in zip(sids, cresiduals)
                        if s in srcToMatch.keys()
                    ])
                    dlat = numpy.array([
                        r[1].asArcseconds() for s, r in zip(sids, cresiduals)
                        if s in srcToMatch.keys()
                    ])
                    idx1 = numpy.where(
                        colors < self.sourceSelector.config.grMin)
                    idx2 = numpy.where(
                        (colors >= self.sourceSelector.config.grMin)
                        & (colors <= self.sourceSelector.config.grMax))
                    idx3 = numpy.where(
                        colors > self.sourceSelector.config.grMax)
                    rms1Long = IqrToSigma * (numpy.percentile(
                        dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25))
                    rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
                                            numpy.percentile(dlat[idx1], 25))
                    rms2Long = IqrToSigma * (numpy.percentile(
                        dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25))
                    rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
                                            numpy.percentile(dlat[idx2], 25))
                    rms3Long = IqrToSigma * (numpy.percentile(
                        dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25))
                    rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
                                            numpy.percentile(dlat[idx3], 25))
                    self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
                                  (numpy.median(dlong[idx1]), rms1Long,
                                   numpy.median(dlat[idx1]), rms1Lat))
                    self.log.info(
                        "Green star offsets'': %.3f %.3f, %.3f %.3f" %
                        (numpy.median(dlong[idx2]), rms2Long,
                         numpy.median(dlat[idx2]), rms2Lat))
                    self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
                                  (numpy.median(dlong[idx3]), rms3Long,
                                   numpy.median(dlat[idx3]), rms3Lat))

                    self.metadata.add("RegisterBlueLongOffsetMedian",
                                      numpy.median(dlong[idx1]))
                    self.metadata.add("RegisterGreenLongOffsetMedian",
                                      numpy.median(dlong[idx2]))
                    self.metadata.add("RegisterRedLongOffsetMedian",
                                      numpy.median(dlong[idx3]))
                    self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
                    self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
                    self.metadata.add("RegisterRedLongOffsetStd", rms3Long)

                    self.metadata.add("RegisterBlueLatOffsetMedian",
                                      numpy.median(dlat[idx1]))
                    self.metadata.add("RegisterGreenLatOffsetMedian",
                                      numpy.median(dlat[idx2]))
                    self.metadata.add("RegisterRedLatOffsetMedian",
                                      numpy.median(dlat[idx3]))
                    self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
                    self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
                    self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)

            # warp template exposure to match exposure,
            # PSF match template exposure to exposure,
            # then return the difference

            #Return warped template...  Construct sourceKernelCand list after subtract
            self.log.info("Subtracting images")
            subtractRes = self.subtract.subtractExposures(
                templateExposure=templateExposure,
                scienceExposure=exposure,
                candidateList=kernelSources,
                convolveTemplate=self.config.convolveTemplate,
                doWarping=not self.config.doUseRegister)
            subtractedExposure = subtractRes.subtractedExposure

            if self.config.doWriteMatchedExp:
                sensorRef.put(subtractRes.matchedExposure,
                              self.config.coaddName + "Diff_matchedExp")

        if self.config.doDetection:
            self.log.info("Computing diffim PSF")
            if subtractedExposure is None:
                subtractedExposure = sensorRef.get(subtractedExposureName)

            # Get Psf from the appropriate input image if it doesn't exist
            if not subtractedExposure.hasPsf():
                if self.config.convolveTemplate:
                    subtractedExposure.setPsf(exposure.getPsf())
                else:
                    if templateExposure is None:
                        template = self.getTemplate.run(
                            exposure, sensorRef, templateIdList=templateIdList)
                    subtractedExposure.setPsf(template.exposure.getPsf())

        # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
        # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
        if self.config.doDecorrelation and self.config.doSubtract:
            decorrResult = self.decorrelate.run(exposure, templateExposure,
                                                subtractedExposure,
                                                subtractRes.psfMatchingKernel)
            subtractedExposure = decorrResult.correctedExposure

        if self.config.doDetection:
            self.log.info("Running diaSource detection")
            # Erase existing detection mask planes
            mask = subtractedExposure.getMaskedImage().getMask()
            mask &= ~(mask.getPlaneBitMask("DETECTED")
                      | mask.getPlaneBitMask("DETECTED_NEGATIVE"))

            table = afwTable.SourceTable.make(self.schema, idFactory)
            table.setMetadata(self.algMetadata)
            results = self.detection.makeSourceCatalog(
                table=table,
                exposure=subtractedExposure,
                doSmooth=not self.config.doPreConvolve)

            if self.config.doMerge:
                fpSet = results.fpSets.positive
                fpSet.merge(results.fpSets.negative, self.config.growFootprint,
                            self.config.growFootprint, False)
                diaSources = afwTable.SourceCatalog(table)
                fpSet.makeSources(diaSources)
                self.log.info("Merging detections into %d sources" %
                              (len(diaSources)))
            else:
                diaSources = results.sources

            if self.config.doMeasurement:
                self.log.info("Running diaSource measurement")
                if not self.config.doDipoleFitting:
                    self.measurement.run(diaSources, subtractedExposure)
                else:
                    if self.config.doSubtract:
                        self.measurement.run(diaSources, subtractedExposure,
                                             exposure,
                                             subtractRes.matchedExposure)
                    else:
                        self.measurement.run(diaSources, subtractedExposure,
                                             exposure)

            # Match with the calexp sources if possible
            if self.config.doMatchSources:
                if sensorRef.datasetExists("src"):
                    # Create key,val pair where key=diaSourceId and val=sourceId
                    matchRadAsec = self.config.diaSourceMatchRadius
                    matchRadPixel = matchRadAsec / exposure.getWcs(
                    ).pixelScale().asArcseconds()

                    srcMatches = afwTable.matchXy(sensorRef.get("src"),
                                                  diaSources, matchRadPixel,
                                                  True)
                    srcMatchDict = dict([(srcMatch.second.getId(),
                                          srcMatch.first.getId())
                                         for srcMatch in srcMatches])
                    self.log.info("Matched %d / %d diaSources to sources" %
                                  (len(srcMatchDict), len(diaSources)))
                else:
                    self.log.warn(
                        "Src product does not exist; cannot match with diaSources"
                    )
                    srcMatchDict = {}

                # Create key,val pair where key=diaSourceId and val=refId
                refAstromConfig = measAstrom.AstrometryConfig()
                refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
                refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
                astromRet = refAstrometer.run(exposure=exposure,
                                              sourceCat=diaSources)
                refMatches = astromRet.matches
                if refMatches is None:
                    self.log.warn(
                        "No diaSource matches with reference catalog")
                    refMatchDict = {}
                else:
                    self.log.info(
                        "Matched %d / %d diaSources to reference catalog" %
                        (len(refMatches), len(diaSources)))
                    refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
                                             refMatch in refMatches])

                # Assign source Ids
                for diaSource in diaSources:
                    sid = diaSource.getId()
                    if srcMatchDict.has_key(sid):
                        diaSource.set("srcMatchId", srcMatchDict[sid])
                    if refMatchDict.has_key(sid):
                        diaSource.set("refMatchId", refMatchDict[sid])

            if diaSources is not None and self.config.doWriteSources:
                sensorRef.put(diaSources,
                              self.config.coaddName + "Diff_diaSrc")

            if self.config.doAddMetrics and self.config.doSelectSources:
                self.log.info("Evaluating metrics and control sample")

                kernelCandList = []
                for cell in subtractRes.kernelCellSet.getCellList():
                    for cand in cell.begin(False):  # include bad candidates
                        kernelCandList.append(cast_KernelCandidateF(cand))

                # Get basis list to build control sample kernels
                basisList = afwMath.cast_LinearCombinationKernel(
                    kernelCandList[0].getKernel(
                        KernelCandidateF.ORIG)).getKernelList()

                controlCandList = \
                    diffimTools.sourceTableToCandidateList(controlSources,
                                                           subtractRes.warpedExposure, exposure,
                                                           self.config.subtract.kernel.active,
                                                           self.config.subtract.kernel.active.detectionConfig,
                                                           self.log, doBuild=True, basisList=basisList)

                kcQa.apply(kernelCandList,
                           subtractRes.psfMatchingKernel,
                           subtractRes.backgroundModel,
                           dof=nparam)
                kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
                           subtractRes.backgroundModel)

                if self.config.doDetection:
                    kcQa.aggregate(selectSources, self.metadata, allresids,
                                   diaSources)
                else:
                    kcQa.aggregate(selectSources, self.metadata, allresids)

                sensorRef.put(selectSources,
                              self.config.coaddName + "Diff_kernelSrc")

        if self.config.doWriteSubtractedExp:
            sensorRef.put(subtractedExposure, subtractedExposureName)

        self.runDebug(exposure, subtractRes, selectSources, kernelSources,
                      diaSources)
        return pipeBase.Struct(
            subtractedExposure=subtractedExposure,
            subtractRes=subtractRes,
            sources=diaSources,
        )
class PcaPsfDeterminer(object):
    """!
    A measurePsfTask psf estimator
    """
    ConfigClass = PcaPsfDeterminerConfig

    def __init__(self, config):
        """!Construct a PCA PSF Fitter

        \param[in] config instance of PcaPsfDeterminerConfig
        """
        self.config = config
        # N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging
        # independent of which determiner is active
        self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
        self.warnLog = pexLog.Log(pexLog.getDefaultLog(),
                                  "meas.algorithms.psfDeterminer")

    def _fitPsf(self, exposure, psfCellSet, kernelSize, nEigenComponents):
        algorithmsLib.PsfCandidateF.setPixelThreshold(
            self.config.pixelThreshold)
        algorithmsLib.PsfCandidateF.setMaskBlends(self.config.doMaskBlends)
        #
        # Loop trying to use nEigenComponents, but allowing smaller numbers if necessary
        #
        for nEigen in range(nEigenComponents, 0, -1):
            # Determine KL components
            try:
                kernel, eigenValues = algorithmsLib.createKernelFromPsfCandidates(
                    psfCellSet, exposure.getDimensions(), exposure.getXY0(),
                    nEigen, self.config.spatialOrder, kernelSize,
                    self.config.nStarPerCell, bool(self.config.constantWeight))

                break  # OK, we can get nEigen components
            except pexExceptions.LengthError as e:
                if nEigen == 1:  # can't go any lower
                    raise IndexError("No viable PSF candidates survive")

                self.warnLog.log(
                    pexLog.Log.WARN,
                    "%s: reducing number of eigen components" % e.what())
        #
        # We got our eigen decomposition so let's use it
        #
        # Express eigenValues in units of reduced chi^2 per star
        size = kernelSize + 2 * self.config.borderWidth
        nu = size * size - 1  # number of degrees of freedom/star for chi^2
        eigenValues = [
            l / float(
                algorithmsLib.countPsfCandidates(
                    psfCellSet, self.config.nStarPerCell) * nu)
            for l in eigenValues
        ]

        # Fit spatial model
        status, chi2 = algorithmsLib.fitSpatialKernelFromPsfCandidates(
            kernel, psfCellSet, bool(self.config.nonLinearSpatialFit),
            self.config.nStarPerCellSpatialFit, self.config.tolerance,
            self.config.lam)

        psf = algorithmsLib.PcaPsf(kernel)

        return psf, eigenValues, nEigen, chi2

    def determinePsf(self,
                     exposure,
                     psfCandidateList,
                     metadata=None,
                     flagKey=None):
        """!Determine a PCA PSF model for an exposure given a list of PSF candidates

        \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
        \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
            typically obtained by detecting sources and then running them through a star selector
        \param[in,out] metadata  a home for interesting tidbits of information
        \param[in] flagKey schema key used to mark sources actually used in PSF determination

        \return a list of
         - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
         - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = lsstDebug.Info(
            __name__).displayExposure  # display the Exposure + spatialCells
        displayPsfCandidates = lsstDebug.Info(
            __name__).displayPsfCandidates  # show the viable candidates
        displayIterations = lsstDebug.Info(
            __name__).displayIterations  # display on each PSF iteration
        displayPsfComponents = lsstDebug.Info(
            __name__).displayPsfComponents  # show the PCA components
        displayResiduals = lsstDebug.Info(
            __name__).displayResiduals  # show residuals
        displayPsfMosaic = lsstDebug.Info(
            __name__).displayPsfMosaic  # show mosaic of reconstructed PSF(x,y)
        matchKernelAmplitudes = lsstDebug.Info(
            __name__).matchKernelAmplitudes  # match Kernel amplitudes
        # for spatial plots
        keepMatplotlibPlots = lsstDebug.Info(
            __name__).keepMatplotlibPlots  # Keep matplotlib alive
        # post mortem
        displayPsfSpatialModel = lsstDebug.Info(
            __name__).displayPsfSpatialModel  # Plot spatial model?
        showBadCandidates = lsstDebug.Info(
            __name__).showBadCandidates  # Include bad candidates
        normalizeResiduals = lsstDebug.Info(
            __name__).normalizeResiduals  # Normalise residuals by
        # object amplitude
        pause = lsstDebug.Info(
            __name__).pause  # Prompt user after each iteration?

        if display > 1:
            pause = True

        mi = exposure.getMaskedImage()

        if len(psfCandidateList) == 0:
            raise RuntimeError("No PSF candidates supplied.")

        # construct and populate a spatial cell set
        bbox = mi.getBBox()
        psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
                                            self.config.sizeCellY)
        sizes = []
        for i, psfCandidate in enumerate(psfCandidateList):
            if psfCandidate.getSource().getPsfFluxFlag():  # bad measurement
                continue

            try:
                psfCellSet.insertCandidate(psfCandidate)
            except Exception, e:
                self.debugLog.debug(
                    2, "Skipping PSF candidate %d of %d: %s" %
                    (i, len(psfCandidateList), e))
                continue
            source = psfCandidate.getSource()

            quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
                                     source.getIxy())
            axes = afwEll.Axes(quad)
            sizes.append(axes.getA())
        if len(sizes) == 0:
            raise RuntimeError("No usable PSF candidates supplied")
        nEigenComponents = self.config.nEigenComponents  # initial version

        if self.config.kernelSize >= 15:
            self.debugLog.debug(1, \
                "WARNING: NOT scaling kernelSize by stellar quadrupole moment " +
                "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead" \
                % (self.config.kernelSize,)
            )
            actualKernelSize = int(self.config.kernelSize)
        else:
            medSize = numpy.median(sizes)
            actualKernelSize = 2 * int(self.config.kernelSize *
                                       math.sqrt(medSize) + 0.5) + 1
            if actualKernelSize < self.config.kernelSizeMin:
                actualKernelSize = self.config.kernelSizeMin
            if actualKernelSize > self.config.kernelSizeMax:
                actualKernelSize = self.config.kernelSizeMax

            if display:
                print "Median size=%s" % (medSize, )
        self.debugLog.debug(3, "Kernel size=%s" % (actualKernelSize, ))

        # Set size of image returned around candidate
        psfCandidateList[0].setHeight(actualKernelSize)
        psfCandidateList[0].setWidth(actualKernelSize)

        if self.config.doRejectBlends:
            # Remove blended candidates completely
            blendedCandidates = [
            ]  # Candidates to remove; can't do it while iterating
            for cell, cand in candidatesIter(psfCellSet, False):
                if len(cand.getSource().getFootprint().getPeaks()) > 1:
                    blendedCandidates.append((cell, cand))
                    continue
            if display:
                print "Removing %d blended Psf candidates" % len(
                    blendedCandidates)
            for cell, cand in blendedCandidates:
                cell.removeCandidate(cand)
            if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
                raise RuntimeError("All PSF candidates removed as blends")

        if display:
            frame = 0
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            self.config.nStarPerCell,
                                            symb="o",
                                            ctype=ds9.CYAN,
                                            ctypeUnused=ds9.YELLOW,
                                            size=4,
                                            frame=frame)

        #
        # Do a PCA decomposition of those PSF candidates
        #
        reply = "y"  # used in interactive mode
        for iter in range(self.config.nIterForPsf):
            if display and displayPsfCandidates:  # Show a mosaic of usable PSF candidates
                #
                import lsst.afw.display.utils as displayUtils

                stamps = []
                for cell in psfCellSet.getCellList():
                    for cand in cell.begin(not showBadCandidates
                                           ):  # maybe include bad candidates
                        cand = algorithmsLib.cast_PsfCandidateF(cand)

                        try:
                            im = cand.getMaskedImage()

                            chi2 = cand.getChi2()
                            if chi2 > 1e100:
                                chi2 = numpy.nan

                            stamps.append(
                                (im, "%d%s" %
                                 (maUtils.splitId(cand.getSource().getId(),
                                                  True)["objId"], chi2),
                                 cand.getStatus()))
                        except Exception, e:
                            continue

                if len(stamps) == 0:
                    print "WARNING: No PSF candidates to show; try setting showBadCandidates=True"
                else:
                    mos = displayUtils.Mosaic()
                    for im, label, status in stamps:
                        im = type(im)(im, True)
                        try:
                            im /= afwMath.makeStatistics(
                                im, afwMath.MAX).getValue()
                        except NotImplementedError:
                            pass

                        mos.append(
                            im, label, ds9.GREEN
                            if status == afwMath.SpatialCellCandidate.GOOD else
                            ds9.YELLOW if status ==
                            afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED)

                    mos.makeMosaic(frame=8, title="Psf Candidates")

            # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
            cleanChi2 = False  # Any naughty (negative/NAN) chi^2 values?
            while not cleanChi2:
                cleanChi2 = True
                #
                # First, estimate the PSF
                #
                psf, eigenValues, nEigenComponents, fitChi2 = \
                    self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
                #
                # In clipping, allow all candidates to be innocent until proven guilty on this iteration.
                # Throw out any prima facie guilty candidates (naughty chi^2 values)
                #
                for cell in psfCellSet.getCellList():
                    awfulCandidates = []
                    for cand in cell.begin(False):  # include bad candidates
                        cand = algorithmsLib.cast_PsfCandidateF(cand)
                        cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN
                                       )  # until proven guilty
                        rchi2 = cand.getChi2()
                        if not numpy.isfinite(rchi2) or rchi2 <= 0:
                            # Guilty prima facie
                            awfulCandidates.append(cand)
                            cleanChi2 = False
                            self.debugLog.debug(
                                2, "chi^2=%s; id=%s" %
                                (cand.getChi2(), cand.getSource().getId()))
                    for cand in awfulCandidates:
                        if display:
                            print "Removing bad candidate: id=%d, chi^2=%f" % \
                                  (cand.getSource().getId(), cand.getChi2())
                        cell.removeCandidate(cand)

            #
            # Clip out bad fits based on reduced chi^2
            #
            badCandidates = list()
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):  # include bad candidates
                    cand = algorithmsLib.cast_PsfCandidateF(cand)
                    rchi2 = cand.getChi2(
                    )  # reduced chi^2 when fitting PSF to candidate
                    assert rchi2 > 0
                    if rchi2 > self.config.reducedChi2ForPsfCandidates:
                        badCandidates.append(cand)

            badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
            numBad = int(
                len(badCandidates) * (iter + 1) / self.config.nIterForPsf +
                0.5)
            for i, c in zip(range(numBad), badCandidates):
                if display:
                    chi2 = c.getChi2()
                    if chi2 > 1e100:
                        chi2 = numpy.nan

                    print "Chi^2 clipping %-4d  %.2g" % (c.getSource().getId(),
                                                         chi2)
                c.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Clip out bad fits based on spatial fitting.
            #
            # This appears to be better at getting rid of sources that have a single dominant kernel component
            # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
            # set the spatial model) don't contain that kernel component, and so the spatial modeling
            # downweights the component.
            #

            residuals = list()
            candidates = list()
            kernel = psf.getKernel()
            noSpatialKernel = afwMath.cast_LinearCombinationKernel(
                psf.getKernel())
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):
                    cand = algorithmsLib.cast_PsfCandidateF(cand)
                    candCenter = afwGeom.PointD(cand.getXCenter(),
                                                cand.getYCenter())
                    try:
                        im = cand.getMaskedImage(kernel.getWidth(),
                                                 kernel.getHeight())
                    except Exception, e:
                        continue

                    fit = algorithmsLib.fitKernelParamsToImage(
                        noSpatialKernel, im, candCenter)
                    params = fit[0]
                    kernels = fit[1]
                    amp = 0.0
                    for p, k in zip(params, kernels):
                        amp += p * afwMath.cast_FixedKernel(k).getSum()

                    predict = [
                        kernel.getSpatialFunction(k)(candCenter.getX(),
                                                     candCenter.getY())
                        for k in range(kernel.getNKernelParameters())
                    ]

                    #print cand.getSource().getId(), [a / amp for a in params], predict

                    residuals.append(
                        [a / amp - p for a, p in zip(params, predict)])
                    candidates.append(cand)
Example #13
0
    def run(self, sensorRef):
        """Subtract an image from a template coadd and measure the result
    
        Steps include:
        - warp template coadd to match WCS of image
        - PSF match image to warped template
        - subtract image from PSF-matched, warped template
        - persist difference image
        - detect sources
        - measure sources
        
        @param sensorRef: sensor-level butler data reference, used for the following data products:
        Input only:
        - calexp
        - psf
        - ccdExposureId
        - ccdExposureId_bits
        - self.config.coaddName + "Coadd_skyMap"
        - self.config.coaddName + "Coadd"
        Input or output, depending on config:
        - self.config.coaddName + "Diff_subtractedExp"
        Output, depending on config:
        - self.config.coaddName + "Diff_matchedExp"
        - self.config.coaddName + "Diff_src"
            
        @return pipe_base Struct containing these fields:
        - subtractedExposure: exposure after subtracting template;
            the unpersisted version if subtraction not run but detection run
            None if neither subtraction nor detection run (i.e. nothing useful done)
        - subtractRes: results of subtraction task; None if subtraction not run
        - sources: detected and possibly measured sources; None if detection not run
        """
        self.log.info("Processing %s" % (sensorRef.dataId))

        # initialize outputs and some intermediate products
        subtractedExposure = None
        subtractRes = None
        selectSources = None
        kernelSources = None
        controlSources = None
        diaSources = None

        # We make one IdFactory that will be used by both icSrc and src datasets;
        # I don't know if this is the way we ultimately want to do things, but at least
        # this ensures the source IDs are fully unique.
        expBits = sensorRef.get("ccdExposureId_bits")
        expId = long(sensorRef.get("ccdExposureId"))
        idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
        
        # Retrieve the science image we wish to analyze
        exposure = sensorRef.get("calexp", immediate=True)
        if self.config.doAddCalexpBackground:
            mi = exposure.getMaskedImage()
            mi += sensorRef.get("calexpBackground").getImage()
        if not exposure.hasPsf():
            raise pipeBase.TaskError("Exposure has no psf")
        sciencePsf = exposure.getPsf()

        if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0:
            self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS")
            bbox = exposure.getBBox(afwImage.PARENT)
            bbox.grow(-self.config.winter2013borderMask)
            self.setEdgeBits(exposure.getMaskedImage(), bbox, 
                             exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA"))
            
        # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
        ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter()
        psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
        scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
        
        subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
        templateExposure = None  # Stitched coadd exposure
        templateSources = None   # Sources on the template image
        if self.config.doSubtract:
            templateExposure, templateSources = self.getTemplate(exposure, sensorRef)

            # sigma of PSF of template image before warping
            ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter()
            psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
            templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)

            # if requested, convolve the science exposure with its PSF
            # (properly, this should be a cross-correlation, but our code does not yet support that)
            # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
            # else sigma of original science exposure
            if self.config.doPreConvolve:
                convControl = afwMath.ConvolutionControl()
                # cannot convolve in place, so make a new MI to receive convolved image
                srcMI = exposure.getMaskedImage()
                destMI = srcMI.Factory(srcMI.getDimensions())
                srcPsf = sciencePsf
                if self.config.useGaussianForPreConvolution:
                    # convolve with a simplified PSF model: a double Gaussian
                    kWidth, kHeight = sciencePsf.getKernel().getDimensions()
                    preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
                else:
                    # convolve with science exposure's PSF model
                    preConvPsf = psf
                afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl)
                exposure.setMaskedImage(destMI)
                scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
            else:
                scienceSigmaPost = scienceSigmaOrig

            # If requested, find sources in the image
            if self.config.doSelectSources:
                if not sensorRef.datasetExists("src"):
                    self.log.warn("Src product does not exist; running detection, measurement, selection")
                    # Run own detection and measurement; necessary in nightly processing
                    selectSources = self.subtract.getSelectSources(
                        exposure, 
                        sigma = scienceSigmaPost, 
                        doSmooth = not self.doPreConvolve,
                        idFactory = idFactory,
                    )
                else:
                    self.log.info("Source selection via src product")
                    # Sources already exist; for data release processing
                    selectSources = sensorRef.get("src")

                # Number of basis functions
                nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
                                                 referenceFwhmPix = scienceSigmaPost * FwhmPerSigma,
                                                 targetFwhmPix = templateSigma * FwhmPerSigma))

                if self.config.doAddMetrics:
                    # Modify the schema of all Sources
                    self.kcQa = diUtils.KernelCandidateQa(nparam, self.log)
                    selectSources = self.kcQa.addToSchema(selectSources)

                astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig())
                astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure)
                matches = astromRet.matches
                kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches)
                random.shuffle(kernelSources, random.random)
                controlSources = kernelSources[::self.config.controlStepSize]
                [kernelSources.remove(x) for x in controlSources]

                self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \
                                  % (len(kernelSources), len(selectSources), len(controlSources)))
            allresids = {}
            if self.config.doUseRegister:
                self.log.info("Registering images")
                
                if templateSources is None:
                    # First step: we need to subtract the background out
                    # for detection and measurement.  Use large binsize
                    # for the background estimation.
                    binsize = self.config.templateBgBinSize
    
                    # Second step: we need to run detection on the
                    # background-subtracted template
                    #
                    # Estimate FWHM for detection
                    templateSources = self.subtract.getSelectSources(
                        templateExposure,
                        sigma = templateSigma,
                        doSmooth = True,
                        idFactory = idFactory,
                        binsize = binsize,
                    )

                # Third step: we need to fit the relative astrometry.
                #
                # One problem is that the SIP fits are w.r.t. CRPIX,
                # and these coadd patches have the CRPIX of the entire
                # tract, i.e. off the image.  This causes
                # register.fitWcs to fail.  A workaround for now is to
                # re-fit the Wcs which returns with a CRPIX that is on
                # the image, and *then* to fit for the relative Wcs.
                #
                # Requires low Sip order to avoid overfitting

                # useWinter2013Hacks includes us using the deep calexp
                # as the template.  In this case we don't need to
                # refit the Wcs.
                if not self.config.useWinter2013Hacks:
                    sipOrder = self.config.templateSipOrder
                    astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder))
                    newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs()
                    results = self.register.run(templateSources, newWcs, 
                                                templateExposure.getBBox(afwImage.PARENT), selectSources)
                else:
                    if self.config.winter2013WcsShift > 0.0:
                        offset = afwGeom.Extent2D(self.config.winter2013WcsShift, 
                                                  self.config.winter2013WcsShift)
                        cKey = templateSources[0].getTable().getCentroidKey()
                        for source in templateSources:
                            centroid = source.get(cKey)
                            source.set(cKey, centroid+offset)
                    elif self.config.winter2013WcsRms > 0.0:
                        cKey = templateSources[0].getTable().getCentroidKey()
                        for source in templateSources:
                            offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(), 
                                                      self.config.winter2013WcsRms*numpy.random.normal())
                            centroid = source.get(cKey)
                            source.set(cKey, centroid+offset)

                    results = self.register.run(templateSources, templateExposure.getWcs(),
                                                templateExposure.getBBox(afwImage.PARENT), selectSources)

                warpedExp = self.register.warpExposure(templateExposure, results.wcs, 
                                            exposure.getWcs(), exposure.getBBox(afwImage.PARENT))
                templateExposure = warpedExp

                # Create debugging outputs on the astrometric
                # residuals as a function of position.  Persistence
                # not yet implemented; expected on (I believe) #2636.
                if self.config.doDebugRegister:
                    refCoordKey = results.matches[0].first.getTable().getCoordKey()
                    inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
                    sids      = [m.first.getId() for m in results.matches]
                    positions = [m.first.get(refCoordKey) for m in results.matches]
                    residuals = [m.first.get(refCoordKey).getOffsetFrom(
                                   results.wcs.pixelToSky(m.second.get(inCentroidKey))) for
                                 m in results.matches]
                    allresids = dict(zip(sids, zip(positions, residuals)))

            # warp template exposure to match exposure,
            # PSF match template exposure to exposure,
            # then return the difference

            #Return warped template...  Construct sourceKernelCand list after subtract
            self.log.info("Subtracting images")
            subtractRes = self.subtract.subtractExposures(
                templateExposure = templateExposure,
                scienceExposure = exposure,
                scienceFwhmPix = scienceSigmaPost * FwhmPerSigma,
                templateFwhmPix = templateSigma * FwhmPerSigma,
                candidateList = kernelSources,
                convolveTemplate = self.config.convolveTemplate,
                doWarping = not self.config.doUseRegister
            )
            subtractedExposure = subtractRes.subtractedExposure

            if self.config.doWriteMatchedExp:
                sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")

        if self.config.doDetection:
            self.log.info("Running diaSource detection")
            if subtractedExposure is None:
                subtractedExposure = sensorRef.get(subtractedExposureName)
            
            # Get Psf from the appropriate input image if it doesn't exist
            if not subtractedExposure.hasPsf():
                if self.config.convolveTemplate:
                    subtractedExposure.setPsf(exposure.getPsf())
                else:
                    if templateExposure is None:
                        templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
                    subtractedExposure.setPsf(templateExposure.getPsf())

            # Erase existing detection mask planes
            mask  = subtractedExposure.getMaskedImage().getMask()
            mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))

            table = afwTable.SourceTable.make(self.schema, idFactory)
            table.setMetadata(self.algMetadata)
            results = self.detection.makeSourceCatalog(
                table = table,
                exposure = subtractedExposure,
                doSmooth = not self.config.doPreConvolve
                )

            if self.config.doMerge:
                fpSet = results.fpSets.positive
                fpSet.merge(results.fpSets.negative, self.config.growFootprint, 
                            self.config.growFootprint, False)
                diaSources = afwTable.SourceCatalog(table)
                fpSet.makeSources(diaSources)
                self.log.info("Merging detections into %d sources" % (len(diaSources)))
            else:
                diaSources = results.sources

            if self.config.doMeasurement:
                self.log.info("Running diaSource measurement")
                if len(diaSources) < self.config.maxDiaSourcesToMeasure:
                    self.dipolemeasurement.run(subtractedExposure, diaSources)
                else:
                    self.measurement.run(subtractedExposure, diaSources)

            # Match with the calexp sources if possible
            if self.config.doMatchSources:
                if sensorRef.datasetExists("src"):
                    # Create key,val pair where key=diaSourceId and val=sourceId
                    matchRadAsec = self.config.diaSourceMatchRadius
                    matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()

                    # This does not do what I expect so I cobbled together a brute force method in python 
                    srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) 
                    srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \
                                             srcMatch in srcMatches])
                    self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), 
                                                                             len(diaSources)))
                else:
                    self.log.warn("Src product does not exist; cannot match with diaSources")
                    srcMatchDict = {}

                # Create key,val pair where key=diaSourceId and val=refId
                astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec))
                astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure)
                refMatches = astromRet.matches
                if refMatches is None:
                    self.log.warn("No diaSource matches with reference catalog")
                    refMatchDict = {}
                else:
                    self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches), 
                                                                                       len(diaSources)))
                    refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
                                             refMatch in refMatches])

                # Assign source Ids
                for diaSource in diaSources:                    
                    sid = diaSource.getId()
                    if srcMatchDict.has_key(sid):
                        diaSource.set("srcMatchId", srcMatchDict[sid])
                    if refMatchDict.has_key(sid):
                        diaSource.set("refMatchId", refMatchDict[sid])
                        
            if diaSources is not None and self.config.doWriteSources:
                sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources
                                    else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
                sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags)

            if self.config.doAddMetrics and self.config.doSelectSources:
                self.log.info("Evaluating metrics and control sample")

                kernelCandList = []
                for cell in subtractRes.kernelCellSet.getCellList():
                    for cand in cell.begin(False): # include bad candidates
                        kernelCandList.append(cast_KernelCandidateF(cand))

                # Get basis list to build control sample kernels
                basisList = afwMath.cast_LinearCombinationKernel(
                    kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()

                controlCandList = \
                    diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure, 
                                                      self.config.subtract.kernel.active, 
                                                      self.config.subtract.kernel.active.detectionConfig, 
                                                      self.log, dobuild=True, basisList=basisList)

                self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, 
                                dof=nparam)
                self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)

                if self.config.doDetection:
                    self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
                else:
                    self.kcQa.aggregate(selectSources, self.metadata, allresids)

                #Persist using butler
                sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")

        if self.config.doWriteSubtractedExp:
            sensorRef.put(subtractedExposure, subtractedExposureName)
 
        self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
        return pipeBase.Struct(
            subtractedExposure = subtractedExposure,
            subtractRes = subtractRes,
            sources = diaSources,
        )
Example #14
0
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True,
                      variance=None, chi=None):
    """Display the PSF candidates.
If psf is provided include PSF model and residuals;  if normalize is true normalize the PSFs (and residuals)
If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
"""
    if chi is None:
        if variance is not None:        # old name for chi
            chi = variance
    #
    # Show us the ccandidates
    #
    mos = displayUtils.Mosaic()
    #
    candidateCenters = []
    candidateCentersBad = []
    candidateIndex = 0

    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False): # include bad candidates
            cand = algorithmsLib.cast_PsfCandidateF(cand)

            rchi2 = cand.getChi2()
            if rchi2 > 1e100:
                rchi2 = numpy.nan

            if not showBadCandidates and cand.isBad():
                continue

            if psf:
                im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")

                try:
                    im = cand.getMaskedImage() # copy of this object's image
                    xc, yc = cand.getXCenter(), cand.getYCenter()

                    margin = 0 if True else 5
                    w, h = im.getDimensions()
                    bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions())

                    if margin > 0:
                        bim = im.Factory(w + 2*margin, h + 2*margin)

                        stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
                        afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                        bim *= stdev
                        var = bim.getVariance(); var.set(stdev**2); del var

                        sbim = im.Factory(bim, bbox)
                        sbim <<= im
                        del sbim
                        im = bim
                        xc += margin; yc += margin

                    im = im.Factory(im, True)
                    im.setXY0(cand.getMaskedImage().getXY0())
                except:
                    continue

                if not variance:
                    im_resid.append(im.Factory(im, True))

                # residuals using spatial model
                chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc)
                
                resid = im
                if variance:
                    resid = resid.getImage()
                    var = im.getVariance()
                    var = var.Factory(var, True)
                    numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
                    resid /= var
                    
                im_resid.append(resid)

                # Fit the PSF components directly to the data (i.e. ignoring the spatial model)
                im = cand.getMaskedImage()

                im = im.Factory(im, True)
                im.setXY0(cand.getMaskedImage().getXY0())

                noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
                candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
                fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
                params = fit[0]
                kernels = afwMath.KernelList(fit[1])
                outputKernel = afwMath.LinearCombinationKernel(kernels, params)

                outImage = afwImage.ImageD(outputKernel.getDimensions())
                outputKernel.computeImage(outImage, False)

                im -= outImage.convertF()
                resid = im

                if margin > 0:
                    bim = im.Factory(w + 2*margin, h + 2*margin)
                    afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
                    bim *= stdev

                    sbim = im.Factory(bim, bbox)
                    sbim <<= resid
                    del sbim
                    resid = bim

                if variance:
                    resid = resid.getImage()
                    resid /= var
                    
                im_resid.append(resid)

                im = im_resid.makeMosaic()
            else:
                im = cand.getMaskedImage()

            if normalize:
                im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()

            objId = splitId(cand.getSource().getId(), True)["objId"]
            if psf:
                lab = "%d chi^2 %.1f" % (objId, rchi2)
                ctype = ds9.RED if cand.isBad() else ds9.GREEN
            else:
                lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux())
                ctype = ds9.GREEN

            mos.append(im, lab, ctype)

            if False and numpy.isnan(rchi2):
                ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1)
                print "amp",  cand.getAmplitude()

            im = cand.getMaskedImage()
            center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
            candidateIndex += 1
            if cand.isBad():
                candidateCentersBad.append(center)
            else:
                candidateCenters.append(center)

    if variance:
        title = "chi(Psf fit)"
    else:
        title = "Stars & residuals"
    mosaicImage = mos.makeMosaic(frame=frame, title=title)

    with ds9.Buffering():
        for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)):
            for cen in centers:
                bbox = mos.getBBox(cen[0])
                ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color)

    return mosaicImage
                                                         chi2)
                c.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Clip out bad fits based on spatial fitting.
            #
            # This appears to be better at getting rid of sources that have a single dominant kernel component
            # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
            # set the spatial model) don't contain that kernel component, and so the spatial modeling
            # downweights the component.
            #

            residuals = list()
            candidates = list()
            kernel = psf.getKernel()
            noSpatialKernel = afwMath.cast_LinearCombinationKernel(
                psf.getKernel())
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):
                    cand = algorithmsLib.cast_PsfCandidateF(cand)
                    candCenter = afwGeom.PointD(cand.getXCenter(),
                                                cand.getYCenter())
                    try:
                        im = cand.getMaskedImage(kernel.getWidth(),
                                                 kernel.getHeight())
                    except Exception, e:
                        continue

                    fit = algorithmsLib.fitKernelParamsToImage(
                        noSpatialKernel, im, candCenter)
                    params = fit[0]
                    kernels = fit[1]
Example #16
0
    def testGetPcaKernel(self):
        """Convert our cellSet to a LinearCombinationKernel"""

        nEigenComponents = 2
        spatialOrder = 1
        kernelSize = 21
        nStarPerCell = 2
        nStarPerCellSpatialFit = 2
        tolerance = 1e-5

        if display:
            ds9.mtv(self.mi, frame=0)
            #
            # Show the candidates we're using
            #
            for cell in self.cellSet.getCellList():
                i = 0
                for cand in cell:
                    i += 1
                    source = algorithms.PsfCandidateF.cast(cand).getSource()

                    xc, yc = source.getXAstrom() - self.mi.getX0(
                    ), source.getYAstrom() - self.mi.getY0()
                    if i <= nStarPerCell:
                        ds9.dot("o", xc, yc, ctype=ds9.GREEN)
                    else:
                        ds9.dot("o", xc, yc, ctype=ds9.YELLOW)

        pair = algorithms.createKernelFromPsfCandidates(
            self.cellSet, self.exposure.getDimensions(),
            self.exposure.getXY0(), nEigenComponents, spatialOrder, kernelSize,
            nStarPerCell)

        kernel, eigenValues = pair[0], pair[1]
        del pair

        print("lambda", " ".join(["%g" % l for l in eigenValues]))

        pair = algorithms.fitSpatialKernelFromPsfCandidates(
            kernel, self.cellSet, nStarPerCellSpatialFit, tolerance)
        status, chi2 = pair[0], pair[1]
        del pair
        print("Spatial fit: %s chi^2 = %.2g" % (status, chi2))

        psf = algorithms.PcaPsf.swigConvert(
            roundTripPsf(5, algorithms.PcaPsf(kernel)))  # Hurrah!

        self.assertIsNone(afwMath.cast_AnalyticKernel(psf.getKernel()))
        self.assertIsNotNone(
            afwMath.cast_LinearCombinationKernel(psf.getKernel()))

        self.checkTablePersistence(psf)

        if display:
            # print psf.getKernel().toString()

            eImages = []
            for k in afwMath.cast_LinearCombinationKernel(
                    psf.getKernel()).getKernelList():
                im = afwImage.ImageD(k.getDimensions())
                k.computeImage(im, False)
                eImages.append(im)

            mos = displayUtils.Mosaic()
            frame = 3
            ds9.mtv(mos.makeMosaic(eImages), frame=frame)
            ds9.dot("Eigen Images", 0, 0, frame=frame)
            #
            # Make a mosaic of PSF candidates
            #
            stamps = []
            stampInfo = []

            for cell in self.cellSet.getCellList():
                for cand in cell:
                    #
                    # Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate;  all
                    # it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates
                    # don't know about getMaskedImage;  so cast the pointer to PsfCandidate
                    #
                    cand = algorithms.PsfCandidateF.cast(cand)
                    s = cand.getSource()

                    im = cand.getMaskedImage()

                    stamps.append(im)
                    stampInfo.append("[%d 0x%x]" %
                                     (s.getId(), s.getFlagForDetection()))

                    mos = displayUtils.Mosaic()
            frame = 1
            ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True)
            for i in range(len(stampInfo)):
                ds9.dot(stampInfo[i],
                        mos.getBBox(i).getX0(),
                        mos.getBBox(i).getY0(),
                        frame=frame,
                        ctype=ds9.RED)

            psfImages = []
            labels = []
            if False:
                nx, ny = 3, 4
                for iy in range(ny):
                    for ix in range(nx):
                        x = int((ix + 0.5) * self.mi.getWidth() / nx)
                        y = int((iy + 0.5) * self.mi.getHeight() / ny)

                        im = psf.getImage(x, y)
                        psfImages.append(im.Factory(im, True))
                        labels.append("PSF(%d,%d)" % (int(x), int(y)))

                        if True:
                            print((x, y, "PSF parameters:",
                                   psf.getKernel().getKernelParameters()))
            else:
                nx, ny = 2, 2
                for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]:

                    im = psf.computeImage(afwGeom.PointD(x, y))
                    psfImages.append(im.Factory(im, True))
                    labels.append("PSF(%d,%d)" % (int(x), int(y)))

                    if True:
                        print(x, y, "PSF parameters:",
                              psf.getKernel().getKernelParameters())

            frame = 2
            mos.makeMosaic(psfImages, frame=frame, mode=nx)
            mos.drawLabels(labels, frame=frame)

        if display:

            ds9.mtv(self.mi, frame=0)

            psfImages = []
            labels = []
            if False:
                nx, ny = 3, 4
                for iy in range(ny):
                    for ix in range(nx):
                        x = int((ix + 0.5) * self.mi.getWidth() / nx)
                        y = int((iy + 0.5) * self.mi.getHeight() / ny)

                        algorithms.subtractPsf(psf, self.mi, x, y)
            else:
                nx, ny = 2, 2
                for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]:

                    if False:  # Test subtraction with non-centered psfs
                        x += 0.5
                        y -= 0.5

                    #algorithms.subtractPsf(psf, self.mi, x, y)

            ds9.mtv(self.mi, frame=1)
    def determinePsf(self,
                     exposure,
                     psfCandidateList,
                     metadata=None,
                     flagKey=None):
        """!Determine a PCA PSF model for an exposure given a list of PSF candidates

        \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
        \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
            typically obtained by detecting sources and then running them through a star selector
        \param[in,out] metadata  a home for interesting tidbits of information
        \param[in] flagKey schema key used to mark sources actually used in PSF determination

        \return a list of
         - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
         - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = lsstDebug.Info(
            __name__).displayExposure  # display the Exposure + spatialCells
        displayPsfCandidates = lsstDebug.Info(
            __name__).displayPsfCandidates  # show the viable candidates
        displayIterations = lsstDebug.Info(
            __name__).displayIterations  # display on each PSF iteration
        displayPsfComponents = lsstDebug.Info(
            __name__).displayPsfComponents  # show the PCA components
        displayResiduals = lsstDebug.Info(
            __name__).displayResiduals  # show residuals
        displayPsfMosaic = lsstDebug.Info(
            __name__).displayPsfMosaic  # show mosaic of reconstructed PSF(x,y)
        # match Kernel amplitudes for spatial plots
        matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
        # Keep matplotlib alive post mortem
        keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
        displayPsfSpatialModel = lsstDebug.Info(
            __name__).displayPsfSpatialModel  # Plot spatial model?
        showBadCandidates = lsstDebug.Info(
            __name__).showBadCandidates  # Include bad candidates
        # Normalize residuals by object amplitude
        normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
        pause = lsstDebug.Info(
            __name__).pause  # Prompt user after each iteration?

        if display > 1:
            pause = True

        mi = exposure.getMaskedImage()

        if len(psfCandidateList) == 0:
            raise RuntimeError("No PSF candidates supplied.")

        # construct and populate a spatial cell set
        bbox = mi.getBBox()
        psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
                                            self.config.sizeCellY)
        sizes = []
        for i, psfCandidate in enumerate(psfCandidateList):
            if psfCandidate.getSource().getPsfFluxFlag():  # bad measurement
                continue

            try:
                psfCellSet.insertCandidate(psfCandidate)
            except Exception as e:
                self.log.debug("Skipping PSF candidate %d of %d: %s", i,
                               len(psfCandidateList), e)
                continue
            source = psfCandidate.getSource()

            quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
                                     source.getIxy())
            axes = afwEll.Axes(quad)
            sizes.append(axes.getA())
        if len(sizes) == 0:
            raise RuntimeError("No usable PSF candidates supplied")
        nEigenComponents = self.config.nEigenComponents  # initial version

        if self.config.kernelSize >= 15:
            self.log.warn(
                "WARNING: NOT scaling kernelSize by stellar quadrupole moment "
                +
                "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead",
                self.config.kernelSize)
            actualKernelSize = int(self.config.kernelSize)
        else:
            medSize = numpy.median(sizes)
            actualKernelSize = 2 * int(self.config.kernelSize *
                                       math.sqrt(medSize) + 0.5) + 1
            if actualKernelSize < self.config.kernelSizeMin:
                actualKernelSize = self.config.kernelSizeMin
            if actualKernelSize > self.config.kernelSizeMax:
                actualKernelSize = self.config.kernelSizeMax

            if display:
                print("Median size=%s" % (medSize, ))
        self.log.trace("Kernel size=%s", actualKernelSize)

        # Set size of image returned around candidate
        psfCandidateList[0].setHeight(actualKernelSize)
        psfCandidateList[0].setWidth(actualKernelSize)

        if self.config.doRejectBlends:
            # Remove blended candidates completely
            blendedCandidates = [
            ]  # Candidates to remove; can't do it while iterating
            for cell, cand in candidatesIter(psfCellSet, False):
                if len(cand.getSource().getFootprint().getPeaks()) > 1:
                    blendedCandidates.append((cell, cand))
                    continue
            if display:
                print("Removing %d blended Psf candidates" %
                      len(blendedCandidates))
            for cell, cand in blendedCandidates:
                cell.removeCandidate(cand)
            if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
                raise RuntimeError("All PSF candidates removed as blends")

        if display:
            frame = 0
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            self.config.nStarPerCell,
                                            symb="o",
                                            ctype=ds9.CYAN,
                                            ctypeUnused=ds9.YELLOW,
                                            size=4,
                                            frame=frame)

        #
        # Do a PCA decomposition of those PSF candidates
        #
        reply = "y"  # used in interactive mode
        for iterNum in range(self.config.nIterForPsf):
            if display and displayPsfCandidates:  # Show a mosaic of usable PSF candidates
                #
                import lsst.afw.display.utils as displayUtils

                stamps = []
                for cell in psfCellSet.getCellList():
                    for cand in cell.begin(not showBadCandidates
                                           ):  # maybe include bad candidates
                        cand = algorithmsLib.PsfCandidateF.cast(cand)

                        try:
                            im = cand.getMaskedImage()

                            chi2 = cand.getChi2()
                            if chi2 > 1e100:
                                chi2 = numpy.nan

                            stamps.append(
                                (im, "%d%s" %
                                 (maUtils.splitId(cand.getSource().getId(),
                                                  True)["objId"], chi2),
                                 cand.getStatus()))
                        except Exception as e:
                            continue

                if len(stamps) == 0:
                    print(
                        "WARNING: No PSF candidates to show; try setting showBadCandidates=True"
                    )
                else:
                    mos = displayUtils.Mosaic()
                    for im, label, status in stamps:
                        im = type(im)(im, True)
                        try:
                            im /= afwMath.makeStatistics(
                                im, afwMath.MAX).getValue()
                        except NotImplementedError:
                            pass

                        mos.append(
                            im, label, ds9.GREEN
                            if status == afwMath.SpatialCellCandidate.GOOD else
                            ds9.YELLOW if status ==
                            afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED)

                    mos.makeMosaic(frame=8, title="Psf Candidates")

            # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
            cleanChi2 = False  # Any naughty (negative/NAN) chi^2 values?
            while not cleanChi2:
                cleanChi2 = True
                #
                # First, estimate the PSF
                #
                psf, eigenValues, nEigenComponents, fitChi2 = \
                    self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
                #
                # In clipping, allow all candidates to be innocent until proven guilty on this iteration.
                # Throw out any prima facie guilty candidates (naughty chi^2 values)
                #
                for cell in psfCellSet.getCellList():
                    awfulCandidates = []
                    for cand in cell.begin(False):  # include bad candidates
                        cand = algorithmsLib.PsfCandidateF.cast(cand)
                        cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN
                                       )  # until proven guilty
                        rchi2 = cand.getChi2()
                        if not numpy.isfinite(rchi2) or rchi2 <= 0:
                            # Guilty prima facie
                            awfulCandidates.append(cand)
                            cleanChi2 = False
                            self.log.debug("chi^2=%s; id=%s", cand.getChi2(),
                                           cand.getSource().getId())
                    for cand in awfulCandidates:
                        if display:
                            print("Removing bad candidate: id=%d, chi^2=%f" % \
                                  (cand.getSource().getId(), cand.getChi2()))
                        cell.removeCandidate(cand)

            #
            # Clip out bad fits based on reduced chi^2
            #
            badCandidates = list()
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):  # include bad candidates
                    cand = algorithmsLib.PsfCandidateF.cast(cand)
                    rchi2 = cand.getChi2(
                    )  # reduced chi^2 when fitting PSF to candidate
                    assert rchi2 > 0
                    if rchi2 > self.config.reducedChi2ForPsfCandidates:
                        badCandidates.append(cand)

            badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
            numBad = numCandidatesToReject(len(badCandidates), iterNum,
                                           self.config.nIterForPsf)
            for i, c in zip(range(numBad), badCandidates):
                if display:
                    chi2 = c.getChi2()
                    if chi2 > 1e100:
                        chi2 = numpy.nan

                    print("Chi^2 clipping %-4d  %.2g" %
                          (c.getSource().getId(), chi2))
                c.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Clip out bad fits based on spatial fitting.
            #
            # This appears to be better at getting rid of sources that have a single dominant kernel component
            # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
            # set the spatial model) don't contain that kernel component, and so the spatial modeling
            # downweights the component.
            #

            residuals = list()
            candidates = list()
            kernel = psf.getKernel()
            noSpatialKernel = afwMath.cast_LinearCombinationKernel(
                psf.getKernel())
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):
                    cand = algorithmsLib.PsfCandidateF.cast(cand)
                    candCenter = afwGeom.PointD(cand.getXCenter(),
                                                cand.getYCenter())
                    try:
                        im = cand.getMaskedImage(kernel.getWidth(),
                                                 kernel.getHeight())
                    except Exception as e:
                        continue

                    fit = algorithmsLib.fitKernelParamsToImage(
                        noSpatialKernel, im, candCenter)
                    params = fit[0]
                    kernels = fit[1]
                    amp = 0.0
                    for p, k in zip(params, kernels):
                        amp += p * afwMath.cast_FixedKernel(k).getSum()

                    predict = [
                        kernel.getSpatialFunction(k)(candCenter.getX(),
                                                     candCenter.getY())
                        for k in range(kernel.getNKernelParameters())
                    ]

                    #print cand.getSource().getId(), [a / amp for a in params], predict

                    residuals.append(
                        [a / amp - p for a, p in zip(params, predict)])
                    candidates.append(cand)

            residuals = numpy.array(residuals)

            for k in range(kernel.getNKernelParameters()):
                if False:
                    # Straight standard deviation
                    mean = residuals[:, k].mean()
                    rms = residuals[:, k].std()
                elif False:
                    # Using interquartile range
                    sr = numpy.sort(residuals[:, k])
                    mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \
                        0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1])
                    rms = 0.74 * (sr[int(0.75 * len(sr))] -
                                  sr[int(0.25 * len(sr))])
                else:
                    stats = afwMath.makeStatistics(
                        residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
                    mean = stats.getValue(afwMath.MEANCLIP)
                    rms = stats.getValue(afwMath.STDEVCLIP)

                rms = max(
                    1.0e-4,
                    rms)  # Don't trust RMS below this due to numerical issues

                if display:
                    print("Mean for component %d is %f" % (k, mean))
                    print("RMS for component %d is %f" % (k, rms))
                badCandidates = list()
                for i, cand in enumerate(candidates):
                    if numpy.fabs(residuals[i, k] -
                                  mean) > self.config.spatialReject * rms:
                        badCandidates.append(i)

                badCandidates.sort(
                    key=lambda x: numpy.fabs(residuals[x, k] - mean),
                    reverse=True)

                numBad = numCandidatesToReject(len(badCandidates), iterNum,
                                               self.config.nIterForPsf)

                for i, c in zip(range(min(len(badCandidates), numBad)),
                                badCandidates):
                    cand = candidates[c]
                    if display:
                        print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \
                              (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
                               residuals[badCandidates[i], k], self.config.spatialReject * rms))
                    cand.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Display results
            #
            if display and displayIterations:
                if displayExposure:
                    if iterNum > 0:
                        ds9.erase(frame=frame)
                    maUtils.showPsfSpatialCells(exposure,
                                                psfCellSet,
                                                self.config.nStarPerCell,
                                                showChi2=True,
                                                symb="o",
                                                size=8,
                                                frame=frame,
                                                ctype=ds9.YELLOW,
                                                ctypeBad=ds9.RED,
                                                ctypeUnused=ds9.MAGENTA)
                    if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
                        maUtils.showPsfSpatialCells(
                            exposure,
                            psfCellSet,
                            self.config.nStarPerCellSpatialFit,
                            symb="o",
                            size=10,
                            frame=frame,
                            ctype=ds9.YELLOW,
                            ctypeBad=ds9.RED)
                if displayResiduals:
                    while True:
                        try:
                            maUtils.showPsfCandidates(
                                exposure,
                                psfCellSet,
                                psf=psf,
                                frame=4,
                                normalize=normalizeResiduals,
                                showBadCandidates=showBadCandidates)
                            maUtils.showPsfCandidates(
                                exposure,
                                psfCellSet,
                                psf=psf,
                                frame=5,
                                normalize=normalizeResiduals,
                                showBadCandidates=showBadCandidates,
                                variance=True)
                        except:
                            if not showBadCandidates:
                                showBadCandidates = True
                                continue
                        break

                if displayPsfComponents:
                    maUtils.showPsf(psf, eigenValues, frame=6)
                if displayPsfMosaic:
                    maUtils.showPsfMosaic(exposure,
                                          psf,
                                          frame=7,
                                          showFwhm=True)
                    ds9.scale('linear', 0, 1, frame=7)
                if displayPsfSpatialModel:
                    maUtils.plotPsfSpatialModel(
                        exposure,
                        psf,
                        psfCellSet,
                        showBadCandidates=True,
                        matchKernelAmplitudes=matchKernelAmplitudes,
                        keepPlots=keepMatplotlibPlots)

                if pause:
                    while True:
                        try:
                            reply = input(
                                "Next iteration? [ynchpqQs] ").strip()
                        except EOFError:
                            reply = "n"

                        reply = reply.split()
                        if reply:
                            reply, args = reply[0], reply[1:]
                        else:
                            reply = ""

                        if reply in ("", "c", "h", "n", "p", "q", "Q", "s",
                                     "y"):
                            if reply == "c":
                                pause = False
                            elif reply == "h":
                                print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \
                                      "s[ave fileName] y[es]")
                                continue
                            elif reply == "p":
                                import pdb
                                pdb.set_trace()
                            elif reply == "q":
                                display = False
                            elif reply == "Q":
                                sys.exit(1)
                            elif reply == "s":
                                fileName = args.pop(0)
                                if not fileName:
                                    print("Please provide a filename")
                                    continue

                                print("Saving to %s" % fileName)
                                maUtils.saveSpatialCellSet(psfCellSet,
                                                           fileName=fileName)
                                continue
                            break
                        else:
                            print("Unrecognised response: %s" % reply,
                                  file=sys.stderr)

                    if reply == "n":
                        break

        # One last time, to take advantage of the last iteration
        psf, eigenValues, nEigenComponents, fitChi2 = \
            self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)

        #
        # Display code for debugging
        #
        if display and reply != "n":
            if displayExposure:
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            self.config.nStarPerCell,
                                            showChi2=True,
                                            symb="o",
                                            ctype=ds9.YELLOW,
                                            ctypeBad=ds9.RED,
                                            size=8,
                                            frame=frame)
                if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
                    maUtils.showPsfSpatialCells(
                        exposure,
                        psfCellSet,
                        self.config.nStarPerCellSpatialFit,
                        symb="o",
                        ctype=ds9.YELLOW,
                        ctypeBad=ds9.RED,
                        size=10,
                        frame=frame)
                if displayResiduals:
                    maUtils.showPsfCandidates(
                        exposure,
                        psfCellSet,
                        psf=psf,
                        frame=4,
                        normalize=normalizeResiduals,
                        showBadCandidates=showBadCandidates)

            if displayPsfComponents:
                maUtils.showPsf(psf, eigenValues, frame=6)

            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
                ds9.scale("linear", 0, 1, frame=7)
            if displayPsfSpatialModel:
                maUtils.plotPsfSpatialModel(
                    exposure,
                    psf,
                    psfCellSet,
                    showBadCandidates=True,
                    matchKernelAmplitudes=matchKernelAmplitudes,
                    keepPlots=keepMatplotlibPlots)
        #
        # Generate some QA information
        #
        # Count PSF stars
        #
        numGoodStars = 0
        numAvailStars = 0

        avgX = 0.0
        avgY = 0.0

        for cell in psfCellSet.getCellList():
            for cand in cell.begin(False):  # don't ignore BAD stars
                numAvailStars += 1

            for cand in cell.begin(True):  # do ignore BAD stars
                cand = algorithmsLib.PsfCandidateF.cast(cand)
                src = cand.getSource()
                if flagKey is not None:
                    src.set(flagKey, True)
                avgX += src.getX()
                avgY += src.getY()
                numGoodStars += 1

        avgX /= numGoodStars
        avgY /= numGoodStars

        if metadata is not None:
            metadata.set("spatialFitChi2", fitChi2)
            metadata.set("numGoodStars", numGoodStars)
            metadata.set("numAvailStars", numAvailStars)
            metadata.set("avgX", avgX)
            metadata.set("avgY", avgY)

        psf = algorithmsLib.PcaPsf(psf.getKernel(),
                                   afwGeom.Point2D(avgX, avgY))

        return psf, psfCellSet
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128,
                        matchKernelAmplitudes=False, keepPlots=True):
    """Plot the PSF spatial model."""

    if not plt:
        print >> sys.stderr, "Unable to import matplotlib"
        return

    noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel())
    candPos = list()
    candFits = list()
    badPos = list()
    badFits = list()
    candAmps = list()
    badAmps = list()
    for cell in psfCellSet.getCellList():
        for cand in cell.begin(False):
            cand = algorithmsLib.cast_PsfCandidateF(cand)
            if not showBadCandidates and cand.isBad():
                continue
            candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
            try:
                im = cand.getMaskedImage()
            except Exception:
                continue

            fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter)
            params = fit[0]
            kernels = fit[1]
            amp = 0.0
            for p, k in zip(params, kernels):
                amp += p * afwMath.cast_FixedKernel(k).getSum()

            targetFits = badFits if cand.isBad() else candFits
            targetPos = badPos if cand.isBad() else candPos
            targetAmps = badAmps if cand.isBad() else candAmps

            targetFits.append([x / amp for x in params])
            targetPos.append(candCenter)
            targetAmps.append(amp)

    numCandidates = len(candFits)
    numBasisFuncs = noSpatialKernel.getNBasisKernels()

    xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0()
    yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0()
    zGood = numpy.array(candFits)
    ampGood = numpy.array(candAmps)

    xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0()
    yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0()
    zBad = numpy.array(badFits)
    ampBad = numpy.array(badAmps)
    numBad = len(badPos)

    xRange = numpy.linspace(0, exposure.getWidth(), num=numSample)
    yRange = numpy.linspace(0, exposure.getHeight(), num=numSample)

    kernel = psf.getKernel()
    nKernelComponents = kernel.getNKernelParameters()
    #
    # Figure out how many panels we'll need
    #
    nPanelX = int(math.sqrt(nKernelComponents))
    nPanelY = nKernelComponents//nPanelX
    while nPanelY*nPanelX < nKernelComponents:
        nPanelX += 1

    fig = plt.figure(1)
    fig.clf()
    try:
        fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
    except:                                 # protect against API changes
        pass
    #
    # Generator for axes arranged in panels
    #
    subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04)

    for k in range(nKernelComponents):
        func = kernel.getSpatialFunction(k)
        dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos])
        yMin = dfGood.min()
        yMax = dfGood.max()
        if numBad > 0:
            dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos])
            yMin = min([yMin, dfBad.min()])
            yMax = max([yMax, dfBad.max()])
        yMin -= 0.05 * (yMax - yMin)
        yMax += 0.05 * (yMax - yMin)

        yMin = -0.01
        yMax = 0.01

        fRange = numpy.ndarray((len(xRange), len(yRange)))
        for j, yVal in enumerate(yRange):
            for i, xVal in enumerate(xRange):
                fRange[j][i] = func(xVal, yVal)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        ax.set_autoscale_on(False)
        ax.set_xbound(lower=0, upper=exposure.getHeight())
        ax.set_ybound(lower=yMin, upper=yMax)
        ax.plot(yGood, dfGood, 'b+')
        if numBad > 0:
            ax.plot(yBad, dfBad, 'r+')
        ax.axhline(0.0)
        ax.set_title('Residuals(y)')

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        if matchKernelAmplitudes and k == 0:
            vmin = 0.0
            vmax = 1.1
        else:
            vmin = fRange.min()
            vmax = fRange.max()

        norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
        im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm,
                       extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1])
        ax.set_title('Spatial poly')
        plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()
        ax.set_autoscale_on(False)
        ax.set_xbound(lower=0, upper=exposure.getWidth())
        ax.set_ybound(lower=yMin, upper=yMax)
        ax.plot(xGood, dfGood, 'b+')
        if numBad > 0:
            ax.plot(xBad, dfBad, 'r+')
        ax.axhline(0.0)
        ax.set_title('K%d Residuals(x)' % k)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

        ax = subplots.next()

        if False:
            ax.scatter(xGood, yGood, c=dfGood, marker='o')
            ax.scatter(xBad, yBad, c=dfBad, marker='x')
            ax.set_xbound(lower=0, upper=exposure.getWidth())
            ax.set_ybound(lower=0, upper=exposure.getHeight())
            ax.set_title('Spatial residuals')
            plt.colorbar(im, orientation='horizontal')
        else:
            calib = exposure.getCalib()
            if calib.getFluxMag0()[0] <= 0:
                calib = type(calib)()
                calib.setFluxMag0(1.0)

            with CalibNoThrow():
                ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+')
                if numBad > 0:
                    ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+')

            ax.set_title('Flux variation')

    fig.show()

    global keptPlots
    if keepPlots and not keptPlots:
        # Keep plots open when done
        def show():
            print "%s: Please close plots when done." % __name__
            try:
                plt.show()
            except:
                pass
            print "Plots closed, exiting..."
        import atexit
        atexit.register(show)
        keptPlots = True