Beispiel #1
0
    def testStretch(self):
        """Test playing with the lookup table"""
        ds9.show()

        ds9.scale("linear", "zscale")
Beispiel #2
0
    def determinePsf(self,
                     exposure,
                     psfCandidateList,
                     metadata=None,
                     flagKey=None):
        """!Determine a PCA PSF model for an exposure given a list of PSF candidates

        \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
        \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
            typically obtained by detecting sources and then running them through a star selector
        \param[in,out] metadata  a home for interesting tidbits of information
        \param[in] flagKey schema key used to mark sources actually used in PSF determination

        \return a list of
         - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
         - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = lsstDebug.Info(
            __name__).displayExposure  # display the Exposure + spatialCells
        displayPsfCandidates = lsstDebug.Info(
            __name__).displayPsfCandidates  # show the viable candidates
        displayIterations = lsstDebug.Info(
            __name__).displayIterations  # display on each PSF iteration
        displayPsfComponents = lsstDebug.Info(
            __name__).displayPsfComponents  # show the PCA components
        displayResiduals = lsstDebug.Info(
            __name__).displayResiduals  # show residuals
        displayPsfMosaic = lsstDebug.Info(
            __name__).displayPsfMosaic  # show mosaic of reconstructed PSF(x,y)
        # match Kernel amplitudes for spatial plots
        matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
        # Keep matplotlib alive post mortem
        keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
        displayPsfSpatialModel = lsstDebug.Info(
            __name__).displayPsfSpatialModel  # Plot spatial model?
        showBadCandidates = lsstDebug.Info(
            __name__).showBadCandidates  # Include bad candidates
        # Normalize residuals by object amplitude
        normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
        pause = lsstDebug.Info(
            __name__).pause  # Prompt user after each iteration?

        if display > 1:
            pause = True

        mi = exposure.getMaskedImage()

        if len(psfCandidateList) == 0:
            raise RuntimeError("No PSF candidates supplied.")

        # construct and populate a spatial cell set
        bbox = mi.getBBox()
        psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
                                            self.config.sizeCellY)
        sizes = []
        for i, psfCandidate in enumerate(psfCandidateList):
            if psfCandidate.getSource().getPsfFluxFlag():  # bad measurement
                continue

            try:
                psfCellSet.insertCandidate(psfCandidate)
            except Exception as e:
                self.log.debug("Skipping PSF candidate %d of %d: %s", i,
                               len(psfCandidateList), e)
                continue
            source = psfCandidate.getSource()

            quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
                                     source.getIxy())
            axes = afwEll.Axes(quad)
            sizes.append(axes.getA())
        if len(sizes) == 0:
            raise RuntimeError("No usable PSF candidates supplied")
        nEigenComponents = self.config.nEigenComponents  # initial version

        if self.config.kernelSize >= 15:
            self.log.warn(
                "WARNING: NOT scaling kernelSize by stellar quadrupole moment "
                +
                "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead",
                self.config.kernelSize)
            actualKernelSize = int(self.config.kernelSize)
        else:
            medSize = numpy.median(sizes)
            actualKernelSize = 2 * int(self.config.kernelSize *
                                       math.sqrt(medSize) + 0.5) + 1
            if actualKernelSize < self.config.kernelSizeMin:
                actualKernelSize = self.config.kernelSizeMin
            if actualKernelSize > self.config.kernelSizeMax:
                actualKernelSize = self.config.kernelSizeMax

            if display:
                print("Median size=%s" % (medSize, ))
        self.log.trace("Kernel size=%s", actualKernelSize)

        # Set size of image returned around candidate
        psfCandidateList[0].setHeight(actualKernelSize)
        psfCandidateList[0].setWidth(actualKernelSize)

        if self.config.doRejectBlends:
            # Remove blended candidates completely
            blendedCandidates = [
            ]  # Candidates to remove; can't do it while iterating
            for cell, cand in candidatesIter(psfCellSet, False):
                if len(cand.getSource().getFootprint().getPeaks()) > 1:
                    blendedCandidates.append((cell, cand))
                    continue
            if display:
                print("Removing %d blended Psf candidates" %
                      len(blendedCandidates))
            for cell, cand in blendedCandidates:
                cell.removeCandidate(cand)
            if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0:
                raise RuntimeError("All PSF candidates removed as blends")

        if display:
            frame = 0
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            self.config.nStarPerCell,
                                            symb="o",
                                            ctype=ds9.CYAN,
                                            ctypeUnused=ds9.YELLOW,
                                            size=4,
                                            frame=frame)

        #
        # Do a PCA decomposition of those PSF candidates
        #
        reply = "y"  # used in interactive mode
        for iterNum in range(self.config.nIterForPsf):
            if display and displayPsfCandidates:  # Show a mosaic of usable PSF candidates
                #
                import lsst.afw.display.utils as displayUtils

                stamps = []
                for cell in psfCellSet.getCellList():
                    for cand in cell.begin(not showBadCandidates
                                           ):  # maybe include bad candidates
                        try:
                            im = cand.getMaskedImage()

                            chi2 = cand.getChi2()
                            if chi2 > 1e100:
                                chi2 = numpy.nan

                            stamps.append(
                                (im, "%d%s" %
                                 (maUtils.splitId(cand.getSource().getId(),
                                                  True)["objId"], chi2),
                                 cand.getStatus()))
                        except Exception as e:
                            continue

                if len(stamps) == 0:
                    print(
                        "WARNING: No PSF candidates to show; try setting showBadCandidates=True"
                    )
                else:
                    mos = displayUtils.Mosaic()
                    for im, label, status in stamps:
                        im = type(im)(im, True)
                        try:
                            im /= afwMath.makeStatistics(
                                im, afwMath.MAX).getValue()
                        except NotImplementedError:
                            pass

                        mos.append(
                            im, label, ds9.GREEN
                            if status == afwMath.SpatialCellCandidate.GOOD else
                            ds9.YELLOW if status ==
                            afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED)

                    mos.makeMosaic(frame=8, title="Psf Candidates")

            # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit
            cleanChi2 = False  # Any naughty (negative/NAN) chi^2 values?
            while not cleanChi2:
                cleanChi2 = True
                #
                # First, estimate the PSF
                #
                psf, eigenValues, nEigenComponents, fitChi2 = \
                    self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
                #
                # In clipping, allow all candidates to be innocent until proven guilty on this iteration.
                # Throw out any prima facie guilty candidates (naughty chi^2 values)
                #
                for cell in psfCellSet.getCellList():
                    awfulCandidates = []
                    for cand in cell.begin(False):  # include bad candidates
                        cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN
                                       )  # until proven guilty
                        rchi2 = cand.getChi2()
                        if not numpy.isfinite(rchi2) or rchi2 <= 0:
                            # Guilty prima facie
                            awfulCandidates.append(cand)
                            cleanChi2 = False
                            self.log.debug("chi^2=%s; id=%s", cand.getChi2(),
                                           cand.getSource().getId())
                    for cand in awfulCandidates:
                        if display:
                            print("Removing bad candidate: id=%d, chi^2=%f" % \
                                  (cand.getSource().getId(), cand.getChi2()))
                        cell.removeCandidate(cand)

            #
            # Clip out bad fits based on reduced chi^2
            #
            badCandidates = list()
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):  # include bad candidates
                    rchi2 = cand.getChi2(
                    )  # reduced chi^2 when fitting PSF to candidate
                    assert rchi2 > 0
                    if rchi2 > self.config.reducedChi2ForPsfCandidates:
                        badCandidates.append(cand)

            badCandidates.sort(key=lambda x: x.getChi2(), reverse=True)
            numBad = numCandidatesToReject(len(badCandidates), iterNum,
                                           self.config.nIterForPsf)
            for i, c in zip(range(numBad), badCandidates):
                if display:
                    chi2 = c.getChi2()
                    if chi2 > 1e100:
                        chi2 = numpy.nan

                    print("Chi^2 clipping %-4d  %.2g" %
                          (c.getSource().getId(), chi2))
                c.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Clip out bad fits based on spatial fitting.
            #
            # This appears to be better at getting rid of sources that have a single dominant kernel component
            # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help
            # set the spatial model) don't contain that kernel component, and so the spatial modeling
            # downweights the component.
            #

            residuals = list()
            candidates = list()
            kernel = psf.getKernel()
            noSpatialKernel = psf.getKernel()
            for cell in psfCellSet.getCellList():
                for cand in cell.begin(False):
                    candCenter = afwGeom.PointD(cand.getXCenter(),
                                                cand.getYCenter())
                    try:
                        im = cand.getMaskedImage(kernel.getWidth(),
                                                 kernel.getHeight())
                    except Exception as e:
                        continue

                    fit = fitKernelParamsToImage(noSpatialKernel, im,
                                                 candCenter)
                    params = fit[0]
                    kernels = fit[1]
                    amp = 0.0
                    for p, k in zip(params, kernels):
                        amp += p * k.getSum()

                    predict = [
                        kernel.getSpatialFunction(k)(candCenter.getX(),
                                                     candCenter.getY())
                        for k in range(kernel.getNKernelParameters())
                    ]

                    #print cand.getSource().getId(), [a / amp for a in params], predict

                    residuals.append(
                        [a / amp - p for a, p in zip(params, predict)])
                    candidates.append(cand)

            residuals = numpy.array(residuals)

            for k in range(kernel.getNKernelParameters()):
                if False:
                    # Straight standard deviation
                    mean = residuals[:, k].mean()
                    rms = residuals[:, k].std()
                elif False:
                    # Using interquartile range
                    sr = numpy.sort(residuals[:, k])
                    mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \
                        0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1])
                    rms = 0.74 * (sr[int(0.75 * len(sr))] -
                                  sr[int(0.25 * len(sr))])
                else:
                    stats = afwMath.makeStatistics(
                        residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
                    mean = stats.getValue(afwMath.MEANCLIP)
                    rms = stats.getValue(afwMath.STDEVCLIP)

                rms = max(
                    1.0e-4,
                    rms)  # Don't trust RMS below this due to numerical issues

                if display:
                    print("Mean for component %d is %f" % (k, mean))
                    print("RMS for component %d is %f" % (k, rms))
                badCandidates = list()
                for i, cand in enumerate(candidates):
                    if numpy.fabs(residuals[i, k] -
                                  mean) > self.config.spatialReject * rms:
                        badCandidates.append(i)

                badCandidates.sort(
                    key=lambda x: numpy.fabs(residuals[x, k] - mean),
                    reverse=True)

                numBad = numCandidatesToReject(len(badCandidates), iterNum,
                                               self.config.nIterForPsf)

                for i, c in zip(range(min(len(badCandidates), numBad)),
                                badCandidates):
                    cand = candidates[c]
                    if display:
                        print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \
                              (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
                               residuals[badCandidates[i], k], self.config.spatialReject * rms))
                    cand.setStatus(afwMath.SpatialCellCandidate.BAD)

            #
            # Display results
            #
            if display and displayIterations:
                if displayExposure:
                    if iterNum > 0:
                        ds9.erase(frame=frame)
                    maUtils.showPsfSpatialCells(exposure,
                                                psfCellSet,
                                                self.config.nStarPerCell,
                                                showChi2=True,
                                                symb="o",
                                                size=8,
                                                frame=frame,
                                                ctype=ds9.YELLOW,
                                                ctypeBad=ds9.RED,
                                                ctypeUnused=ds9.MAGENTA)
                    if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
                        maUtils.showPsfSpatialCells(
                            exposure,
                            psfCellSet,
                            self.config.nStarPerCellSpatialFit,
                            symb="o",
                            size=10,
                            frame=frame,
                            ctype=ds9.YELLOW,
                            ctypeBad=ds9.RED)
                if displayResiduals:
                    while True:
                        try:
                            maUtils.showPsfCandidates(
                                exposure,
                                psfCellSet,
                                psf=psf,
                                frame=4,
                                normalize=normalizeResiduals,
                                showBadCandidates=showBadCandidates)
                            maUtils.showPsfCandidates(
                                exposure,
                                psfCellSet,
                                psf=psf,
                                frame=5,
                                normalize=normalizeResiduals,
                                showBadCandidates=showBadCandidates,
                                variance=True)
                        except:
                            if not showBadCandidates:
                                showBadCandidates = True
                                continue
                        break

                if displayPsfComponents:
                    maUtils.showPsf(psf, eigenValues, frame=6)
                if displayPsfMosaic:
                    maUtils.showPsfMosaic(exposure,
                                          psf,
                                          frame=7,
                                          showFwhm=True)
                    ds9.scale('linear', 0, 1, frame=7)
                if displayPsfSpatialModel:
                    maUtils.plotPsfSpatialModel(
                        exposure,
                        psf,
                        psfCellSet,
                        showBadCandidates=True,
                        matchKernelAmplitudes=matchKernelAmplitudes,
                        keepPlots=keepMatplotlibPlots)

                if pause:
                    while True:
                        try:
                            reply = input(
                                "Next iteration? [ynchpqQs] ").strip()
                        except EOFError:
                            reply = "n"

                        reply = reply.split()
                        if reply:
                            reply, args = reply[0], reply[1:]
                        else:
                            reply = ""

                        if reply in ("", "c", "h", "n", "p", "q", "Q", "s",
                                     "y"):
                            if reply == "c":
                                pause = False
                            elif reply == "h":
                                print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \
                                      "s[ave fileName] y[es]")
                                continue
                            elif reply == "p":
                                import pdb
                                pdb.set_trace()
                            elif reply == "q":
                                display = False
                            elif reply == "Q":
                                sys.exit(1)
                            elif reply == "s":
                                fileName = args.pop(0)
                                if not fileName:
                                    print("Please provide a filename")
                                    continue

                                print("Saving to %s" % fileName)
                                maUtils.saveSpatialCellSet(psfCellSet,
                                                           fileName=fileName)
                                continue
                            break
                        else:
                            print("Unrecognised response: %s" % reply,
                                  file=sys.stderr)

                    if reply == "n":
                        break

        # One last time, to take advantage of the last iteration
        psf, eigenValues, nEigenComponents, fitChi2 = \
            self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)

        #
        # Display code for debugging
        #
        if display and reply != "n":
            if displayExposure:
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            self.config.nStarPerCell,
                                            showChi2=True,
                                            symb="o",
                                            ctype=ds9.YELLOW,
                                            ctypeBad=ds9.RED,
                                            size=8,
                                            frame=frame)
                if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
                    maUtils.showPsfSpatialCells(
                        exposure,
                        psfCellSet,
                        self.config.nStarPerCellSpatialFit,
                        symb="o",
                        ctype=ds9.YELLOW,
                        ctypeBad=ds9.RED,
                        size=10,
                        frame=frame)
                if displayResiduals:
                    maUtils.showPsfCandidates(
                        exposure,
                        psfCellSet,
                        psf=psf,
                        frame=4,
                        normalize=normalizeResiduals,
                        showBadCandidates=showBadCandidates)

            if displayPsfComponents:
                maUtils.showPsf(psf, eigenValues, frame=6)

            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
                ds9.scale("linear", 0, 1, frame=7)
            if displayPsfSpatialModel:
                maUtils.plotPsfSpatialModel(
                    exposure,
                    psf,
                    psfCellSet,
                    showBadCandidates=True,
                    matchKernelAmplitudes=matchKernelAmplitudes,
                    keepPlots=keepMatplotlibPlots)
        #
        # Generate some QA information
        #
        # Count PSF stars
        #
        numGoodStars = 0
        numAvailStars = 0

        avgX = 0.0
        avgY = 0.0

        for cell in psfCellSet.getCellList():
            for cand in cell.begin(False):  # don't ignore BAD stars
                numAvailStars += 1

            for cand in cell.begin(True):  # do ignore BAD stars
                src = cand.getSource()
                if flagKey is not None:
                    src.set(flagKey, True)
                avgX += src.getX()
                avgY += src.getY()
                numGoodStars += 1

        avgX /= numGoodStars
        avgY /= numGoodStars

        if metadata is not None:
            metadata.set("spatialFitChi2", fitChi2)
            metadata.set("numGoodStars", numGoodStars)
            metadata.set("numAvailStars", numAvailStars)
            metadata.set("avgX", avgX)
            metadata.set("avgY", avgY)

        psf = PcaPsf(psf.getKernel(), afwGeom.Point2D(avgX, avgY))

        return psf, psfCellSet
                        action="store_true",
                        help="Load debug.py?",
                        default=False)
    parser.add_argument('--ds9',
                        action="store_true",
                        help="Display the result?",
                        default=False)
    parser.add_argument('--write',
                        '-w',
                        action="store_true",
                        help="Write the result?",
                        default=False)
    args = parser.parse_args()

    if args.debug:
        try:
            import debug
        except ImportError as e:
            print(e, file=sys.stderr)

    exposure = runIsr()

    if args.ds9:
        im = exposure.getMaskedImage().getImage()
        im_median = numpy.median(im.getArray())
        ds9.mtv(im)
        ds9.scale(min=im_median * 0.90, max=im_median * 1.1, type='SQRT')

    if args.write:
        exposure.writeFits("postISRCCD.fits")
Beispiel #4
0
    flatExposure = exampleUtils.makeFlat(GRADIENT)
    rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)

    output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
    return output.exposure

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Demonstrate the use of IsrTask")
    parser.add_argument('--debug', '-d', action="store_true", help="Load debug.py?", default=False)
    parser.add_argument('--ds9', action="store_true", help="Display the result?", default=False)
    parser.add_argument('--write', '-w', action="store_true", help="Write the result?", default=False)
    args = parser.parse_args()

    if args.debug:
        try:
            import debug
        except ImportError as e:
            print >> sys.stderr, e

    exposure = runIsr()

    if args.ds9:
        im = exposure.getMaskedImage().getImage()
        im_median = numpy.median(im.getArray())
        ds9.mtv(im)
        ds9.scale(min=im_median*0.90, max=im_median*1.1, type='SQRT')

    if args.write:
        exposure.writeFits("postISRCCD.fits")
                                                      showBadCandidates=showBadCandidates)
                            maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=5,
                                                      normalize=normalizeResiduals,
                                                      showBadCandidates=showBadCandidates,
                                                      variance=True)
                        except:
                            if not showBadCandidates:
                                showBadCandidates = True
                                continue
                        break

                if displayPsfComponents:
                    maUtils.showPsf(psf, eigenValues, frame=6)
                if displayPsfMosaic:
                    maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
                    ds9.scale('linear', 0, 1, frame=7)
                if displayPsfSpatialModel:
                    maUtils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True,
                                                matchKernelAmplitudes=matchKernelAmplitudes,
                                                keepPlots=keepMatplotlibPlots)

                if pause:
                    while True:
                        try:
                            reply = raw_input("Next iteration? [ynchpqQs] ").strip()
                        except EOFError:
                            reply = "n"

                        reply = reply.split()
                        if reply:
                            reply, args = reply[0], reply[1:]
Beispiel #6
0
    def run(self, exposure, sources, expId=0, matches=None):
        """!Measure the PSF

        \param[in,out]   exposure   Exposure to process; measured PSF will be added.
        \param[in,out]   sources    Measured sources on exposure; flag fields will be set marking
                                    stars chosen by the star selector and the PSF determiner if a schema
                                    was passed to the task constructor.
        \param[in]       expId      Exposure id used for generating random seed.
        \param[in]  matches         A list of lsst.afw.table.ReferenceMatch objects
                                    (\em i.e. of lsst.afw.table.Match
                                    with \c first being of type lsst.afw.table.SimpleRecord and \c second
                                    type lsst.afw.table.SourceRecord --- the reference object and detected
                                    object respectively) as returned by \em e.g. the AstrometryTask.
                                    Used by star selectors that choose to refer to an external catalog.

        \return a pipe.base.Struct with fields:
         - psf: The measured PSF (also set in the input exposure)
         - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
            as returned by the psf determiner.
        """
        self.log.info("Measuring PSF")

        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = lsstDebug.Info(
            __name__).displayExposure  # display the Exposure + spatialCells
        displayPsfMosaic = lsstDebug.Info(
            __name__).displayPsfMosaic  # show mosaic of reconstructed PSF(x,y)
        displayPsfCandidates = lsstDebug.Info(
            __name__).displayPsfCandidates  # show mosaic of candidates
        displayResiduals = lsstDebug.Info(
            __name__).displayResiduals  # show residuals
        showBadCandidates = lsstDebug.Info(
            __name__).showBadCandidates  # include bad candidates
        normalizeResiduals = lsstDebug.Info(
            __name__).normalizeResiduals  # normalise residuals by object peak

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
        #
        # Run star selector
        #
        selectionResult = self.starSelector.run(exposure=exposure,
                                                sourceCat=sources,
                                                matches=matches)
        reserveResult = self.reserve.run(selectionResult.starCat, expId=expId)
        psfCandidateList = [
            cand for cand, use in zip(selectionResult.psfCandidates,
                                      reserveResult.use) if use
        ]

        if psfCandidateList and self.candidateKey is not None:
            for cand in psfCandidateList:
                source = cand.getSource()
                source.set(self.candidateKey, True)

        self.log.info("PSF star selector found %d candidates" %
                      len(psfCandidateList))

        if display:
            frame = display
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
        #
        # Determine PSF
        #
        psf, cellSet = self.psfDeterminer.determinePsf(exposure,
                                                       psfCandidateList,
                                                       self.metadata,
                                                       flagKey=self.usedKey)
        self.log.info("PSF determination using %d/%d stars." %
                      (self.metadata.get("numGoodStars"),
                       self.metadata.get("numAvailStars")))

        exposure.setPsf(psf)

        if display:
            frame = display
            if displayExposure:
                showPsfSpatialCells(exposure,
                                    cellSet,
                                    showBadCandidates,
                                    frame=frame)
                frame += 1

            if displayPsfCandidates:  # Show a mosaic of  PSF candidates
                plotPsfCandidates(cellSet, showBadCandidates, frame)
                frame += 1

            if displayResiduals:
                frame = plotResiduals(exposure,
                                      cellSet,
                                      showBadCandidates=showBadCandidates,
                                      normalizeResiduals=normalizeResiduals,
                                      frame=frame)
            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure,
                                      psf,
                                      frame=frame,
                                      showFwhm=True)
                ds9.scale(0, 1, "linear", frame=frame)
                frame += 1

        return pipeBase.Struct(
            psf=psf,
            cellSet=cellSet,
        )
Beispiel #7
0
    def run(self, exposure, sources, expId=0, matches=None):
        """!Measure the PSF

        \param[in,out]   exposure   Exposure to process; measured PSF will be added.
        \param[in,out]   sources    Measured sources on exposure; flag fields will be set marking
                                    stars chosen by the star selector and the PSF determiner if a schema
                                    was passed to the task constructor.
        \param[in]       expId      Exposure id used for generating random seed.
        \param[in]  matches         A list of lsst.afw.table.ReferenceMatch objects
                                    (\em i.e. of lsst.afw.table.Match
                                    with \c first being of type lsst.afw.table.SimpleRecord and \c second
                                    type lsst.afw.table.SourceRecord --- the reference object and detected
                                    object respectively) as returned by \em e.g. the AstrometryTask.
                                    Used by star selectors that choose to refer to an external catalog.

        \return a pipe.base.Struct with fields:
         - psf: The measured PSF (also set in the input exposure)
         - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
            as returned by the psf determiner.
        """
        self.log.info("Measuring PSF")

        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = lsstDebug.Info(__name__).displayExposure     # display the Exposure + spatialCells
        displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y)
        displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show mosaic of candidates
        displayResiduals = lsstDebug.Info(__name__).displayResiduals   # show residuals
        showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # include bad candidates
        normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals # normalise residuals by object peak

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
        #
        # Run star selector
        #
        psfCandidateList = self.starSelector.run(exposure=exposure, sourceCat=sources,
            matches=matches).psfCandidates
        reserveList = []
        
        if self.config.reserveFraction > 0 :
            random.seed(self.config.reserveSeed*expId)
            reserveList = random.sample(psfCandidateList,
                                        int((self.config.reserveFraction)*len(psfCandidateList)))

            for cand in reserveList:
                psfCandidateList.remove(cand)

            if reserveList and self.reservedKey is not None:
                for cand in reserveList:
                    source = cand.getSource()
                    source.set(self.reservedKey,True)
            
        if psfCandidateList and self.candidateKey is not None:
            for cand in psfCandidateList:
                source = cand.getSource()
                source.set(self.candidateKey, True)

        self.log.info("PSF star selector found %d candidates" % len(psfCandidateList))
        if self.config.reserveFraction > 0 :
            self.log.info("Reserved %d candidates from the fitting" % len(reserveList))

        if display:
            frame = display
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
        #
        # Determine PSF
        #
        psf, cellSet = self.psfDeterminer.determinePsf(exposure, psfCandidateList, self.metadata,
                                                       flagKey=self.usedKey)
        self.log.info("PSF determination using %d/%d stars." %
                     (self.metadata.get("numGoodStars"), self.metadata.get("numAvailStars")))

        exposure.setPsf(psf)

        if display:
            frame = display
            if displayExposure:
                showPsfSpatialCells(exposure, cellSet, showBadCandidates, frame=frame)
                frame += 1

            if displayPsfCandidates:    # Show a mosaic of  PSF candidates
                plotPsfCandidates(cellSet, showBadCandidates, frame)
                frame += 1

            if displayResiduals:
                frame = plotResiduals(exposure, cellSet,
                                      showBadCandidates=showBadCandidates,
                                      normalizeResiduals=normalizeResiduals,
                                      frame=frame)
            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure, psf, frame=frame, showFwhm=True)
                ds9.scale(0, 1, "linear", frame=frame)
                frame += 1

        return pipeBase.Struct(
            psf = psf,
            cellSet = cellSet,
        )
Beispiel #8
0
    def runDataRef(self, sensorRef):
        self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
        ccdExposure = sensorRef.get('raw')

        if self.config.removePcCards:  # Remove any PC00N00M cards in the header
            raw_md = sensorRef.get("raw_md")
            nPc = 0
            for i in (
                    1,
                    2,
            ):
                for j in (
                        1,
                        2,
                ):
                    k = "PC%03d%03d" % (i, j)
                    for md in (raw_md, ccdExposure.getMetadata()):
                        if md.exists(k):
                            md.remove(k)
                            nPc += 1

            if nPc:
                self.log.info("Recreating Wcs after stripping PC00n00m" %
                              (sensorRef.dataId))
                ccdExposure.setWcs(afwImage.makeWcs(raw_md))

        ccdExposure = self.convertIntToFloat(ccdExposure)
        ccd = ccdExposure.getDetector()

        # Read in defects to check for any dead amplifiers (entire amp is within defect region)
        defects = sensorRef.get("defects", immediate=True)

        for amp in ccd:
            # Check if entire amp region is defined as defect (need to use amp.getBBox() for correct
            # comparison with current defects definition.
            badAmp = bool(
                sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
            # In the case of bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
            # association with pixels in current ccdExposure).
            if badAmp:
                dataView = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
                                                 amp.getRawBBox(),
                                                 afwImage.PARENT)
                maskView = dataView.getMask()
                maskView |= maskView.getPlaneBitMask("BAD")
                del maskView

            self.measureOverscan(ccdExposure, amp)
            if self.config.doSaturation and not badAmp:
                self.saturationDetection(ccdExposure, amp)
            if self.config.doSuspect and not badAmp:
                self.suspectDetection(ccdExposure, amp)
            ampMask = afwImage.Mask(ccdExposure.getMaskedImage().getMask(),
                                    amp.getRawDataBBox(), afwImage.PARENT)
            maskVal = ampMask.getPlaneBitMask(
                [self.config.saturatedMaskName, self.config.suspectMaskName])
            if numpy.all(ampMask.getArray() & maskVal > 0):
                badAmp = True
            if self.config.doOverscan and not badAmp:
                ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
                                                 amp.getRawDataBBox(),
                                                 afwImage.PARENT)
                overscan = afwImage.MaskedImageF(
                    ccdExposure.getMaskedImage(),
                    amp.getRawHorizontalOverscanBBox(), afwImage.PARENT)
                overscanArray = overscan.getImage().getArray()
                median = numpy.ma.median(
                    numpy.ma.masked_where(overscan.getMask().getArray(),
                                          overscanArray))
                bad = numpy.where(
                    numpy.abs(overscanArray -
                              median) > self.config.overscanMaxDev)
                overscan.getMask().getArray()[bad] = overscan.getMask(
                ).getPlaneBitMask("SAT")

                statControl = afwMath.StatisticsControl()
                statControl.setAndMask(
                    ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
                        "SAT"))
                lsstIsr.overscanCorrection(
                    ampMaskedImage=ampImage,
                    overscanImage=overscan,
                    fitType=self.config.overscanFitType,
                    order=self.config.overscanOrder,
                    collapseRej=self.config.overscanRej,
                    statControl=statControl,
                )

        ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
        ccd = ccdExposure.getDetector()

        if self.config.qa.doWriteOss:
            sensorRef.put(ccdExposure, "ossImage")
        if self.config.qa.doThumbnailOss:
            self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)

        if self.config.doBias:
            biasExposure = self.getIsrExposure(sensorRef, "bias")
            self.biasCorrection(ccdExposure, biasExposure)
        if self.config.doVariance:
            for amp in ccd:
                ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox(),
                                                  afwImage.PARENT)
                self.updateVariance(ampExposure, amp)
        if self.doLinearize(ccd):
            # immediate=True required for functors and linearizers are functors; see ticket DM-6515
            linearizer = sensorRef.get("linearizer", immediate=True)
            linearizer(image=ccdExposure.getMaskedImage().getImage(),
                       detector=ccd,
                       log=self.log)
        if self.config.doCrosstalk:
            self.crosstalk.run(ccdExposure)

        if self.config.doWidenSaturationTrails:
            self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())

        interpolationDone = False

        darkExposure = self.getIsrExposure(
            sensorRef, "dark") if self.config.doDark else None
        flatExposure = self.getIsrExposure(
            sensorRef, "flat") if self.config.doFlat else None

        if self.config.doBrighterFatter:
            # We need to apply flats and darks before we can interpolate, and we
            # need to interpolate before we do B-F, but we do B-F without the
            # flats and darks applied so we can work in units of electrons or holes.
            # This context manager applies and then removes the darks and flats.
            with self.flatContext(ccdExposure, flatExposure, darkExposure):
                if self.config.doDefect:
                    self.maskAndInterpDefect(ccdExposure, defects)
                if self.config.doSaturationInterpolation:
                    self.saturationInterpolation(ccdExposure)
                self.maskAndInterpNan(ccdExposure)
                interpolationDone = True

            brighterFatterKernel = sensorRef.get('bfKernel')
            self.brighterFatterCorrection(
                ccdExposure,
                brighterFatterKernel,
                self.config.brighterFatterMaxIter,
                self.config.brighterFatterThreshold,
                self.config.brighterFatterApplyGain,
            )
        if self.config.doDark:
            self.darkCorrection(ccdExposure, darkExposure)

        if self.config.doStrayLight:
            self.strayLight.run(sensorRef, ccdExposure)

        if self.config.doFlat:
            self.flatCorrection(ccdExposure,
                                flatExposure,
                                doTweakFlat=self.config.doTweakFlat)

        if self.config.doDefect and not interpolationDone:
            self.maskAndInterpDefect(ccdExposure, defects)

        if self.config.doApplyGains:
            self.applyGains(ccdExposure, self.config.normalizeGains)
        if self.config.doSaturation and not interpolationDone:
            self.saturationInterpolation(ccdExposure)

        if self.config.doFringe:
            self.fringe.runDataRef(ccdExposure, sensorRef)
        if self.config.doSetBadRegions:
            self.setBadRegions(ccdExposure)

        if not interpolationDone or self.config.doNanInterpAfterFlat:
            self.maskAndInterpNan(ccdExposure)

        if self.config.qa.doWriteFlattened:
            sensorRef.put(ccdExposure, "flattenedImage")
        if self.config.qa.doThumbnailFlattened:
            self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)

        self.measureBackground(ccdExposure)

        if self.config.doGuider:
            self.guider(ccdExposure)

        self.roughZeroPoint(ccdExposure)

        if self.config.doWriteVignettePolygon:
            self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)

        if self.config.doWrite:
            sensorRef.put(ccdExposure, "postISRCCD")

        if self._display:
            im = ccdExposure.getMaskedImage().getImage()
            im_median = float(numpy.median(im.getArray()))
            ds9.mtv(im)
            ds9.scale(min=im_median * 0.95, max=im_median * 1.15)

        return Struct(exposure=ccdExposure)
                                showBadCandidates=showBadCandidates,
                                variance=True)
                        except:
                            if not showBadCandidates:
                                showBadCandidates = True
                                continue
                        break

                if displayPsfComponents:
                    maUtils.showPsf(psf, eigenValues, frame=6)
                if displayPsfMosaic:
                    maUtils.showPsfMosaic(exposure,
                                          psf,
                                          frame=7,
                                          showFwhm=True)
                    ds9.scale('linear', 0, 1, frame=7)
                if displayPsfSpatialModel:
                    maUtils.plotPsfSpatialModel(
                        exposure,
                        psf,
                        psfCellSet,
                        showBadCandidates=True,
                        matchKernelAmplitudes=matchKernelAmplitudes,
                        keepPlots=keepMatplotlibPlots)

                if pause:
                    while True:
                        try:
                            reply = raw_input(
                                "Next iteration? [ynchpqQs] ").strip()
                        except EOFError:
Beispiel #10
0
    def runDataRef(self, sensorRef):
        self.log.log(self.log.INFO,
                     "Performing ISR on sensor %s" % (sensorRef.dataId))
        ccdExposure = sensorRef.get('raw')

        if self.config.removePcCards:  # Remove any PC00N00M cards in the header
            raw_md = sensorRef.get("raw_md")
            nPc = 0
            for i in (
                    1,
                    2,
            ):
                for j in (
                        1,
                        2,
                ):
                    k = "PC%03d%03d" % (i, j)
                    for md in (raw_md, ccdExposure.getMetadata()):
                        if md.exists(k):
                            md.remove(k)
                            nPc += 1

            if nPc:
                self.log.log(
                    self.log.INFO, "Recreating Wcs after stripping PC00n00m" %
                    (sensorRef.dataId))
                ccdExposure.setWcs(afwImage.makeWcs(raw_md))

        ccdExposure = self.convertIntToFloat(ccdExposure)
        ccd = ccdExposure.getDetector()

        for amp in ccd:
            self.measureOverscan(ccdExposure, amp)
            if self.config.doSaturation:
                self.saturationDetection(ccdExposure, amp)
            if self.config.doOverscan:
                ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
                                                 amp.getRawDataBBox(),
                                                 afwImage.PARENT)
                overscan = afwImage.MaskedImageF(
                    ccdExposure.getMaskedImage(),
                    amp.getRawHorizontalOverscanBBox(), afwImage.PARENT)
                overscanArray = overscan.getImage().getArray()
                median = numpy.ma.median(
                    numpy.ma.masked_where(overscan.getMask().getArray(),
                                          overscanArray))
                bad = numpy.where(
                    numpy.abs(overscanArray -
                              median) > self.config.overscanMaxDev)
                overscan.getMask().getArray()[bad] = overscan.getMask(
                ).getPlaneBitMask("SAT")

                statControl = afwMath.StatisticsControl()
                statControl.setAndMask(
                    ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
                        "SAT"))
                lsstIsr.overscanCorrection(
                    ampMaskedImage=ampImage,
                    overscanImage=overscan,
                    fitType=self.config.overscanFitType,
                    order=self.config.overscanOrder,
                    collapseRej=self.config.overscanRej,
                    statControl=statControl,
                )

            if self.config.doVariance:
                # Ideally, this should be done after bias subtraction,
                # but CCD assembly demands a variance plane
                ampExposure = ccdExposure.Factory(ccdExposure,
                                                  amp.getRawDataBBox(),
                                                  afwImage.PARENT)
                self.updateVariance(ampExposure, amp)

        ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
        ccd = ccdExposure.getDetector()

        doRotateCalib = False  # Rotate calib images for bias/dark/flat correction?
        nQuarter = ccd.getOrientation().getNQuarter()
        if nQuarter != 0:
            doRotateCalib = True

        if self.config.doDefect:
            defects = sensorRef.get('defects', immediate=True)
            self.maskAndInterpDefect(ccdExposure, defects)

        if self.config.qa.doWriteOss:
            sensorRef.put(ccdExposure, "ossImage")
        if self.config.qa.doThumbnailOss:
            self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)

        if self.config.doBias:
            biasExposure = self.getIsrExposure(sensorRef, "bias")
            if not doRotateCalib:
                self.biasCorrection(ccdExposure, biasExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.biasCorrection(exp, biasExposure)
        if self.config.doLinearize:
            self.linearize(ccdExposure)
        if self.config.doCrosstalk:
            self.crosstalk.run(ccdExposure)
        if self.config.doDark:
            darkExposure = self.getIsrExposure(sensorRef, "dark")
            if not doRotateCalib:
                self.darkCorrection(ccdExposure, darkExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.darkCorrection(exp, darkExposure)
        if self.config.doFlat:
            flatExposure = self.getIsrExposure(sensorRef, "flat")
            if not doRotateCalib:
                self.flatCorrection(ccdExposure, flatExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.flatCorrection(exp, flatExposure)

        if self.config.doApplyGains:
            self.applyGains(ccdExposure, self.config.normalizeGains)
        if self.config.doWidenSaturationTrails:
            self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
        if self.config.doSaturation:
            self.saturationInterpolation(ccdExposure)

        if self.config.doFringe:
            self.fringe.runDataRef(ccdExposure, sensorRef)
        if self.config.doSetBadRegions:
            self.setBadRegions(ccdExposure)

        self.maskAndInterpNan(ccdExposure)

        if self.config.qa.doWriteFlattened:
            sensorRef.put(ccdExposure, "flattenedImage")
        if self.config.qa.doThumbnailFlattened:
            self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)

        self.measureBackground(ccdExposure)

        if self.config.doGuider:
            self.guider(ccdExposure)

        self.roughZeroPoint(ccdExposure)

        if self.config.doWriteVignettePolygon:
            self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)

        if self.config.doWrite:
            sensorRef.put(ccdExposure, "postISRCCD")

        if self._display:
            im = ccdExposure.getMaskedImage().getImage()
            im_median = float(numpy.median(im.getArray()))
            ds9.mtv(im)
            ds9.scale(min=im_median * 0.95, max=im_median * 1.15)

        return Struct(exposure=ccdExposure)
    def determinePsf(self,
                     exposure,
                     psfCandidateList,
                     metadata=None,
                     flagKey=None):
        """Determine a PSFEX PSF model for an exposure given a list of PSF candidates

        @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure)
        @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
            typically obtained by detecting sources and then running them through a star selector
        @param[in,out] metadata  a home for interesting tidbits of information
        @param[in] flagKey: schema key used to mark sources actually used in PSF determination

        @return psf: a meas.extensions.psfex.PsfexPsf
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = display and \
            lsstDebug.Info(__name__).displayExposure      # display the Exposure + spatialCells
        displayPsfComponents = display and \
            lsstDebug.Info(__name__).displayPsfComponents  # show the basis functions
        showBadCandidates = display and \
            lsstDebug.Info(__name__).showBadCandidates    # Include bad candidates (meaningless, methinks)
        displayResiduals = display and \
            lsstDebug.Info(__name__).displayResiduals     # show residuals
        displayPsfMosaic = display and \
            lsstDebug.Info(__name__).displayPsfMosaic     # show mosaic of reconstructed PSF(x,y)
        normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
        # Normalise residuals by object amplitude

        mi = exposure.getMaskedImage()

        nCand = len(psfCandidateList)
        if nCand == 0:
            raise RuntimeError("No PSF candidates supplied.")
        #
        # How big should our PSF models be?
        #
        if display:  # only needed for debug plots
            # construct and populate a spatial cell set
            bbox = mi.getBBox(afwImage.PARENT)
            psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX,
                                                self.config.sizeCellY)
        else:
            psfCellSet = None

        sizes = np.empty(nCand)
        for i, psfCandidate in enumerate(psfCandidateList):
            try:
                if psfCellSet:
                    psfCellSet.insertCandidate(psfCandidate)
            except Exception as e:
                self.log.debug("Skipping PSF candidate %d of %d: %s", i,
                               len(psfCandidateList), e)
                continue

            source = psfCandidate.getSource()
            quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(),
                                     source.getIxy())
            rmsSize = quad.getTraceRadius()
            sizes[i] = rmsSize

        if self.config.kernelSize >= 15:
            self.log.warn(
                "NOT scaling kernelSize by stellar quadrupole moment, but using absolute value"
            )
            actualKernelSize = int(self.config.kernelSize)
        else:
            actualKernelSize = 2 * int(self.config.kernelSize *
                                       np.sqrt(np.median(sizes)) + 0.5) + 1
            if actualKernelSize < self.config.kernelSizeMin:
                actualKernelSize = self.config.kernelSizeMin
            if actualKernelSize > self.config.kernelSizeMax:
                actualKernelSize = self.config.kernelSizeMax
            if display:
                rms = np.median(sizes)
                print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" %
                      (rms, 2 * np.sqrt(2 * np.log(2)) * rms))

        # If we manually set the resolution then we need the size in pixel units
        pixKernelSize = actualKernelSize
        if self.config.samplingSize > 0:
            pixKernelSize = int(actualKernelSize * self.config.samplingSize)
            if pixKernelSize % 2 == 0:
                pixKernelSize += 1
        self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f",
                       actualKernelSize, pixKernelSize)
        psfCandidateList[0].setHeight(pixKernelSize)
        psfCandidateList[0].setWidth(pixKernelSize)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
        #
        # Insert the good candidates into the set
        #
        defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"],
                                    "config", "default-lsst.psfex")
        args_md = dafBase.PropertySet()
        args_md.set("BASIS_TYPE", str(self.config.psfexBasis))
        args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
        args_md.set("PSF_SIZE", str(actualKernelSize))
        args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
        prefs = psfex.Prefs(defaultsFile, args_md)
        prefs.setCommandLine([])
        prefs.addCatalog("psfexPsfDeterminer")

        prefs.use()
        principalComponentExclusionFlag = bool(
            bool(psfex.Context.REMOVEHIDDEN) if False else psfex.Context.
            KEEPHIDDEN)
        context = psfex.Context(prefs.getContextName(),
                                prefs.getContextGroup(), prefs.getGroupDeg(),
                                principalComponentExclusionFlag)
        set = psfex.Set(context)
        set.setVigSize(pixKernelSize, pixKernelSize)
        set.setFwhm(2 * np.sqrt(2 * np.log(2)) * np.median(sizes))
        set.setRecentroid(self.config.recentroid)

        catindex, ext = 0, 0
        backnoise2 = afwMath.makeStatistics(mi.getImage(),
                                            afwMath.VARIANCECLIP).getValue()
        ccd = exposure.getDetector()
        if ccd:
            gain = np.mean(np.array([a.getGain() for a in ccd]))
        else:
            gain = 1.0
            self.log.warn("Setting gain to %g" % (gain, ))

        contextvalp = []
        for i, key in enumerate(context.getName()):
            if context.getPcflag(i):
                contextvalp.append(pcval[pc])
                pc += 1
            elif key[0] == ':':
                try:
                    contextvalp.append(exposure.getMetadata().get(key[1:]))
                except KeyError:
                    raise RuntimeError(
                        "*Error*: %s parameter not found in the header of %s" %
                        (key[1:], prefs.getContextName()))
            else:
                try:
                    contextvalp.append(
                        np.array([
                            psfCandidateList[_].getSource().get(key)
                            for _ in range(nCand)
                        ]))
                except KeyError:
                    raise RuntimeError("*Error*: %s parameter not found" %
                                       (key, ))
                set.setContextname(i, key)

        if display:
            frame = 0
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")

        badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
        fluxName = prefs.getPhotfluxRkey()
        fluxFlagName = "base_" + fluxName + "_flag"

        xpos, ypos = [], []
        for i, psfCandidate in enumerate(psfCandidateList):
            source = psfCandidate.getSource()
            xc, yc = source.getX(), source.getY()
            try:
                int(xc), int(yc)
            except ValueError:
                continue

            try:
                pstamp = psfCandidate.getMaskedImage().clone()
            except Exception as e:
                continue

            if fluxFlagName in source.schema and source.get(fluxFlagName):
                continue

            flux = source.get(fluxName)
            if flux < 0 or np.isnan(flux):
                continue

            # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate).
            # Having created the sample, we must proceed to configure it, and then fini (finalize),
            # or it will be malformed.
            try:
                sample = set.newSample()
                sample.setCatindex(catindex)
                sample.setExtindex(ext)
                sample.setObjindex(i)

                imArray = pstamp.getImage().getArray()
                imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
                    -2*psfex.BIG
                sample.setVig(imArray)

                sample.setNorm(flux)
                sample.setBacknoise2(backnoise2)
                sample.setGain(gain)
                sample.setX(xc)
                sample.setY(yc)
                sample.setFluxrad(sizes[i])

                for j in range(set.getNcontext()):
                    sample.setContext(j, float(contextvalp[j][i]))
            except Exception as e:
                self.log.debug(
                    "Exception when processing sample at (%f,%f): %s", xc, yc,
                    e)
                continue
            else:
                set.finiSample(sample)

            xpos.append(xc)  # for QA
            ypos.append(yc)

        if displayExposure:
            with ds9.Buffering():
                ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame)

        if set.getNsample() == 0:
            raise RuntimeError("No good PSF candidates to pass to PSFEx")

        #---- Update min and max and then the scaling
        for i in range(set.getNcontext()):
            cmin = contextvalp[i].min()
            cmax = contextvalp[i].max()
            set.setContextScale(i, cmax - cmin)
            set.setContextOffset(i, (cmin + cmax) / 2.0)

        # Don't waste memory!
        set.trimMemory()

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
        #
        # Do a PSFEX decomposition of those PSF candidates
        #
        fields = []
        field = psfex.Field("Unknown")
        field.addExt(exposure.getWcs(), exposure.getWidth(),
                     exposure.getHeight(), set.getNsample())
        field.finalize()

        fields.append(field)

        sets = []
        sets.append(set)

        psfex.makeit(fields, sets)
        psfs = field.getPsfs()

        # Flag which objects were actually used in psfex by
        good_indices = []
        for i in range(sets[0].getNsample()):
            index = sets[0].getSample(i).getObjindex()
            if index > -1:
                good_indices.append(index)

        if flagKey is not None:
            for i, psfCandidate in enumerate(psfCandidateList):
                source = psfCandidate.getSource()
                if i in good_indices:
                    source.set(flagKey, True)

        xpos = np.array(xpos)
        ypos = np.array(ypos)
        numGoodStars = len(good_indices)
        avgX, avgY = np.mean(xpos), np.mean(ypos)

        psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY))

        if False and (displayResiduals or displayPsfMosaic):
            ext = 0
            frame = 1
            diagnostics = True
            catDir = "."
            title = "psfexPsfDeterminer"
            psfex.psfex.showPsf(psfs,
                                set,
                                ext, [(exposure.getWcs(), exposure.getWidth(),
                                       exposure.getHeight())],
                                nspot=3,
                                trim=5,
                                frame=frame,
                                diagnostics=diagnostics,
                                outDir=catDir,
                                title=title)
        #
        # Display code for debugging
        #
        if display:
            assert psfCellSet is not None

            if displayExposure:
                maUtils.showPsfSpatialCells(exposure,
                                            psfCellSet,
                                            showChi2=True,
                                            symb="o",
                                            ctype=ds9.YELLOW,
                                            ctypeBad=ds9.RED,
                                            size=8,
                                            frame=frame)
            if displayResiduals:
                maUtils.showPsfCandidates(exposure,
                                          psfCellSet,
                                          psf=psf,
                                          frame=4,
                                          normalize=normalizeResiduals,
                                          showBadCandidates=showBadCandidates)
            if displayPsfComponents:
                maUtils.showPsf(psf, frame=6)
            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
                ds9.scale('linear', 0, 1, frame=7)
        #
        # Generate some QA information
        #
        # Count PSF stars
        #
        if metadata is not None:
            metadata.set("spatialFitChi2", np.nan)
            metadata.set("numAvailStars", nCand)
            metadata.set("numGoodStars", numGoodStars)
            metadata.set("avgX", avgX)
            metadata.set("avgY", avgY)

        psfCellSet = None
        return psf, psfCellSet
Beispiel #12
0
    def testStretch(self):
        """Test playing with the lookup table"""
        ds9.show()

        ds9.scale("linear", "zscale")
Beispiel #13
0
    def runDataRef(self, sensorRef):
        self.log.log(self.log.INFO, "Performing ISR on sensor %s" % (sensorRef.dataId))
        ccdExposure = sensorRef.get('raw')

        if self.config.removePcCards: # Remove any PC00N00M cards in the header
            raw_md = sensorRef.get("raw_md")
            nPc = 0
            for i in (1, 2,):
                for j in (1, 2,):
                    k = "PC%03d%03d" % (i, j)
                    for md in (raw_md, ccdExposure.getMetadata()):
                        if md.exists(k):
                            md.remove(k)
                            nPc += 1

            if nPc:
                self.log.log(self.log.INFO, "Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId))
                ccdExposure.setWcs(afwImage.makeWcs(raw_md))

        ccdExposure = self.convertIntToFloat(ccdExposure)
        ccd = ccdExposure.getDetector()

        # Read in defects to check for any dead amplifiers (entire amp is within defect region)
        defectsRaw = sensorRef.get("defects", immediate=True)
        # Need to rotate defects bbox if we are dealing with a rotated ccd as they are defined assuming
        # that (0, 0) is the lower-left corner (LLC).  Rotation needs to be done in trimmed/assembled image
        # dimensions as that is the frame for which the defects are defined.  This is computed here as
        # bboxTrimmed.  Also need to accommodate even and odd number of 90deg rotations.
        bboxTrimmed = afwGeom.Box2I()
        for amp in ccd:
            bboxTrimmed.include(amp.getBBox())
        nQuarter = ccd.getOrientation().getNQuarter()
        defects = []
        if nQuarter != 0:
            for v in defectsRaw:
                rotBox = afwImage.imageLib.DefectBase(
                    afwCG.rotateBBoxBy90(v.getBBox(), 4 - nQuarter,
                                         afwGeom.Extent2I(bboxTrimmed.getHeight(), bboxTrimmed.getHeight())))
                # We have rotated about the center of a square with ccd height dimension on a side.
                # rotateBBoxBy90 rotates CCW, so using 4 - nQuarter we are effectively rotating
                # CW nQuarter turns.  So, for nQuarter = 2 or 3, the rotated LLC will be shifted
                # in x by width - height pixels wrt the LLC of the rotation square.  Thus defects
                # for all nQuarter > 1 ccds need to be shifted back in x by this amount.
                if nQuarter > 1:
                    shiftBoxX0 = bboxTrimmed.getWidth() - bboxTrimmed.getHeight()
                    shiftBoxY0 = 0
                    rotBox.shift(afwGeom.Extent2I(shiftBoxX0, shiftBoxY0))
                defects.append(rotBox)
        else:
            defects = defectsRaw

        for amp in ccd:
            # Check if entire amp region is defined as defect (need to use amp.getBBox() for correct
            # comparison with current defects definition.
            badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
            # In the case of bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
            # association with pixels in current ccdExposure).
            if badAmp:
                dataView = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawBBox(),
                                                 afwImage.PARENT)
                maskView = dataView.getMask()
                maskView |= maskView.getPlaneBitMask("BAD")
                del maskView

            self.measureOverscan(ccdExposure, amp)
            if self.config.doSaturation and not badAmp:
                self.saturationDetection(ccdExposure, amp)
            if self.config.doOverscan and not badAmp:
                ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
                                                 afwImage.PARENT)
                overscan = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
                                                 amp.getRawHorizontalOverscanBBox(), afwImage.PARENT)
                overscanArray = overscan.getImage().getArray()
                median = numpy.ma.median(numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray))
                bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
                overscan.getMask().getArray()[bad] = overscan.getMask().getPlaneBitMask("SAT")

                statControl = afwMath.StatisticsControl()
                statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
                lsstIsr.overscanCorrection(ampMaskedImage=ampImage, overscanImage=overscan,
                                           fitType=self.config.overscanFitType,
                                           order=self.config.overscanOrder,
                                           collapseRej=self.config.overscanRej,
                                           statControl=statControl,
                                   )

            if self.config.doVariance:
                # Ideally, this should be done after bias subtraction,
                # but CCD assembly demands a variance plane
                ampExposure = ccdExposure.Factory(ccdExposure, amp.getRawDataBBox(), afwImage.PARENT)
                self.updateVariance(ampExposure, amp)

        ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
        ccd = ccdExposure.getDetector()

        doRotateCalib = False   # Rotate calib images for bias/dark/flat correction?
        if nQuarter != 0:
            doRotateCalib = True

        if self.config.doDefect:
            self.maskAndInterpDefect(ccdExposure, defects)

        if self.config.qa.doWriteOss:
            sensorRef.put(ccdExposure, "ossImage")
        if self.config.qa.doThumbnailOss:
            self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)

        if self.config.doBias:
            biasExposure = self.getIsrExposure(sensorRef, "bias")
            if not doRotateCalib:
                self.biasCorrection(ccdExposure, biasExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.biasCorrection(exp, biasExposure)
        if self.config.doLinearize:
            self.linearize(ccdExposure)
        if self.config.doCrosstalk:
            self.crosstalk.run(ccdExposure)

        if self.config.doBrighterFatter:
            brighterFatterKernel = sensorRef.get('bfKernel')
            self.brighterFatterCorrection(ccdExposure, brighterFatterKernel,
                                          self.config.brighterFatterMaxIter,
                                          self.config.brighterFatterThreshold,
                                          self.config.brighterFatterApplyGain,
                                          )
        if self.config.doDark:
            darkExposure = self.getIsrExposure(sensorRef, "dark")
            if not doRotateCalib:
                self.darkCorrection(ccdExposure, darkExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.darkCorrection(exp, darkExposure)
        if self.config.doFlat:
            flatExposure = self.getIsrExposure(sensorRef, "flat")
            if not doRotateCalib:
                self.flatCorrection(ccdExposure, flatExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.flatCorrection(exp, flatExposure)

        if self.config.doApplyGains:
            self.applyGains(ccdExposure, self.config.normalizeGains)
        if self.config.doWidenSaturationTrails:
            self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
        if self.config.doSaturation:
            self.saturationInterpolation(ccdExposure)

        if self.config.doFringe:
            self.fringe.runDataRef(ccdExposure, sensorRef)
        if self.config.doSetBadRegions:
            self.setBadRegions(ccdExposure)

        self.maskAndInterpNan(ccdExposure)

        if self.config.qa.doWriteFlattened:
            sensorRef.put(ccdExposure, "flattenedImage")
        if self.config.qa.doThumbnailFlattened:
            self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)

        self.measureBackground(ccdExposure)

        if self.config.doGuider:
            self.guider(ccdExposure)

        self.roughZeroPoint(ccdExposure)

        if self.config.doWriteVignettePolygon:
            self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)

        if self.config.doWrite:
            sensorRef.put(ccdExposure, "postISRCCD")

        if self._display:
            im = ccdExposure.getMaskedImage().getImage()
            im_median = float(numpy.median(im.getArray()))
            ds9.mtv(im)
            ds9.scale(min=im_median*0.95, max=im_median*1.15)

        return Struct(exposure=ccdExposure)
Beispiel #14
0
    def runDataRef(self, sensorRef):
        self.log.log(self.log.INFO, "Performing ISR on sensor %s" % (sensorRef.dataId))
        ccdExposure = sensorRef.get("raw")

        if self.config.removePcCards:  # Remove any PC00N00M cards in the header
            raw_md = sensorRef.get("raw_md")
            nPc = 0
            for i in (1, 2):
                for j in (1, 2):
                    k = "PC%03d%03d" % (i, j)
                    for md in (raw_md, ccdExposure.getMetadata()):
                        if md.exists(k):
                            md.remove(k)
                            nPc += 1

            if nPc:
                self.log.log(self.log.INFO, "Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId))
                ccdExposure.setWcs(afwImage.makeWcs(raw_md))

        ccdExposure = self.convertIntToFloat(ccdExposure)
        ccd = ccdExposure.getDetector()

        for amp in ccd:
            self.measureOverscan(ccdExposure, amp)
            if self.config.doSaturation:
                self.saturationDetection(ccdExposure, amp)
            if self.config.doOverscan:
                ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(), afwImage.PARENT)
                overscan = afwImage.MaskedImageF(
                    ccdExposure.getMaskedImage(), amp.getRawHorizontalOverscanBBox(), afwImage.PARENT
                )
                overscanArray = overscan.getImage().getArray()
                median = numpy.ma.median(numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray))
                bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
                overscan.getMask().getArray()[bad] = overscan.getMask().getPlaneBitMask("SAT")

                statControl = afwMath.StatisticsControl()
                statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
                lsstIsr.overscanCorrection(
                    ampMaskedImage=ampImage,
                    overscanImage=overscan,
                    fitType=self.config.overscanFitType,
                    order=self.config.overscanOrder,
                    collapseRej=self.config.overscanRej,
                    statControl=statControl,
                )

            if self.config.doVariance:
                # Ideally, this should be done after bias subtraction,
                # but CCD assembly demands a variance plane
                ampExposure = ccdExposure.Factory(ccdExposure, amp.getRawDataBBox(), afwImage.PARENT)
                self.updateVariance(ampExposure, amp)

        ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
        ccd = ccdExposure.getDetector()

        doRotateCalib = False  # Rotate calib images for bias/dark/flat correction?
        nQuarter = ccd.getOrientation().getNQuarter()
        if nQuarter != 0:
            doRotateCalib = True

        if self.config.doDefect:
            defects = sensorRef.get("defects", immediate=True)
            self.maskAndInterpDefect(ccdExposure, defects)

        if self.config.qa.doWriteOss:
            sensorRef.put(ccdExposure, "ossImage")
        if self.config.qa.doThumbnailOss:
            self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)

        if self.config.doBias:
            biasExposure = self.getIsrExposure(sensorRef, "bias")
            if not doRotateCalib:
                self.biasCorrection(ccdExposure, biasExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.biasCorrection(exp, biasExposure)
        if self.config.doLinearize:
            self.linearize(ccdExposure)
        if self.config.doCrosstalk:
            self.crosstalk.run(ccdExposure)
        if self.config.doDark:
            darkExposure = self.getIsrExposure(sensorRef, "dark")
            if not doRotateCalib:
                self.darkCorrection(ccdExposure, darkExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.darkCorrection(exp, darkExposure)
        if self.config.doFlat:
            flatExposure = self.getIsrExposure(sensorRef, "flat")
            if not doRotateCalib:
                self.flatCorrection(ccdExposure, flatExposure)
            else:
                with self.rotated(ccdExposure) as exp:
                    self.flatCorrection(exp, flatExposure)

        if self.config.doApplyGains:
            self.applyGains(ccdExposure, self.config.normalizeGains)
        if self.config.doWidenSaturationTrails:
            self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
        if self.config.doSaturation:
            self.saturationInterpolation(ccdExposure)

        if self.config.doFringe:
            self.fringe.runDataRef(ccdExposure, sensorRef)
        if self.config.doSetBadRegions:
            self.setBadRegions(ccdExposure)

        self.maskAndInterpNan(ccdExposure)

        if self.config.qa.doWriteFlattened:
            sensorRef.put(ccdExposure, "flattenedImage")
        if self.config.qa.doThumbnailFlattened:
            self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)

        self.measureBackground(ccdExposure)

        if self.config.doGuider:
            self.guider(ccdExposure)

        self.roughZeroPoint(ccdExposure)

        if self.config.doWriteVignettePolygon:
            self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)

        if self.config.doWrite:
            sensorRef.put(ccdExposure, "postISRCCD")

        if self._display:
            im = ccdExposure.getMaskedImage().getImage()
            im_median = float(numpy.median(im.getArray()))
            ds9.mtv(im)
            ds9.scale(min=im_median * 0.95, max=im_median * 1.15)

        return Struct(exposure=ccdExposure)
    def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
        """Determine a PSFEX PSF model for an exposure given a list of PSF candidates

        @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure)
        @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
            typically obtained by detecting sources and then running them through a star selector
        @param[in,out] metadata  a home for interesting tidbits of information
        @param[in] flagKey: schema key used to mark sources actually used in PSF determination

        @return psf: a meas.extensions.psfex.PsfexPsf
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayExposure = display and \
            lsstDebug.Info(__name__).displayExposure      # display the Exposure + spatialCells
        displayPsfComponents = display and \
            lsstDebug.Info(__name__).displayPsfComponents  # show the basis functions
        showBadCandidates = display and \
            lsstDebug.Info(__name__).showBadCandidates    # Include bad candidates (meaningless, methinks)
        displayResiduals = display and \
            lsstDebug.Info(__name__).displayResiduals     # show residuals
        displayPsfMosaic = display and \
            lsstDebug.Info(__name__).displayPsfMosaic     # show mosaic of reconstructed PSF(x,y)
        normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
        # Normalise residuals by object amplitude

        mi = exposure.getMaskedImage()

        nCand = len(psfCandidateList)
        if nCand == 0:
            raise RuntimeError("No PSF candidates supplied.")
        #
        # How big should our PSF models be?
        #
        if display:                     # only needed for debug plots
            # construct and populate a spatial cell set
            bbox = mi.getBBox(afwImage.PARENT)
            psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
        else:
            psfCellSet = None

        sizes = np.empty(nCand)
        for i, psfCandidate in enumerate(psfCandidateList):
            try:
                if psfCellSet:
                    psfCellSet.insertCandidate(psfCandidate)
            except Exception as e:
                self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
                continue

            source = psfCandidate.getSource()
            quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
            rmsSize = quad.getTraceRadius()
            sizes[i] = rmsSize

        if self.config.kernelSize >= 15:
            self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value")
            actualKernelSize = int(self.config.kernelSize)
        else:
            actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1
            if actualKernelSize < self.config.kernelSizeMin:
                actualKernelSize = self.config.kernelSizeMin
            if actualKernelSize > self.config.kernelSizeMax:
                actualKernelSize = self.config.kernelSizeMax
            if display:
                rms = np.median(sizes)
                print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms))

        # If we manually set the resolution then we need the size in pixel units
        pixKernelSize = actualKernelSize
        if self.config.samplingSize > 0:
            pixKernelSize = int(actualKernelSize*self.config.samplingSize)
            if pixKernelSize % 2 == 0:
                pixKernelSize += 1
        self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize)
        psfCandidateList[0].setHeight(pixKernelSize)
        psfCandidateList[0].setWidth(pixKernelSize)

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX
        #
        # Insert the good candidates into the set
        #
        defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex")
        args_md = dafBase.PropertySet()
        args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder))
        args_md.set("PSF_SIZE", str(actualKernelSize))
        args_md.set("PSF_SAMPLING", str(self.config.samplingSize))
        prefs = psfex.Prefs(defaultsFile, args_md)
        prefs.setCommandLine([])
        prefs.addCatalog("psfexPsfDeterminer")

        prefs.use()
        principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN)
                                               if False else psfex.Context.KEEPHIDDEN)
        context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(),
                                prefs.getGroupDeg(), principalComponentExclusionFlag)
        set = psfex.Set(context)
        set.setVigSize(pixKernelSize, pixKernelSize)
        set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes))
        set.setRecentroid(self.config.recentroid)

        catindex, ext = 0, 0
        backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
        ccd = exposure.getDetector()
        if ccd:
            gain = np.mean(np.array([a.getGain() for a in ccd]))
        else:
            gain = 1.0
            self.log.warn("Setting gain to %g" % (gain,))

        contextvalp = []
        for i, key in enumerate(context.getName()):
            if context.getPcflag(i):
                contextvalp.append(pcval[pc])
                pc += 1
            elif key[0] == ':':
                try:
                    contextvalp.append(exposure.getMetadata().get(key[1:]))
                except KeyError:
                    raise RuntimeError("*Error*: %s parameter not found in the header of %s" %
                                       (key[1:], prefs.getContextName()))
            else:
                try:
                    contextvalp.append(np.array([psfCandidateList[_].getSource().get(key)
                                                 for _ in range(nCand)]))
                except KeyError:
                    raise RuntimeError("*Error*: %s parameter not found" % (key,))
                set.setContextname(i, key)

        if display:
            frame = 0
            if displayExposure:
                ds9.mtv(exposure, frame=frame, title="psf determination")

        badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits)
        fluxName = prefs.getPhotfluxRkey()
        fluxFlagName = "base_" + fluxName + "_flag"

        xpos, ypos = [], []
        for i, psfCandidate in enumerate(psfCandidateList):
            source = psfCandidate.getSource()
            xc, yc = source.getX(), source.getY()
            try:
                int(xc), int(yc)
            except ValueError:
                continue

            try:
                pstamp = psfCandidate.getMaskedImage().clone()
            except Exception as e:
                continue

            if fluxFlagName in source.schema and source.get(fluxFlagName):
                continue

            flux = source.get(fluxName)
            if flux < 0 or np.isnan(flux):
                continue

            # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate).
            # Having created the sample, we must proceed to configure it, and then fini (finalize),
            # or it will be malformed.
            try:
                sample = set.newSample()
                sample.setCatindex(catindex)
                sample.setExtindex(ext)
                sample.setObjindex(i)

                imArray = pstamp.getImage().getArray()
                imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \
                    -2*psfex.cvar.BIG
                sample.setVig(imArray)

                sample.setNorm(flux)
                sample.setBacknoise2(backnoise2)
                sample.setGain(gain)
                sample.setX(xc)
                sample.setY(yc)
                sample.setFluxrad(sizes[i])

                for j in range(set.getNcontext()):
                    sample.setContext(j, float(contextvalp[j][i]))
            except Exception as e:
                self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e)
                continue
            else:
                set.finiSample(sample)

            xpos.append(xc)  # for QA
            ypos.append(yc)

        if displayExposure:
            with ds9.Buffering():
                ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame)

        if set.getNsample() == 0:
            raise RuntimeError("No good PSF candidates to pass to PSFEx")

        #---- Update min and max and then the scaling
        for i in range(set.getNcontext()):
            cmin = contextvalp[i].min()
            cmax = contextvalp[i].max()
            set.setContextScale(i, cmax - cmin)
            set.setContextOffset(i, (cmin + cmax)/2.0)

        # Don't waste memory!
        set.trimMemory()

        #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX
        #
        # Do a PSFEX decomposition of those PSF candidates
        #
        fields = psfex.vectorField()
        field = psfex.Field("Unknown")
        field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample())
        field.finalize()

        fields.append(field)

        sets = psfex.vectorSet()
        sets.append(set)

        psfex.makeit(fields, sets)
        psfs = field.getPsfs()

        # Flag which objects were actually used in psfex by
        good_indices = []
        for i in range(sets[0].getNsample()):
            index = sets[0].getSample(i).getObjindex()
            if index > -1:
                good_indices.append(index)

        if flagKey is not None:
            for i, psfCandidate in enumerate(psfCandidateList):
                source = psfCandidate.getSource()
                if i in good_indices:
                    source.set(flagKey, True)

        xpos = np.array(xpos)
        ypos = np.array(ypos)
        numGoodStars = len(good_indices)
        avgX, avgY = np.mean(xpos), np.mean(ypos)

        psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY))

        if False and (displayResiduals or displayPsfMosaic):
            ext = 0
            frame = 1
            diagnostics = True
            catDir = "."
            title = "psfexPsfDeterminer"
            psfex.psfex.showPsf(psfs, set, ext,
                                [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())],
                                nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir,
                                title=title)
        #
        # Display code for debugging
        #
        if display:
            assert psfCellSet is not None

            if displayExposure:
                maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True,
                                            symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame)
            if displayResiduals:
                maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4,
                                          normalize=normalizeResiduals,
                                          showBadCandidates=showBadCandidates)
            if displayPsfComponents:
                maUtils.showPsf(psf, frame=6)
            if displayPsfMosaic:
                maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True)
                ds9.scale('linear', 0, 1, frame=7)
        #
        # Generate some QA information
        #
        # Count PSF stars
        #
        if metadata is not None:
            metadata.set("spatialFitChi2", np.nan)
            metadata.set("numAvailStars", nCand)
            metadata.set("numGoodStars", numGoodStars)
            metadata.set("avgX", avgX)
            metadata.set("avgY", avgY)

        psfCellSet = None
        return psf, psfCellSet