Esempio n. 1
0
def main():
    
    mimg          = afwImage.MaskedImageF(afwGeom.Extent2I(100, 100))
    mimValue      = (2, 0x0, 1)
    mimg.set(mimValue)

    # call with the factory function ... should get stats on the image plane
    fmt = "%-40s %-16s %3.1f\n"
    print fmt % ("Using makeStatistics:",  "(should be " + str(mimValue[0]) + ")",
                 afwMath.makeStatistics(mimg, afwMath.MEAN).getValue()),
    
    # call the constructor directly ... once for image plane, then for variance
    # - make sure we're not using weighted stats for this
    sctrl = afwMath.StatisticsControl()
    sctrl.setWeighted(False)
    print fmt % ("Using Statistics on getImage():", "(should be " + str(mimValue[0]) + ")",
                 afwMath.StatisticsF(mimg.getImage(), mimg.getMask(), mimg.getVariance(),
                                     afwMath.MEAN, sctrl).getValue()),
    print fmt % ("Using Statistics on getVariance():", "(should be " + str(mimValue[2]) + ")",
                 afwMath.StatisticsF(mimg.getVariance(), mimg.getMask(), mimg.getVariance(),
                                     afwMath.MEAN, sctrl).getValue()),

    # call makeStatistics as a front-end for the constructor
    print fmt % ("Using makeStatistics on getImage():", "(should be " + str(mimValue[0]) + ")",
                 afwMath.makeStatistics(mimg.getImage(), mimg.getMask(), afwMath.MEAN).getValue()),
    print fmt % ("Using makeStatistics on getVariance():", "(should be " + str(mimValue[2]) + ")",
                 afwMath.makeStatistics(mimg.getVariance(), mimg.getMask(), afwMath.MEAN).getValue()),
Esempio n. 2
0
    def testRunDataRef(self, pedestal=0.0, stddevMax=1.0e-4):
        """Test the .runDataRef method for complete test converage.

        Paramters
        ---------
        pedestal : `float`, optional
           Pedestal to add into fringe frame.
        stddevMax : `float`, optional
           Maximum allowable standard deviation.
        """
        xFreq = np.pi/10.0
        xOffset = 1.0
        yFreq = np.pi/15.0
        yOffset = 0.5
        scale = 1.0
        fringe = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
        fMi = fringe.getMaskedImage()
        fMi += pedestal
        exp = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
        eMi = exp.getMaskedImage()
        eMi *= scale

        task = FringeTask(name="fringe", config=self.config)
        dataRef = FringeDataRef(fringe)
        task.runDataRef(exp, dataRef)

        mi = exp.getMaskedImage()
        mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
        self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), stddevMax)
Esempio n. 3
0
    def checkFringe(self, task, exp, fringes, precision):
        """Run fringe subtraction and verify

        @param task         Task to run
        @param exp          Science exposure
        @param dataRef      Data reference that will provide the fringes
        @param precision    Precision for assertAlmostEqual
        """
        if debug:
            global frame
            ds9.mtv(exp, frame=frame, title="Science exposure")
            frame += 1
            fringe = dataRef.get()
            if not isinstance(fringe, list):
                fringe = [fringe]
            for i, f in enumerate(fringe):
                ds9.mtv(f, frame=frame, title="Fringe frame %d" % (i+1))
                frame += 1

        task.run(exp, fringes)

        mi = exp.getMaskedImage()

        if debug:
            ds9.mtv(exp, frame=frame, title="Subtracted")
            frame += 1

        mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
        self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), precision)
    def testNaN(self):

        # get the stats for the image with two values
        stats = afwMath.makeStatistics(
            self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
        mean = 0.5*(self.valL + self.valR)
        nL, nR = self.mimgL.getWidth()*self.mimgL.getHeight(), self.mimgR.getWidth() * \
            self.mimgR.getHeight()
        stdev = ((nL*(self.valL - mean)**2 + nR *
                  (self.valR - mean)**2)/(nL + nR - 1))**0.5

        self.assertEqual(stats.getValue(afwMath.NPOINT), self.n)
        self.assertEqual(stats.getValue(afwMath.MEAN), mean)
        self.assertEqual(stats.getValue(afwMath.STDEV), stdev)

        # set the right side to NaN and stats should be just for the left side
        self.mimgR.set(np.nan, 0x0, self.valR)

        statsNaN = afwMath.makeStatistics(
            self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
        mean = self.valL
        stdev = 0.0
        self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL)
        self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean)
        self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
Esempio n. 5
0
    def testRunDataRef(self, pedestal=0.0, precision=1.0e-4):
        """Test the .runDataRef method for complete test converage

        @param pedestal    Pedestal to add into fringe frame
        @param precision   Precision for assertAlmostEqual
        """
        xFreq = numpy.pi / 10.0
        xOffset = 1.0
        yFreq = numpy.pi / 15.0
        yOffset = 0.5
        scale = 1.0
        fringe = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
        fMi = fringe.getMaskedImage()
        fMi += pedestal
        exp = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
        eMi = exp.getMaskedImage()
        eMi *= scale

        task = FringeTask(name="fringe", config=self.config)
        dataRef = FringeDataRef(fringe)
        task.runDataRef(exp, dataRef)

        mi = exp.getMaskedImage()
        mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
        self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), precision)
Esempio n. 6
0
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0):
    """Apply flat correction in place

    @param[in,out] maskedImage  afw.image.MaskedImage to correct
    @param[in] flatMaskedImage  flat field afw.image.MaskedImage
    @param[in] scalingType  how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
    @param[in] userScale  scale to use if scalingType is 'USER', else ignored
    """
    if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
        raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" % \
            (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))

    # Figure out scale from the data
    # Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
    # in the case that the flat is created by some other mechanism.
    if scalingType == 'MEAN':
        flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEAN).getValue(afwMath.MEAN)
    elif scalingType == 'MEDIAN':
        flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEDIAN).getValue(afwMath.MEDIAN)
    elif scalingType == 'USER':
        flatScale = userScale
    else:
        raise pexExcept.Exception, '%s : %s not implemented' % ("flatCorrection", scalingType)

    maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
Esempio n. 7
0
def cr(infn, crfn, maskfn):
	exposure = afwImage.ExposureF(infn) #'850994p-21.fits'
	print 'exposure', exposure
	print 'w,h', exposure.getWidth(), exposure.getHeight()
	W,H = exposure.getWidth(), exposure.getHeight()

	#var = exposure.getMaskedImage().getVariance()
	#print 'Variance', var.get(0,0)

	wcs = exposure.getWcs()
	print 'wcs', wcs
	pixscale = wcs.pixelScale().asArcseconds()
	psf = getFakePsf(pixscale)

	# CRs
	mask = exposure.getMaskedImage().getMask()
	crBit = mask.getMaskPlane("CR")
	mask.clearMaskPlane(crBit)
	mi = exposure.getMaskedImage()
	bg = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
	print 'bg', bg

	varval = afwMath.makeStatistics(mi, afwMath.VARIANCE).getValue()
	print 'variance', varval, 'std', math.sqrt(varval)
	varval = afwMath.makeStatistics(mi, afwMath.VARIANCECLIP).getValue()
	print 'clipped variance', varval, 'std', math.sqrt(varval)
	var = exposure.getMaskedImage().getVariance()
	var.set(varval)

	var = exposure.getMaskedImage().getVariance()
	print 'Variance:', var.get(0,0)

	keepCRs = False
	policy = pexPolicy.Policy()
	# policy.add('minSigma', 6.)
	# policy.add('min_DN', 150.)
	# policy.add('cond3_fac', 2.5)
	# policy.add('cond3_fac2', 0.6)
	# policy.add('niteration', 3)
	# policy.add('nCrPixelMax', 200000)

	policy.add('minSigma', 10.)
	policy.add('min_DN', 500.)
	policy.add('cond3_fac', 2.5)
	policy.add('cond3_fac2', 0.6)
	policy.add('niteration', 1)
	policy.add('nCrPixelMax', 100000)

	#psfimg = psf.computeImage(afwGeom.Point2D(W/2., H/2.))
	#psfimg.writeFits('psf.fits')

	print 'Finding cosmics...'
	crs = measAlg.findCosmicRays(mi, psf, bg, policy, keepCRs)
	print 'got', len(crs), 'cosmic rays',

	mask = mi.getMask()
	crBit = mask.getPlaneBitMask("CR")
	afwDet.setMaskFromFootprintList(mask, crs, crBit)
	mask.writeFits(maskfn)
	exposure.writeFits(crfn)
 def __init__(self, img):
     """
     img: an afwImage.ImageF
     """
     self.mim = afwImage.MaskedImageF(img)
     self.mean = afwMath.makeStatistics(img, afwMath.MEAN)
     self.std = afwMath.makeStatistics(img, afwMath.STDEV)
Esempio n. 9
0
    def testComputeImage(self):
        """Test the computation of the PSF's image at a point"""

        ccdXY = afwGeom.Point2D(0, 0)
        kIm = self.psf.computeImage(ccdXY)

        if False:
            ds9.mtv(kIm)        

        self.assertTrue(kIm.getWidth() == self.ksize)
        #
        # Check that the image is as expected. 
        #
        xcen, ycen = self.ksize/2, self.ksize/2
        I0 = kIm.get(xcen, ycen)
        self.assertAlmostEqual(kIm.get(xcen + 1, ycen + 1),
                               I0*self.psf.computeImage().get(xcen + 1, ycen + 1))
        #
        # Is image normalised to a peak value of unity?
        #
        self.assertAlmostEqual(afwMath.makeStatistics(kIm, afwMath.MAX).getValue(), 1.0)
        #
        # Now create a normalised version
        #
        kIm = self.psf.computeImage(ccdXY, False)

        self.assertAlmostEqual(afwMath.makeStatistics(kIm, afwMath.SUM).getValue(), 1.0)
Esempio n. 10
0
def main():
    
    gaussFunction = afwMath.GaussianFunction2D(3, 2, 0.5)
    gaussKernel   = afwMath.AnalyticKernel(10, 10, gaussFunction)
    inImage       = afwImage.ImageF(afwGeom.Extent2I(100, 100))
    inImage.set(1)
    if disp:
        ds9.mtv(inImage, frame = 0)
    
    # works
    outImage      = afwImage.ImageF(afwGeom.Extent2I(100, 100))
    afwMath.convolve(outImage, inImage, gaussKernel, False, True)
    if disp:
        ds9.mtv(outImage, frame = 1)
    print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.MEAN).getValue()
    print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.STDEV).getValue()
    
    # not works ... now does work
    outImage      = afwImage.ImageF(afwGeom.Extent2I(100, 100))
    afwMath.convolve(outImage, inImage, gaussKernel, False, False)
    if disp:
        ds9.mtv(outImage, frame = 2)
    print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.MEAN).getValue()
    print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.STDEV).getValue()

    # This will print nan
    sctrl = afwMath.StatisticsControl()
    sctrl.setNanSafe(False)
    print ("Should be a nan (nanSafe set to False): " +
           str(afwMath.makeStatistics(outImage, afwMath.MEAN, sctrl).getValue()))
    def testMasked(self):

        # get the stats for the image with two values
        self.mimgR.set(self.valR, 0x0, self.valR)
        stats = afwMath.makeStatistics(
            self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
        mean = 0.5*(self.valL + self.valR)
        nL, nR = self.mimgL.getWidth()*self.mimgL.getHeight(), self.mimgR.getWidth() * \
            self.mimgR.getHeight()
        stdev = ((nL*(self.valL - mean)**2 + nR *
                  (self.valR - mean)**2)/(nL + nR - 1))**0.5

        self.assertEqual(stats.getValue(afwMath.NPOINT), self.n)
        self.assertEqual(stats.getValue(afwMath.MEAN), mean)
        self.assertEqual(stats.getValue(afwMath.STDEV), stdev)

        # set the right side Mask and the StatisticsControl andMask to 0x1
        #  Stats should be just for the left side!
        maskBit = 0x1
        self.mimgR.getMask().set(maskBit)

        sctrl = afwMath.StatisticsControl()
        sctrl.setAndMask(maskBit)
        statsNaN = afwMath.makeStatistics(
            self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl)

        mean = self.valL
        stdev = 0.0

        self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL)
        self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean)
        self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
Esempio n. 12
0
    def _testBadValue(self, badVal):
        """Test that we can handle an instance of `badVal` in the data correctly

        Note that we only test ImageF here (as ImageI can't contain a NaN)
        """
        mean = self.images[0][1]
        x, y = 10, 10
        for useImage in [True, False]:
            if useImage:
                image = afwImage.ImageF(100, 100)
                image.set(mean)
                image[x, y] = badVal
            else:
                image = afwImage.MaskedImageF(100, 100)
                image.set(mean, 0x0, 1.0)
                image[x, y] = (badVal, 0x0, 1.0)

            self.assertEqual(afwMath.makeStatistics(image, afwMath.MAX).getValue(), mean)
            self.assertEqual(afwMath.makeStatistics(image, afwMath.MEAN).getValue(), mean)

            sctrl = afwMath.StatisticsControl()

            sctrl.setNanSafe(False)
            self.assertFalse(np.isfinite(afwMath.makeStatistics(image, afwMath.MAX, sctrl).getValue()))
            self.assertFalse(np.isfinite(afwMath.makeStatistics(image, afwMath.MEAN, sctrl).getValue()))
Esempio n. 13
0
    def testStatsZebra(self):
        """Add 1 to every other row"""
        for image, isInt, mean, median, std in self.images:
            image2 = image.clone()
            #
            # Add 1 to every other row, so the variance is increased by 1/4
            #
            self.assertEqual(image2.getHeight() % 2, 0)
            width = image2.getWidth()
            for y in range(1, image2.getHeight(), 2):
                sim = image2[lsst.geom.Box2I(lsst.geom.Point2I(0, y), lsst.geom.Extent2I(width, 1))]
                sim += 1

            if display:
                afwDisplay.Display(frame=0).mtv(image, "Image 1")
                afwDisplay.Display(frame=1).mtv(image2, "Image 2 (var inc by 1/4)")

            stats = afwMath.makeStatistics(image2,
                                           afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.ERRORS)
            meanRes = stats.getResult(afwMath.MEAN)
            n = stats.getValue(afwMath.NPOINT)
            sd = stats.getValue(afwMath.STDEV)

            self.assertAlmostEqual(meanRes[0], mean + 0.5, delta=self.delta("mean", isInt))
            self.assertAlmostEqual(sd, np.hypot(std, 1/math.sqrt(4.0)*math.sqrt(n/(n - 1))),
                                   delta=0.00011)
            self.assertAlmostEqual(meanRes[1], sd/math.sqrt(image2.getWidth()*image2.getHeight()), 10)

            meanSquare = afwMath.makeStatistics(image2, afwMath.MEANSQUARE).getValue()
            self.assertAlmostEqual(meanSquare, 0.5*(mean**2 + (mean + 1)**2) + std**2,
                                   delta=0.00025 if isInt else 0.00006)
Esempio n. 14
0
    def testStatsZebra(self):
        """Add 1 to every other row"""
        image2 = self.image.Factory(self.image, True)
        #
        # Add 1 to every other row, so the variance is 1/4
        #
        self.assertEqual(image2.getHeight()%2, 0)
        width = image2.getWidth()
        for y in range(1, image2.getHeight(), 2):
            sim = image2.Factory(image2, afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(width, 1)),
                                 afwImage.LOCAL)
            sim += 1

        if display:
            ds9.mtv(self.image, frame = 0)
            ds9.mtv(image2, frame = 1)

        stats = afwMath.makeStatistics(image2,
                                       afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.ERRORS)
        mean = stats.getResult(afwMath.MEAN)
        n = stats.getValue(afwMath.NPOINT)
        sd = stats.getValue(afwMath.STDEV)

        self.assertEqual(mean[0],  image2.get(0, 0) + 0.5)
        self.assertEqual(sd, 1/math.sqrt(4.0)*math.sqrt(n/(n - 1)))
        self.assertAlmostEqual(mean[1], sd/math.sqrt(image2.getWidth()*image2.getHeight()), 10)

        meanSquare = afwMath.makeStatistics(image2, afwMath.MEANSQUARE).getValue()
        self.assertEqual(meanSquare, 0.5*(image2.get(0, 0)**2 + image2.get(0, 1)**2))
Esempio n. 15
0
    def testTicket1123(self):
        """
        Ticket #1123 reported that the Statistics stack routine throws an exception
        when all pixels in a stack are masked.  Returning a NaN pixel in the stack is preferred
        """

        ctrl = afwMath.StatisticsControl()
        ctrl.setAndMask(~0x0)
        
        mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
        mimg.set([self.val, 0x1, self.val])

        # test the case with no valid pixels ... both mean and stdev should be nan
        stat  = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
        mean  = stat.getValue(afwMath.MEAN)
        stdev = stat.getValue(afwMath.STDEV)
        self.assertNotEqual(mean, mean)   # NaN does not equal itself
        self.assertNotEqual(stdev, stdev) # NaN does not equal itself
        
        # test the case with one valid pixel ... mean is ok, but stdev should still be nan
        mimg.getMask().set(1, 1, 0x0)
        stat  = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
        mean  = stat.getValue(afwMath.MEAN)
        stdev = stat.getValue(afwMath.STDEV)
        self.assertEqual(mean, self.val)
        self.assertNotEqual(stdev, stdev) # NaN does not equal itself

        # test the case with two valid pixels ... both mean and stdev are ok
        mimg.getMask().set(1, 2, 0x0)
        stat  = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
        mean  = stat.getValue(afwMath.MEAN)
        stdev = stat.getValue(afwMath.STDEV)
        self.assertEqual(mean, self.val)
        self.assertEqual(stdev, 0.0)
Esempio n. 16
0
    def _setFallbackValue(self, mi=None):
        """Set the edge fallbackValue for interpolation

        \param[in] mi  input maksedImage on which to calculate the statistics
                       Must be provided if fallbackValueType != "USER".

        \return fallbackValue  The value set/computed based on the fallbackValueType
                               and negativeFallbackAllowed config parameters
        """
        if self.config.fallbackValueType != "USER":
            assert mi, "No maskedImage provided"
        if self.config.fallbackValueType == "MEAN":
            fallbackValue = afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
        elif self.config.fallbackValueType == "MEDIAN":
            fallbackValue = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
        elif self.config.fallbackValueType == "MEANCLIP":
            fallbackValue = afwMath.makeStatistics(mi, afwMath.MEANCLIP).getValue()
        elif self.config.fallbackValueType == "USER":
            fallbackValue = self.config.fallbackUserValue
        else:
            raise NotImplementedError("%s : %s not implemented" % ("fallbackValueType", self.config.fallbackValueType))

        if not self.config.negativeFallbackAllowed and fallbackValue < 0.0:
            self.log.warn(
                "Negative interpolation edge fallback value computed but "
                "negativeFallbackAllowed is False: setting fallbackValue to 0.0"
            )
            fallbackValue = max(fallbackValue, 0.0)

        self.log.info("fallbackValueType %s has been set to %.4f" % (self.config.fallbackValueType, fallbackValue))

        return fallbackValue
 def addNoise(self, mi):
     img       = mi.getImage()
     seed      = int(afwMath.makeStatistics(mi.getVariance(), afwMath.MEDIAN).getValue())
     rdm       = afwMath.Random(afwMath.Random.MT19937, seed)
     rdmImage  = img.Factory(img.getDimensions())
     afwMath.randomGaussianImage(rdmImage, rdm)
     img      += rdmImage
     return afwMath.makeStatistics(rdmImage, afwMath.MEAN).getValue(afwMath.MEAN)
Esempio n. 18
0
    def testMedian(self):
        """Test the median code"""
        for image, isInt, mean, median, std in self.images:
            med = afwMath.makeStatistics(image, afwMath.MEDIAN).getValue()
            self.assertAlmostEqual(med, median, delta=self.delta("median", isInt))

            values = [1.0, 2.0, 3.0, 2.0]
            self.assertEqual(afwMath.makeStatistics(values, afwMath.MEDIAN).getValue(), 2.0)
Esempio n. 19
0
    def testMedian(self):
        """Test the median code"""
        stats = afwMath.makeStatistics(self.image, afwMath.MEDIAN)

        self.assertEqual(stats.getValue(afwMath.MEDIAN), self.val)

        values = [1.0, 2.0, 3.0, 2.0 ]
        self.assertEqual(afwMath.makeStatistics(values, afwMath.MEDIAN).getValue(), 2.0)
Esempio n. 20
0
    def testVarianceClip(self):
        """Test the 3-sigma clipped standard deviation and variance"""
        for image, isInt, mean, median, std in self.images:
            delta = 0.0006 if isInt else 0.0014
            stdevClip = afwMath.makeStatistics(image, afwMath.STDEVCLIP).getValue()
            self.assertAlmostEqual(stdevClip, math.sqrt(self.clippedVariance3)*std, delta=delta)

            varianceClip = afwMath.makeStatistics(image, afwMath.VARIANCECLIP).getValue()
            self.assertAlmostEqual(varianceClip, self.clippedVariance3*std**2, delta=2*delta)
Esempio n. 21
0
 def _checkMaskedImage(self, mim, width, height, val1, val2, val3):
     # Check that the input image has dimensions width & height and that the image, mask and
     # variance have mean val1, val2 & val3 respectively.
     self.assertEqual(mim.getWidth(), width)
     self.assertEqual(mim.getHeight(), width)
     self.assertEqual(afwMath.makeStatistics(mim.getImage(), afwMath.MEAN).getValue(), val1)
     s = afwMath.makeStatistics(mim.getMask(), afwMath.SUM | afwMath.NPOINT)
     self.assertEqual(float(s.getValue(afwMath.SUM)) / s.getValue(afwMath.NPOINT), val2)
     self.assertEqual(afwMath.makeStatistics(mim.getVariance(), afwMath.MEAN).getValue(), val3)
Esempio n. 22
0
 def scaleVariance(self, exposure):
     ctrl = afwMath.StatisticsControl()
     ctrl.setAndMask(~0x0)
     var    = exposure.getMaskedImage().getVariance()
     mask   = exposure.getMaskedImage().getMask()
     dstats = afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.VARIANCECLIP, ctrl).getValue(afwMath.VARIANCECLIP)
     vstats = afwMath.makeStatistics(var, mask, afwMath.MEANCLIP, ctrl).getValue(afwMath.MEANCLIP)
     vrat   = dstats / vstats
     self.log.info("Renormalising variance by %f" % (vrat))
     var   *= vrat
Esempio n. 23
0
def fitRadialParabola(mi, niter=3, tol=1e-5, nsigma=5, returnResidualImage=False):
    """Fit a radial parabola (centered at (0, 0) to the image im using a linear fit with an nsigma clip"""

    width, height = mi.getDimensions()
    BAD = afwImage.MaskU.getPlaneBitMask("BAD")

    X, Y = np.meshgrid(np.arange(width), np.arange(height))
    R = np.hypot(X + mi.getX0(), Y + mi.getY0())
    R = R.flatten()
    one = np.ones_like(R)

    A = np.array([one, R, R**2]).transpose()
    isMaskedImage = hasattr(mi, "getImage")
    im = mi.getImage() if isMaskedImage else mi
    ima = im.getArray().flatten()

    sctrl = afwMath.StatisticsControl()
    sctrl.setAndMask(afwImage.MaskU.getPlaneBitMask("BAD"))
    c0, c1, c2 = afwMath.makeStatistics(mi, afwMath.MEANCLIP, sctrl).getValue(), 0, 0 # initial guess
    c00 = c0

    returnResidualImage = True
    if returnResidualImage:
        res = mi.clone()                # residuals from plane fit
        resim = res.getImage() if isMaskedImage else res

    for i in range(niter):
        fit = c0 + c1*R + c2*R**2

        resim.getArray()[:] = im.getArray() - fit.reshape(height, width)

        stats = afwMath.makeStatistics(res, afwMath.STDEVCLIP, sctrl)
        stdev = stats.getValue(afwMath.STDEVCLIP)
        good = np.abs(resim.getArray().flatten()) < nsigma*stdev

        if isMaskedImage:
            good = np.where(np.logical_and(good,
                                           np.bitwise_and(res.getMask().getArray().flatten(), BAD) == 0))[0]
        else:
            good = np.where(np.logical_and(good, np.isfinite(ima)))[0]

        b, residuals = np.linalg.lstsq(A[good], ima[good])[:2]

        if np.all(residuals < tol):
            break

        c0, c1, c2 = b

    if returnResidualImage:             # need to update with latest values
        fit = (c0 - c00) + c1*R + c2*R**2 # n.b. not c0
        resim.getArray()[:] = im.getArray() - fit.reshape(height, width)

        return b, res
    else:
        return b, None
Esempio n. 24
0
 def testNMasked(self):
     """Test that NMASKED works"""
     maskVal = 0xBE
     ctrl = afwMath.StatisticsControl()
     ctrl.setAndMask(maskVal)
     for image, isInt, mean, median, std in self.images:
         mask = afwImage.Mask(image.getBBox())
         mask.set(0)
         self.assertEqual(afwMath.makeStatistics(image, mask, afwMath.NMASKED, ctrl).getValue(), 0)
         mask[1, 1] = maskVal
         self.assertEqual(afwMath.makeStatistics(image, mask, afwMath.NMASKED, ctrl).getValue(), 1)
Esempio n. 25
0
def calcEffectiveGain(maskedImage):
    """Calculate effective gain

    @param[in] maskedImage  afw.image.MaskedImage to process
    @return (median gain, mean gain) in e-/ADU
    """
    im = afwImage.ImageF(maskedImage.getImage(), True)
    var = maskedImage.getVariance()
    im /= var
    medgain = afwMath.makeStatistics(im, afwMath.MEDIAN).getValue()
    meangain = afwMath.makeStatistics(im, afwMath.MEANCLIP).getValue()
    return medgain, meangain
Esempio n. 26
0
    def testStatsStdevclip(self):
        """Test STDEVCLIP; cf. #611"""
        image2 = self.image.Factory(self.image, True)

        stats = afwMath.makeStatistics(image2, afwMath.STDEVCLIP | afwMath.NPOINT | afwMath.SUM)
        self.assertEqual(stats.getValue(afwMath.STDEVCLIP), 0)
        #
        # Check we get the correct sum even when clipping
        #
        self.assertEqual(stats.getValue(afwMath.NPOINT)*
                         afwMath.makeStatistics(image2, afwMath.MEAN).getValue(),
                         stats.getValue(afwMath.SUM))
Esempio n. 27
0
    def testMeanClip(self):
        """Verify that the 3-sigma clipped mean doesn't not return NaN for a single value."""
        stats = afwMath.makeStatistics(self.image, afwMath.MEANCLIP)
        self.assertEqual(stats.getValue(afwMath.MEANCLIP), self.val)

        # this bug was caused by the iterative nature of the MEANCLIP.
        # With only one point, the sample variance returns NaN to avoid a divide by zero error
        # Thus, on the second iteration, the clip width (based on _variance) is NaN and corrupts
        #   all further calculations.
        img = afwImage.ImageF(afwGeom.Extent2I(1, 1))
        img.set(0)
        stats = afwMath.makeStatistics(img, afwMath.MEANCLIP)
        self.assertEqual(stats.getValue(), 0)
Esempio n. 28
0
    def measureBackground(self, exposure):
        statsControl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
                                                 self.config.qa.flatness.nIter)
        maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
        statsControl.setAndMask(maskVal)
        maskedImage = exposure.getMaskedImage()
        stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
        skyLevel = stats.getValue(afwMath.MEDIAN)
        skySigma = stats.getValue(afwMath.STDEVCLIP)
        self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
        metadata = exposure.getMetadata()
        metadata.set('SKYLEVEL', skyLevel)
        metadata.set('SKYSIGMA', skySigma)

        # calcluating flatlevel over the subgrids
        stat = afwMath.MEANCLIP if self.config.qa.flatness.doClip else afwMath.MEAN
        meshXHalf = int(self.config.qa.flatness.meshX/2.)
        meshYHalf = int(self.config.qa.flatness.meshY/2.)
        nX = int((exposure.getWidth() + meshXHalf) / self.config.qa.flatness.meshX)
        nY = int((exposure.getHeight() + meshYHalf) / self.config.qa.flatness.meshY)
        skyLevels = numpy.zeros((nX,nY))

        for j in range(nY):
            yc = meshYHalf + j * self.config.qa.flatness.meshY
            for i in range(nX):
                xc = meshXHalf + i * self.config.qa.flatness.meshX

                xLLC = xc - meshXHalf
                yLLC = yc - meshYHalf
                xURC = xc + meshXHalf - 1
                yURC = yc + meshYHalf - 1

                bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
                miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)

                skyLevels[i,j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()

        good = numpy.where(numpy.isfinite(skyLevels))
        skyMedian = numpy.median(skyLevels[good])
        flatness =  (skyLevels[good] - skyMedian) / skyMedian
        flatness_rms = numpy.std(flatness)
        flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan

        self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
        self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" % (nX, nY, flatness_pp, flatness_rms))

        metadata.set('FLATNESS_PP', float(flatness_pp))
        metadata.set('FLATNESS_RMS', float(flatness_rms))
        metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
        metadata.set('FLATNESS_MESHX', self.config.qa.flatness.meshX)
        metadata.set('FLATNESS_MESHY', self.config.qa.flatness.meshY)
Esempio n. 29
0
def fitPlane(mi, niter=3, tol=1e-5, nsigma=5, returnResidualImage=False):
    """Fit a plane to the image im using a linear fit with an nsigma clip"""

    width, height = mi.getDimensions()
    BAD = afwImage.MaskU.getPlaneBitMask("BAD")

    X, Y = np.meshgrid(np.arange(width), np.arange(height))
    X, Y = X.flatten(), Y.flatten()
    one = np.ones_like(X)

    A = np.array([one, X, Y]).transpose()
    isMaskedImage = hasattr(mi, "getImage")
    im = mi.getImage() if isMaskedImage else mi
    ima = im.getArray().flatten()

    sctrl = afwMath.StatisticsControl()
    sctrl.setAndMask(afwImage.MaskU.getPlaneBitMask("BAD"))
    z, dzdx, dzdy = afwMath.makeStatistics(mi, afwMath.MEANCLIP, sctrl).getValue(), 0, 0 # initial guess

    returnResidualImage = True
    if returnResidualImage:
        res = mi.clone()                # residuals from plane fit
        resim = res.getImage() if isMaskedImage else res

    for i in range(niter):
        plane = z + dzdx*X + dzdy*Y

        resim.getArray()[:] = im.getArray() - plane.reshape(height, width)

        if isMaskedImage:
            stats = afwMath.makeStatistics(res, afwMath.STDEVCLIP, sctrl)
            stdev = stats.getValue(afwMath.STDEVCLIP)
            good = np.where(np.logical_and(np.abs(res.getImage().getArray().flatten()) < nsigma*stdev,
                                           np.bitwise_and(res.getMask().getArray().flatten(), BAD) == 0))[0]
            b, residuals = np.linalg.lstsq(A[good], ima[good])[:2]
        else:
            b, residuals = np.linalg.lstsq(A, ima)[:2]

        if np.all(residuals < tol):
            break

        z, dzdx, dzdy = b

    if returnResidualImage:             # need to update with latest values
        plane = z + dzdx*X + dzdy*Y
        resim.getArray()[:] = im.getArray() - plane.reshape(height, width)

        return b, res
    else:
        return b, None
Esempio n. 30
0
    def scaleVariance(self, maskedImage):
        """Scale the variance in a maskedImage

        Scales the variance plane to match the measured variance.
        """
        ctrl = afwMath.StatisticsControl()
        ctrl.setAndMask(~0x0)
        var    = maskedImage.getVariance()
        mask   = maskedImage.getMask()
        dstats = afwMath.makeStatistics(maskedImage, afwMath.VARIANCECLIP, ctrl).getValue()
        vstats = afwMath.makeStatistics(var, mask, afwMath.MEANCLIP, ctrl).getValue()
        vrat   = dstats / vstats
        self.log.info("Renormalising variance by %f" % (vrat))
        var   *= vrat
Esempio n. 31
0
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
    """Apply flat correction in place.

    Parameters
    ----------
    maskedImage : `lsst.afw.image.MaskedImage`
        Image to process.  The image is modified.
    flatMaskedImage : `lsst.afw.image.MaskedImage`
        Flat image of the same size as ``maskedImage``
    scalingType : str
        Flat scale computation method.  Allowed values are 'MEAN',
        'MEDIAN', or 'USER'.
    userScale : scalar, optional
        Scale to use if ``scalingType``='USER'.
    invert : `Bool`, optional
        If True, unflatten an already flattened image.
    trimToFit : `Bool`, optional
        If True, raw data is symmetrically trimmed to match
        calibration size.

    Raises
    ------
    RuntimeError
        Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
        the same size or if ``scalingType`` is not an allowed value.
    """
    if trimToFit:
        maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)

    if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
        raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
                           (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))

    # Figure out scale from the data
    # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
    # in the case that the flat is created by some other mechanism.
    if scalingType in ('MEAN', 'MEDIAN'):
        scalingType = afwMath.stringToStatisticsProperty(scalingType)
        flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
    elif scalingType == 'USER':
        flatScale = userScale
    else:
        raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))

    if not invert:
        maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
    else:
        maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
Esempio n. 32
0
    def cosmicray(self, exposure, psf):
        """Cosmic ray masking

        @param exposure Exposure to process
        @param psf PSF
        """
        import lsstDebug
        display = lsstDebug.Info(__name__).display
        displayCR = lsstDebug.Info(__name__).displayCR

        assert exposure, "No exposure provided"
        assert psf, "No psf provided"
        # Blow away old mask
        try:
            mask = exposure.getMaskedImage().getMask()
            crBit = mask.getMaskPlane("CR")
            mask.clearMaskPlane(crBit)
        except:
            pass

        if display and displayCR:
            ds9.incrDefaultFrame()
            ds9.mtv(exposure, title="Pre-CR")

        policy = self.config['cosmicray'].getPolicy()
        mi = exposure.getMaskedImage()
        bg = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
        crs = measAlg.findCosmicRays(mi, psf, bg, policy, self._keepCRs)
        num = 0
        if crs is not None:
            mask = mi.getMask()
            crBit = mask.getPlaneBitMask("CR")
            afwDet.setMaskFromFootprintList(mask, crs, crBit)
            num = len(crs)

            if display and displayCR:
                ds9.incrDefaultFrame()
                ds9.mtv(exposure, title="Post-CR")

                ds9.cmdBuffer.pushSize()

                for cr in crs:
                    displayUtils.drawBBox(cr.getBBox(), borderWidth=0.55)

                ds9.cmdBuffer.popSize()

        self.log.log(self.log.INFO, "Identified %d cosmic rays." % num)
        return
Esempio n. 33
0
def makeExposure(inputFile, weightFile, varianceFile, badPixelValue, variance):
    exposure = afwImage.ExposureF(inputFile)

    if np.isfinite(badPixelValue):
        mi = exposure.getMaskedImage()
        bad = mi.getImage().getArray() == badPixelValue
        mi.getMask().getArray()[bad] |= mi.getMask().getPlaneBitMask("BAD")
        del bad
        del mi

    if weightFile or varianceFile:
        assert (not (weightFile and varianceFile))  # we checked this earlier

        assert not np.isfinite(variance), \
            "Please don't specify a variance and %s file" % ("weight" if weightFile else "variance")

        mi = exposure.getMaskedImage()

        if weightFile:
            variance = afwImage.ImageF(weightFile)

            varr = variance.getArray()
            bad = (varr == 0)
            varr[bad] = np.inf  # avoid numpy warning
            varr[:] = 1 / varr
        else:
            variance = afwImage.ImageF(varianceFile)
            bad = np.logical_not(np.isfinite(mi.getImage().getArray()))

        mi.getMask().getArray()[bad] |= mi.getMask().getPlaneBitMask("BAD")
        del bad

        mi.getVariance()[:] = variance
        del mi
    else:
        if not np.isfinite(variance) or variance <= 0:
            mi = exposure.getMaskedImage()

            sctrl = afwMath.StatisticsControl()
            sctrl.setAndMask(mi.getMask().getPlaneBitMask("BAD"))
            variance = afwMath.makeStatistics(mi, afwMath.VARIANCECLIP,
                                              sctrl).getValue()
            del sctrl
            del mi

        exposure.getMaskedImage().getVariance()[:] = variance

    return exposure
Esempio n. 34
0
    def interpolate(self, exposure, psf, defects):
        """Interpolate over defects

        @param exposure Exposure to process
        @param psf PSF for interpolation
        @param defects Defect list
        """
        assert exposure, "No exposure provided"
        assert defects is not None, "No defects provided"
        assert psf, "No psf provided"
        mi = exposure.getMaskedImage()
        fallbackValue = afwMath.makeStatistics(mi, afwMath.MEANCLIP).getValue()
        measAlg.interpolateOverDefects(mi, psf, defects, fallbackValue)
        self.log.log(self.log.INFO,
                     "Interpolated over %d defects." % len(defects))
        return
Esempio n. 35
0
 def testMakeMatchStatisticsInPixels(self):
     """Test testMakeMatchStatisticsInPixels
     """
     np.random.seed(164)
     offList = [lsst.geom.Extent2D(val) for val in (np.random.random_sample([self.numMatches, 2])-0.5)*10]
     for off, match in zip(offList, self.matchList):
         centroid = match.second.get(self.sourceCentroidKey)
         offCentroid = centroid + off
         match.second.set(self.sourceCentroidKey, offCentroid)
     itemList = (afwMath.MEDIAN, afwMath.MEAN, afwMath.STDEVCLIP)
     itemMask = reduce(lambda a, b: a | b, itemList)
     distStats = measAstrom.makeMatchStatisticsInPixels(self.wcs, self.matchList, itemMask)
     distList = [math.hypot(*val) for val in offList]
     directStats = afwMath.makeStatistics(distList, itemMask)
     for item in itemList:
         self.assertAlmostEqual(distStats.getValue(item), directStats.getValue(item))
Esempio n. 36
0
 def _inverted_image(self, offset=None):
     """
     Return a masked image which is the trimmed and unbiased amp
     image subtracted from an offset value.  This enables regions
     below a specified threshold value to be found using afwDetect.
     If offset is None, then use the median of the unmasked pixels.
     """
     my_image = self.ccd.unbiased_and_trimmed_image(self.amp).clone()
     if offset is None:
         median = afwMath.makeStatistics(my_image, afwMath.MEDIAN,
                                         self.ccd.stat_ctrl).getValue()
     else:
         median = offset
     imarr = my_image.getImage().getArray()  # a view into the image data
     imarr[:] = median - imarr[:]
     return my_image, median
Esempio n. 37
0
    def setVariance(self, exposure):
        mi = exposure.getMaskedImage()
        image = mi.getImage().getArray()
        variance = mi.getVariance().getArray()
        if self.config.isBackgroundSubtracted:
            bkgdVariance = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
            self.log.info("Setting variance: background variance = %g ADU" % (bkgdVariance))
        else:
            self.log.info("Setting variance: noise=%g ADU" % (self.config.noise))
            bkgdVariance = self.config.noise**2

        variance[:] = bkgdVariance

        if self.config.gain > 0.0:
            self.log.info("Setting variance: gain=%g e/ADU" % (self.config.gain))
            variance[:] += image/self.config.gain
Esempio n. 38
0
    def testComputeImage(self):
        """Test the computation of the PSF's image at a point."""

        for psf in [self.psfDg, self.psfSg]:
            ccdXY = lsst.geom.Point2D(0, 0)
            kIm = psf.computeImage(ccdXY)

            if False:
                afwDisplay.Display(frame=1).mtv(kIm,
                                                title=self._testMethodName +
                                                ": kIm")

            self.assertEqual(kIm.getWidth(), self.ksize)
            kIm = psf.computeImage(ccdXY)
            self.assertAlmostEqual(
                afwMath.makeStatistics(kIm, afwMath.SUM).getValue(), 1.0)
Esempio n. 39
0
    def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]):
        """Average the amplifier level kernels to create a detector level
        kernel.
        """
        inKernels = np.array([self.ampKernels[amp] for amp in
                              self.ampKernels if amp not in ampsToExclude])
        averagingList = np.transpose(inKernels)
        avgKernel = np.zeros_like(inKernels[0])
        sctrl = afwMath.StatisticsControl()
        sctrl.setNumSigmaClip(5.0)
        for i in range(np.shape(avgKernel)[0]):
            for j in range(np.shape(avgKernel)[1]):
                avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j],
                                                         afwMath.MEANCLIP, sctrl).getValue()

        self.detKernels[detectorName] = avgKernel
Esempio n. 40
0
    def testTicket1125(self):
        """Ticket 1125 reported that the clipped routines were aborting when called with no valid pixels. """
        mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
        mimg.set([self.val, 0x1, self.val])

        ctrl = afwMath.StatisticsControl()
        ctrl.setAndMask(~0x0)

        # test the case with no valid pixels ... try MEANCLIP and STDEVCLIP
        stat = afwMath.makeStatistics(mimg,
                                      afwMath.MEANCLIP | afwMath.STDEVCLIP,
                                      ctrl)
        mean = stat.getValue(afwMath.MEANCLIP)
        stdev = stat.getValue(afwMath.STDEVCLIP)
        self.assertNotEqual(mean, mean)  # NaN does not equal itself
        self.assertNotEqual(stdev, stdev)  # NaN does not equal itself
Esempio n. 41
0
    def testMask(self):
        mask = afwImage.MaskU(afwGeom.Extent2I(10, 10))
        mask.set(0x0)

        mask.set(1, 1, 0x10)
        mask.set(3, 1, 0x08)
        mask.set(5, 4, 0x08)
        mask.set(4, 5, 0x02)

        stats = afwMath.makeStatistics(mask, afwMath.SUM | afwMath.NPOINT)
        self.assertEqual(mask.getWidth()*mask.getHeight(), stats.getValue(afwMath.NPOINT))
        self.assertEqual(0x1a, stats.getValue(afwMath.SUM))

        def tst():
            stats = afwMath.makeStatistics(mask, afwMath.MEAN)
        self.assertRaises(lsst.pex.exceptions.InvalidParameterError, tst)
    def testRunDataRef(self):
        """Test LsstSimIsrTask on amp-sized images in tests/data/

        applyToSensorRef is not intended to take single amp-sized exposures, but will
        run if the doAssembleCcd config parameter is False.
        However, the exposure is not trimmed, and gain not reset.
        """
        config = LsstSimIsrTask.ConfigClass()
        config.doDark = False
        config.doFringe = False
        config.doAssembleCcd = False
        config.doSnapCombine = False
        lsstIsrTask = LsstSimIsrTask(config=config)
        exposure = lsstIsrTask.runDataRef(self.ampRef).exposure
        self.assertAlmostEqual(afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.MEAN).getValue(),
                               2.855780, places=3)
Esempio n. 43
0
    def amplifierStats(self,
                       exposure,
                       keywordDict,
                       statControl,
                       failAll=False):
        """Measure amplifier level statistics from the exposure.

        Parameters
        ----------
        exposure : `lsst.afw.image.Exposure`
            The exposure to measure.
        keywordDict : `dict` [`str`, `str`]
            A dictionary of keys to use in the output results, with
            values the string name associated with the
            `lsst.afw.math.statistics.Property` to measure.
        statControl : `lsst.afw.math.StatisticsControl`
            Statistics control object with parameters defined by
            the config.
        failAll : `bool`, optional
            If True, all tests will be set as failed.

        Returns
        -------
        ampStats : `dict` [`str`, `dict` [`str`, scalar]]
            A dictionary indexed by the amplifier name, containing
            dictionaries of the statistics measured and their values.
        """
        ampStats = {}

        statisticToRun, statAccessor = self._configHelper(keywordDict)

        # Measure stats on all amplifiers.
        for ampIdx, amp in enumerate(exposure.getDetector()):
            ampName = amp.getName()
            theseStats = {}
            ampExp = exposure.Factory(exposure, amp.getBBox())
            stats = afwMath.makeStatistics(ampExp.getMaskedImage(),
                                           statisticToRun, statControl)

            for k, v in statAccessor.items():
                theseStats[k] = stats.getValue(v)

            if failAll:
                theseStats['FORCE_FAILURE'] = failAll
            ampStats[ampName] = theseStats

        return ampStats
Esempio n. 44
0
def setBadRegions(exposure, badStatistic="MEDIAN"):
    """Set all BAD areas of the chip to the average of the rest of the exposure

    Parameters
    ----------
    exposure : `lsst.afw.image.Exposure`
        Exposure to mask.  The exposure mask is modified.
    badStatistic : `str`, optional
        Statistic to use to generate the replacement value from the
        image data.  Allowed values are 'MEDIAN' or 'MEANCLIP'.

    Returns
    -------
    badPixelCount : scalar
        Number of bad pixels masked.
    badPixelValue : scalar
        Value substituted for bad pixels.

    Raises
    ------
    RuntimeError
        Raised if `badStatistic` is not an allowed value.
    """
    if badStatistic == "MEDIAN":
        statistic = afwMath.MEDIAN
    elif badStatistic == "MEANCLIP":
        statistic = afwMath.MEANCLIP
    else:
        raise RuntimeError("Impossible method %s of bad region correction" %
                           badStatistic)

    mi = exposure.getMaskedImage()
    mask = mi.getMask()
    BAD = mask.getPlaneBitMask("BAD")
    INTRP = mask.getPlaneBitMask("INTRP")

    sctrl = afwMath.StatisticsControl()
    sctrl.setAndMask(BAD)
    value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()

    maskArray = mask.getArray()
    imageArray = mi.getImage().getArray()
    badPixels = numpy.logical_and((maskArray & BAD) > 0,
                                  (maskArray & INTRP) == 0)
    imageArray[:] = numpy.where(badPixels, value, imageArray)

    return badPixels.sum(), value
Esempio n. 45
0
def addCosmicRays(image, nCR=100, emin=800, emax=1000, seed=None):
    """Add nCR fake cosmic rays to a frame, with pixel values between emin and emax (and some extra associated fainter pixels too)"""
    #
    if seed is None:
        seed = int(afwMath.makeStatistics(image, afwMath.MAX).getValue())
    if seed == 0:
        seed = 1

    width = image.getWidth()
    height = image.getHeight()
    
    rand = afwMath.Random(afwMath.Random.RANLUX, seed)

    for i in range(nCR):
        #
        # Initial point in CR
        #
        x = rand.uniformInt(width)
        y = rand.uniformInt(height)
        amp = emin + rand.uniformInt(emax - emin)
        #
        # Extra contamination at about the initial amplitude
        #
        badPixels = []
        while True:
            badPixels.append([x, y, amp + 0.1*(emax - emin)*rand.uniform()])
            
            if rand.uniform() > 0.5:
                break

            x += rand.uniformInt(3) - 1
            y += rand.uniformInt(3) - 1
        #
        # Add a little extra CR flux to the pixels surrounding badPixels
        #
        for x, y, amp in badPixels:
            while rand.uniform() < 0.5:
                image.set(x, y, amp*rand.uniform())
                
                x += rand.uniformInt(3) - 1
                y += rand.uniformInt(3) - 1
        #
        # And set the initial badPixels themselves
        #
        for x, y, amp in badPixels:
            if x >= 0 and x < width and y >= 0 and y < height:
                image.set(x, y, amp)
Esempio n. 46
0
    def makePsfCandidates(self, exposure, starCat):
        """!Make a list of PSF candidates from a star catalog

        @param[in] exposure  the exposure containing the sources
        @param[in] starCat  catalog of stars (an lsst.afw.table.SourceCatalog),
                            e.g. as returned by the run or selectStars method

        @return an lsst.pipe.base.Struct with fields:
        - psfCandidates  list of PSF candidates (lsst.meas.algorithms.PsfCandidate)
        - goodStarCat  catalog of stars that were successfully made into PSF candidates (a subset of starCat)
        """
        goodStarCat = SourceCatalog(starCat.schema)

        psfCandidateList = []
        didSetSize = False
        for star in starCat:
            try:
                psfCandidate = algorithmsLib.makePsfCandidate(star, exposure)

                # The setXXX methods are class static, but it's convenient to call them on
                # an instance as we don't know Exposure's pixel type
                # (and hence psfCandidate's exact type)
                if not didSetSize:
                    psfCandidate.setBorderWidth(self.config.borderWidth)
                    psfCandidate.setWidth(self.config.kernelSize +
                                          2 * self.config.borderWidth)
                    psfCandidate.setHeight(self.config.kernelSize +
                                           2 * self.config.borderWidth)
                    didSetSize = True

                im = psfCandidate.getMaskedImage().getImage()
            except Exception as err:
                self.log.debug(
                    "Failed to make a psfCandidate from star %d: %s",
                    star.getId(), err)
                continue

            vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
            if not np.isfinite(vmax):
                continue
            psfCandidateList.append(psfCandidate)
            goodStarCat.append(star)

        return pipeBase.Struct(
            psfCandidates=psfCandidateList,
            goodStarCat=goodStarCat,
        )
Esempio n. 47
0
    def addExposure(self, exposure, weightFactor=1.0):
        """Add an Exposure to the coadd

        Parameters
        ----------
        exposure: `afwImage.Exposure`
            Exposure to add to coadd; this should be:
            - background-subtracted or background-matched to the other images
                being coadded
            - psf-matched to the desired PSF model (optional)
            - warped to match the coadd
            - photometrically scaled to the desired flux magnitude
        weightFactor : `float`
            the extra weight factor for this exposure

        Returns
        --------
        overlapBBox, weight : `afwGeom.Box2I`, `float`
            the region of overlap between exposure and coadd in  parent
            coordinates. weight with which exposure was added to coadd;
            weight = weightFactor / clipped mean variance
            Subclasses may override to preprocess the exposure or change
            the way it is added to the coadd.

        """
        maskedImage = exposure.getMaskedImage()

        # compute the weight
        statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
                                         afwMath.MEANCLIP, self._statsControl)
        meanVar = statObj.getResult(afwMath.MEANCLIP)[0]
        weight = weightFactor / float(meanVar)
        if math.isnan(weight):
            raise RuntimeError("Weight is NaN (weightFactor=%s; mean variance=%s)" % (weightFactor, meanVar))

        # save filter info
        filter = exposure.getFilter()
        self._filterDict.setdefault(filter.getName(), filter)

        self._log.info("Add exposure to coadd with weight=%0.3g", weight)

        overlapBBox = coadd_utils.addToCoadd(self._coadd.getMaskedImage(),
                                             self._weightMap, maskedImage,
                                             self._badPixelMask, weight)

        return overlapBBox, weight
Esempio n. 48
0
 def __init__(self, ccd, amp, bg_reg=(10, 10)):
     self.ccd = ccd
     self.amp = amp
     self.ccdtemp = ccd.md.get('CCDTEMP')
     self.fe55_yield = Fe55Yield(self.ccdtemp)
     raw_image = ccd[amp]
     try:
         self.imarr = raw_image.getArray()
     except AttributeError:
         self.imarr = raw_image.getImage().getArray()
     self.image = imutils.trim(raw_image, imaging=ccd.amp_geom.imaging)
     self.image -= self._bg_image(*bg_reg)
     flags = afwMath.MEANCLIP | afwMath.STDEVCLIP
     stats = afwMath.makeStatistics(self.image, flags, self.ccd.stat_ctrl)
     self.mean = stats.getValue(afwMath.MEANCLIP)
     self.stdev = stats.getValue(afwMath.STDEVCLIP)
     self.footprint_signal = self._footprint_signal_spans
Esempio n. 49
0
 def _generate_stats(self, amp):
     #
     # Extract the imaging region of the masked image for the
     # specified amplifier.
     #
     image = imutils.trim(self.ccd[amp])
     #
     # Store numpy array of full segment for
     # self._footprint_signal, since footprint spans use full
     # segment image pixel coordinates.
     #
     self.arr = self.ccd[amp].getImage().getArray()
     stats = afwMath.makeStatistics(image,
                                    afwMath.STDEVCLIP | afwMath.MEDIAN,
                                    self.ccd.stat_ctrl)
     self.noise = stats.getValue(afwMath.STDEVCLIP)
     self.median = stats.getValue(afwMath.MEDIAN)
Esempio n. 50
0
 def testMakeMatchStatisticsInRadians(self):
     """Test makeMatchStatisticsInRadians
     """
     np.random.seed(164)
     offLenList = [val*lsst.geom.radians for val in np.random.random_sample([self.numMatches])]
     offDirList = [val*lsst.geom.radians for val in np.random.random_sample([self.numMatches])*math.pi*2]
     for offLen, offDir, match in zip(offLenList, offDirList, self.matchList):
         coord = match.first.get(self.refCoordKey)
         offsetCoord = coord.offset(offDir, offLen)
         match.first.set(self.refCoordKey, offsetCoord)
     itemList = (afwMath.MEDIAN, afwMath.MEANCLIP, afwMath.IQRANGE)
     itemMask = reduce(lambda a, b: a | b, itemList)
     distStats = measAstrom.makeMatchStatisticsInRadians(self.wcs, self.matchList, itemMask)
     distRadiansList = [val.asRadians() for val in offLenList]
     directStats = afwMath.makeStatistics(distRadiansList, itemMask)
     for item in itemList:
         self.assertAlmostEqual(distStats.getValue(item), directStats.getValue(item))
Esempio n. 51
0
    def testFootprintToBBoxList(self):
        """Test footprintToBBoxList"""
        region = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(12, 10))
        foot = afwDetect.Footprint(0, region)
        for y, x0, x1 in [
            (3, 3, 5),
            (3, 7, 7),
            (4, 2, 3),
            (4, 5, 7),
            (5, 2, 3),
            (5, 5, 8),
            (6, 3, 5),
        ]:
            foot.addSpan(y, x0, x1)

        idImage = afwImage.ImageU(region.getDimensions())
        idImage.set(0)

        foot.insertIntoImage(idImage, 1)
        if display:
            ds9.mtv(idImage)

        idImageFromBBox = idImage.Factory(idImage, True)
        idImageFromBBox.set(0)
        bboxes = afwDetect.footprintToBBoxList(foot)
        for bbox in bboxes:
            x0, y0, x1, y1 = bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(
            ), bbox.getMaxY()

            for y in range(y0, y1 + 1):
                for x in range(x0, x1 + 1):
                    idImageFromBBox.set(x, y, 1)

            if display:
                x0 -= 0.5
                y0 -= 0.5
                x1 += 0.5
                y1 += 0.5

                ds9.line([(x0, y0), (x1, y0), (x1, y1), (x0, y1), (x0, y0)],
                         ctype=ds9.RED)

        idImageFromBBox -= idImage  # should be blank
        stats = afwMath.makeStatistics(idImageFromBBox, afwMath.MAX)

        self.assertEqual(stats.getValue(), 0)
Esempio n. 52
0
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
    """Interpolate over defects specified in a defect list

    @param[in,out] maskedImage  masked image to process
    @param[in] defectList  defect list
    @param[in] fwhm  FWHM of double Gaussian smoothing kernel
    @param[in] fallbackValue  fallback value if an interpolated value cannot be determined;
                              if None then use clipped mean image value
    """
    psf = createPsf(fwhm)
    if fallbackValue is None:
        fallbackValue = afwMath.makeStatistics(maskedImage.getImage(),
                                               afwMath.MEANCLIP).getValue()
    if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
        maskedImage.getMask.addMaskPlane('INTRP')
    measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue,
                                   True)
Esempio n. 53
0
def computeVarianceMean(exposure,
                        actuallyDoImage=False,
                        statToDo=afwMath.MEANCLIP):
    statsControl = afwMath.StatisticsControl()
    statsControl.setNumSigmaClip(3.)
    statsControl.setNumIter(3)
    ignoreMaskPlanes = ("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD",
                        "NO_DATA", "DETECTED_NEGATIVE")
    statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(ignoreMaskPlanes))
    imToDo = exposure.getMaskedImage().getVariance()
    if actuallyDoImage:
        imToDo = exposure.getMaskedImage().getImage()
    statObj = afwMath.makeStatistics(imToDo,
                                     exposure.getMaskedImage().getMask(),
                                     statToDo, statsControl)
    var = statObj.getValue(statToDo)
    return var
Esempio n. 54
0
    def _i_scale(self, algorithm, minval, maxval, unit, *args, **kwargs):

        maskedPixels = kwargs.get("maskedPixels", [])
        if isinstance(maskedPixels, str):
            maskedPixels = [maskedPixels]
        bitmask = afwImage.Mask.getPlaneBitMask(maskedPixels)

        sctrl = afwMath.StatisticsControl()
        sctrl.setAndMask(bitmask)

        if minval == "minmax":
            if self._image is None:
                raise RuntimeError("You may only use minmax if an image is loaded into the display")

            mi = afwImage.makeMaskedImage(self._image, self._mask)
            stats = afwMath.makeStatistics(mi, afwMath.MIN | afwMath.MAX, sctrl)
            minval = stats.getValue(afwMath.MIN)
            maxval = stats.getValue(afwMath.MAX)
        elif minval == "zscale":
            if bitmask:
                print("scale(..., 'zscale', maskedPixels=...) is not yet implemented")

        if algorithm is None:
            self._normalize = None
        elif algorithm == "asinh":
            if minval == "zscale":
                if self._image is None:
                    raise RuntimeError("You may only use zscale if an image is loaded into the display")

                self._normalize = AsinhZScaleNormalize(image=self._image, Q=kwargs.get("Q", 8.0))
            else:
                self._normalize = AsinhNormalize(minimum=minval,
                                                 dataRange=maxval - minval, Q=kwargs.get("Q", 8.0))
        elif algorithm == "linear":
            if minval == "zscale":
                if self._image is None:
                    raise RuntimeError("You may only use zscale if an image is loaded into the display")

                self._normalize = ZScaleNormalize(image=self._image,
                                                  nSamples=kwargs.get("nSamples", 1000),
                                                  contrast=kwargs.get("contrast", 0.25))
            else:
                self._normalize = LinearNormalize(minimum=minval, maximum=maxval)
        else:
            raise RuntimeError("Unsupported stretch algorithm \"%s\"" % algorithm)
Esempio n. 55
0
    def testBadAreaFailsSpline(self):
        """Check that a NaN in the stats image doesn't cause spline interpolation to fail (#2734)"""
        image = afwImage.ImageF(15, 9)
        for y in range(image.getHeight()):
            for x in range(image.getWidth()):
                # n.b. linear, which is what the interpolation will fall back
                # to
                image[x, y, afwImage.LOCAL] = 1 + 2 * y

        # Set the right corner to NaN.  This will mean that we have too few
        # points for a spline interpolator
        binSize = 3
        image[-binSize:, -binSize:, afwImage.LOCAL] = np.nan

        nx = image.getWidth() // binSize
        ny = image.getHeight() // binSize

        sctrl = afwMath.StatisticsControl()
        bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)

        bkgd = afwMath.makeBackground(image, bctrl)
        if debugMode:
            afwDisplay.Display(frame=0).mtv(image,
                                            title=self._testMethodName +
                                            " image")
            afwDisplay.Display(frame=1).mtv(bkgd.getStatsImage(),
                                            title=self._testMethodName +
                                            " bkgd StatsImage")
        # Should throw if we don't permit REDUCE_INTERP_ORDER
        self.assertRaises(lsst.pex.exceptions.OutOfRangeError, bkgd.getImageF,
                          afwMath.Interpolate.NATURAL_SPLINE)
        # The interpolation should fall back to linear for the right part of the image
        # where the NaNs don't permit spline interpolation (n.b. this happens
        # to be exact)
        bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE,
                                   afwMath.REDUCE_INTERP_ORDER)

        if debugMode:
            afwDisplay.Display(frame=2).mtv(bkgdImage,
                                            title=self._testMethodName +
                                            " bkgdImage")

        image -= bkgdImage
        self.assertEqual(
            afwMath.makeStatistics(image, afwMath.MEAN).getValue(), 0.0)
Esempio n. 56
0
    def testNormalize(self):
        """Test Footprint.normalize"""
        w, h = 12, 10
        region = afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(w,h))
        im = afwImage.ImageU(afwGeom.Extent2I(w, h))
        im.set(0)
        #
        # Create a footprint;  note that these Spans overlap
        #
        for spans, box in (([(3, 5, 6),
                             (4, 7, 7), ], afwGeom.Box2I(afwGeom.Point2I(5,3), afwGeom.Point2I(7,4))),
                           ([(3, 3, 5), (3, 6, 9),
                             (4, 2, 3), (4, 5, 7), (4, 8, 8),
                             (5, 2, 3), (5, 5, 8), (5, 6, 7),
                             (6, 3, 5),
                             ], afwGeom.Box2I(afwGeom.Point2I(2,3), afwGeom.Point2I(9,6)))
                      ):

            foot = afwDetect.Footprint(0, region)
            for y, x0, x1 in spans:
                foot.addSpan(y, x0, x1)

                for x in range(x0, x1 + 1): # also insert into im
                    im.set(x, y, 1)

            idImage = afwImage.ImageU(afwGeom.Extent2I(w, h))
            idImage.set(0)

            foot.insertIntoImage(idImage, 1)
            if display:             # overlaping pixels will be > 1
                ds9.mtv(idImage)
            #
            # Normalise the Footprint, removing overlapping spans
            #
            foot.normalize()

            idImage.set(0)
            foot.insertIntoImage(idImage, 1)
            if display:
                ds9.mtv(idImage, frame=1)

            idImage -= im

            self.assertTrue(box == foot.getBBox())
            self.assertEqual(afwMath.makeStatistics(idImage, afwMath.MAX).getValue(), 0)
Esempio n. 57
0
    def flatCorrection(self, exposure, flatExposure):
        """Apply flat correction in-place

        This version allows tweaking the flat-field to match the observed
        ratios of background flux in the amplifiers.  This may be necessary
        if the gains drift or the levels are slightly affected by non-linearity
        or similar.  Note that this tweak may not work if there is significant
        structure in the image, and especially if the structure varies over
        the amplifiers.

        @param[in,out]  exposure        exposure to process
        @param[in]      flatExposure    flatfield exposure of same size as exposure
        """

        if self.config.doTweakFlat:
            data = []
            flatAmpList = []
            bad = exposure.getMaskedImage().getMask().getPlaneBitMask(
                ["BAD", "SAT"])
            stats = afwMath.StatisticsControl()
            stats.setAndMask(bad)
            for amp in exposure.getDetector():
                box = amp.getDataSec(True)
                dataAmp = afwImage.MaskedImageF(exposure.getMaskedImage(), box,
                                                afwImage.LOCAL).clone()
                flatAmp = afwImage.MaskedImageF(flatExposure.getMaskedImage(),
                                                box, afwImage.LOCAL)
                flatAmpList.append(flatAmp)
                dataAmp /= flatAmp
                data.append(
                    afwMath.makeStatistics(dataAmp, afwMath.MEDIAN,
                                           stats).getValue())

            data = numpy.array(data)
            tweak = data / data.sum() * len(data)
            self.log.warn(
                "Tweaking flat-field to match observed amplifier ratios: %s" %
                tweak)
            for i, flat in enumerate(flatAmpList):
                flat *= tweak[i]

        lsstIsr.flatCorrection(exposure.getMaskedImage(),
                               flatExposure.getMaskedImage(),
                               scalingType="USER",
                               userScale=1.0)
Esempio n. 58
0
def get_stats(image, nsigma=10):
    """
    Parameters
    ----------
    image: lsst.afw.image.Image
        Image object for which to compute the clipped mean and clipped
        stdev.
    nsigma: int [10]
        Value to use for sigma clipping.

    Returns
    -------
    (float, float): Tuple of the (mean, stdev) of the pixel values.
    """
    stat_ctrl = afwMath.StatisticsControl(numSigmaClip=nsigma)
    flags = afwMath.MEANCLIP | afwMath.STDEVCLIP
    stats = afwMath.makeStatistics(image, flags=flags, sctrl=stat_ctrl)
    return stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
    def testWeightedVector(self):
        """Test std::vector, but with weights"""
        sctrl = afwMath.StatisticsControl()

        nval = len(self.vecList[0])
        weight = 10
        weights = [i * weight / float(nval - 1) for i in range(nval)]

        for vec in self.vecList:
            stats = afwMath.makeStatistics(
                vec, weights,
                afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.SUM,
                sctrl)

            self.assertAlmostEqual(
                0.5 * weight * sum(vec) / stats.getValue(afwMath.SUM), 1.0)
            self.assertAlmostEqual(
                sum(vec) / len(vec), stats.getValue(afwMath.MEAN))
Esempio n. 60
0
    def __init__(self, minimum=None, maximum=None, image=None):
        if minimum is None or maximum is None:
            assert image is not None, "You must provide an image if you don't set both minimum and maximum"

            stats = afwMath.makeStatistics(image, afwMath.MIN | afwMath.MAX)
            if minimum is None:
                minimum = stats.getValue(afwMath.MIN)
            if maximum is None:
                maximum = stats.getValue(afwMath.MAX)

        Mapping.__init__(self, minimum, image)
        self.maximum = maximum

        if maximum is None:
            self._range = None
        else:
            assert maximum - minimum != 0, "minimum and maximum values must not be equal"
            self._range = float(maximum - minimum)