Ejemplo n.º 1
0
    def testDot(self):
        """Test HeavyFootprint::dot"""
        size = 20, 20
        for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]:
            mi1 = afwImage.MaskedImageF(*size)
            mi2 = afwImage.MaskedImageF(*size)
            mi1.set(0)
            mi2.set(0)

            spanList1 = []
            spanList2 = []
            for y, x0, x1 in [(5, 3, 7),
                              (6, 3, 4),
                              (6, 6, 7),
                              (7, 3, 7), ]:
                spanList1.append(afwGeom.Span(y, x0, x1))
                spanList2.append(afwGeom.Span(y + yOffset, x0 + xOffset,
                                              x1 + xOffset))
                for x in range(x0, x1 + 1):
                    value = (x + y, 0, 1.0)
                    mi1[x, y, afwImage.LOCAL] = value
                    mi2[x + xOffset, y + yOffset, afwImage.LOCAL] = value

            fp1 = afwDetect.Footprint(afwGeom.SpanSet(spanList1))
            fp2 = afwDetect.Footprint(afwGeom.SpanSet(spanList2))

            hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1)
            hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2)

            dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray())
            self.assertEqual(hfp1.dot(hfp2), dot)
            self.assertEqual(hfp2.dot(hfp1), dot)
Ejemplo n.º 2
0
    def testDot(self):
        """Test HeavyFootprint::dot"""
        size = 20, 20
        for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]:
            mi1 = afwImage.MaskedImageF(*size)
            mi2 = afwImage.MaskedImageF(*size)
            mi1.set(0)
            mi2.set(0)

            fp1 = afwDetect.Footprint()
            fp2 = afwDetect.Footprint()
            for y, x0, x1 in [(5, 3, 7),
                              (6, 3, 4),
                              (6, 6, 7),
                              (7, 3, 7),]:
                fp1.addSpan(y, x0, x1)
                fp2.addSpan(y + yOffset, x0 + xOffset, x1 + xOffset)
                for x in range(x0, x1 + 1):
                    value = (x + y, 0, 1.0)
                    mi1.set(x, y, value)
                    mi2.set(x + xOffset, y + yOffset, value)

            hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1)
            hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2)
            hfp1.normalize()
            hfp2.normalize()

            dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray())
            self.assertEqual(hfp1.dot(hfp2), dot)
            self.assertEqual(hfp2.dot(hfp1), dot)
Ejemplo n.º 3
0
    def testDot(self):
        """Test HeavyFootprint::dot"""
        size = 20, 20
        for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]:
            mi1 = afwImage.MaskedImageF(*size)
            mi2 = afwImage.MaskedImageF(*size)
            mi1.set(0)
            mi2.set(0)

            spanList1 = []
            spanList2 = []
            for y, x0, x1 in [
                (5, 3, 7),
                (6, 3, 4),
                (6, 6, 7),
                (7, 3, 7),
            ]:
                spanList1.append(afwGeom.Span(y, x0, x1))
                spanList2.append(
                    afwGeom.Span(y + yOffset, x0 + xOffset, x1 + xOffset))
                for x in range(x0, x1 + 1):
                    value = (x + y, 0, 1.0)
                    mi1[x, y, afwImage.LOCAL] = value
                    mi2[x + xOffset, y + yOffset, afwImage.LOCAL] = value

            fp1 = afwDetect.Footprint(afwGeom.SpanSet(spanList1))
            fp2 = afwDetect.Footprint(afwGeom.SpanSet(spanList2))

            hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1)
            hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2)

            dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray())
            self.assertEqual(hfp1.dot(hfp2), dot)
            self.assertEqual(hfp2.dot(hfp1), dot)
Ejemplo n.º 4
0
def footprintToImage(fp, mi=None, mask=False):
    if fp.isHeavy():
        try:
            fp = afwDet.cast_HeavyFootprintF(fp) # not needed with pybind11
        except AttributeError:
            pass
        pass
    elif mi is None:
        print("Unable to make a HeavyFootprint as image is None", file=sys.stderr)
    else:
        fp = afwDet.makeHeavyFootprint(fp, mi)
    bb = fp.getBBox()
    if mask:
        im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight())
    else:
        im = afwImage.ImageF(bb.getWidth(), bb.getHeight())
    im.setXY0(bb.getMinX(), bb.getMinY())

    try:
        fp.insert(im)
    except AttributeError:              # we failed to make it heavy
        assert not mi
        pass
    
    if mask:
        im = im.getMask()
    return im
def morphToHeavy(source, peakSchema, xy0=Point2I()):
    """Convert the morphology to a `HeavyFootprint`

    Parameters
    ----------
    source : `scarlet.Component`
        The scarlet source with a morphology to convert to
        a `HeavyFootprint`.
    peakSchema : `lsst.daf.butler.Schema`
        The schema for the `PeakCatalog` of the `HeavyFootprint`.
    xy0 : `tuple`
        `(x,y)` coordinates of the bounding box containing the
        `HeavyFootprint`.

    Returns
    -------
    heavy : `lsst.afw.detection.HeavyFootprint`
    """
    mask = afwImage.MaskX(np.array(source.morph > 0, dtype=np.int32), xy0=xy0)
    ss = SpanSet.fromMask(mask)

    if len(ss) == 0:
        return None

    tfoot = afwDet.Footprint(ss, peakSchema=peakSchema)
    cy, cx = source.pixel_center
    xmin, ymin = xy0
    # HeavyFootprints are not defined for 64 bit floats
    morph = source.morph.astype(np.float32)
    peakFlux = morph[cy, cx]
    tfoot.addPeak(cx + xmin, cy + ymin, peakFlux)
    timg = afwImage.ImageF(morph, xy0=xy0)
    timg = timg[tfoot.getBBox()]
    heavy = afwDet.makeHeavyFootprint(tfoot, afwImage.MaskedImageF(timg))
    return heavy
Ejemplo n.º 6
0
def footprintToImage(fp, mi=None, mask=False):
    if fp.isHeavy():
        try:
            fp = afwDet.cast_HeavyFootprintF(fp)  # not needed with pybind11
        except AttributeError:
            pass
        pass
    elif mi is None:
        print >> sys.stderr, "Unable to make a HeavyFootprint as image is None"
    else:
        fp = afwDet.makeHeavyFootprint(fp, mi)
    bb = fp.getBBox()
    if mask:
        im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight())
    else:
        im = afwImage.ImageF(bb.getWidth(), bb.getHeight())
    im.setXY0(bb.getMinX(), bb.getMinY())

    try:
        fp.insert(im)
    except AttributeError:  # we failed to make it heavy
        assert not mi
        pass

    if mask:
        im = im.getMask()
    return im
Ejemplo n.º 7
0
    def testSetFootprint(self):
        """Check that we can create a HeavyFootprint and set the pixels under it"""

        ctrl = afwDetect.HeavyFootprintCtrl()
        ctrl.setModifySource(afwDetect.HeavyFootprintCtrl.SET)  # clear the pixels in the Footprint
        ctrl.setMaskVal(self.objectPixelVal[1])

        afwDetect.makeHeavyFootprint(self.foot, self.mi, ctrl)
        #
        # Check that we cleared all the pixels
        #
        self.assertEqual(np.min(self.mi.getImage().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getImage().getArray()), 0.0)
        self.assertEqual(np.min(self.mi.getMask().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getMask().getArray()), 0.0)
        self.assertEqual(np.min(self.mi.getVariance().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getVariance().getArray()), 0.0)
Ejemplo n.º 8
0
    def testSetFootprint(self):
        """Check that we can create a HeavyFootprint and set the pixels under it"""

        ctrl = afwDetect.HeavyFootprintCtrl()
        ctrl.setModifySource(afwDetect.HeavyFootprintCtrl.SET)  # clear the pixels in the Footprint
        ctrl.setMaskVal(self.objectPixelVal[1])

        afwDetect.makeHeavyFootprint(self.foot, self.mi, ctrl)
        #
        # Check that we cleared all the pixels
        #
        self.assertEqual(np.min(self.mi.getImage().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getImage().getArray()), 0.0)
        self.assertEqual(np.min(self.mi.getMask().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getMask().getArray()), 0.0)
        self.assertEqual(np.min(self.mi.getVariance().getArray()), 0.0)
        self.assertEqual(np.max(self.mi.getVariance().getArray()), 0.0)
Ejemplo n.º 9
0
    def testCast_HeavyFootprint(self):
        """Test that we can cast a Footprint to a HeavyFootprint"""

        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi)

        ctrl = afwDetect.HeavyFootprintCtrl(afwDetect.HeavyFootprintCtrl.NONE)
        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi, ctrl)
        #
        # This isn't quite a full test, as hfoot is already a HeavyFootprint,
        # the complete test is in testMakeHeavy
        #        
        self.assertNotEqual(afwDetect.cast_HeavyFootprint(hfoot, self.mi), None,
                            "Cast to the right sort of HeavyFootprint")
        self.assertNotEqual(afwDetect.HeavyFootprintF.cast(hfoot), None,
                            "Cast to the right sort of HeavyFootprint")

        self.assertEqual(afwDetect.cast_HeavyFootprint(self.foot, self.mi), None,
                         "Can't cast a Footprint to a HeavyFootprint")
        self.assertEqual(afwDetect.HeavyFootprintI.cast(hfoot), None,
                         "Cast to the wrong sort of HeavyFootprint")
Ejemplo n.º 10
0
    def test1(self):
        im = afwImage.ImageF(100, 100)
        im += 42.
        fp = afwDet.Footprint(afwGeom.Point2I(50,50), 10.)
        #seed = 42
        #rand = afwMath.Random(afwMath.Random.MT19937, seed)
        #afwMath.randomGaussianImage(im, rand)
        mi = afwImage.MaskedImageF(im)
        # set a mask bit before grabbing the heavyfootprint
        mi.getMask().set(50, 50, 1)
        heavy = afwDet.makeHeavyFootprint(fp, mi)
        # reset it
        mi.getMask().set(50, 50, 0)

        schema = afwTable.SourceTable.makeMinimalSchema()
        table = afwTable.SourceTable.make(schema)
        table.preallocate(10)
        catalog = afwTable.SourceCatalog(table)
        catalog.addNew()
        # This used to segfault
        catalog[0].setFootprint(heavy)

        # However, we still have to up-cast
        fp = catalog[0].getFootprint()
        hfp = afwDet.cast_HeavyFootprintF(fp)
        # change one pixel...
        self.assertEqual(mi.getImage().get(50, 50), 42)
        self.assertEqual(mi.getMask().get(50, 50), 0)
        mi.getImage().set(50, 50, 100)
        mi.getMask().set(50, 50, 2)
        mi.getMask().set(51, 50, 2)
        self.assertEqual(mi.getImage().get(50, 50), 100)
        self.assertEqual(mi.getMask().get(50, 50), 2)
        self.assertEqual(mi.getMask().get(51, 50), 2)
        # reinsert the heavy footprint; it should reset the pixel value.
        # insert(MaskedImage)
        hfp.insert(mi)
        self.assertEqual(mi.getImage().get(50, 50), 42)
        self.assertEqual(mi.getMask().get(50, 50), 1)
        self.assertEqual(mi.getMask().get(51, 50), 0)

        # Also test insert(Image)
        im = mi.getImage()
        self.assertEqual(im.get(50, 50), 42)
        im.set(50, 50, 100)
        self.assertEqual(im.get(50, 50), 100)
        self.assertEqual(mi.getImage().get(50, 50), 100)
        # reinsert the heavy footprint; it should reset the pixel value.
        hfp.insert(im)
        self.assertEqual(im.get(50, 50), 42)
        self.assertEqual(mi.getImage().get(50, 50), 42)
        self.assertEqual(mi.getMask().get(50, 50), 1)
        self.assertEqual(mi.getMask().get(51, 50), 0)
Ejemplo n.º 11
0
def footprintToImage(fp, mi=None, mask=False):
    if not fp.isHeavy():
        fp = afwDet.makeHeavyFootprint(fp, mi)
    bb = fp.getBBox()
    if mask:
        im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight())
    else:
        im = afwImage.ImageF(bb.getWidth(), bb.getHeight())
    im.setXY0(bb.getMinX(), bb.getMinY())
    fp.insert(im)
    if mask:
        im = im.getMask()
    return im
Ejemplo n.º 12
0
def footprintToImage(fp, mi=None, mask=False):
    if not fp.isHeavy():
        fp = afwDet.makeHeavyFootprint(fp, mi)
    bb = fp.getBBox()
    if mask:
        im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight())
    else:
        im = afwImage.ImageF(bb.getWidth(), bb.getHeight())
    im.setXY0(bb.getMinX(), bb.getMinY())
    fp.insert(im)
    if mask:
        im = im.getMask()
    return im
Ejemplo n.º 13
0
    def getFluxPortion(self, strayFlux=True):
        """!
        Return a HeavyFootprint containing the flux apportioned to this peak.

        @param[in]     strayFlux   include stray flux also?
        """
        if self.templateFootprint is None or self.fluxPortion is None:
            return None
        heavy = afwDet.makeHeavyFootprint(self.templateFootprint, self.fluxPortion)
        if strayFlux:
            if self.strayFlux is not None:
                heavy = afwDet.mergeHeavyFootprints(heavy, self.strayFlux)

        return heavy
Ejemplo n.º 14
0
    def getFluxPortion(self, strayFlux=True):
        """!
        Return a HeavyFootprint containing the flux apportioned to this peak.

        @param[in]     strayFlux   include stray flux also?
        """
        if self.templateFootprint is None or self.fluxPortion is None:
            return None
        heavy = afwDet.makeHeavyFootprint(self.templateFootprint, self.fluxPortion)
        if strayFlux:
            if self.strayFlux is not None:
                heavy = afwDet.mergeHeavyFootprints(heavy, self.strayFlux)

        return heavy
Ejemplo n.º 15
0
    def test1(self):
        im = afwImage.ImageF(100, 100)
        im += 42.
        spanSet = afwGeom.SpanSet.fromShape(10).shiftedBy(50, 50)
        fp = afwDet.Footprint(spanSet)
        mi = afwImage.MaskedImageF(im)
        # set a mask bit before grabbing the heavyfootprint
        mi.mask[50, 50, afwImage.LOCAL] = 1
        heavy = afwDet.makeHeavyFootprint(fp, mi)
        # reset it
        mi.mask[50, 50, afwImage.LOCAL] = 0

        schema = afwTable.SourceTable.makeMinimalSchema()
        table = afwTable.SourceTable.make(schema)
        table.preallocate(10)
        catalog = afwTable.SourceCatalog(table)
        catalog.addNew()
        # This used to segfault
        catalog[0].setFootprint(heavy)

        fp = catalog[0].getFootprint()
        # change one pixel...
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 0)
        mi.image[50, 50, afwImage.LOCAL] = 100
        mi.mask[50, 50, afwImage.LOCAL] = 2
        mi.mask[51, 50, afwImage.LOCAL] = 2
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 2)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 2)
        # reinsert the heavy footprint; it should reset the pixel value.
        # insert(MaskedImage)
        fp.insert(mi)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0)

        # Also test insert(Image)
        im = mi.image
        self.assertEqual(im[50, 50, afwImage.LOCAL], 42)
        im[50, 50, afwImage.LOCAL] = 100
        self.assertEqual(im[50, 50, afwImage.LOCAL], 100)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100)
        # reinsert the heavy footprint; it should reset the pixel value.
        fp.insert(im)
        self.assertEqual(im[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0)
Ejemplo n.º 16
0
    def test1(self):
        im = afwImage.ImageF(100, 100)
        im += 42.
        spanSet = afwGeom.SpanSet.fromShape(10).shiftedBy(50, 50)
        fp = afwDet.Footprint(spanSet)
        mi = afwImage.MaskedImageF(im)
        # set a mask bit before grabbing the heavyfootprint
        mi.mask[50, 50, afwImage.LOCAL] = 1
        heavy = afwDet.makeHeavyFootprint(fp, mi)
        # reset it
        mi.mask[50, 50, afwImage.LOCAL] = 0

        schema = afwTable.SourceTable.makeMinimalSchema()
        table = afwTable.SourceTable.make(schema)
        table.preallocate(10)
        catalog = afwTable.SourceCatalog(table)
        catalog.addNew()
        # This used to segfault
        catalog[0].setFootprint(heavy)

        fp = catalog[0].getFootprint()
        # change one pixel...
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 0)
        mi.image[50, 50, afwImage.LOCAL] = 100
        mi.mask[50, 50, afwImage.LOCAL] = 2
        mi.mask[51, 50, afwImage.LOCAL] = 2
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 2)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 2)
        # reinsert the heavy footprint; it should reset the pixel value.
        # insert(MaskedImage)
        fp.insert(mi)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0)

        # Also test insert(Image)
        im = mi.image
        self.assertEqual(im[50, 50, afwImage.LOCAL], 42)
        im[50, 50, afwImage.LOCAL] = 100
        self.assertEqual(im[50, 50, afwImage.LOCAL], 100)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100)
        # reinsert the heavy footprint; it should reset the pixel value.
        fp.insert(im)
        self.assertEqual(im[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42)
        self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1)
        self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0)
Ejemplo n.º 17
0
    def testCreate(self):
        """Check that we can create a HeavyFootprint"""

        imi = self.mi.Factory(self.mi, True)  # copy of input image

        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        self.assertNotEqual(hfoot.getId(),
                            None)  # check we can call a base-class method
        #
        # Check we didn't modify the input image
        #
        self.assertTrue(
            np.all(
                np.equal(self.mi.getImage().getArray(),
                         imi.getImage().getArray())))

        omi = self.mi.Factory(self.mi.getDimensions())
        omi.set((1, 0x4, 0.1))
        hfoot.insert(omi)

        if display:
            ds9.mtv(imi, frame=0, title="input")
            ds9.mtv(omi, frame=1, title="output")

        for s in self.foot.getSpans():
            y = s.getY()
            for x in range(s.getX0(), s.getX1() + 1):
                self.assertEqual(imi.get(x, y), omi.get(x, y))

        # Check that we can call getImageArray(), etc
        arr = hfoot.getImageArray()
        print arr
        # Check that it's iterable
        for x in arr:
            pass
        arr = hfoot.getMaskArray()
        print arr
        for x in arr:
            pass
        arr = hfoot.getVarianceArray()
        print arr
        # Check that it's iterable
        for x in arr:
            pass
Ejemplo n.º 18
0
def readCatalog(sourcefn,
                heavypat,
                ndeblends=0,
                dataref=None,
                keepids=None,
                keepxys=None,
                patargs=dict()):
    if sourcefn is None:
        cat = dataref.get('src')
        try:
            if not cat:
                return None
        except:
            return None
    else:
        if not os.path.exists(sourcefn):
            print('No source catalog:', sourcefn)
            return None
        print('Reading catalog:', sourcefn)
        cat = afwTable.SourceCatalog.readFits(sourcefn)
        print(len(cat), 'sources')
    cat.sort()
    cat.defineCentroid('base_SdssCentroid')

    if ndeblends or keepids or keepxys:
        cat = cutCatalog(cat, ndeblends, keepids, keepxys)
        print('Cut to', len(cat), 'sources')

    if heavypat is not None:
        print('Reading heavyFootprints...')
        for src in cat:
            if not src.getParent():
                continue
            dd = patargs.copy()
            dd.update(id=src.getId())
            heavyfn = heavypat % dd
            if not os.path.exists(heavyfn):
                print('No heavy footprint:', heavyfn)
                return None
            mim = afwImage.MaskedImageF(heavyfn)
            heavy = afwDet.makeHeavyFootprint(src.getFootprint(), mim)
            src.setFootprint(heavy)
    return cat
Ejemplo n.º 19
0
    def testCreate(self):
        """Check that we can create a HeavyFootprint"""

        imi = self.mi.Factory(self.mi, True)  # copy of input image

        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        # check we can call a base-class method
        self.assertTrue(hfoot.isHeavy())
        #
        # Check we didn't modify the input image
        #
        self.assertFloatsEqual(self.mi.getImage().getArray(),
                               imi.getImage().getArray())

        omi = self.mi.Factory(self.mi.getDimensions())
        omi.set((1, 0x4, 0.1))
        hfoot.insert(omi)

        if display:
            afwDisplay.Display(frame=0).mtv(imi, title="testCreate input")
            afwDisplay.Display(frame=1).mtv(omi, title="testCreate output")

        for s in self.foot.getSpans():
            y = s.getY()
            for x in range(s.getX0(), s.getX1() + 1):
                self.assertEqual(imi[x, y, afwImage.LOCAL],
                                 omi[x, y, afwImage.LOCAL])

        # Check that we can call getImageArray(), etc
        arr = hfoot.getImageArray()
        # Check that it's iterable
        for x in arr:
            pass
        arr = hfoot.getMaskArray()
        for x in arr:
            pass
        arr = hfoot.getVarianceArray()
        # Check that it's iterable
        for x in arr:
            pass
Ejemplo n.º 20
0
    def testCreate(self):
        """Check that we can create a HeavyFootprint"""

        imi = self.mi.Factory(self.mi, True) # copy of input image

        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        self.assertNotEqual(hfoot.getId(), None) # check we can call a base-class method
        #
        # Check we didn't modify the input image
        #
        self.assertTrue(np.all(np.equal(self.mi.getImage().getArray(), imi.getImage().getArray())))
        
        omi = self.mi.Factory(self.mi.getDimensions())
        omi.set((1, 0x4, 0.1))
        hfoot.insert(omi)

        if display:
            ds9.mtv(imi, frame=0, title="input")
            ds9.mtv(omi, frame=1, title="output")

        for s in self.foot.getSpans():
            y = s.getY()
            for x in range(s.getX0(), s.getX1() + 1):
                self.assertEqual(imi.get(x, y), omi.get(x, y))

        # Check that we can call getImageArray(), etc
        arr = hfoot.getImageArray()
        print arr
        # Check that it's iterable
        for x in arr:
            pass
        arr = hfoot.getMaskArray()
        print arr
        for x in arr:
            pass
        arr = hfoot.getVarianceArray()
        print arr
        # Check that it's iterable
        for x in arr:
            pass
Ejemplo n.º 21
0
def readCatalog(sourcefn, heavypat, ndeblends=0, dataref=None,
                keepids=None, keepxys=None,
                patargs=dict()):
    if sourcefn is None:
        cat = dataref.get('src')
        try:
            if not cat:
                return None
        except Exception:
            return None
    else:
        if not os.path.exists(sourcefn):
            print('No source catalog:', sourcefn)
            return None
        print('Reading catalog:', sourcefn)
        cat = afwTable.SourceCatalog.readFits(sourcefn)
        print(len(cat), 'sources')
    cat.sort()
    cat.defineCentroid('base_SdssCentroid')

    if ndeblends or keepids or keepxys:
        cat = cutCatalog(cat, ndeblends, keepids, keepxys)
        print('Cut to', len(cat), 'sources')

    if heavypat is not None:
        print('Reading heavyFootprints...')
        for src in cat:
            if not src.getParent():
                continue
            dd = patargs.copy()
            dd.update(id=src.getId())
            heavyfn = heavypat % dd
            if not os.path.exists(heavyfn):
                print('No heavy footprint:', heavyfn)
                return None
            mim = afwImage.MaskedImageF(heavyfn)
            heavy = afwDet.makeHeavyFootprint(src.getFootprint(), mim)
            src.setFootprint(heavy)
    return cat
Ejemplo n.º 22
0
    def testCreate(self):
        """Check that we can create a HeavyFootprint"""

        imi = self.mi.Factory(self.mi, True)  # copy of input image

        hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        # check we can call a base-class method
        self.assertNotEqual(hfoot.getId(), None)
        #
        # Check we didn't modify the input image
        #
        self.assertFloatsEqual(
            self.mi.getImage().getArray(), imi.getImage().getArray())

        omi = self.mi.Factory(self.mi.getDimensions())
        omi.set((1, 0x4, 0.1))
        hfoot.insert(omi)

        if display:
            afwDisplay.Display(frame=0).mtv(imi, title="testCreate input")
            afwDisplay.Display(frame=1).mtv(omi, title="testCreate output")

        for s in self.foot.getSpans():
            y = s.getY()
            for x in range(s.getX0(), s.getX1() + 1):
                self.assertEqual(imi[x, y, afwImage.LOCAL], omi[x, y, afwImage.LOCAL])

        # Check that we can call getImageArray(), etc
        arr = hfoot.getImageArray()
        # Check that it's iterable
        for x in arr:
            pass
        arr = hfoot.getMaskArray()
        for x in arr:
            pass
        arr = hfoot.getVarianceArray()
        # Check that it's iterable
        for x in arr:
            pass
Ejemplo n.º 23
0
 def setUp(self):
     np.random.seed(1)
     self.spans = SpanSet.fromShape(2, Stencil.CIRCLE)
     self.footprint = Footprint(self.spans)
     self.footprint.addPeak(3, 4, 10)
     self.footprint.addPeak(8, 1, 2)
     fp = Footprint(self.spans)
     for peak in self.footprint.getPeaks():
         fp.addPeak(peak["f_x"], peak["f_y"], peak["peakValue"])
     self.peaks = fp.getPeaks()
     self.bbox = self.footprint.getBBox()
     self.filters = ("G", "R", "I")
     singles = []
     images = []
     for n, f in enumerate(self.filters):
         image = ImageF(self.spans.getBBox())
         image.set(n)
         images.append(image.array)
         maskedImage = MaskedImageF(image)
         heavy = makeHeavyFootprint(self.footprint, maskedImage)
         singles.append(heavy)
     self.image = np.array(images)
     self.mFoot = MultibandFootprint(self.filters, singles)
Ejemplo n.º 24
0
 def setUp(self):
     np.random.seed(1)
     self.spans = SpanSet.fromShape(2, Stencil.CIRCLE)
     self.footprint = Footprint(self.spans)
     self.footprint.addPeak(3, 4, 10)
     self.footprint.addPeak(8, 1, 2)
     fp = Footprint(self.spans)
     for peak in self.footprint.getPeaks():
         fp.addPeak(peak["f_x"], peak["f_y"], peak["peakValue"])
     self.peaks = fp.getPeaks()
     self.bbox = self.footprint.getBBox()
     self.filters = ("G", "R", "I")
     singles = []
     images = []
     for n, f in enumerate(self.filters):
         image = ImageF(self.spans.getBBox())
         image.set(n)
         images.append(image.array)
         maskedImage = MaskedImageF(image)
         heavy = makeHeavyFootprint(self.footprint, maskedImage)
         singles.append(heavy)
     self.image = np.array(images)
     self.mFoot = MultibandFootprint(self.filters, singles)
Ejemplo n.º 25
0
    def testMergeHeavyFootprints(self):
        mi = afwImage.MaskedImageF(20, 10)
        objectPixelVal = (42, 0x9, 400)
        
        foot = afwDetect.Footprint()
        for y, x0, x1 in [(1, 9, 12),
                          (2, 12, 13),
                          (3, 11, 15)]:
            foot.addSpan(y, x0, x1)
            for x in range(x0, x1 + 1):
                mi.set(x, y, objectPixelVal)

        hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        hfoot2 = afwDetect.makeHeavyFootprint(foot, mi)

        hfoot1.normalize()
        hfoot2.normalize()
        hsum = afwDetect.mergeHeavyFootprintsF(hfoot1, hfoot2)
        
        bb = hsum.getBBox()
        self.assertEquals(bb.getMinX(), 9)
        self.assertEquals(bb.getMaxX(), 15)
        self.assertEquals(bb.getMinY(), 1)
        self.assertEquals(bb.getMaxY(), 3)

        msum = afwImage.MaskedImageF(20,10)
        hsum.insert(msum)

        sa = msum.getImage().getArray()

        self.assertTrue(np.all(sa[1, 9:13] == objectPixelVal[0]))
        self.assertTrue(np.all(sa[2, 12:14] == objectPixelVal[0] + self.objectPixelVal[0]))
        self.assertTrue(np.all(sa[2, 10:12] == self.objectPixelVal[0]))

        sv = msum.getVariance().getArray()

        self.assertTrue(np.all(sv[1, 9:13] == objectPixelVal[2]))
        self.assertTrue(np.all(sv[2, 12:14] == objectPixelVal[2] + self.objectPixelVal[2]))
        self.assertTrue(np.all(sv[2, 10:12] == self.objectPixelVal[2]))

        sm = msum.getMask().getArray()

        self.assertTrue(np.all(sm[1, 9:13] == objectPixelVal[1]))
        self.assertTrue(np.all(sm[2, 12:14] == objectPixelVal[1] | self.objectPixelVal[1]))
        self.assertTrue(np.all(sm[2, 10:12] == self.objectPixelVal[1]))


        if False:
            import matplotlib
            matplotlib.use('Agg')
            import pylab as plt
            im1 = afwImage.ImageF(bb)
            hfoot1.insert(im1)
            im2 = afwImage.ImageF(bb)
            hfoot2.insert(im2)
            im3 = afwImage.ImageF(bb)
            hsum.insert(im3)
            plt.clf()
            plt.subplot(1,3,1)
            plt.imshow(im1.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1,3,2)
            plt.imshow(im2.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1,3,3)
            plt.imshow(im3.getArray(), interpolation='nearest', origin='lower')
            plt.savefig('merge.png')
Ejemplo n.º 26
0
    def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None):
        """!
        Initialize the NoiseReplacer.

        @param[in]      config       instance of NoiseReplacerConfig
        @param[in,out]  exposure     Exposure to be noise replaced. (All sources replaced on return)
        @param[in]      footprints   dict of {id: (parent, footprint)};
        @param[in]      noiseImage   an afw.image.ImageF used as a predictable noise replacement source
                                     (for tests only)
        @param[in]      log          pex.logging.Log object to use for status messages; no status messages
                                     will be printed if None

        'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the
        source ID, but in forced photometry, this will be the reference ID, as that's what we used to
        determine the deblend families.  This routine should create HeavyFootprints for any non-Heavy
        Footprints, and replace them in the dict.  It should then create a dict of HeavyFootprints
        containing noise, but only for parent objects, then replace all sources with noise.
        This should ignore any footprints that lay outside the bounding box of the exposure,
        and clip those that lie on the border.

        NOTE: as the code currently stands, the heavy footprint for a deblended object must be available
        from the input catalog.  If it is not, it cannot be reproduced here.  In that case, the
        topmost parent in the objects parent chain must be used.  The heavy footprint for that source
        is created in this class from the masked image.
        """
        noiseMeanVar=None
        self.noiseSource = config.noiseSource
        self.noiseOffset = config.noiseOffset
        self.noiseSeedMultiplier = config.noiseSeedMultiplier
        self.noiseGenMean = None
        self.noiseGenStd = None
        self.log = log

        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        self.exposure = exposure
        self.footprints = footprints
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()
        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            try:
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                if self.log: self.log.logdebug('Mask plane "%s" already existed' % maskname)
            except:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            if self.log: self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x'
                % (maskname, plane, bitmask, bitmask))
        self.thisbitmask,self.otherbitmask = bitmasks
        del bitmasks
        self.heavies = {}
        # Start by creating HeavyFootprints for each source which has no parent
        # and just use them for children which do not already have heavy footprints.
        # If a heavy footprint is available for a child, we will use it. Otherwise,
        # we use the first parent in the parent chain which has a heavy footprint,
        # which with the one level deblender will alway be the topmost parent
        # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py,
        # so they are never available for forced measurements.

        # Create in the dict heavies = {id:heavyfootprint}
        for id in footprints.keys():
            fp = footprints[id]
            if fp[1].isHeavy():
                self.heavies[id] = afwDet.cast_HeavyFootprintF(fp[1])
            elif fp[0] == 0:
                self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)

        ### FIXME: the heavy footprint includes the mask
        ### and variance planes, which we shouldn't need
        ### (I don't think we ever want to modify them in
        ### the input image).  Copying them around is
        ### wasteful.

        # We now create a noise HeavyFootprint for each source with has a heavy footprint.
        # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId)
        #  The noiseGenMean and Std are used by the unit tests
        self.noiseGenMean = noisegen.mean
        self.noiseGenStd = noisegen.std
        if self.log: self.log.logdebug('Using noise generator: %s' % (str(noisegen)))
        for id in self.heavies.keys():
            fp = footprints[id][1]
            noiseFp = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[id] = noiseFp
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            noiseFp.insert(im)
            # Also set the OTHERDET bit
            afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
Ejemplo n.º 27
0
    def deblend(self, exposure, srcs, psf):
        """!
        Deblend.

        @param[in]     exposure Exposure to process
        @param[in,out] srcs     SourceCatalog containing sources detected on this exposure.
        @param[in]     psf      PSF

        @return None
        """
        self.log.info("Deblending %d sources" % len(srcs))

        from lsst.meas.deblender.baseline import deblend

        # find the median stdev in the image...
        mi = exposure.getMaskedImage()
        statsCtrl = afwMath.StatisticsControl()
        statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(
            self.config.maskPlanes))
        stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(),
                                       afwMath.MEDIAN, statsCtrl)
        sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
        self.log.trace('sigma1: %g', sigma1)

        n0 = len(srcs)
        nparents = 0
        for i, src in enumerate(srcs):
            # t0 = time.clock()

            fp = src.getFootprint()
            pks = fp.getPeaks()

            # Since we use the first peak for the parent object, we should propagate its flags
            # to the parent source.
            src.assign(pks[0], self.peakSchemaMapper)

            if len(pks) < 2:
                continue

            if self.isLargeFootprint(fp):
                src.set(self.tooBigKey, True)
                self.skipParent(src, mi.getMask())
                self.log.warn('Parent %i: skipping large footprint (area: %i)',
                              int(src.getId()), int(fp.getArea()))
                continue
            if self.isMasked(fp, exposure.getMaskedImage().getMask()):
                src.set(self.maskedKey, True)
                self.skipParent(src, mi.getMask())
                self.log.warn(
                    'Parent %i: skipping masked footprint (area: %i)',
                    int(src.getId()), int(fp.getArea()))
                continue

            nparents += 1
            bb = fp.getBBox()
            psf_fwhm = self._getPsfFwhm(psf, bb)

            self.log.trace('Parent %i: deblending %i peaks', int(src.getId()),
                           len(pks))

            self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm,
                                      sigma1)
            npre = len(srcs)

            # This should really be set in deblend, but deblend doesn't have access to the src
            src.set(self.tooManyPeaksKey,
                    len(fp.getPeaks()) > self.config.maxNumberOfPeaks)

            try:
                res = deblend(
                    fp,
                    mi,
                    psf,
                    psf_fwhm,
                    sigma1=sigma1,
                    psfChisqCut1=self.config.psfChisq1,
                    psfChisqCut2=self.config.psfChisq2,
                    psfChisqCut2b=self.config.psfChisq2b,
                    maxNumberOfPeaks=self.config.maxNumberOfPeaks,
                    strayFluxToPointSources=self.config.
                    strayFluxToPointSources,
                    assignStrayFlux=self.config.assignStrayFlux,
                    strayFluxAssignment=self.config.strayFluxRule,
                    rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
                    patchEdges=(self.config.edgeHandling == 'noclip'),
                    tinyFootprintSize=self.config.tinyFootprintSize,
                    clipStrayFluxFraction=self.config.clipStrayFluxFraction,
                    weightTemplates=self.config.weightTemplates,
                    removeDegenerateTemplates=self.config.
                    removeDegenerateTemplates,
                    maxTempDotProd=self.config.maxTempDotProd,
                    medianSmoothTemplate=self.config.medianSmoothTemplate)
                if self.config.catchFailures:
                    src.set(self.deblendFailedKey, False)
            except Exception as e:
                if self.config.catchFailures:
                    self.log.warn("Unable to deblend source %d: %s" %
                                  (src.getId(), e))
                    src.set(self.deblendFailedKey, True)
                    import traceback
                    traceback.print_exc()
                    continue
                else:
                    raise

            kids = []
            nchild = 0
            for j, peak in enumerate(res.deblendedParents[0].peaks):
                heavy = peak.getFluxPortion()
                if heavy is None or peak.skip:
                    src.set(self.deblendSkippedKey, True)
                    if not self.config.propagateAllPeaks:
                        # Don't care
                        continue
                    # We need to preserve the peak: make sure we have enough info to create a minimal
                    # child src
                    self.log.trace(
                        "Peak at (%i,%i) failed.  Using minimal default info for child.",
                        pks[j].getIx(), pks[j].getIy())
                    if heavy is None:
                        # copy the full footprint and strip out extra peaks
                        foot = afwDet.Footprint(src.getFootprint())
                        peakList = foot.getPeaks()
                        peakList.clear()
                        peakList.append(peak.peak)
                        zeroMimg = afwImage.MaskedImageF(foot.getBBox())
                        heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
                    if peak.deblendedAsPsf:
                        if peak.psfFitFlux is None:
                            peak.psfFitFlux = 0.0
                        if peak.psfFitCenter is None:
                            peak.psfFitCenter = (peak.peak.getIx(),
                                                 peak.peak.getIy())

                assert (len(heavy.getPeaks()) == 1)

                src.set(self.deblendSkippedKey, False)
                child = srcs.addNew()
                nchild += 1
                for key in self.toCopyFromParent:
                    child.set(key, src.get(key))
                child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
                child.setParent(src.getId())
                child.setFootprint(heavy)
                child.set(self.psfKey, peak.deblendedAsPsf)
                child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
                if peak.deblendedAsPsf:
                    (cx, cy) = peak.psfFitCenter
                    child.set(self.psfCenterKey, geom.Point2D(cx, cy))
                    child.set(self.psfFluxKey, peak.psfFitFlux)
                child.set(self.deblendRampedTemplateKey,
                          peak.hasRampedTemplate)
                child.set(self.deblendPatchedTemplateKey, peak.patched)

                # Set the position of the peak from the parent footprint
                # This will make it easier to match the same source across
                # deblenders and across observations, where the peak
                # position is unlikely to change unless enough time passes
                # for a source to move on the sky.
                child.set(self.peakCenter,
                          geom.Point2I(pks[j].getIx(), pks[j].getIy()))
                child.set(self.peakIdKey, pks[j].getId())

                # The children have a single peak
                child.set(self.nPeaksKey, 1)
                # Set the number of peaks in the parent
                child.set(self.parentNPeaksKey, len(pks))

                kids.append(child)

            # Child footprints may extend beyond the full extent of their parent's which
            # results in a failure of the replace-by-noise code to reinstate these pixels
            # to their original values.  The following updates the parent footprint
            # in-place to ensure it contains the full union of itself and all of its
            # children's footprints.
            spans = src.getFootprint().spans
            for child in kids:
                spans = spans.union(child.getFootprint().spans)
            src.getFootprint().setSpans(spans)

            src.set(self.nChildKey, nchild)

            self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf,
                                       psf_fwhm, sigma1, res)
            # print('Deblending parent id', src.getId(), 'took', time.clock() - t0)

        n1 = len(srcs)
        self.log.info(
            'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
            % (n0, nparents, n1 - n0, n1))
    def deblend(self, mExposure, sources):
        """Deblend a data cube of multiband images

        Parameters
        ----------
        mExposure : `MultibandExposure`
            The exposures should be co-added images of the same
            shape and region of the sky.
        sources : `SourceCatalog`
            The merged `SourceCatalog` that contains parent footprints
            to (potentially) deblend.

        Returns
        -------
        fluxCatalogs : dict or None
            Keys are the names of the filters and the values are
            `lsst.afw.table.source.source.SourceCatalog`'s.
            These are the flux-conserved catalogs with heavy footprints with
            the image data weighted by the multiband templates.
            If `self.config.conserveFlux` is `False`, then this item will be
            None
        templateCatalogs : dict or None
            Keys are the names of the filters and the values are
            `lsst.afw.table.source.source.SourceCatalog`'s.
            These are catalogs with heavy footprints that are the templates
            created by the multiband templates.
            If `self.config.saveTemplates` is `False`, then this item will be
            None
        """
        import time

        filters = mExposure.filters
        self.log.info("Deblending {0} sources in {1} exposure bands".format(
            len(sources), len(mExposure)))

        # Create the output catalogs
        templateCatalogs = {}
        # This must be returned but is not calculated right now, setting it to
        # None to be consistent with doc string
        fluxCatalogs = None
        for f in filters:
            _catalog = afwTable.SourceCatalog(sources.table.clone())
            _catalog.extend(sources)
            templateCatalogs[f] = _catalog

        n0 = len(sources)
        nparents = 0
        for pk, src in enumerate(sources):
            foot = src.getFootprint()
            bbox = foot.getBBox()
            logger.info("id: {0}".format(src["id"]))
            peaks = foot.getPeaks()

            # Since we use the first peak for the parent object, we should
            # propagate its flags to the parent source.
            src.assign(peaks[0], self.peakSchemaMapper)

            # Block of Skipping conditions
            if len(peaks) < 2 and not self.config.processSingles:
                for f in filters:
                    templateCatalogs[f][pk].set(self.runtimeKey, 0)
                continue
            if self._isLargeFootprint(foot):
                src.set(self.tooBigKey, True)
                self._skipParent(src, mExposure.mask)
                self.log.trace('Parent %i: skipping large footprint',
                               int(src.getId()))
                continue
            if self._isMasked(foot, mExposure):
                src.set(self.maskedKey, True)
                mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array,
                                            axis=0)
                mask = afwImage.MaskX(mask, xy0=bbox.getMin())
                self._skipParent(src, mask)
                self.log.trace('Parent %i: skipping masked footprint',
                               int(src.getId()))
                continue
            if len(peaks) > self.config.maxNumberOfPeaks:
                src.set(self.tooManyPeaksKey, True)
                msg = 'Parent {0}: Too many peaks, using the first {1} peaks'
                self.log.trace(
                    msg.format(int(src.getId()), self.config.maxNumberOfPeaks))

            nparents += 1
            self.log.trace('Parent %i: deblending %i peaks', int(src.getId()),
                           len(peaks))
            # Run the deblender
            try:
                t0 = time.time()
                # Build the parameter lists with the same ordering
                blend, skipped = deblend(mExposure, foot, self.config)
                tf = time.time()
                runtime = (tf - t0) * 1000
                src.set(self.deblendFailedKey, False)
                src.set(self.runtimeKey, runtime)
                converged = checkBlendConvergence(blend,
                                                  self.config.relativeError)
                src.set(self.blendConvergenceFailedFlagKey, converged)
                sources = [src for src in blend.sources]
                # Re-insert place holders for skipped sources
                # to propagate them in the catalog so
                # that the peaks stay consistent
                for k in skipped:
                    sources.insert(k, None)
            except Exception as e:
                if self.config.catchFailures:
                    self.log.warn("Unable to deblend source %d: %s" %
                                  (src.getId(), e))
                    src.set(self.deblendFailedKey, True)
                    src.set(self.runtimeKey, 0)
                    import traceback
                    traceback.print_exc()
                    continue
                else:
                    raise

            # Add the merged source as a parent in the catalog for each band
            templateParents = {}
            parentId = src.getId()
            for f in filters:
                templateParents[f] = templateCatalogs[f][pk]
                templateParents[f].set(self.runtimeKey, runtime)
                templateParents[f].set(self.iterKey, len(blend.loss))

            # Add each source to the catalogs in each band
            templateSpans = {f: afwGeom.SpanSet() for f in filters}
            nchild = 0
            for k, source in enumerate(sources):
                # Skip any sources with no flux or that scarlet skipped because
                # it could not initialize
                if k in skipped:
                    if not self.config.propagateAllPeaks:
                        # We don't care
                        continue
                    # We need to preserve the peak: make sure we have enough
                    # info to create a minimal child src
                    msg = "Peak at {0} failed deblending.  Using minimal default info for child."
                    self.log.trace(msg.format(src.getFootprint().peaks[k]))
                    # copy the full footprint and strip out extra peaks
                    foot = afwDet.Footprint(src.getFootprint())
                    peakList = foot.getPeaks()
                    peakList.clear()
                    peakList.append(src.peaks[k])
                    zeroMimg = afwImage.MaskedImageF(foot.getBBox())
                    heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
                    models = afwDet.MultibandFootprint(
                        mExposure.filters, [heavy] * len(mExposure.filters))
                else:
                    src.set(self.deblendSkippedKey, False)
                    models = modelToHeavy(source,
                                          filters,
                                          xy0=bbox.getMin(),
                                          observation=blend.observations[0])
                # TODO: We should eventually write the morphology and SED to
                # the catalog
                # morph = source.morphToHeavy(xy0=bbox.getMin())
                # sed = source.sed / source.sed.sum()

                for f in filters:
                    if len(models[f].getPeaks()) != 1:
                        err = "Heavy footprint should have a single peak, got {0}"
                        raise ValueError(err.format(len(models[f].peaks)))
                    cat = templateCatalogs[f]
                    child = self._addChild(parentId,
                                           cat,
                                           models[f],
                                           source,
                                           converged,
                                           xy0=bbox.getMin())
                    if parentId == 0:
                        child.setId(src.getId())
                        child.set(self.runtimeKey, runtime)
                    else:
                        templateSpans[f] = templateSpans[f].union(
                            models[f].getSpans())
                nchild += 1

            # Child footprints may extend beyond the full extent of their
            # parent's which results in a failure of the replace-by-noise code
            # to reinstate these pixels to their original values.  The
            # following updates the parent footprint in-place to ensure it
            # contains the full union of itself and all of its
            # children's footprints.
            for f in filters:
                templateParents[f].set(self.nChildKey, nchild)
                templateParents[f].getFootprint().setSpans(templateSpans[f])

        K = len(list(templateCatalogs.values())[0])
        self.log.info(
            'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
            % (n0, nparents, K - n0, K))
        return fluxCatalogs, templateCatalogs
Ejemplo n.º 29
0
    def testMergeHeavyFootprints(self):
        mi = afwImage.MaskedImageF(20, 10)
        objectPixelVal = (42, 0x9, 400)

        spanList = []
        for y, x0, x1 in [(1, 9, 12),
                          (2, 12, 13),
                          (3, 11, 15)]:
            spanList.append(afwGeom.Span(y, x0, x1))
            for x in range(x0, x1 + 1):
                mi[x, y, afwImage.LOCAL] = objectPixelVal

        foot = afwDetect.Footprint(afwGeom.SpanSet(spanList))

        hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        hfoot2 = afwDetect.makeHeavyFootprint(foot, mi)

        hsum = afwDetect.mergeHeavyFootprints(hfoot1, hfoot2)

        bb = hsum.getBBox()
        self.assertEqual(bb.getMinX(), 9)
        self.assertEqual(bb.getMaxX(), 15)
        self.assertEqual(bb.getMinY(), 1)
        self.assertEqual(bb.getMaxY(), 3)

        msum = afwImage.MaskedImageF(20, 10)
        hsum.insert(msum)

        sa = msum.getImage().getArray()

        self.assertFloatsEqual(sa[1, 9:13], objectPixelVal[0])
        self.assertFloatsEqual(
            sa[2, 12:14], objectPixelVal[0] + self.objectPixelVal[0])
        self.assertFloatsEqual(sa[2, 10:12], self.objectPixelVal[0])

        sv = msum.getVariance().getArray()

        self.assertFloatsEqual(sv[1, 9:13], objectPixelVal[2])
        self.assertFloatsEqual(
            sv[2, 12:14], objectPixelVal[2] + self.objectPixelVal[2])
        self.assertFloatsEqual(sv[2, 10:12], self.objectPixelVal[2])

        sm = msum.getMask().getArray()

        self.assertFloatsEqual(sm[1, 9:13], objectPixelVal[1])
        self.assertFloatsEqual(
            sm[2, 12:14], objectPixelVal[1] | self.objectPixelVal[1])
        self.assertFloatsEqual(sm[2, 10:12], self.objectPixelVal[1])

        if False:
            import matplotlib
            matplotlib.use('Agg')
            import pylab as plt
            im1 = afwImage.ImageF(bb)
            hfoot1.insert(im1)
            im2 = afwImage.ImageF(bb)
            hfoot2.insert(im2)
            im3 = afwImage.ImageF(bb)
            hsum.insert(im3)
            plt.clf()
            plt.subplot(1, 3, 1)
            plt.imshow(im1.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1, 3, 2)
            plt.imshow(im2.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1, 3, 3)
            plt.imshow(im3.getArray(), interpolation='nearest', origin='lower')
            plt.savefig('merge.png')
Ejemplo n.º 30
0
    def test2(self):
        # Check that doReplaceWithNoise works with deblended source
        # hierarchies.
        seed = 42
        rand = afwMath.Random(afwMath.Random.MT19937, seed)

        psf = self.getpsf()
        im = afwImage.ImageF(200, 50)
        skystd = 100
        afwMath.randomGaussianImage(im, rand)
        im *= skystd
        imorig = afwImage.ImageF(im, True)
        noiseim = imorig

        mi = afwImage.MaskedImageF(im)
        mi.getVariance().set(skystd**2)
        exposure = afwImage.makeExposure(mi)
        exposure.setPsf(psf)

        detconf = measAlg.SourceDetectionConfig()
        detconf.returnOriginalFootprints = True
        detconf.reEstimateBackground = False
        measconf = measAlg.SourceMeasurementConfig()
        measconf.doReplaceWithNoise = True
        measconf.replaceWithNoise.noiseSeed = 42

        schema = afwTable.SourceTable.makeMinimalSchema()
        detect = measAlg.SourceDetectionTask(config=detconf, schema=schema)
        measure = MySourceMeasurementTask(config=measconf, schema=schema,
                                          doplot=plots)
        table = afwTable.SourceTable.make(schema)
        table.preallocate(10)

        # We're going to fake up a perfect deblend hierarchy here, by
        # creating individual images containing single sources and
        # measuring them, and then creating a deblend hierarchy where
        # the children have the correct HeavyFootprints.  We want to
        # find that the measurements on the deblend hierarchy and the
        # blended image are equal to the individual images.
        #
        # Note that in the normal setup we don't expect the
        # measurements to be *identical* because of the faint wings of
        # the objects; when measuring a deblended child, we pick up
        # the wings of the other objects.
        #
        # In order to get exactly equal measurements, we'll fake some
        # sources that have no wings -- we'll copy just the source
        # pixels within the footprint.  This means that all the
        # footprints are the same, and the pixels inside the footprint
        # are the same.
        
        fullim = None
        sources = None
        # "normal" measurements
        xx0,yy0,vx0,vy0 = [],[],[],[]
        # "no-wing" measurements
        xx1,yy1,vx1,vy1 = [],[],[],[]

        y = 25
        for i in range(5):
            # no-noise source image
            sim = afwImage.ImageF(imorig.getWidth(), imorig.getHeight())
            # Put all four sources in the parent (i==0), and one
            # source in each child (i=[1 to 4])
            if i in [0,1]:
                addPsf(sim, psf, 20, y, 1000)
            if i in [0,2]:
                addGaussian(sim, 40, y, 10, 3, 2e5)
            if i in [0,3]:
                addGaussian(sim, 75, y, 10, 3, 2e5)
            if i in [0,4]:
                addPsf(sim, psf, 95, y, 1000)
            imcopy = afwImage.ImageF(imorig, True)
            imcopy += sim
            # copy the pixels into the exposure object
            im <<= imcopy

            if i == 0:
                detected = detect.makeSourceCatalog(table, exposure)
                sources = detected.sources
                print 'detected', len(sources), 'sources'
                self.assertEqual(len(sources), 1)
            else:
                fpSets = detect.detectFootprints(exposure)
                print 'detected', fpSets.numPos, 'sources'
                fpSets.positive.makeSources(sources)
                self.assertEqual(fpSets.numPos, 1)
                print len(sources), 'sources total'

            measure.plotpat = 'single-%i.png' % i
            measure.run(exposure, sources[-1:])
            s = sources[-1]
            fp = s.getFootprint()
            if i == 0:
                # This is the blended image
                fullim = imcopy
            else:
                print 'Creating heavy footprint...'
                heavy = afwDet.makeHeavyFootprint(fp, mi)
                s.setFootprint(heavy)

            # Record the single-source measurements.
            xx0.append(s.getX())
            yy0.append(s.getY())
            vx0.append(s.getIxx())
            vy0.append(s.getIyy())

            # "no-wings": add just the source pixels within the footprint
            im <<= sim
            h = afwDet.makeHeavyFootprint(fp, mi)
            sim2 = afwImage.ImageF(imorig.getWidth(), imorig.getHeight())
            h.insert(sim2)
            imcopy = afwImage.ImageF(imorig, True)
            imcopy += sim2
            im <<= imcopy
            measure.plotpat = 'single2-%i.png' % i
            measure.run(exposure, sources[i:i+1], noiseImage=noiseim)
            s = sources[i]
            xx1.append(s.getX())
            yy1.append(s.getY())
            vx1.append(s.getIxx())
            vy1.append(s.getIyy())
            if i == 0:
                fullim2 = imcopy

        # Now we'll build the fake deblended hierarchy.
        parent = sources[0]
        kids = sources[1:]
        # Ensure that the parent footprint contains all the child footprints
        pfp = parent.getFootprint()
        for s in kids:
            for span in s.getFootprint().getSpans():
                pfp.addSpan(span)
        pfp.normalize()
        #parent.setFootprint(pfp)
        # The parent-child relationship is established through the IDs
        parentid = parent.getId()
        for s in kids:
            s.setParent(parentid)

        # Reset all the measurements
        shkey = sources.getTable().getShapeKey()
        ckey = sources.getTable().getCentroidKey()
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Measure the "deblended" normal sources
        im <<= fullim
        measure.plotpat = 'joint-%(sourcenum)i.png'
        measure.run(exposure, sources)
        xx2,yy2,vx2,vy2 = [],[],[],[]
        for s in sources:
            xx2.append(s.getX())
            yy2.append(s.getY())
            vx2.append(s.getIxx())
            vy2.append(s.getIyy())

        # Measure the "deblended" no-wings sources
        im <<= fullim2
        measure.plotpat = 'joint2-%(sourcenum)i.png'
        measure.run(exposure, sources, noiseImage=noiseim)
        xx3,yy3,vx3,vy3 = [],[],[],[]
        for s in sources:
            xx3.append(s.getX())
            yy3.append(s.getY())
            vx3.append(s.getIxx())
            vy3.append(s.getIyy())

        print 'Normal:'
        print 'xx  ', xx0
        print '  vs', xx2
        print 'yy  ', yy0
        print '  vs', yy2
        print 'vx  ', vx0
        print '  vs', vx2
        print 'vy  ', vy0
        print '  vs', vy2

        print 'No wings:'
        print 'xx  ', xx1
        print '  vs', xx3
        print 'yy  ', yy1
        print '  vs', yy3
        print 'vx  ', vx1
        print '  vs', vx3
        print 'vy  ', vy1
        print '  vs', vy3

        # These "normal" tests are not very stringent.
        # 0.1-pixel centroids
        self.assertTrue(all([abs(v1-v2) < 0.1 for v1,v2 in zip(xx0,xx2)]))
        self.assertTrue(all([abs(v1-v2) < 0.1 for v1,v2 in zip(yy0,yy2)]))
        # 10% variances
        self.assertTrue(all([abs(v1-v2)/((v1+v2)/2.) < 0.1 for v1,v2 in zip(vx0,vx2)]))
        self.assertTrue(all([abs(v1-v2)/((v1+v2)/2.) < 0.1 for v1,v2 in zip(vy0,vy2)]))

        # The "no-wings" tests should be exact.
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)

        # Reset sources
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Test that the parent/child order is unimportant.
        im <<= fullim2
        measure.doplot = False
        sources2 = sources.copy()
        perm = [2,1,0,3,4]
        for i,j in enumerate(perm):
            sources2[i] = sources[j]
            # I'm not convinced that HeavyFootprints get copied correctly...
            sources2[i].setFootprint(sources[j].getFootprint())
        measure.run(exposure, sources2, noiseImage=noiseim)
        # "measure.run" reorders the sources!
        xx3,yy3,vx3,vy3 = [],[],[],[]
        for s in sources:
            xx3.append(s.getX())
            yy3.append(s.getY())
            vx3.append(s.getIxx())
            vy3.append(s.getIyy())
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)

        # Reset sources
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Test that it still works when the parent ID falls in the middle of
        # the child IDs.
        im <<= fullim2
        measure.doplot = False
        sources2 = sources.copy()
        parentid = 3
        ids = [parentid, 1,2,4,5]
        for i,s in enumerate(sources2):
            s.setId(ids[i])
            if i != 0:
                s.setParent(parentid)
            s.setFootprint(sources[i].getFootprint())
            
        measure.run(exposure, sources2, noiseImage=noiseim)
        # The sources get reordered!
        xx3,yy3,vx3,vy3 = [],[],[],[]
        xx3,yy3,vx3,vy3 = [0]*5,[0]*5,[0]*5,[0]*5
        for i,j in enumerate(ids):
            xx3[i] = sources2[j-1].getX()
            yy3[i] = sources2[j-1].getY()
            vx3[i] = sources2[j-1].getIxx()
            vy3[i] = sources2[j-1].getIyy()
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)
Ejemplo n.º 31
0
    def deblend(self, exposure, srcs, psf):
        """!
        Deblend.

        @param[in]     exposure Exposure to process
        @param[in,out] srcs     SourceCatalog containing sources detected on this exposure.
        @param[in]     psf      PSF

        @return None
        """
        self.log.info("Deblending %d sources" % len(srcs))

        from lsst.meas.deblender.baseline import deblend

        # find the median stdev in the image...
        mi = exposure.getMaskedImage()
        statsCtrl = afwMath.StatisticsControl()
        statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
        stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
        sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
        self.log.logdebug('sigma1: %g' % sigma1)

        n0 = len(srcs)
        nparents = 0
        for i,src in enumerate(srcs):
            #t0 = time.clock()

            fp = src.getFootprint()
            pks = fp.getPeaks()

            # Since we use the first peak for the parent object, we should propagate its flags
            # to the parent source.
            src.assign(pks[0], self.peakSchemaMapper)

            if len(pks) < 2:
                continue

            if self.isLargeFootprint(fp):
                src.set(self.tooBigKey, True)
                self.skipParent(src, mi.getMask())
                self.log.logdebug('Parent %i: skipping large footprint' % (int(src.getId()),))
                continue
            if self.isMasked(fp, exposure.getMaskedImage().getMask()):
                src.set(self.maskedKey, True)
                self.skipParent(src, mi.getMask())
                self.log.logdebug('Parent %i: skipping masked footprint' % (int(src.getId()),))
                continue

            nparents += 1
            bb = fp.getBBox()
            psf_fwhm = self._getPsfFwhm(psf, bb)

            self.log.logdebug('Parent %i: deblending %i peaks' % (int(src.getId()), len(pks)))

            self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
            npre = len(srcs)

            # This should really be set in deblend, but deblend doesn't have access to the src
            src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)

            try:
                res = deblend(
                    fp, mi, psf, psf_fwhm, sigma1=sigma1,
                    psfChisqCut1 = self.config.psfChisq1,
                    psfChisqCut2 = self.config.psfChisq2,
                    psfChisqCut2b= self.config.psfChisq2b,
                    maxNumberOfPeaks=self.config.maxNumberOfPeaks,
                    strayFluxToPointSources=self.config.strayFluxToPointSources,
                    assignStrayFlux=self.config.assignStrayFlux,
                    findStrayFlux=(self.config.assignStrayFlux or self.config.findStrayFlux),
                    strayFluxAssignment=self.config.strayFluxRule,
                    rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
                    patchEdges=(self.config.edgeHandling == 'noclip'),
                    tinyFootprintSize=self.config.tinyFootprintSize,
                    clipStrayFluxFraction=self.config.clipStrayFluxFraction,
                    )
                if self.config.catchFailures:
                    src.set(self.deblendFailedKey, False)
            except Exception as e:
                if self.config.catchFailures:
                    self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
                    src.set(self.deblendFailedKey, True)
                    import traceback
                    traceback.print_exc()
                    continue
                else:
                    raise

            kids = []
            nchild = 0
            for j, peak in enumerate(res.peaks):
                heavy = peak.getFluxPortion()
                if heavy is None or peak.skip:
                    src.set(self.deblendSkippedKey, True)
                    if not self.config.propagateAllPeaks:
                        # Don't care
                        continue
                    # We need to preserve the peak: make sure we have enough info to create a minimal child src
                    self.log.logdebug("Peak at (%i,%i) failed.  Using minimal default info for child." %
                                      (pks[j].getIx(), pks[j].getIy()))
                    if heavy is None:
                        # copy the full footprint and strip out extra peaks
                        foot = afwDet.Footprint(src.getFootprint())
                        peakList = foot.getPeaks()
                        peakList.clear()
                        peakList.append(peak.peak)
                        zeroMimg = afwImage.MaskedImageF(foot.getBBox())
                        heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
                    if peak.deblendedAsPsf:
                        if peak.psfFitFlux is None:
                            peak.psfFitFlux = 0.0
                        if peak.psfFitCenter is None:
                            peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())

                assert(len(heavy.getPeaks()) == 1)

                src.set(self.deblendSkippedKey, False)
                child = srcs.addNew(); nchild += 1
                child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
                child.setParent(src.getId())
                child.setFootprint(heavy)
                child.set(self.psfKey, peak.deblendedAsPsf)
                child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
                if peak.deblendedAsPsf:
                    (cx,cy) = peak.psfFitCenter
                    child.set(self.psfCenterKey, afwGeom.Point2D(cx, cy))
                    child.set(self.psfFluxKey, peak.psfFitFlux)
                child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
                child.set(self.deblendPatchedTemplateKey, peak.patched)
                kids.append(child)

            # Child footprints may extend beyond the full extent of their parent's which
            # results in a failure of the replace-by-noise code to reinstate these pixels
            # to their original values.  The following updates the parent footprint
            # in-place to ensure it contains the full union of itself and all of its
            # children's footprints.
            src.getFootprint().include([child.getFootprint() for child in kids])

            src.set(self.nChildKey, nchild)

            self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
            #print 'Deblending parent id', src.getId(), 'took', time.clock() - t0


        n1 = len(srcs)
        self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
                      % (n0, nparents, n1-n0, n1))
Ejemplo n.º 32
0
    def __init__(self,
                 config,
                 exposure,
                 footprints,
                 noiseImage=None,
                 exposureId=None,
                 log=None):
        noiseMeanVar = None
        self.noiseSource = config.noiseSource
        self.noiseOffset = config.noiseOffset
        self.noiseSeedMultiplier = config.noiseSeedMultiplier
        self.noiseGenMean = None
        self.noiseGenStd = None
        self.log = log

        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        self.exposure = exposure
        self.footprints = footprints
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()
        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            if maskname in mask.getMaskPlaneDict():
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                if self.log:
                    self.log.debug('Mask plane "%s" already existed', maskname)
            else:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            if self.log:
                self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x',
                               maskname, plane, bitmask, bitmask)
        self.thisbitmask, self.otherbitmask = bitmasks
        del bitmasks
        self.heavies = {}
        # Start by creating HeavyFootprints for each source which has no parent
        # and just use them for children which do not already have heavy footprints.
        # If a heavy footprint is available for a child, we will use it. Otherwise,
        # we use the first parent in the parent chain which has a heavy footprint,
        # which with the one level deblender will alway be the topmost parent
        # NOTE: heavy footprints get destroyed by the transform process in forcedPhotCcd.py
        # or forcedPhotCoadd.py so they are never available for forced measurements.

        # Create in the dict heavies = {id:heavyfootprint}
        for id, fp in footprints.items():
            if fp[1].isHeavy():
                self.heavies[id] = fp[1]
            elif fp[0] == 0:
                self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)

        # ## FIXME: the heavy footprint includes the mask
        # ## and variance planes, which we shouldn't need
        # ## (I don't think we ever want to modify them in
        # ## the input image).  Copying them around is
        # ## wasteful.

        # We now create a noise HeavyFootprint for each source with has a heavy footprint.
        # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure,
                                          noiseImage,
                                          noiseMeanVar,
                                          exposureId=exposureId)
        if self.log:
            self.log.debug('Using noise generator: %s', str(noisegen))
        for id in self.heavies:
            fp = footprints[id][1]
            noiseFp = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[id] = noiseFp
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            noiseFp.insert(im)
            # Also set the OTHERDET bit
            fp.spans.setMask(mask, self.otherbitmask)
Ejemplo n.º 33
0
    def __init__(self,
                 config,
                 exposure,
                 footprints,
                 noiseImage=None,
                 exposureId=None,
                 log=None):
        """!
        Initialize the NoiseReplacer.

        @param[in]      config       instance of NoiseReplacerConfig
        @param[in,out]  exposure     Exposure to be noise replaced. (All sources replaced on return)
        @param[in]      footprints   dict of {id: (parent, footprint)};
        @param[in]      noiseImage   an afw.image.ImageF used as a predictable noise replacement source
                                     (for tests only)
        @param[in]      log          pex.logging.Log object to use for status messages; no status messages
                                     will be printed if None

        'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the
        source ID, but in forced photometry, this will be the reference ID, as that's what we used to
        determine the deblend families.  This routine should create HeavyFootprints for any non-Heavy
        Footprints, and replace them in the dict.  It should then create a dict of HeavyFootprints
        containing noise, but only for parent objects, then replace all sources with noise.
        This should ignore any footprints that lay outside the bounding box of the exposure,
        and clip those that lie on the border.

        NOTE: as the code currently stands, the heavy footprint for a deblended object must be available
        from the input catalog.  If it is not, it cannot be reproduced here.  In that case, the
        topmost parent in the objects parent chain must be used.  The heavy footprint for that source
        is created in this class from the masked image.
        """
        noiseMeanVar = None
        self.noiseSource = config.noiseSource
        self.noiseOffset = config.noiseOffset
        self.noiseSeedMultiplier = config.noiseSeedMultiplier
        self.noiseGenMean = None
        self.noiseGenStd = None
        self.log = log

        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        self.exposure = exposure
        self.footprints = footprints
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()
        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            try:
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                if self.log:
                    self.log.logdebug('Mask plane "%s" already existed' %
                                      maskname)
            except:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            if self.log:
                self.log.logdebug(
                    'Mask plane "%s": plane %i, bitmask %i = 0x%x' %
                    (maskname, plane, bitmask, bitmask))
        self.thisbitmask, self.otherbitmask = bitmasks
        del bitmasks
        self.heavies = {}
        # Start by creating HeavyFootprints for each source which has no parent
        # and just use them for children which do not already have heavy footprints.
        # If a heavy footprint is available for a child, we will use it. Otherwise,
        # we use the first parent in the parent chain which has a heavy footprint,
        # which with the one level deblender will alway be the topmost parent
        # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py,
        # so they are never available for forced measurements.

        # Create in the dict heavies = {id:heavyfootprint}
        for id in footprints.keys():
            fp = footprints[id]
            if fp[1].isHeavy():
                self.heavies[id] = afwDet.cast_HeavyFootprintF(fp[1])
            elif fp[0] == 0:
                self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)

        ### FIXME: the heavy footprint includes the mask
        ### and variance planes, which we shouldn't need
        ### (I don't think we ever want to modify them in
        ### the input image).  Copying them around is
        ### wasteful.

        # We now create a noise HeavyFootprint for each source with has a heavy footprint.
        # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure,
                                          noiseImage,
                                          noiseMeanVar,
                                          exposureId=exposureId)
        #  The noiseGenMean and Std are used by the unit tests
        self.noiseGenMean = noisegen.mean
        self.noiseGenStd = noisegen.std
        if self.log:
            self.log.logdebug('Using noise generator: %s' % (str(noisegen)))
        for id in self.heavies.keys():
            fp = footprints[id][1]
            noiseFp = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[id] = noiseFp
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            noiseFp.insert(im)
            # Also set the OTHERDET bit
            afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
Ejemplo n.º 34
0
    def test2(self):
        # Check that doReplaceWithNoise works with deblended source
        # hierarchies.
        seed = 42
        rand = afwMath.Random(afwMath.Random.MT19937, seed)

        psf = self.getpsf()
        im = afwImage.ImageF(200, 50)
        skystd = 100
        afwMath.randomGaussianImage(im, rand)
        im *= skystd
        imorig = afwImage.ImageF(im, True)
        noiseim = imorig

        mi = afwImage.MaskedImageF(im)
        mi.getVariance().set(skystd**2)
        exposure = afwImage.makeExposure(mi)
        exposure.setPsf(psf)

        detconf = measAlg.SourceDetectionConfig()
        detconf.returnOriginalFootprints = True
        detconf.reEstimateBackground = False
        measconf = measAlg.SourceMeasurementConfig()
        measconf.doReplaceWithNoise = True
        measconf.replaceWithNoise.noiseSeed = 42

        schema = afwTable.SourceTable.makeMinimalSchema()
        detect = measAlg.SourceDetectionTask(config=detconf, schema=schema)
        measure = MySourceMeasurementTask(config=measconf,
                                          schema=schema,
                                          doplot=plots)
        table = afwTable.SourceTable.make(schema)
        table.preallocate(10)

        # We're going to fake up a perfect deblend hierarchy here, by
        # creating individual images containing single sources and
        # measuring them, and then creating a deblend hierarchy where
        # the children have the correct HeavyFootprints.  We want to
        # find that the measurements on the deblend hierarchy and the
        # blended image are equal to the individual images.
        #
        # Note that in the normal setup we don't expect the
        # measurements to be *identical* because of the faint wings of
        # the objects; when measuring a deblended child, we pick up
        # the wings of the other objects.
        #
        # In order to get exactly equal measurements, we'll fake some
        # sources that have no wings -- we'll copy just the source
        # pixels within the footprint.  This means that all the
        # footprints are the same, and the pixels inside the footprint
        # are the same.

        fullim = None
        sources = None
        # "normal" measurements
        xx0, yy0, vx0, vy0 = [], [], [], []
        # "no-wing" measurements
        xx1, yy1, vx1, vy1 = [], [], [], []

        y = 25
        for i in range(5):
            # no-noise source image
            sim = afwImage.ImageF(imorig.getWidth(), imorig.getHeight())
            # Put all four sources in the parent (i==0), and one
            # source in each child (i=[1 to 4])
            if i in [0, 1]:
                addPsf(sim, psf, 20, y, 1000)
            if i in [0, 2]:
                addGaussian(sim, 40, y, 10, 3, 2e5)
            if i in [0, 3]:
                addGaussian(sim, 75, y, 10, 3, 2e5)
            if i in [0, 4]:
                addPsf(sim, psf, 95, y, 1000)
            imcopy = afwImage.ImageF(imorig, True)
            imcopy += sim
            # copy the pixels into the exposure object
            im <<= imcopy

            if i == 0:
                detected = detect.makeSourceCatalog(table, exposure)
                sources = detected.sources
                print 'detected', len(sources), 'sources'
                self.assertEqual(len(sources), 1)
            else:
                fpSets = detect.detectFootprints(exposure)
                print 'detected', fpSets.numPos, 'sources'
                fpSets.positive.makeSources(sources)
                self.assertEqual(fpSets.numPos, 1)
                print len(sources), 'sources total'

            measure.plotpat = 'single-%i.png' % i
            measure.run(exposure, sources[-1:])
            s = sources[-1]
            fp = s.getFootprint()
            if i == 0:
                # This is the blended image
                fullim = imcopy
            else:
                print 'Creating heavy footprint...'
                heavy = afwDet.makeHeavyFootprint(fp, mi)
                s.setFootprint(heavy)

            # Record the single-source measurements.
            xx0.append(s.getX())
            yy0.append(s.getY())
            vx0.append(s.getIxx())
            vy0.append(s.getIyy())

            # "no-wings": add just the source pixels within the footprint
            im <<= sim
            h = afwDet.makeHeavyFootprint(fp, mi)
            sim2 = afwImage.ImageF(imorig.getWidth(), imorig.getHeight())
            h.insert(sim2)
            imcopy = afwImage.ImageF(imorig, True)
            imcopy += sim2
            im <<= imcopy
            measure.plotpat = 'single2-%i.png' % i
            measure.run(exposure, sources[i:i + 1], noiseImage=noiseim)
            s = sources[i]
            xx1.append(s.getX())
            yy1.append(s.getY())
            vx1.append(s.getIxx())
            vy1.append(s.getIyy())
            if i == 0:
                fullim2 = imcopy

        # Now we'll build the fake deblended hierarchy.
        parent = sources[0]
        kids = sources[1:]
        # Ensure that the parent footprint contains all the child footprints
        pfp = parent.getFootprint()
        for s in kids:
            for span in s.getFootprint().getSpans():
                pfp.addSpan(span)
        pfp.normalize()
        #parent.setFootprint(pfp)
        # The parent-child relationship is established through the IDs
        parentid = parent.getId()
        for s in kids:
            s.setParent(parentid)

        # Reset all the measurements
        shkey = sources.getTable().getShapeKey()
        ckey = sources.getTable().getCentroidKey()
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Measure the "deblended" normal sources
        im <<= fullim
        measure.plotpat = 'joint-%(sourcenum)i.png'
        measure.run(exposure, sources)
        xx2, yy2, vx2, vy2 = [], [], [], []
        for s in sources:
            xx2.append(s.getX())
            yy2.append(s.getY())
            vx2.append(s.getIxx())
            vy2.append(s.getIyy())

        # Measure the "deblended" no-wings sources
        im <<= fullim2
        measure.plotpat = 'joint2-%(sourcenum)i.png'
        measure.run(exposure, sources, noiseImage=noiseim)
        xx3, yy3, vx3, vy3 = [], [], [], []
        for s in sources:
            xx3.append(s.getX())
            yy3.append(s.getY())
            vx3.append(s.getIxx())
            vy3.append(s.getIyy())

        print 'Normal:'
        print 'xx  ', xx0
        print '  vs', xx2
        print 'yy  ', yy0
        print '  vs', yy2
        print 'vx  ', vx0
        print '  vs', vx2
        print 'vy  ', vy0
        print '  vs', vy2

        print 'No wings:'
        print 'xx  ', xx1
        print '  vs', xx3
        print 'yy  ', yy1
        print '  vs', yy3
        print 'vx  ', vx1
        print '  vs', vx3
        print 'vy  ', vy1
        print '  vs', vy3

        # These "normal" tests are not very stringent.
        # 0.1-pixel centroids
        self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(xx0, xx2)]))
        self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(yy0, yy2)]))
        # 10% variances
        self.assertTrue(
            all([
                abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1
                for v1, v2 in zip(vx0, vx2)
            ]))
        self.assertTrue(
            all([
                abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1
                for v1, v2 in zip(vy0, vy2)
            ]))

        # The "no-wings" tests should be exact.
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)

        # Reset sources
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Test that the parent/child order is unimportant.
        im <<= fullim2
        measure.doplot = False
        sources2 = sources.copy()
        perm = [2, 1, 0, 3, 4]
        for i, j in enumerate(perm):
            sources2[i] = sources[j]
            # I'm not convinced that HeavyFootprints get copied correctly...
            sources2[i].setFootprint(sources[j].getFootprint())
        measure.run(exposure, sources2, noiseImage=noiseim)
        # "measure.run" reorders the sources!
        xx3, yy3, vx3, vy3 = [], [], [], []
        for s in sources:
            xx3.append(s.getX())
            yy3.append(s.getY())
            vx3.append(s.getIxx())
            vy3.append(s.getIyy())
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)

        # Reset sources
        for s in sources:
            sh = s.get(shkey)
            sh.setIxx(np.nan)
            sh.setIyy(np.nan)
            sh.setIxy(np.nan)
            s.set(shkey, sh)
            c = s.get(ckey)
            c.setX(np.nan)
            c.setY(np.nan)
            s.set(ckey, c)

        # Test that it still works when the parent ID falls in the middle of
        # the child IDs.
        im <<= fullim2
        measure.doplot = False
        sources2 = sources.copy()
        parentid = 3
        ids = [parentid, 1, 2, 4, 5]
        for i, s in enumerate(sources2):
            s.setId(ids[i])
            if i != 0:
                s.setParent(parentid)
            s.setFootprint(sources[i].getFootprint())

        measure.run(exposure, sources2, noiseImage=noiseim)
        # The sources get reordered!
        xx3, yy3, vx3, vy3 = [], [], [], []
        xx3, yy3, vx3, vy3 = [0] * 5, [0] * 5, [0] * 5, [0] * 5
        for i, j in enumerate(ids):
            xx3[i] = sources2[j - 1].getX()
            yy3[i] = sources2[j - 1].getY()
            vx3[i] = sources2[j - 1].getIxx()
            vy3[i] = sources2[j - 1].getIyy()
        self.assertTrue(xx1 == xx3)
        self.assertTrue(yy1 == yy3)
        self.assertTrue(vx1 == vx3)
        self.assertTrue(vy1 == vy3)
Ejemplo n.º 35
0
    def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None):
        noiseMeanVar = None
        self.noiseSource = config.noiseSource
        self.noiseOffset = config.noiseOffset
        self.noiseSeedMultiplier = config.noiseSeedMultiplier
        self.noiseGenMean = None
        self.noiseGenStd = None
        self.log = log

        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        self.exposure = exposure
        self.footprints = footprints
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()
        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            try:
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                if self.log:
                    self.log.debug('Mask plane "%s" already existed', maskname)
            except Exception:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            if self.log:
                self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x',
                               maskname, plane, bitmask, bitmask)
        self.thisbitmask, self.otherbitmask = bitmasks
        del bitmasks
        self.heavies = {}
        # Start by creating HeavyFootprints for each source which has no parent
        # and just use them for children which do not already have heavy footprints.
        # If a heavy footprint is available for a child, we will use it. Otherwise,
        # we use the first parent in the parent chain which has a heavy footprint,
        # which with the one level deblender will alway be the topmost parent
        # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py,
        # so they are never available for forced measurements.

        # Create in the dict heavies = {id:heavyfootprint}
        for id, fp in footprints.items():
            if fp[1].isHeavy():
                self.heavies[id] = fp[1]
            elif fp[0] == 0:
                self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi)

        # ## FIXME: the heavy footprint includes the mask
        # ## and variance planes, which we shouldn't need
        # ## (I don't think we ever want to modify them in
        # ## the input image).  Copying them around is
        # ## wasteful.

        # We now create a noise HeavyFootprint for each source with has a heavy footprint.
        # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint}
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId)
        #  The noiseGenMean and Std are used by the unit tests
        self.noiseGenMean = noisegen.mean
        self.noiseGenStd = noisegen.std
        if self.log:
            self.log.debug('Using noise generator: %s', str(noisegen))
        for id in self.heavies:
            fp = footprints[id][1]
            noiseFp = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[id] = noiseFp
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            noiseFp.insert(im)
            # Also set the OTHERDET bit
            fp.spans.setMask(mask, self.otherbitmask)
Ejemplo n.º 36
0
 def getHeavyFootprint(self, fp):
     bb = fp.getBBox()
     mim = self.getMaskedImage(bb)
     return afwDet.makeHeavyFootprint(fp, mim)
Ejemplo n.º 37
0
    def begin(self, exposure, sources, noiseImage=None, noiseMeanVar=None):
        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        # (sources.find() below)
        if not sources.isSorted():
            sources.sort()
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()

        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            try:
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                self.log.logdebug('Mask plane "%s" already existed' % maskname)
            except:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x' %
                              (maskname, plane, bitmask, bitmask))
        self.thisbitmask, self.otherbitmask = bitmasks
        del bitmasks

        # Start by creating HeavyFootprints for each source.
        #
        # The "getParent()" checks are here because top-level
        # sources (ie, those with no parents) are not supposed to
        # have HeavyFootprints, but child sources (ie, those that
        # have been deblended) should have HeavyFootprints
        # already.
        self.heavies = []
        for source in sources:
            fp = source.getFootprint()
            hfp = afwDet.cast_HeavyFootprintF(fp)
            if source.getParent() and hfp is not None:
                # this source has been deblended; "fp" should
                # already be a HeavyFootprint (but may not be
                # if read from disk).
                # Swig downcasts it to Footprint, so we have to re-cast.
                self.heavies.append(hfp)
            else:
                # top-level source: copy pixels from the input
                # image.
                ### FIXME: the heavy footprint includes the mask
                ### and variance planes, which we shouldn't need
                ### (I don't think we ever want to modify them in
                ### the input image).  Copying them around is
                ### wasteful.
                heavy = afwDet.makeHeavyFootprint(fp, mi)
                self.heavies.append(heavy)

        # We now create a noise HeavyFootprint for each top-level Source.
        # We'll put the noisy footprints in a map from id -> HeavyFootprint:
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar)
        self.log.logdebug('Using noise generator: %s' % (str(noisegen)))
        for source in sources:
            if source.getParent():
                continue
            fp = source.getFootprint()
            heavy = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[source.getId()] = heavy
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            heavy.insert(im)
            # Also set the OTHERDET bit
            afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
Ejemplo n.º 38
0
    def testMergeHeavyFootprints(self):
        mi = afwImage.MaskedImageF(20, 10)
        objectPixelVal = (42, 0x9, 400)

        spanList = []
        for y, x0, x1 in [(1, 9, 12), (2, 12, 13), (3, 11, 15)]:
            spanList.append(afwGeom.Span(y, x0, x1))
            for x in range(x0, x1 + 1):
                mi[x, y, afwImage.LOCAL] = objectPixelVal

        foot = afwDetect.Footprint(afwGeom.SpanSet(spanList))

        hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi)
        hfoot2 = afwDetect.makeHeavyFootprint(foot, mi)

        hsum = afwDetect.mergeHeavyFootprints(hfoot1, hfoot2)

        bb = hsum.getBBox()
        self.assertEqual(bb.getMinX(), 9)
        self.assertEqual(bb.getMaxX(), 15)
        self.assertEqual(bb.getMinY(), 1)
        self.assertEqual(bb.getMaxY(), 3)

        msum = afwImage.MaskedImageF(20, 10)
        hsum.insert(msum)

        sa = msum.getImage().getArray()

        self.assertFloatsEqual(sa[1, 9:13], objectPixelVal[0])
        self.assertFloatsEqual(sa[2, 12:14],
                               objectPixelVal[0] + self.objectPixelVal[0])
        self.assertFloatsEqual(sa[2, 10:12], self.objectPixelVal[0])

        sv = msum.getVariance().getArray()

        self.assertFloatsEqual(sv[1, 9:13], objectPixelVal[2])
        self.assertFloatsEqual(sv[2, 12:14],
                               objectPixelVal[2] + self.objectPixelVal[2])
        self.assertFloatsEqual(sv[2, 10:12], self.objectPixelVal[2])

        sm = msum.getMask().getArray()

        self.assertFloatsEqual(sm[1, 9:13], objectPixelVal[1])
        self.assertFloatsEqual(sm[2, 12:14],
                               objectPixelVal[1] | self.objectPixelVal[1])
        self.assertFloatsEqual(sm[2, 10:12], self.objectPixelVal[1])

        if False:
            import matplotlib
            matplotlib.use('Agg')
            import pylab as plt
            im1 = afwImage.ImageF(bb)
            hfoot1.insert(im1)
            im2 = afwImage.ImageF(bb)
            hfoot2.insert(im2)
            im3 = afwImage.ImageF(bb)
            hsum.insert(im3)
            plt.clf()
            plt.subplot(1, 3, 1)
            plt.imshow(im1.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1, 3, 2)
            plt.imshow(im2.getArray(), interpolation='nearest', origin='lower')
            plt.subplot(1, 3, 3)
            plt.imshow(im3.getArray(), interpolation='nearest', origin='lower')
            plt.savefig('merge.png')
Ejemplo n.º 39
0
 def getHeavyFootprint(self, fp):
     bb = fp.getBBox()
     mim = self.getMaskedImage(bb)
     return afwDet.makeHeavyFootprint(fp, mim)
Ejemplo n.º 40
0
def makeplots(butler,
              dataId,
              ps,
              sources=None,
              pids=None,
              minsize=0,
              maxpeaks=10):
    calexp = butler.get("calexp", **dataId)
    if sources is None:
        ss = butler.get('src', **dataId)
    else:
        ss = sources

    # print('Sources', ss)
    # print('Calexp', calexp)
    # print(dir(ss))

    srcs = {}
    families = {}
    for src in ss:
        sid = src.getId()
        srcs[sid] = src
        parent = src.getParent()
        if parent == 0:
            continue
        if parent not in families:
            families[parent] = []
        families[parent].append(src)
        # print 'Source', src
        # print '  ', dir(src)
        # print '  parent', src.getParent()
        # print '  footprint', src.getFootprint()

    print()
    lsstimg = calexp.getMaskedImage().getImage()
    img = lsstimg.getArray()
    schema = ss.getSchema()
    psfkey = schema.find("deblend_deblendedAsPsf").key
    nchildkey = schema.find("deblend_nChild").key
    toomanykey = schema.find("deblend_tooManyPeaks").key
    failedkey = schema.find("deblend_failed").key

    def getFlagString(src):
        ss = ['Nchild: %i' % src.get(nchildkey)]
        for key, s in [(psfkey, 'PSF'), (toomanykey, 'TooMany'),
                       (failedkey, 'Failed')]:
            if src.get(key):
                ss.append(s)
        return ', '.join(ss)

    plt.subplots_adjust(left=0.05,
                        right=0.95,
                        bottom=0.05,
                        top=0.9,
                        hspace=0.2,
                        wspace=0.3)

    sig1 = np.sqrt(
        np.median(calexp.getMaskedImage().getVariance().getArray().ravel()))
    pp = (img /
          np.sqrt(calexp.getMaskedImage().getVariance().getArray())).ravel()
    plt.clf()
    lo, hi = -4, 4
    n, b, p = plt.hist(img.ravel() / sig1,
                       100,
                       range=(lo, hi),
                       histtype='step',
                       color='b')
    plt.hist(pp, 100, range=(lo, hi), histtype='step', color='g')
    xx = np.linspace(lo, hi, 200)
    yy = 1. / (np.sqrt(2. * np.pi)) * np.exp(-0.5 * xx**2)
    yy *= sum(n) * (b[1] - b[0])
    plt.plot(xx, yy, 'k-', alpha=0.5)
    plt.xlim(lo, hi)
    plt.title('image-wide sig1: %.1f' % sig1)
    ps.savefig()

    for ifam, (p, kids) in enumerate(families.items()):

        parent = srcs[p]
        pid = parent.getId() & 0xffff
        if len(pids) and pid not in pids:
            # print('Skipping pid', pid)
            continue

        if len(kids) < minsize:
            print('Skipping parent', pid, ': n kids', len(kids))
            continue

        # if len(kids) < 5:
        #     print 'Skipping family with', len(kids)
        #     continue
        # print 'ifam', ifam
        # if ifam != 18:
        #     print 'skipping'
        #     continue

        print('Parent', parent)
        print('Kids', kids)

        print('Parent', parent.getId())
        print('Kids', [k.getId() for k in kids])

        pfoot = parent.getFootprint()
        bb = pfoot.getBBox()

        y0, y1, x0, x1 = bb.getMinY(), bb.getMaxY(), bb.getMinX(), bb.getMaxX()
        slc = slice(y0, y1 + 1), slice(x0, x1 + 1)

        ima = dict(interpolation='nearest',
                   origin='lower',
                   cmap='gray',
                   vmin=-10,
                   vmax=40)
        mn, mx = ima['vmin'], ima['vmax']

        if False:
            plt.clf()
            plt.imshow(img[slc], extent=bb_to_ext(bb), **ima)
            plt.title('Parent %i, %s' %
                      (parent.getId(), getFlagString(parent)))
            ax = plt.axis()
            x, y = bb_to_xy(bb)
            plt.plot(x, y, 'r-', lw=2)
            for i, kid in enumerate(kids):
                kfoot = kid.getFootprint()
                kbb = kfoot.getBBox()
                kx, ky = bb_to_xy(kbb, margin=0.4)
                plt.plot(kx, ky, 'm-')
            for pk in pfoot.getPeaks():
                plt.plot(pk.getIx(), pk.getIy(), 'r+', ms=10, mew=3)
            plt.axis(ax)
            ps.savefig()

        print('parent footprint:', pfoot)
        print('heavy?', pfoot.isHeavy())
        plt.clf()
        pimg, h = foot_to_img(pfoot, lsstimg)

        plt.imshow(img_to_rgb(pimg.getArray(), mn, mx),
                   extent=bb_to_ext(bb),
                   **ima)
        tt = 'Parent %i' % parent.getId()
        if not h:
            tt += ', no HFoot'
        tt += ', ' + getFlagString(parent)
        plt.title(tt)
        ax = plt.axis()
        plt.plot([x0, x0, x1, x1, x0], [y0, y1, y1, y0, y0], 'r-', lw=2)
        for i, kid in enumerate(kids):
            kfoot = kid.getFootprint()
            kbb = kfoot.getBBox()
            kx, ky = bb_to_xy(kbb, margin=-0.1)
            plt.plot(kx, ky, 'm-', lw=1.5)
        for pk in pfoot.getPeaks():
            plt.plot(pk.getIx(), pk.getIy(), 'r+', ms=10, mew=3)
        plt.axis(ax)
        ps.savefig()

        cols = int(np.ceil(np.sqrt(len(kids))))
        rows = int(np.ceil(len(kids) / float(cols)))

        if False:
            plt.clf()
            for i, kid in enumerate(kids):
                plt.subplot(rows, cols, 1 + i)
                kfoot = kid.getFootprint()
                print('kfoot:', kfoot)
                print('heavy?', kfoot.isHeavy())
                # print(dir(kid))
                kbb = kfoot.getBBox()
                ky0, ky1, kx0, kx1 = kbb.getMinY(), kbb.getMaxY(), kbb.getMinX(
                ), kbb.getMaxX()
                kslc = slice(ky0, ky1 + 1), slice(kx0, kx1 + 1)
                plt.imshow(img[kslc], extent=bb_to_ext(kbb), **ima)
                plt.title('Child %i' % kid.getId())
                plt.axis(ax)
            ps.savefig()

        plt.clf()
        for i, kid in enumerate(kids):
            plt.subplot(rows, cols, 1 + i)
            kfoot = kid.getFootprint()
            kbb = kfoot.getBBox()
            kimg, h = foot_to_img(kfoot, lsstimg)
            tt = getFlagString(kid)
            if not h:
                tt += ', no HFoot'
            plt.title('%s' % tt)
            if kimg is None:
                plt.axis(ax)
                continue
            plt.imshow(img_to_rgb(kimg.getArray(), mn, mx),
                       extent=bb_to_ext(kbb),
                       **ima)
            for pk in kfoot.getPeaks():
                plt.plot(pk.getIx(), pk.getIy(), 'g+', ms=10, mew=3)
            plt.axis(ax)
        plt.suptitle('Child HeavyFootprints')
        ps.savefig()

        print()
        print('Re-running deblender...')
        psf = calexp.getPsf()
        psf_fwhm = psf.computeShape().getDeterminantRadius() * 2.35
        deb = deblend(
            pfoot,
            calexp.getMaskedImage(),
            psf,
            psf_fwhm,
            verbose=True,
            maxNumberOfPeaks=maxpeaks,
            rampFluxAtEdge=True,
            clipStrayFluxFraction=0.01,
        )
        print('Got', deb)

        def getDebFlagString(kid):
            ss = []
            for k in [
                    'skip', 'outOfBounds', 'tinyFootprint', 'noValidPixels',
                ('deblendedAsPsf', 'PSF'), 'psfFitFailed', 'psfFitBadDof',
                    'psfFitBigDecenter', 'psfFitWithDecenter',
                    'failedSymmetricTemplate', 'hasRampedTemplate', 'patched'
            ]:
                if len(k) == 2:
                    k, s = k
                else:
                    s = k
                if getattr(kid, k):
                    ss.append(s)
            return ', '.join(ss)

        N = len(deb.peaks)
        cols = int(np.ceil(np.sqrt(N)))
        rows = int(np.ceil(N / float(cols)))

        for plotnum in range(4):
            plt.clf()
            for i, kid in enumerate(deb.peaks):
                # print 'child', kid
                # print '  flags:', getDebFlagString(kid)

                kfoot = None
                if plotnum == 0:
                    kfoot = kid.getFluxPortion(strayFlux=False)
                    supt = 'flux portion'
                elif plotnum == 1:
                    kfoot = kid.getFluxPortion(strayFlux=True)
                    supt = 'flux portion + stray'
                elif plotnum == 2:
                    kfoot = afwDet.makeHeavyFootprint(kid.templateFootprint,
                                                      kid.templateImage)
                    supt = 'template'
                elif plotnum == 3:
                    if kid.deblendedAsPsf:
                        kfoot = afwDet.makeHeavyFootprint(
                            kid.psfFootprint, kid.psfTemplate)
                        kfoot.normalize()
                        kfoot.clipToNonzero(kid.psfTemplate.getImage())
                        # print 'kfoot BB:', kfoot.getBBox()
                        # print 'Img bb:', kid.psfTemplate.getImage().getBBox()
                        # for sp in kfoot.getSpans():
                        #     print '  span', sp
                    else:
                        kfoot = afwDet.makeHeavyFootprint(
                            kid.templateFootprint, kid.templateImage)
                    supt = 'psf template'

                kimg, h = foot_to_img(kfoot, None)
                tt = 'kid %i: %s' % (i, getDebFlagString(kid))
                if not h:
                    tt += ', no HFoot'
                plt.subplot(rows, cols, 1 + i)
                plt.title('%s' % tt, fontsize=8)
                if kimg is None:
                    plt.axis(ax)
                    continue
                kbb = kfoot.getBBox()

                plt.imshow(img_to_rgb(kimg.getArray(), mn, mx),
                           extent=bb_to_ext(kbb),
                           **ima)

                # plt.imshow(kimg.getArray(), extent=bb_to_ext(kbb), **ima)

                plt.axis(ax)

            plt.suptitle(supt)
            ps.savefig()

        for i, kid in enumerate(deb.peaks):
            if not kid.deblendedAsPsf:
                continue
            plt.clf()

            ima = dict(interpolation='nearest', origin='lower', cmap='gray')
            # vmin=0, vmax=kid.psfFitFlux)

            plt.subplot(2, 4, 1)
            # plt.title('fit psf 0')
            # plt.imshow(kid.psfFitDebugPsf0Img.getArray(), **ima)
            # plt.colorbar()
            # plt.title('valid pixels')
            # plt.imshow(kid.psfFitDebugValidPix, vmin=0, vmax=1, **ima)
            plt.title('weights')
            plt.imshow(kid.psfFitDebugWeight, vmin=0, **ima)
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            plt.subplot(2, 4, 7)
            plt.title('valid pixels')
            plt.imshow(kid.psfFitDebugValidPix, vmin=0, vmax=1, **ima)
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            plt.subplot(2, 4, 2)
            # plt.title('ramp weights')
            # plt.imshow(kid.psfFitDebugRampWeight, vmin=0, vmax=1, **ima)
            # plt.colorbar()
            sig = np.sqrt(kid.psfFitDebugVar.getArray())
            data = kid.psfFitDebugStamp.getArray()
            model = kid.psfFitDebugPsfModel.getArray()
            chi = ((data - model) / sig)
            valid = kid.psfFitDebugValidPix

            plt.hist(np.clip((data / sig)[valid], -5, 5),
                     20,
                     range=(-5, 5),
                     histtype='step',
                     color='m')
            plt.hist(np.clip((model / sig)[valid], -5, 5),
                     20,
                     range=(-5, 5),
                     histtype='step',
                     color='r')
            plt.hist(np.clip(chi.ravel(), -5, 5),
                     20,
                     range=(-5, 5),
                     histtype='step',
                     color='g')
            n, b, p = plt.hist(np.clip(chi[valid], -5, 5),
                               20,
                               range=(-5, 5),
                               histtype='step',
                               color='b')

            xx = np.linspace(-5, 5, 200)
            yy = 1. / (np.sqrt(2. * np.pi)) * np.exp(-0.5 * xx**2)
            yy *= sum(n) * (b[1] - b[0])
            plt.plot(xx, yy, 'k-', alpha=0.5)

            plt.xlim(-5, 5)

            print('Sum of ramp weights:', np.sum(kid.psfFitDebugRampWeight))
            print('Quadrature sum of ramp weights:',
                  np.sqrt(np.sum(kid.psfFitDebugRampWeight**2)))
            print('Number of valid pix:', np.sum(kid.psfFitDebugValidPix))
            rw = kid.psfFitDebugRampWeight
            valid = kid.psfFitDebugValidPix
            # print 'valid values:', np.unique(valid)
            print('rw[valid]', np.sum(rw[valid]))
            print('rw range', rw.min(), rw.max())
            # print 'rw', rw.shape, rw.dtype
            # print 'valid', valid.shape, valid.dtype
            # print 'rw[valid]:', rw[valid]

            myresid = np.sum(kid.psfFitDebugValidPix *
                             kid.psfFitDebugRampWeight *
                             ((kid.psfFitDebugStamp.getArray() -
                               kid.psfFitDebugPsfModel.getArray()) /
                              np.sqrt(kid.psfFitDebugVar.getArray()))**2)
            print('myresid:', myresid)

            plt.subplot(2, 4, 8)
            N = 20000
            rwv = rw[valid]
            print('rwv', rwv)
            x = np.random.normal(size=(N, len(rwv)))
            ss = np.sum(rwv * x**2, axis=1)
            plt.hist(ss, 25)
            chi, dof = kid.psfFitBest
            plt.axvline(chi, color='r')

            mx = kid.psfFitDebugPsfModel.getArray().max()

            plt.subplot(2, 4, 3)
            # plt.title('fit psf')
            # plt.imshow(kid.psfFitDebugPsfImg.getArray(), **ima)
            # plt.colorbar()
            # plt.title('variance')
            # plt.imshow(kid.psfFitDebugVar.getArray(), vmin=0, **ima)
            # plt.colorbar()
            plt.title('model+noise')
            plt.imshow((kid.psfFitDebugPsfModel.getArray() +
                        sig * np.random.normal(size=sig.shape)) * valid,
                       vmin=0,
                       vmax=mx,
                       **ima)
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            plt.subplot(2, 4, 4)
            plt.title('fit psf model')
            plt.imshow(kid.psfFitDebugPsfModel.getArray(),
                       vmin=0,
                       vmax=mx,
                       **ima)
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            plt.subplot(2, 4, 5)
            plt.title('fit psf image')
            plt.imshow(kid.psfFitDebugStamp.getArray(), vmin=0, vmax=mx, **ima)
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            chi = (kid.psfFitDebugValidPix *
                   (kid.psfFitDebugStamp.getArray() -
                    kid.psfFitDebugPsfModel.getArray()) /
                   np.sqrt(kid.psfFitDebugVar.getArray()))

            plt.subplot(2, 4, 6)
            plt.title('fit psf chi')
            plt.imshow(-chi,
                       vmin=-3,
                       vmax=3,
                       interpolation='nearest',
                       origin='lower',
                       cmap='RdBu')
            plt.xticks([])
            plt.yticks([])
            plt.colorbar()

            params = kid.psfFitParams
            (flux, sky, skyx, skyy) = params[:4]

            print('Model sum:', model.sum())
            print('- sky', model.sum() - np.sum(valid) * sky)

            sig1 = np.median(sig)

            chi, dof = kid.psfFitBest
            plt.suptitle(
                'PSF kid %i: flux %.1f, sky %.1f, sig1 %.1f' %
                (i, flux, sky, sig1))  # : chisq %g, dof %i' % (i, chi, dof))

            ps.savefig()
Ejemplo n.º 41
0
    def begin(self, exposure, sources, noiseImage=None, noiseMeanVar=None):
        # creates heavies, replaces all footprints with noise
        # We need the source table to be sorted by ID to do the parent lookups
        # (sources.find() below)
        if not sources.isSorted():
            sources.sort()
        mi = exposure.getMaskedImage()
        im = mi.getImage()
        mask = mi.getMask()

        # Add temporary Mask planes for THISDET and OTHERDET
        self.removeplanes = []
        bitmasks = []
        for maskname in ['THISDET', 'OTHERDET']:
            try:
                # does it already exist?
                plane = mask.getMaskPlane(maskname)
                self.log.logdebug('Mask plane "%s" already existed' % maskname)
            except:
                # if not, add it; we should delete it when done.
                plane = mask.addMaskPlane(maskname)
                self.removeplanes.append(maskname)
            mask.clearMaskPlane(plane)
            bitmask = mask.getPlaneBitMask(maskname)
            bitmasks.append(bitmask)
            self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x' %
                              (maskname, plane, bitmask, bitmask))
        self.thisbitmask,self.otherbitmask = bitmasks
        del bitmasks

        # Start by creating HeavyFootprints for each source.
        #
        # The "getParent()" checks are here because top-level
        # sources (ie, those with no parents) are not supposed to
        # have HeavyFootprints, but child sources (ie, those that
        # have been deblended) should have HeavyFootprints
        # already.
        self.heavies = []
        for source in sources:
            fp = source.getFootprint()
            hfp = afwDet.cast_HeavyFootprintF(fp)
            if source.getParent() and hfp is not None:
                # this source has been deblended; "fp" should
                # already be a HeavyFootprint (but may not be
                # if read from disk).
                # Swig downcasts it to Footprint, so we have to re-cast.
                self.heavies.append(hfp)
            else:
                # top-level source: copy pixels from the input
                # image.
                ### FIXME: the heavy footprint includes the mask
                ### and variance planes, which we shouldn't need
                ### (I don't think we ever want to modify them in
                ### the input image).  Copying them around is
                ### wasteful.
                heavy = afwDet.makeHeavyFootprint(fp, mi)
                self.heavies.append(heavy)

        # We now create a noise HeavyFootprint for each top-level Source.
        # We'll put the noisy footprints in a map from id -> HeavyFootprint:
        self.heavyNoise = {}
        noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar)
        self.log.logdebug('Using noise generator: %s' % (str(noisegen)))
        for source in sources:
            if source.getParent():
                continue
            fp = source.getFootprint()
            heavy = noisegen.getHeavyFootprint(fp)
            self.heavyNoise[source.getId()] = heavy
            # Also insert the noisy footprint into the image now.
            # Notice that we're just inserting it into "im", ie,
            # the Image, not the MaskedImage.
            heavy.insert(im)
            # Also set the OTHERDET bit
            afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)