def testDot(self): """Test HeavyFootprint::dot""" size = 20, 20 for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]: mi1 = afwImage.MaskedImageF(*size) mi2 = afwImage.MaskedImageF(*size) mi1.set(0) mi2.set(0) spanList1 = [] spanList2 = [] for y, x0, x1 in [(5, 3, 7), (6, 3, 4), (6, 6, 7), (7, 3, 7), ]: spanList1.append(afwGeom.Span(y, x0, x1)) spanList2.append(afwGeom.Span(y + yOffset, x0 + xOffset, x1 + xOffset)) for x in range(x0, x1 + 1): value = (x + y, 0, 1.0) mi1[x, y, afwImage.LOCAL] = value mi2[x + xOffset, y + yOffset, afwImage.LOCAL] = value fp1 = afwDetect.Footprint(afwGeom.SpanSet(spanList1)) fp2 = afwDetect.Footprint(afwGeom.SpanSet(spanList2)) hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1) hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2) dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray()) self.assertEqual(hfp1.dot(hfp2), dot) self.assertEqual(hfp2.dot(hfp1), dot)
def testDot(self): """Test HeavyFootprint::dot""" size = 20, 20 for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]: mi1 = afwImage.MaskedImageF(*size) mi2 = afwImage.MaskedImageF(*size) mi1.set(0) mi2.set(0) fp1 = afwDetect.Footprint() fp2 = afwDetect.Footprint() for y, x0, x1 in [(5, 3, 7), (6, 3, 4), (6, 6, 7), (7, 3, 7),]: fp1.addSpan(y, x0, x1) fp2.addSpan(y + yOffset, x0 + xOffset, x1 + xOffset) for x in range(x0, x1 + 1): value = (x + y, 0, 1.0) mi1.set(x, y, value) mi2.set(x + xOffset, y + yOffset, value) hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1) hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2) hfp1.normalize() hfp2.normalize() dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray()) self.assertEqual(hfp1.dot(hfp2), dot) self.assertEqual(hfp2.dot(hfp1), dot)
def testDot(self): """Test HeavyFootprint::dot""" size = 20, 20 for xOffset, yOffset in [(0, 0), (0, 3), (3, 0), (2, 2)]: mi1 = afwImage.MaskedImageF(*size) mi2 = afwImage.MaskedImageF(*size) mi1.set(0) mi2.set(0) spanList1 = [] spanList2 = [] for y, x0, x1 in [ (5, 3, 7), (6, 3, 4), (6, 6, 7), (7, 3, 7), ]: spanList1.append(afwGeom.Span(y, x0, x1)) spanList2.append( afwGeom.Span(y + yOffset, x0 + xOffset, x1 + xOffset)) for x in range(x0, x1 + 1): value = (x + y, 0, 1.0) mi1[x, y, afwImage.LOCAL] = value mi2[x + xOffset, y + yOffset, afwImage.LOCAL] = value fp1 = afwDetect.Footprint(afwGeom.SpanSet(spanList1)) fp2 = afwDetect.Footprint(afwGeom.SpanSet(spanList2)) hfp1 = afwDetect.makeHeavyFootprint(fp1, mi1) hfp2 = afwDetect.makeHeavyFootprint(fp2, mi2) dot = np.vdot(mi1.getImage().getArray(), mi2.getImage().getArray()) self.assertEqual(hfp1.dot(hfp2), dot) self.assertEqual(hfp2.dot(hfp1), dot)
def footprintToImage(fp, mi=None, mask=False): if fp.isHeavy(): try: fp = afwDet.cast_HeavyFootprintF(fp) # not needed with pybind11 except AttributeError: pass pass elif mi is None: print("Unable to make a HeavyFootprint as image is None", file=sys.stderr) else: fp = afwDet.makeHeavyFootprint(fp, mi) bb = fp.getBBox() if mask: im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight()) else: im = afwImage.ImageF(bb.getWidth(), bb.getHeight()) im.setXY0(bb.getMinX(), bb.getMinY()) try: fp.insert(im) except AttributeError: # we failed to make it heavy assert not mi pass if mask: im = im.getMask() return im
def morphToHeavy(source, peakSchema, xy0=Point2I()): """Convert the morphology to a `HeavyFootprint` Parameters ---------- source : `scarlet.Component` The scarlet source with a morphology to convert to a `HeavyFootprint`. peakSchema : `lsst.daf.butler.Schema` The schema for the `PeakCatalog` of the `HeavyFootprint`. xy0 : `tuple` `(x,y)` coordinates of the bounding box containing the `HeavyFootprint`. Returns ------- heavy : `lsst.afw.detection.HeavyFootprint` """ mask = afwImage.MaskX(np.array(source.morph > 0, dtype=np.int32), xy0=xy0) ss = SpanSet.fromMask(mask) if len(ss) == 0: return None tfoot = afwDet.Footprint(ss, peakSchema=peakSchema) cy, cx = source.pixel_center xmin, ymin = xy0 # HeavyFootprints are not defined for 64 bit floats morph = source.morph.astype(np.float32) peakFlux = morph[cy, cx] tfoot.addPeak(cx + xmin, cy + ymin, peakFlux) timg = afwImage.ImageF(morph, xy0=xy0) timg = timg[tfoot.getBBox()] heavy = afwDet.makeHeavyFootprint(tfoot, afwImage.MaskedImageF(timg)) return heavy
def footprintToImage(fp, mi=None, mask=False): if fp.isHeavy(): try: fp = afwDet.cast_HeavyFootprintF(fp) # not needed with pybind11 except AttributeError: pass pass elif mi is None: print >> sys.stderr, "Unable to make a HeavyFootprint as image is None" else: fp = afwDet.makeHeavyFootprint(fp, mi) bb = fp.getBBox() if mask: im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight()) else: im = afwImage.ImageF(bb.getWidth(), bb.getHeight()) im.setXY0(bb.getMinX(), bb.getMinY()) try: fp.insert(im) except AttributeError: # we failed to make it heavy assert not mi pass if mask: im = im.getMask() return im
def testSetFootprint(self): """Check that we can create a HeavyFootprint and set the pixels under it""" ctrl = afwDetect.HeavyFootprintCtrl() ctrl.setModifySource(afwDetect.HeavyFootprintCtrl.SET) # clear the pixels in the Footprint ctrl.setMaskVal(self.objectPixelVal[1]) afwDetect.makeHeavyFootprint(self.foot, self.mi, ctrl) # # Check that we cleared all the pixels # self.assertEqual(np.min(self.mi.getImage().getArray()), 0.0) self.assertEqual(np.max(self.mi.getImage().getArray()), 0.0) self.assertEqual(np.min(self.mi.getMask().getArray()), 0.0) self.assertEqual(np.max(self.mi.getMask().getArray()), 0.0) self.assertEqual(np.min(self.mi.getVariance().getArray()), 0.0) self.assertEqual(np.max(self.mi.getVariance().getArray()), 0.0)
def testCast_HeavyFootprint(self): """Test that we can cast a Footprint to a HeavyFootprint""" hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi) ctrl = afwDetect.HeavyFootprintCtrl(afwDetect.HeavyFootprintCtrl.NONE) hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi, ctrl) # # This isn't quite a full test, as hfoot is already a HeavyFootprint, # the complete test is in testMakeHeavy # self.assertNotEqual(afwDetect.cast_HeavyFootprint(hfoot, self.mi), None, "Cast to the right sort of HeavyFootprint") self.assertNotEqual(afwDetect.HeavyFootprintF.cast(hfoot), None, "Cast to the right sort of HeavyFootprint") self.assertEqual(afwDetect.cast_HeavyFootprint(self.foot, self.mi), None, "Can't cast a Footprint to a HeavyFootprint") self.assertEqual(afwDetect.HeavyFootprintI.cast(hfoot), None, "Cast to the wrong sort of HeavyFootprint")
def test1(self): im = afwImage.ImageF(100, 100) im += 42. fp = afwDet.Footprint(afwGeom.Point2I(50,50), 10.) #seed = 42 #rand = afwMath.Random(afwMath.Random.MT19937, seed) #afwMath.randomGaussianImage(im, rand) mi = afwImage.MaskedImageF(im) # set a mask bit before grabbing the heavyfootprint mi.getMask().set(50, 50, 1) heavy = afwDet.makeHeavyFootprint(fp, mi) # reset it mi.getMask().set(50, 50, 0) schema = afwTable.SourceTable.makeMinimalSchema() table = afwTable.SourceTable.make(schema) table.preallocate(10) catalog = afwTable.SourceCatalog(table) catalog.addNew() # This used to segfault catalog[0].setFootprint(heavy) # However, we still have to up-cast fp = catalog[0].getFootprint() hfp = afwDet.cast_HeavyFootprintF(fp) # change one pixel... self.assertEqual(mi.getImage().get(50, 50), 42) self.assertEqual(mi.getMask().get(50, 50), 0) mi.getImage().set(50, 50, 100) mi.getMask().set(50, 50, 2) mi.getMask().set(51, 50, 2) self.assertEqual(mi.getImage().get(50, 50), 100) self.assertEqual(mi.getMask().get(50, 50), 2) self.assertEqual(mi.getMask().get(51, 50), 2) # reinsert the heavy footprint; it should reset the pixel value. # insert(MaskedImage) hfp.insert(mi) self.assertEqual(mi.getImage().get(50, 50), 42) self.assertEqual(mi.getMask().get(50, 50), 1) self.assertEqual(mi.getMask().get(51, 50), 0) # Also test insert(Image) im = mi.getImage() self.assertEqual(im.get(50, 50), 42) im.set(50, 50, 100) self.assertEqual(im.get(50, 50), 100) self.assertEqual(mi.getImage().get(50, 50), 100) # reinsert the heavy footprint; it should reset the pixel value. hfp.insert(im) self.assertEqual(im.get(50, 50), 42) self.assertEqual(mi.getImage().get(50, 50), 42) self.assertEqual(mi.getMask().get(50, 50), 1) self.assertEqual(mi.getMask().get(51, 50), 0)
def footprintToImage(fp, mi=None, mask=False): if not fp.isHeavy(): fp = afwDet.makeHeavyFootprint(fp, mi) bb = fp.getBBox() if mask: im = afwImage.MaskedImageF(bb.getWidth(), bb.getHeight()) else: im = afwImage.ImageF(bb.getWidth(), bb.getHeight()) im.setXY0(bb.getMinX(), bb.getMinY()) fp.insert(im) if mask: im = im.getMask() return im
def getFluxPortion(self, strayFlux=True): """! Return a HeavyFootprint containing the flux apportioned to this peak. @param[in] strayFlux include stray flux also? """ if self.templateFootprint is None or self.fluxPortion is None: return None heavy = afwDet.makeHeavyFootprint(self.templateFootprint, self.fluxPortion) if strayFlux: if self.strayFlux is not None: heavy = afwDet.mergeHeavyFootprints(heavy, self.strayFlux) return heavy
def test1(self): im = afwImage.ImageF(100, 100) im += 42. spanSet = afwGeom.SpanSet.fromShape(10).shiftedBy(50, 50) fp = afwDet.Footprint(spanSet) mi = afwImage.MaskedImageF(im) # set a mask bit before grabbing the heavyfootprint mi.mask[50, 50, afwImage.LOCAL] = 1 heavy = afwDet.makeHeavyFootprint(fp, mi) # reset it mi.mask[50, 50, afwImage.LOCAL] = 0 schema = afwTable.SourceTable.makeMinimalSchema() table = afwTable.SourceTable.make(schema) table.preallocate(10) catalog = afwTable.SourceCatalog(table) catalog.addNew() # This used to segfault catalog[0].setFootprint(heavy) fp = catalog[0].getFootprint() # change one pixel... self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42) self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 0) mi.image[50, 50, afwImage.LOCAL] = 100 mi.mask[50, 50, afwImage.LOCAL] = 2 mi.mask[51, 50, afwImage.LOCAL] = 2 self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100) self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 2) self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 2) # reinsert the heavy footprint; it should reset the pixel value. # insert(MaskedImage) fp.insert(mi) self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42) self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1) self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0) # Also test insert(Image) im = mi.image self.assertEqual(im[50, 50, afwImage.LOCAL], 42) im[50, 50, afwImage.LOCAL] = 100 self.assertEqual(im[50, 50, afwImage.LOCAL], 100) self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 100) # reinsert the heavy footprint; it should reset the pixel value. fp.insert(im) self.assertEqual(im[50, 50, afwImage.LOCAL], 42) self.assertEqual(mi.image[50, 50, afwImage.LOCAL], 42) self.assertEqual(mi.mask[50, 50, afwImage.LOCAL], 1) self.assertEqual(mi.mask[51, 50, afwImage.LOCAL], 0)
def testCreate(self): """Check that we can create a HeavyFootprint""" imi = self.mi.Factory(self.mi, True) # copy of input image hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi) self.assertNotEqual(hfoot.getId(), None) # check we can call a base-class method # # Check we didn't modify the input image # self.assertTrue( np.all( np.equal(self.mi.getImage().getArray(), imi.getImage().getArray()))) omi = self.mi.Factory(self.mi.getDimensions()) omi.set((1, 0x4, 0.1)) hfoot.insert(omi) if display: ds9.mtv(imi, frame=0, title="input") ds9.mtv(omi, frame=1, title="output") for s in self.foot.getSpans(): y = s.getY() for x in range(s.getX0(), s.getX1() + 1): self.assertEqual(imi.get(x, y), omi.get(x, y)) # Check that we can call getImageArray(), etc arr = hfoot.getImageArray() print arr # Check that it's iterable for x in arr: pass arr = hfoot.getMaskArray() print arr for x in arr: pass arr = hfoot.getVarianceArray() print arr # Check that it's iterable for x in arr: pass
def readCatalog(sourcefn, heavypat, ndeblends=0, dataref=None, keepids=None, keepxys=None, patargs=dict()): if sourcefn is None: cat = dataref.get('src') try: if not cat: return None except: return None else: if not os.path.exists(sourcefn): print('No source catalog:', sourcefn) return None print('Reading catalog:', sourcefn) cat = afwTable.SourceCatalog.readFits(sourcefn) print(len(cat), 'sources') cat.sort() cat.defineCentroid('base_SdssCentroid') if ndeblends or keepids or keepxys: cat = cutCatalog(cat, ndeblends, keepids, keepxys) print('Cut to', len(cat), 'sources') if heavypat is not None: print('Reading heavyFootprints...') for src in cat: if not src.getParent(): continue dd = patargs.copy() dd.update(id=src.getId()) heavyfn = heavypat % dd if not os.path.exists(heavyfn): print('No heavy footprint:', heavyfn) return None mim = afwImage.MaskedImageF(heavyfn) heavy = afwDet.makeHeavyFootprint(src.getFootprint(), mim) src.setFootprint(heavy) return cat
def testCreate(self): """Check that we can create a HeavyFootprint""" imi = self.mi.Factory(self.mi, True) # copy of input image hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi) # check we can call a base-class method self.assertTrue(hfoot.isHeavy()) # # Check we didn't modify the input image # self.assertFloatsEqual(self.mi.getImage().getArray(), imi.getImage().getArray()) omi = self.mi.Factory(self.mi.getDimensions()) omi.set((1, 0x4, 0.1)) hfoot.insert(omi) if display: afwDisplay.Display(frame=0).mtv(imi, title="testCreate input") afwDisplay.Display(frame=1).mtv(omi, title="testCreate output") for s in self.foot.getSpans(): y = s.getY() for x in range(s.getX0(), s.getX1() + 1): self.assertEqual(imi[x, y, afwImage.LOCAL], omi[x, y, afwImage.LOCAL]) # Check that we can call getImageArray(), etc arr = hfoot.getImageArray() # Check that it's iterable for x in arr: pass arr = hfoot.getMaskArray() for x in arr: pass arr = hfoot.getVarianceArray() # Check that it's iterable for x in arr: pass
def testCreate(self): """Check that we can create a HeavyFootprint""" imi = self.mi.Factory(self.mi, True) # copy of input image hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi) self.assertNotEqual(hfoot.getId(), None) # check we can call a base-class method # # Check we didn't modify the input image # self.assertTrue(np.all(np.equal(self.mi.getImage().getArray(), imi.getImage().getArray()))) omi = self.mi.Factory(self.mi.getDimensions()) omi.set((1, 0x4, 0.1)) hfoot.insert(omi) if display: ds9.mtv(imi, frame=0, title="input") ds9.mtv(omi, frame=1, title="output") for s in self.foot.getSpans(): y = s.getY() for x in range(s.getX0(), s.getX1() + 1): self.assertEqual(imi.get(x, y), omi.get(x, y)) # Check that we can call getImageArray(), etc arr = hfoot.getImageArray() print arr # Check that it's iterable for x in arr: pass arr = hfoot.getMaskArray() print arr for x in arr: pass arr = hfoot.getVarianceArray() print arr # Check that it's iterable for x in arr: pass
def readCatalog(sourcefn, heavypat, ndeblends=0, dataref=None, keepids=None, keepxys=None, patargs=dict()): if sourcefn is None: cat = dataref.get('src') try: if not cat: return None except Exception: return None else: if not os.path.exists(sourcefn): print('No source catalog:', sourcefn) return None print('Reading catalog:', sourcefn) cat = afwTable.SourceCatalog.readFits(sourcefn) print(len(cat), 'sources') cat.sort() cat.defineCentroid('base_SdssCentroid') if ndeblends or keepids or keepxys: cat = cutCatalog(cat, ndeblends, keepids, keepxys) print('Cut to', len(cat), 'sources') if heavypat is not None: print('Reading heavyFootprints...') for src in cat: if not src.getParent(): continue dd = patargs.copy() dd.update(id=src.getId()) heavyfn = heavypat % dd if not os.path.exists(heavyfn): print('No heavy footprint:', heavyfn) return None mim = afwImage.MaskedImageF(heavyfn) heavy = afwDet.makeHeavyFootprint(src.getFootprint(), mim) src.setFootprint(heavy) return cat
def testCreate(self): """Check that we can create a HeavyFootprint""" imi = self.mi.Factory(self.mi, True) # copy of input image hfoot = afwDetect.makeHeavyFootprint(self.foot, self.mi) # check we can call a base-class method self.assertNotEqual(hfoot.getId(), None) # # Check we didn't modify the input image # self.assertFloatsEqual( self.mi.getImage().getArray(), imi.getImage().getArray()) omi = self.mi.Factory(self.mi.getDimensions()) omi.set((1, 0x4, 0.1)) hfoot.insert(omi) if display: afwDisplay.Display(frame=0).mtv(imi, title="testCreate input") afwDisplay.Display(frame=1).mtv(omi, title="testCreate output") for s in self.foot.getSpans(): y = s.getY() for x in range(s.getX0(), s.getX1() + 1): self.assertEqual(imi[x, y, afwImage.LOCAL], omi[x, y, afwImage.LOCAL]) # Check that we can call getImageArray(), etc arr = hfoot.getImageArray() # Check that it's iterable for x in arr: pass arr = hfoot.getMaskArray() for x in arr: pass arr = hfoot.getVarianceArray() # Check that it's iterable for x in arr: pass
def setUp(self): np.random.seed(1) self.spans = SpanSet.fromShape(2, Stencil.CIRCLE) self.footprint = Footprint(self.spans) self.footprint.addPeak(3, 4, 10) self.footprint.addPeak(8, 1, 2) fp = Footprint(self.spans) for peak in self.footprint.getPeaks(): fp.addPeak(peak["f_x"], peak["f_y"], peak["peakValue"]) self.peaks = fp.getPeaks() self.bbox = self.footprint.getBBox() self.filters = ("G", "R", "I") singles = [] images = [] for n, f in enumerate(self.filters): image = ImageF(self.spans.getBBox()) image.set(n) images.append(image.array) maskedImage = MaskedImageF(image) heavy = makeHeavyFootprint(self.footprint, maskedImage) singles.append(heavy) self.image = np.array(images) self.mFoot = MultibandFootprint(self.filters, singles)
def testMergeHeavyFootprints(self): mi = afwImage.MaskedImageF(20, 10) objectPixelVal = (42, 0x9, 400) foot = afwDetect.Footprint() for y, x0, x1 in [(1, 9, 12), (2, 12, 13), (3, 11, 15)]: foot.addSpan(y, x0, x1) for x in range(x0, x1 + 1): mi.set(x, y, objectPixelVal) hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi) hfoot2 = afwDetect.makeHeavyFootprint(foot, mi) hfoot1.normalize() hfoot2.normalize() hsum = afwDetect.mergeHeavyFootprintsF(hfoot1, hfoot2) bb = hsum.getBBox() self.assertEquals(bb.getMinX(), 9) self.assertEquals(bb.getMaxX(), 15) self.assertEquals(bb.getMinY(), 1) self.assertEquals(bb.getMaxY(), 3) msum = afwImage.MaskedImageF(20,10) hsum.insert(msum) sa = msum.getImage().getArray() self.assertTrue(np.all(sa[1, 9:13] == objectPixelVal[0])) self.assertTrue(np.all(sa[2, 12:14] == objectPixelVal[0] + self.objectPixelVal[0])) self.assertTrue(np.all(sa[2, 10:12] == self.objectPixelVal[0])) sv = msum.getVariance().getArray() self.assertTrue(np.all(sv[1, 9:13] == objectPixelVal[2])) self.assertTrue(np.all(sv[2, 12:14] == objectPixelVal[2] + self.objectPixelVal[2])) self.assertTrue(np.all(sv[2, 10:12] == self.objectPixelVal[2])) sm = msum.getMask().getArray() self.assertTrue(np.all(sm[1, 9:13] == objectPixelVal[1])) self.assertTrue(np.all(sm[2, 12:14] == objectPixelVal[1] | self.objectPixelVal[1])) self.assertTrue(np.all(sm[2, 10:12] == self.objectPixelVal[1])) if False: import matplotlib matplotlib.use('Agg') import pylab as plt im1 = afwImage.ImageF(bb) hfoot1.insert(im1) im2 = afwImage.ImageF(bb) hfoot2.insert(im2) im3 = afwImage.ImageF(bb) hsum.insert(im3) plt.clf() plt.subplot(1,3,1) plt.imshow(im1.getArray(), interpolation='nearest', origin='lower') plt.subplot(1,3,2) plt.imshow(im2.getArray(), interpolation='nearest', origin='lower') plt.subplot(1,3,3) plt.imshow(im3.getArray(), interpolation='nearest', origin='lower') plt.savefig('merge.png')
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None): """! Initialize the NoiseReplacer. @param[in] config instance of NoiseReplacerConfig @param[in,out] exposure Exposure to be noise replaced. (All sources replaced on return) @param[in] footprints dict of {id: (parent, footprint)}; @param[in] noiseImage an afw.image.ImageF used as a predictable noise replacement source (for tests only) @param[in] log pex.logging.Log object to use for status messages; no status messages will be printed if None 'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the source ID, but in forced photometry, this will be the reference ID, as that's what we used to determine the deblend families. This routine should create HeavyFootprints for any non-Heavy Footprints, and replace them in the dict. It should then create a dict of HeavyFootprints containing noise, but only for parent objects, then replace all sources with noise. This should ignore any footprints that lay outside the bounding box of the exposure, and clip those that lie on the border. NOTE: as the code currently stands, the heavy footprint for a deblended object must be available from the input catalog. If it is not, it cannot be reproduced here. In that case, the topmost parent in the objects parent chain must be used. The heavy footprint for that source is created in this class from the masked image. """ noiseMeanVar=None self.noiseSource = config.noiseSource self.noiseOffset = config.noiseOffset self.noiseSeedMultiplier = config.noiseSeedMultiplier self.noiseGenMean = None self.noiseGenStd = None self.log = log # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups self.exposure = exposure self.footprints = footprints mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: try: # does it already exist? plane = mask.getMaskPlane(maskname) if self.log: self.log.logdebug('Mask plane "%s" already existed' % maskname) except: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) if self.log: self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x' % (maskname, plane, bitmask, bitmask)) self.thisbitmask,self.otherbitmask = bitmasks del bitmasks self.heavies = {} # Start by creating HeavyFootprints for each source which has no parent # and just use them for children which do not already have heavy footprints. # If a heavy footprint is available for a child, we will use it. Otherwise, # we use the first parent in the parent chain which has a heavy footprint, # which with the one level deblender will alway be the topmost parent # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py, # so they are never available for forced measurements. # Create in the dict heavies = {id:heavyfootprint} for id in footprints.keys(): fp = footprints[id] if fp[1].isHeavy(): self.heavies[id] = afwDet.cast_HeavyFootprintF(fp[1]) elif fp[0] == 0: self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi) ### FIXME: the heavy footprint includes the mask ### and variance planes, which we shouldn't need ### (I don't think we ever want to modify them in ### the input image). Copying them around is ### wasteful. # We now create a noise HeavyFootprint for each source with has a heavy footprint. # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint} self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId) # The noiseGenMean and Std are used by the unit tests self.noiseGenMean = noisegen.mean self.noiseGenStd = noisegen.std if self.log: self.log.logdebug('Using noise generator: %s' % (str(noisegen))) for id in self.heavies.keys(): fp = footprints[id][1] noiseFp = noisegen.getHeavyFootprint(fp) self.heavyNoise[id] = noiseFp # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. noiseFp.insert(im) # Also set the OTHERDET bit afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
def deblend(self, exposure, srcs, psf): """! Deblend. @param[in] exposure Exposure to process @param[in,out] srcs SourceCatalog containing sources detected on this exposure. @param[in] psf PSF @return None """ self.log.info("Deblending %d sources" % len(srcs)) from lsst.meas.deblender.baseline import deblend # find the median stdev in the image... mi = exposure.getMaskedImage() statsCtrl = afwMath.StatisticsControl() statsCtrl.setAndMask(mi.getMask().getPlaneBitMask( self.config.maskPlanes)) stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) self.log.trace('sigma1: %g', sigma1) n0 = len(srcs) nparents = 0 for i, src in enumerate(srcs): # t0 = time.clock() fp = src.getFootprint() pks = fp.getPeaks() # Since we use the first peak for the parent object, we should propagate its flags # to the parent source. src.assign(pks[0], self.peakSchemaMapper) if len(pks) < 2: continue if self.isLargeFootprint(fp): src.set(self.tooBigKey, True) self.skipParent(src, mi.getMask()) self.log.warn('Parent %i: skipping large footprint (area: %i)', int(src.getId()), int(fp.getArea())) continue if self.isMasked(fp, exposure.getMaskedImage().getMask()): src.set(self.maskedKey, True) self.skipParent(src, mi.getMask()) self.log.warn( 'Parent %i: skipping masked footprint (area: %i)', int(src.getId()), int(fp.getArea())) continue nparents += 1 bb = fp.getBBox() psf_fwhm = self._getPsfFwhm(psf, bb) self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks)) self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1) npre = len(srcs) # This should really be set in deblend, but deblend doesn't have access to the src src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks) try: res = deblend( fp, mi, psf, psf_fwhm, sigma1=sigma1, psfChisqCut1=self.config.psfChisq1, psfChisqCut2=self.config.psfChisq2, psfChisqCut2b=self.config.psfChisq2b, maxNumberOfPeaks=self.config.maxNumberOfPeaks, strayFluxToPointSources=self.config. strayFluxToPointSources, assignStrayFlux=self.config.assignStrayFlux, strayFluxAssignment=self.config.strayFluxRule, rampFluxAtEdge=(self.config.edgeHandling == 'ramp'), patchEdges=(self.config.edgeHandling == 'noclip'), tinyFootprintSize=self.config.tinyFootprintSize, clipStrayFluxFraction=self.config.clipStrayFluxFraction, weightTemplates=self.config.weightTemplates, removeDegenerateTemplates=self.config. removeDegenerateTemplates, maxTempDotProd=self.config.maxTempDotProd, medianSmoothTemplate=self.config.medianSmoothTemplate) if self.config.catchFailures: src.set(self.deblendFailedKey, False) except Exception as e: if self.config.catchFailures: self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) src.set(self.deblendFailedKey, True) import traceback traceback.print_exc() continue else: raise kids = [] nchild = 0 for j, peak in enumerate(res.deblendedParents[0].peaks): heavy = peak.getFluxPortion() if heavy is None or peak.skip: src.set(self.deblendSkippedKey, True) if not self.config.propagateAllPeaks: # Don't care continue # We need to preserve the peak: make sure we have enough info to create a minimal # child src self.log.trace( "Peak at (%i,%i) failed. Using minimal default info for child.", pks[j].getIx(), pks[j].getIy()) if heavy is None: # copy the full footprint and strip out extra peaks foot = afwDet.Footprint(src.getFootprint()) peakList = foot.getPeaks() peakList.clear() peakList.append(peak.peak) zeroMimg = afwImage.MaskedImageF(foot.getBBox()) heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) if peak.deblendedAsPsf: if peak.psfFitFlux is None: peak.psfFitFlux = 0.0 if peak.psfFitCenter is None: peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy()) assert (len(heavy.getPeaks()) == 1) src.set(self.deblendSkippedKey, False) child = srcs.addNew() nchild += 1 for key in self.toCopyFromParent: child.set(key, src.get(key)) child.assign(heavy.getPeaks()[0], self.peakSchemaMapper) child.setParent(src.getId()) child.setFootprint(heavy) child.set(self.psfKey, peak.deblendedAsPsf) child.set(self.hasStrayFluxKey, peak.strayFlux is not None) if peak.deblendedAsPsf: (cx, cy) = peak.psfFitCenter child.set(self.psfCenterKey, geom.Point2D(cx, cy)) child.set(self.psfFluxKey, peak.psfFitFlux) child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) child.set(self.deblendPatchedTemplateKey, peak.patched) # Set the position of the peak from the parent footprint # This will make it easier to match the same source across # deblenders and across observations, where the peak # position is unlikely to change unless enough time passes # for a source to move on the sky. child.set(self.peakCenter, geom.Point2I(pks[j].getIx(), pks[j].getIy())) child.set(self.peakIdKey, pks[j].getId()) # The children have a single peak child.set(self.nPeaksKey, 1) # Set the number of peaks in the parent child.set(self.parentNPeaksKey, len(pks)) kids.append(child) # Child footprints may extend beyond the full extent of their parent's which # results in a failure of the replace-by-noise code to reinstate these pixels # to their original values. The following updates the parent footprint # in-place to ensure it contains the full union of itself and all of its # children's footprints. spans = src.getFootprint().spans for child in kids: spans = spans.union(child.getFootprint().spans) src.getFootprint().setSpans(spans) src.set(self.nChildKey, nchild) self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res) # print('Deblending parent id', src.getId(), 'took', time.clock() - t0) n1 = len(srcs) self.log.info( 'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' % (n0, nparents, n1 - n0, n1))
def deblend(self, mExposure, sources): """Deblend a data cube of multiband images Parameters ---------- mExposure : `MultibandExposure` The exposures should be co-added images of the same shape and region of the sky. sources : `SourceCatalog` The merged `SourceCatalog` that contains parent footprints to (potentially) deblend. Returns ------- fluxCatalogs : dict or None Keys are the names of the filters and the values are `lsst.afw.table.source.source.SourceCatalog`'s. These are the flux-conserved catalogs with heavy footprints with the image data weighted by the multiband templates. If `self.config.conserveFlux` is `False`, then this item will be None templateCatalogs : dict or None Keys are the names of the filters and the values are `lsst.afw.table.source.source.SourceCatalog`'s. These are catalogs with heavy footprints that are the templates created by the multiband templates. If `self.config.saveTemplates` is `False`, then this item will be None """ import time filters = mExposure.filters self.log.info("Deblending {0} sources in {1} exposure bands".format( len(sources), len(mExposure))) # Create the output catalogs templateCatalogs = {} # This must be returned but is not calculated right now, setting it to # None to be consistent with doc string fluxCatalogs = None for f in filters: _catalog = afwTable.SourceCatalog(sources.table.clone()) _catalog.extend(sources) templateCatalogs[f] = _catalog n0 = len(sources) nparents = 0 for pk, src in enumerate(sources): foot = src.getFootprint() bbox = foot.getBBox() logger.info("id: {0}".format(src["id"])) peaks = foot.getPeaks() # Since we use the first peak for the parent object, we should # propagate its flags to the parent source. src.assign(peaks[0], self.peakSchemaMapper) # Block of Skipping conditions if len(peaks) < 2 and not self.config.processSingles: for f in filters: templateCatalogs[f][pk].set(self.runtimeKey, 0) continue if self._isLargeFootprint(foot): src.set(self.tooBigKey, True) self._skipParent(src, mExposure.mask) self.log.trace('Parent %i: skipping large footprint', int(src.getId())) continue if self._isMasked(foot, mExposure): src.set(self.maskedKey, True) mask = np.bitwise_or.reduce(mExposure.mask[:, bbox].array, axis=0) mask = afwImage.MaskX(mask, xy0=bbox.getMin()) self._skipParent(src, mask) self.log.trace('Parent %i: skipping masked footprint', int(src.getId())) continue if len(peaks) > self.config.maxNumberOfPeaks: src.set(self.tooManyPeaksKey, True) msg = 'Parent {0}: Too many peaks, using the first {1} peaks' self.log.trace( msg.format(int(src.getId()), self.config.maxNumberOfPeaks)) nparents += 1 self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(peaks)) # Run the deblender try: t0 = time.time() # Build the parameter lists with the same ordering blend, skipped = deblend(mExposure, foot, self.config) tf = time.time() runtime = (tf - t0) * 1000 src.set(self.deblendFailedKey, False) src.set(self.runtimeKey, runtime) converged = checkBlendConvergence(blend, self.config.relativeError) src.set(self.blendConvergenceFailedFlagKey, converged) sources = [src for src in blend.sources] # Re-insert place holders for skipped sources # to propagate them in the catalog so # that the peaks stay consistent for k in skipped: sources.insert(k, None) except Exception as e: if self.config.catchFailures: self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) src.set(self.deblendFailedKey, True) src.set(self.runtimeKey, 0) import traceback traceback.print_exc() continue else: raise # Add the merged source as a parent in the catalog for each band templateParents = {} parentId = src.getId() for f in filters: templateParents[f] = templateCatalogs[f][pk] templateParents[f].set(self.runtimeKey, runtime) templateParents[f].set(self.iterKey, len(blend.loss)) # Add each source to the catalogs in each band templateSpans = {f: afwGeom.SpanSet() for f in filters} nchild = 0 for k, source in enumerate(sources): # Skip any sources with no flux or that scarlet skipped because # it could not initialize if k in skipped: if not self.config.propagateAllPeaks: # We don't care continue # We need to preserve the peak: make sure we have enough # info to create a minimal child src msg = "Peak at {0} failed deblending. Using minimal default info for child." self.log.trace(msg.format(src.getFootprint().peaks[k])) # copy the full footprint and strip out extra peaks foot = afwDet.Footprint(src.getFootprint()) peakList = foot.getPeaks() peakList.clear() peakList.append(src.peaks[k]) zeroMimg = afwImage.MaskedImageF(foot.getBBox()) heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) models = afwDet.MultibandFootprint( mExposure.filters, [heavy] * len(mExposure.filters)) else: src.set(self.deblendSkippedKey, False) models = modelToHeavy(source, filters, xy0=bbox.getMin(), observation=blend.observations[0]) # TODO: We should eventually write the morphology and SED to # the catalog # morph = source.morphToHeavy(xy0=bbox.getMin()) # sed = source.sed / source.sed.sum() for f in filters: if len(models[f].getPeaks()) != 1: err = "Heavy footprint should have a single peak, got {0}" raise ValueError(err.format(len(models[f].peaks))) cat = templateCatalogs[f] child = self._addChild(parentId, cat, models[f], source, converged, xy0=bbox.getMin()) if parentId == 0: child.setId(src.getId()) child.set(self.runtimeKey, runtime) else: templateSpans[f] = templateSpans[f].union( models[f].getSpans()) nchild += 1 # Child footprints may extend beyond the full extent of their # parent's which results in a failure of the replace-by-noise code # to reinstate these pixels to their original values. The # following updates the parent footprint in-place to ensure it # contains the full union of itself and all of its # children's footprints. for f in filters: templateParents[f].set(self.nChildKey, nchild) templateParents[f].getFootprint().setSpans(templateSpans[f]) K = len(list(templateCatalogs.values())[0]) self.log.info( 'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' % (n0, nparents, K - n0, K)) return fluxCatalogs, templateCatalogs
def testMergeHeavyFootprints(self): mi = afwImage.MaskedImageF(20, 10) objectPixelVal = (42, 0x9, 400) spanList = [] for y, x0, x1 in [(1, 9, 12), (2, 12, 13), (3, 11, 15)]: spanList.append(afwGeom.Span(y, x0, x1)) for x in range(x0, x1 + 1): mi[x, y, afwImage.LOCAL] = objectPixelVal foot = afwDetect.Footprint(afwGeom.SpanSet(spanList)) hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi) hfoot2 = afwDetect.makeHeavyFootprint(foot, mi) hsum = afwDetect.mergeHeavyFootprints(hfoot1, hfoot2) bb = hsum.getBBox() self.assertEqual(bb.getMinX(), 9) self.assertEqual(bb.getMaxX(), 15) self.assertEqual(bb.getMinY(), 1) self.assertEqual(bb.getMaxY(), 3) msum = afwImage.MaskedImageF(20, 10) hsum.insert(msum) sa = msum.getImage().getArray() self.assertFloatsEqual(sa[1, 9:13], objectPixelVal[0]) self.assertFloatsEqual( sa[2, 12:14], objectPixelVal[0] + self.objectPixelVal[0]) self.assertFloatsEqual(sa[2, 10:12], self.objectPixelVal[0]) sv = msum.getVariance().getArray() self.assertFloatsEqual(sv[1, 9:13], objectPixelVal[2]) self.assertFloatsEqual( sv[2, 12:14], objectPixelVal[2] + self.objectPixelVal[2]) self.assertFloatsEqual(sv[2, 10:12], self.objectPixelVal[2]) sm = msum.getMask().getArray() self.assertFloatsEqual(sm[1, 9:13], objectPixelVal[1]) self.assertFloatsEqual( sm[2, 12:14], objectPixelVal[1] | self.objectPixelVal[1]) self.assertFloatsEqual(sm[2, 10:12], self.objectPixelVal[1]) if False: import matplotlib matplotlib.use('Agg') import pylab as plt im1 = afwImage.ImageF(bb) hfoot1.insert(im1) im2 = afwImage.ImageF(bb) hfoot2.insert(im2) im3 = afwImage.ImageF(bb) hsum.insert(im3) plt.clf() plt.subplot(1, 3, 1) plt.imshow(im1.getArray(), interpolation='nearest', origin='lower') plt.subplot(1, 3, 2) plt.imshow(im2.getArray(), interpolation='nearest', origin='lower') plt.subplot(1, 3, 3) plt.imshow(im3.getArray(), interpolation='nearest', origin='lower') plt.savefig('merge.png')
def test2(self): # Check that doReplaceWithNoise works with deblended source # hierarchies. seed = 42 rand = afwMath.Random(afwMath.Random.MT19937, seed) psf = self.getpsf() im = afwImage.ImageF(200, 50) skystd = 100 afwMath.randomGaussianImage(im, rand) im *= skystd imorig = afwImage.ImageF(im, True) noiseim = imorig mi = afwImage.MaskedImageF(im) mi.getVariance().set(skystd**2) exposure = afwImage.makeExposure(mi) exposure.setPsf(psf) detconf = measAlg.SourceDetectionConfig() detconf.returnOriginalFootprints = True detconf.reEstimateBackground = False measconf = measAlg.SourceMeasurementConfig() measconf.doReplaceWithNoise = True measconf.replaceWithNoise.noiseSeed = 42 schema = afwTable.SourceTable.makeMinimalSchema() detect = measAlg.SourceDetectionTask(config=detconf, schema=schema) measure = MySourceMeasurementTask(config=measconf, schema=schema, doplot=plots) table = afwTable.SourceTable.make(schema) table.preallocate(10) # We're going to fake up a perfect deblend hierarchy here, by # creating individual images containing single sources and # measuring them, and then creating a deblend hierarchy where # the children have the correct HeavyFootprints. We want to # find that the measurements on the deblend hierarchy and the # blended image are equal to the individual images. # # Note that in the normal setup we don't expect the # measurements to be *identical* because of the faint wings of # the objects; when measuring a deblended child, we pick up # the wings of the other objects. # # In order to get exactly equal measurements, we'll fake some # sources that have no wings -- we'll copy just the source # pixels within the footprint. This means that all the # footprints are the same, and the pixels inside the footprint # are the same. fullim = None sources = None # "normal" measurements xx0,yy0,vx0,vy0 = [],[],[],[] # "no-wing" measurements xx1,yy1,vx1,vy1 = [],[],[],[] y = 25 for i in range(5): # no-noise source image sim = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) # Put all four sources in the parent (i==0), and one # source in each child (i=[1 to 4]) if i in [0,1]: addPsf(sim, psf, 20, y, 1000) if i in [0,2]: addGaussian(sim, 40, y, 10, 3, 2e5) if i in [0,3]: addGaussian(sim, 75, y, 10, 3, 2e5) if i in [0,4]: addPsf(sim, psf, 95, y, 1000) imcopy = afwImage.ImageF(imorig, True) imcopy += sim # copy the pixels into the exposure object im <<= imcopy if i == 0: detected = detect.makeSourceCatalog(table, exposure) sources = detected.sources print 'detected', len(sources), 'sources' self.assertEqual(len(sources), 1) else: fpSets = detect.detectFootprints(exposure) print 'detected', fpSets.numPos, 'sources' fpSets.positive.makeSources(sources) self.assertEqual(fpSets.numPos, 1) print len(sources), 'sources total' measure.plotpat = 'single-%i.png' % i measure.run(exposure, sources[-1:]) s = sources[-1] fp = s.getFootprint() if i == 0: # This is the blended image fullim = imcopy else: print 'Creating heavy footprint...' heavy = afwDet.makeHeavyFootprint(fp, mi) s.setFootprint(heavy) # Record the single-source measurements. xx0.append(s.getX()) yy0.append(s.getY()) vx0.append(s.getIxx()) vy0.append(s.getIyy()) # "no-wings": add just the source pixels within the footprint im <<= sim h = afwDet.makeHeavyFootprint(fp, mi) sim2 = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) h.insert(sim2) imcopy = afwImage.ImageF(imorig, True) imcopy += sim2 im <<= imcopy measure.plotpat = 'single2-%i.png' % i measure.run(exposure, sources[i:i+1], noiseImage=noiseim) s = sources[i] xx1.append(s.getX()) yy1.append(s.getY()) vx1.append(s.getIxx()) vy1.append(s.getIyy()) if i == 0: fullim2 = imcopy # Now we'll build the fake deblended hierarchy. parent = sources[0] kids = sources[1:] # Ensure that the parent footprint contains all the child footprints pfp = parent.getFootprint() for s in kids: for span in s.getFootprint().getSpans(): pfp.addSpan(span) pfp.normalize() #parent.setFootprint(pfp) # The parent-child relationship is established through the IDs parentid = parent.getId() for s in kids: s.setParent(parentid) # Reset all the measurements shkey = sources.getTable().getShapeKey() ckey = sources.getTable().getCentroidKey() for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Measure the "deblended" normal sources im <<= fullim measure.plotpat = 'joint-%(sourcenum)i.png' measure.run(exposure, sources) xx2,yy2,vx2,vy2 = [],[],[],[] for s in sources: xx2.append(s.getX()) yy2.append(s.getY()) vx2.append(s.getIxx()) vy2.append(s.getIyy()) # Measure the "deblended" no-wings sources im <<= fullim2 measure.plotpat = 'joint2-%(sourcenum)i.png' measure.run(exposure, sources, noiseImage=noiseim) xx3,yy3,vx3,vy3 = [],[],[],[] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) print 'Normal:' print 'xx ', xx0 print ' vs', xx2 print 'yy ', yy0 print ' vs', yy2 print 'vx ', vx0 print ' vs', vx2 print 'vy ', vy0 print ' vs', vy2 print 'No wings:' print 'xx ', xx1 print ' vs', xx3 print 'yy ', yy1 print ' vs', yy3 print 'vx ', vx1 print ' vs', vx3 print 'vy ', vy1 print ' vs', vy3 # These "normal" tests are not very stringent. # 0.1-pixel centroids self.assertTrue(all([abs(v1-v2) < 0.1 for v1,v2 in zip(xx0,xx2)])) self.assertTrue(all([abs(v1-v2) < 0.1 for v1,v2 in zip(yy0,yy2)])) # 10% variances self.assertTrue(all([abs(v1-v2)/((v1+v2)/2.) < 0.1 for v1,v2 in zip(vx0,vx2)])) self.assertTrue(all([abs(v1-v2)/((v1+v2)/2.) < 0.1 for v1,v2 in zip(vy0,vy2)])) # The "no-wings" tests should be exact. self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that the parent/child order is unimportant. im <<= fullim2 measure.doplot = False sources2 = sources.copy() perm = [2,1,0,3,4] for i,j in enumerate(perm): sources2[i] = sources[j] # I'm not convinced that HeavyFootprints get copied correctly... sources2[i].setFootprint(sources[j].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # "measure.run" reorders the sources! xx3,yy3,vx3,vy3 = [],[],[],[] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that it still works when the parent ID falls in the middle of # the child IDs. im <<= fullim2 measure.doplot = False sources2 = sources.copy() parentid = 3 ids = [parentid, 1,2,4,5] for i,s in enumerate(sources2): s.setId(ids[i]) if i != 0: s.setParent(parentid) s.setFootprint(sources[i].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # The sources get reordered! xx3,yy3,vx3,vy3 = [],[],[],[] xx3,yy3,vx3,vy3 = [0]*5,[0]*5,[0]*5,[0]*5 for i,j in enumerate(ids): xx3[i] = sources2[j-1].getX() yy3[i] = sources2[j-1].getY() vx3[i] = sources2[j-1].getIxx() vy3[i] = sources2[j-1].getIyy() self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3)
def deblend(self, exposure, srcs, psf): """! Deblend. @param[in] exposure Exposure to process @param[in,out] srcs SourceCatalog containing sources detected on this exposure. @param[in] psf PSF @return None """ self.log.info("Deblending %d sources" % len(srcs)) from lsst.meas.deblender.baseline import deblend # find the median stdev in the image... mi = exposure.getMaskedImage() statsCtrl = afwMath.StatisticsControl() statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes)) stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl) sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN)) self.log.logdebug('sigma1: %g' % sigma1) n0 = len(srcs) nparents = 0 for i,src in enumerate(srcs): #t0 = time.clock() fp = src.getFootprint() pks = fp.getPeaks() # Since we use the first peak for the parent object, we should propagate its flags # to the parent source. src.assign(pks[0], self.peakSchemaMapper) if len(pks) < 2: continue if self.isLargeFootprint(fp): src.set(self.tooBigKey, True) self.skipParent(src, mi.getMask()) self.log.logdebug('Parent %i: skipping large footprint' % (int(src.getId()),)) continue if self.isMasked(fp, exposure.getMaskedImage().getMask()): src.set(self.maskedKey, True) self.skipParent(src, mi.getMask()) self.log.logdebug('Parent %i: skipping masked footprint' % (int(src.getId()),)) continue nparents += 1 bb = fp.getBBox() psf_fwhm = self._getPsfFwhm(psf, bb) self.log.logdebug('Parent %i: deblending %i peaks' % (int(src.getId()), len(pks))) self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1) npre = len(srcs) # This should really be set in deblend, but deblend doesn't have access to the src src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks) try: res = deblend( fp, mi, psf, psf_fwhm, sigma1=sigma1, psfChisqCut1 = self.config.psfChisq1, psfChisqCut2 = self.config.psfChisq2, psfChisqCut2b= self.config.psfChisq2b, maxNumberOfPeaks=self.config.maxNumberOfPeaks, strayFluxToPointSources=self.config.strayFluxToPointSources, assignStrayFlux=self.config.assignStrayFlux, findStrayFlux=(self.config.assignStrayFlux or self.config.findStrayFlux), strayFluxAssignment=self.config.strayFluxRule, rampFluxAtEdge=(self.config.edgeHandling == 'ramp'), patchEdges=(self.config.edgeHandling == 'noclip'), tinyFootprintSize=self.config.tinyFootprintSize, clipStrayFluxFraction=self.config.clipStrayFluxFraction, ) if self.config.catchFailures: src.set(self.deblendFailedKey, False) except Exception as e: if self.config.catchFailures: self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e)) src.set(self.deblendFailedKey, True) import traceback traceback.print_exc() continue else: raise kids = [] nchild = 0 for j, peak in enumerate(res.peaks): heavy = peak.getFluxPortion() if heavy is None or peak.skip: src.set(self.deblendSkippedKey, True) if not self.config.propagateAllPeaks: # Don't care continue # We need to preserve the peak: make sure we have enough info to create a minimal child src self.log.logdebug("Peak at (%i,%i) failed. Using minimal default info for child." % (pks[j].getIx(), pks[j].getIy())) if heavy is None: # copy the full footprint and strip out extra peaks foot = afwDet.Footprint(src.getFootprint()) peakList = foot.getPeaks() peakList.clear() peakList.append(peak.peak) zeroMimg = afwImage.MaskedImageF(foot.getBBox()) heavy = afwDet.makeHeavyFootprint(foot, zeroMimg) if peak.deblendedAsPsf: if peak.psfFitFlux is None: peak.psfFitFlux = 0.0 if peak.psfFitCenter is None: peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy()) assert(len(heavy.getPeaks()) == 1) src.set(self.deblendSkippedKey, False) child = srcs.addNew(); nchild += 1 child.assign(heavy.getPeaks()[0], self.peakSchemaMapper) child.setParent(src.getId()) child.setFootprint(heavy) child.set(self.psfKey, peak.deblendedAsPsf) child.set(self.hasStrayFluxKey, peak.strayFlux is not None) if peak.deblendedAsPsf: (cx,cy) = peak.psfFitCenter child.set(self.psfCenterKey, afwGeom.Point2D(cx, cy)) child.set(self.psfFluxKey, peak.psfFitFlux) child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate) child.set(self.deblendPatchedTemplateKey, peak.patched) kids.append(child) # Child footprints may extend beyond the full extent of their parent's which # results in a failure of the replace-by-noise code to reinstate these pixels # to their original values. The following updates the parent footprint # in-place to ensure it contains the full union of itself and all of its # children's footprints. src.getFootprint().include([child.getFootprint() for child in kids]) src.set(self.nChildKey, nchild) self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res) #print 'Deblending parent id', src.getId(), 'took', time.clock() - t0 n1 = len(srcs) self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources' % (n0, nparents, n1-n0, n1))
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None): noiseMeanVar = None self.noiseSource = config.noiseSource self.noiseOffset = config.noiseOffset self.noiseSeedMultiplier = config.noiseSeedMultiplier self.noiseGenMean = None self.noiseGenStd = None self.log = log # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups self.exposure = exposure self.footprints = footprints mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: if maskname in mask.getMaskPlaneDict(): # does it already exist? plane = mask.getMaskPlane(maskname) if self.log: self.log.debug('Mask plane "%s" already existed', maskname) else: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) if self.log: self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x', maskname, plane, bitmask, bitmask) self.thisbitmask, self.otherbitmask = bitmasks del bitmasks self.heavies = {} # Start by creating HeavyFootprints for each source which has no parent # and just use them for children which do not already have heavy footprints. # If a heavy footprint is available for a child, we will use it. Otherwise, # we use the first parent in the parent chain which has a heavy footprint, # which with the one level deblender will alway be the topmost parent # NOTE: heavy footprints get destroyed by the transform process in forcedPhotCcd.py # or forcedPhotCoadd.py so they are never available for forced measurements. # Create in the dict heavies = {id:heavyfootprint} for id, fp in footprints.items(): if fp[1].isHeavy(): self.heavies[id] = fp[1] elif fp[0] == 0: self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi) # ## FIXME: the heavy footprint includes the mask # ## and variance planes, which we shouldn't need # ## (I don't think we ever want to modify them in # ## the input image). Copying them around is # ## wasteful. # We now create a noise HeavyFootprint for each source with has a heavy footprint. # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint} self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId) if self.log: self.log.debug('Using noise generator: %s', str(noisegen)) for id in self.heavies: fp = footprints[id][1] noiseFp = noisegen.getHeavyFootprint(fp) self.heavyNoise[id] = noiseFp # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. noiseFp.insert(im) # Also set the OTHERDET bit fp.spans.setMask(mask, self.otherbitmask)
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None): """! Initialize the NoiseReplacer. @param[in] config instance of NoiseReplacerConfig @param[in,out] exposure Exposure to be noise replaced. (All sources replaced on return) @param[in] footprints dict of {id: (parent, footprint)}; @param[in] noiseImage an afw.image.ImageF used as a predictable noise replacement source (for tests only) @param[in] log pex.logging.Log object to use for status messages; no status messages will be printed if None 'footprints' is a dict of {id: (parent, footprint)}; when used in SFM, the ID will be the source ID, but in forced photometry, this will be the reference ID, as that's what we used to determine the deblend families. This routine should create HeavyFootprints for any non-Heavy Footprints, and replace them in the dict. It should then create a dict of HeavyFootprints containing noise, but only for parent objects, then replace all sources with noise. This should ignore any footprints that lay outside the bounding box of the exposure, and clip those that lie on the border. NOTE: as the code currently stands, the heavy footprint for a deblended object must be available from the input catalog. If it is not, it cannot be reproduced here. In that case, the topmost parent in the objects parent chain must be used. The heavy footprint for that source is created in this class from the masked image. """ noiseMeanVar = None self.noiseSource = config.noiseSource self.noiseOffset = config.noiseOffset self.noiseSeedMultiplier = config.noiseSeedMultiplier self.noiseGenMean = None self.noiseGenStd = None self.log = log # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups self.exposure = exposure self.footprints = footprints mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: try: # does it already exist? plane = mask.getMaskPlane(maskname) if self.log: self.log.logdebug('Mask plane "%s" already existed' % maskname) except: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) if self.log: self.log.logdebug( 'Mask plane "%s": plane %i, bitmask %i = 0x%x' % (maskname, plane, bitmask, bitmask)) self.thisbitmask, self.otherbitmask = bitmasks del bitmasks self.heavies = {} # Start by creating HeavyFootprints for each source which has no parent # and just use them for children which do not already have heavy footprints. # If a heavy footprint is available for a child, we will use it. Otherwise, # we use the first parent in the parent chain which has a heavy footprint, # which with the one level deblender will alway be the topmost parent # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py, # so they are never available for forced measurements. # Create in the dict heavies = {id:heavyfootprint} for id in footprints.keys(): fp = footprints[id] if fp[1].isHeavy(): self.heavies[id] = afwDet.cast_HeavyFootprintF(fp[1]) elif fp[0] == 0: self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi) ### FIXME: the heavy footprint includes the mask ### and variance planes, which we shouldn't need ### (I don't think we ever want to modify them in ### the input image). Copying them around is ### wasteful. # We now create a noise HeavyFootprint for each source with has a heavy footprint. # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint} self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId) # The noiseGenMean and Std are used by the unit tests self.noiseGenMean = noisegen.mean self.noiseGenStd = noisegen.std if self.log: self.log.logdebug('Using noise generator: %s' % (str(noisegen))) for id in self.heavies.keys(): fp = footprints[id][1] noiseFp = noisegen.getHeavyFootprint(fp) self.heavyNoise[id] = noiseFp # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. noiseFp.insert(im) # Also set the OTHERDET bit afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
def test2(self): # Check that doReplaceWithNoise works with deblended source # hierarchies. seed = 42 rand = afwMath.Random(afwMath.Random.MT19937, seed) psf = self.getpsf() im = afwImage.ImageF(200, 50) skystd = 100 afwMath.randomGaussianImage(im, rand) im *= skystd imorig = afwImage.ImageF(im, True) noiseim = imorig mi = afwImage.MaskedImageF(im) mi.getVariance().set(skystd**2) exposure = afwImage.makeExposure(mi) exposure.setPsf(psf) detconf = measAlg.SourceDetectionConfig() detconf.returnOriginalFootprints = True detconf.reEstimateBackground = False measconf = measAlg.SourceMeasurementConfig() measconf.doReplaceWithNoise = True measconf.replaceWithNoise.noiseSeed = 42 schema = afwTable.SourceTable.makeMinimalSchema() detect = measAlg.SourceDetectionTask(config=detconf, schema=schema) measure = MySourceMeasurementTask(config=measconf, schema=schema, doplot=plots) table = afwTable.SourceTable.make(schema) table.preallocate(10) # We're going to fake up a perfect deblend hierarchy here, by # creating individual images containing single sources and # measuring them, and then creating a deblend hierarchy where # the children have the correct HeavyFootprints. We want to # find that the measurements on the deblend hierarchy and the # blended image are equal to the individual images. # # Note that in the normal setup we don't expect the # measurements to be *identical* because of the faint wings of # the objects; when measuring a deblended child, we pick up # the wings of the other objects. # # In order to get exactly equal measurements, we'll fake some # sources that have no wings -- we'll copy just the source # pixels within the footprint. This means that all the # footprints are the same, and the pixels inside the footprint # are the same. fullim = None sources = None # "normal" measurements xx0, yy0, vx0, vy0 = [], [], [], [] # "no-wing" measurements xx1, yy1, vx1, vy1 = [], [], [], [] y = 25 for i in range(5): # no-noise source image sim = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) # Put all four sources in the parent (i==0), and one # source in each child (i=[1 to 4]) if i in [0, 1]: addPsf(sim, psf, 20, y, 1000) if i in [0, 2]: addGaussian(sim, 40, y, 10, 3, 2e5) if i in [0, 3]: addGaussian(sim, 75, y, 10, 3, 2e5) if i in [0, 4]: addPsf(sim, psf, 95, y, 1000) imcopy = afwImage.ImageF(imorig, True) imcopy += sim # copy the pixels into the exposure object im <<= imcopy if i == 0: detected = detect.makeSourceCatalog(table, exposure) sources = detected.sources print 'detected', len(sources), 'sources' self.assertEqual(len(sources), 1) else: fpSets = detect.detectFootprints(exposure) print 'detected', fpSets.numPos, 'sources' fpSets.positive.makeSources(sources) self.assertEqual(fpSets.numPos, 1) print len(sources), 'sources total' measure.plotpat = 'single-%i.png' % i measure.run(exposure, sources[-1:]) s = sources[-1] fp = s.getFootprint() if i == 0: # This is the blended image fullim = imcopy else: print 'Creating heavy footprint...' heavy = afwDet.makeHeavyFootprint(fp, mi) s.setFootprint(heavy) # Record the single-source measurements. xx0.append(s.getX()) yy0.append(s.getY()) vx0.append(s.getIxx()) vy0.append(s.getIyy()) # "no-wings": add just the source pixels within the footprint im <<= sim h = afwDet.makeHeavyFootprint(fp, mi) sim2 = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) h.insert(sim2) imcopy = afwImage.ImageF(imorig, True) imcopy += sim2 im <<= imcopy measure.plotpat = 'single2-%i.png' % i measure.run(exposure, sources[i:i + 1], noiseImage=noiseim) s = sources[i] xx1.append(s.getX()) yy1.append(s.getY()) vx1.append(s.getIxx()) vy1.append(s.getIyy()) if i == 0: fullim2 = imcopy # Now we'll build the fake deblended hierarchy. parent = sources[0] kids = sources[1:] # Ensure that the parent footprint contains all the child footprints pfp = parent.getFootprint() for s in kids: for span in s.getFootprint().getSpans(): pfp.addSpan(span) pfp.normalize() #parent.setFootprint(pfp) # The parent-child relationship is established through the IDs parentid = parent.getId() for s in kids: s.setParent(parentid) # Reset all the measurements shkey = sources.getTable().getShapeKey() ckey = sources.getTable().getCentroidKey() for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Measure the "deblended" normal sources im <<= fullim measure.plotpat = 'joint-%(sourcenum)i.png' measure.run(exposure, sources) xx2, yy2, vx2, vy2 = [], [], [], [] for s in sources: xx2.append(s.getX()) yy2.append(s.getY()) vx2.append(s.getIxx()) vy2.append(s.getIyy()) # Measure the "deblended" no-wings sources im <<= fullim2 measure.plotpat = 'joint2-%(sourcenum)i.png' measure.run(exposure, sources, noiseImage=noiseim) xx3, yy3, vx3, vy3 = [], [], [], [] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) print 'Normal:' print 'xx ', xx0 print ' vs', xx2 print 'yy ', yy0 print ' vs', yy2 print 'vx ', vx0 print ' vs', vx2 print 'vy ', vy0 print ' vs', vy2 print 'No wings:' print 'xx ', xx1 print ' vs', xx3 print 'yy ', yy1 print ' vs', yy3 print 'vx ', vx1 print ' vs', vx3 print 'vy ', vy1 print ' vs', vy3 # These "normal" tests are not very stringent. # 0.1-pixel centroids self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(xx0, xx2)])) self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(yy0, yy2)])) # 10% variances self.assertTrue( all([ abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1 for v1, v2 in zip(vx0, vx2) ])) self.assertTrue( all([ abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1 for v1, v2 in zip(vy0, vy2) ])) # The "no-wings" tests should be exact. self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that the parent/child order is unimportant. im <<= fullim2 measure.doplot = False sources2 = sources.copy() perm = [2, 1, 0, 3, 4] for i, j in enumerate(perm): sources2[i] = sources[j] # I'm not convinced that HeavyFootprints get copied correctly... sources2[i].setFootprint(sources[j].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # "measure.run" reorders the sources! xx3, yy3, vx3, vy3 = [], [], [], [] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that it still works when the parent ID falls in the middle of # the child IDs. im <<= fullim2 measure.doplot = False sources2 = sources.copy() parentid = 3 ids = [parentid, 1, 2, 4, 5] for i, s in enumerate(sources2): s.setId(ids[i]) if i != 0: s.setParent(parentid) s.setFootprint(sources[i].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # The sources get reordered! xx3, yy3, vx3, vy3 = [], [], [], [] xx3, yy3, vx3, vy3 = [0] * 5, [0] * 5, [0] * 5, [0] * 5 for i, j in enumerate(ids): xx3[i] = sources2[j - 1].getX() yy3[i] = sources2[j - 1].getY() vx3[i] = sources2[j - 1].getIxx() vy3[i] = sources2[j - 1].getIyy() self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3)
def __init__(self, config, exposure, footprints, noiseImage=None, exposureId=None, log=None): noiseMeanVar = None self.noiseSource = config.noiseSource self.noiseOffset = config.noiseOffset self.noiseSeedMultiplier = config.noiseSeedMultiplier self.noiseGenMean = None self.noiseGenStd = None self.log = log # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups self.exposure = exposure self.footprints = footprints mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: try: # does it already exist? plane = mask.getMaskPlane(maskname) if self.log: self.log.debug('Mask plane "%s" already existed', maskname) except Exception: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) if self.log: self.log.debug('Mask plane "%s": plane %i, bitmask %i = 0x%x', maskname, plane, bitmask, bitmask) self.thisbitmask, self.otherbitmask = bitmasks del bitmasks self.heavies = {} # Start by creating HeavyFootprints for each source which has no parent # and just use them for children which do not already have heavy footprints. # If a heavy footprint is available for a child, we will use it. Otherwise, # we use the first parent in the parent chain which has a heavy footprint, # which with the one level deblender will alway be the topmost parent # NOTE: heavy footprints get destroyed by the transform process in forcedPhotImage.py, # so they are never available for forced measurements. # Create in the dict heavies = {id:heavyfootprint} for id, fp in footprints.items(): if fp[1].isHeavy(): self.heavies[id] = fp[1] elif fp[0] == 0: self.heavies[id] = afwDet.makeHeavyFootprint(fp[1], mi) # ## FIXME: the heavy footprint includes the mask # ## and variance planes, which we shouldn't need # ## (I don't think we ever want to modify them in # ## the input image). Copying them around is # ## wasteful. # We now create a noise HeavyFootprint for each source with has a heavy footprint. # We'll put the noise footprints in a dict heavyNoise = {id:heavyNoiseFootprint} self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar, exposureId=exposureId) # The noiseGenMean and Std are used by the unit tests self.noiseGenMean = noisegen.mean self.noiseGenStd = noisegen.std if self.log: self.log.debug('Using noise generator: %s', str(noisegen)) for id in self.heavies: fp = footprints[id][1] noiseFp = noisegen.getHeavyFootprint(fp) self.heavyNoise[id] = noiseFp # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. noiseFp.insert(im) # Also set the OTHERDET bit fp.spans.setMask(mask, self.otherbitmask)
def getHeavyFootprint(self, fp): bb = fp.getBBox() mim = self.getMaskedImage(bb) return afwDet.makeHeavyFootprint(fp, mim)
def begin(self, exposure, sources, noiseImage=None, noiseMeanVar=None): # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups # (sources.find() below) if not sources.isSorted(): sources.sort() mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: try: # does it already exist? plane = mask.getMaskPlane(maskname) self.log.logdebug('Mask plane "%s" already existed' % maskname) except: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x' % (maskname, plane, bitmask, bitmask)) self.thisbitmask, self.otherbitmask = bitmasks del bitmasks # Start by creating HeavyFootprints for each source. # # The "getParent()" checks are here because top-level # sources (ie, those with no parents) are not supposed to # have HeavyFootprints, but child sources (ie, those that # have been deblended) should have HeavyFootprints # already. self.heavies = [] for source in sources: fp = source.getFootprint() hfp = afwDet.cast_HeavyFootprintF(fp) if source.getParent() and hfp is not None: # this source has been deblended; "fp" should # already be a HeavyFootprint (but may not be # if read from disk). # Swig downcasts it to Footprint, so we have to re-cast. self.heavies.append(hfp) else: # top-level source: copy pixels from the input # image. ### FIXME: the heavy footprint includes the mask ### and variance planes, which we shouldn't need ### (I don't think we ever want to modify them in ### the input image). Copying them around is ### wasteful. heavy = afwDet.makeHeavyFootprint(fp, mi) self.heavies.append(heavy) # We now create a noise HeavyFootprint for each top-level Source. # We'll put the noisy footprints in a map from id -> HeavyFootprint: self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar) self.log.logdebug('Using noise generator: %s' % (str(noisegen))) for source in sources: if source.getParent(): continue fp = source.getFootprint() heavy = noisegen.getHeavyFootprint(fp) self.heavyNoise[source.getId()] = heavy # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. heavy.insert(im) # Also set the OTHERDET bit afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)
def testMergeHeavyFootprints(self): mi = afwImage.MaskedImageF(20, 10) objectPixelVal = (42, 0x9, 400) spanList = [] for y, x0, x1 in [(1, 9, 12), (2, 12, 13), (3, 11, 15)]: spanList.append(afwGeom.Span(y, x0, x1)) for x in range(x0, x1 + 1): mi[x, y, afwImage.LOCAL] = objectPixelVal foot = afwDetect.Footprint(afwGeom.SpanSet(spanList)) hfoot1 = afwDetect.makeHeavyFootprint(self.foot, self.mi) hfoot2 = afwDetect.makeHeavyFootprint(foot, mi) hsum = afwDetect.mergeHeavyFootprints(hfoot1, hfoot2) bb = hsum.getBBox() self.assertEqual(bb.getMinX(), 9) self.assertEqual(bb.getMaxX(), 15) self.assertEqual(bb.getMinY(), 1) self.assertEqual(bb.getMaxY(), 3) msum = afwImage.MaskedImageF(20, 10) hsum.insert(msum) sa = msum.getImage().getArray() self.assertFloatsEqual(sa[1, 9:13], objectPixelVal[0]) self.assertFloatsEqual(sa[2, 12:14], objectPixelVal[0] + self.objectPixelVal[0]) self.assertFloatsEqual(sa[2, 10:12], self.objectPixelVal[0]) sv = msum.getVariance().getArray() self.assertFloatsEqual(sv[1, 9:13], objectPixelVal[2]) self.assertFloatsEqual(sv[2, 12:14], objectPixelVal[2] + self.objectPixelVal[2]) self.assertFloatsEqual(sv[2, 10:12], self.objectPixelVal[2]) sm = msum.getMask().getArray() self.assertFloatsEqual(sm[1, 9:13], objectPixelVal[1]) self.assertFloatsEqual(sm[2, 12:14], objectPixelVal[1] | self.objectPixelVal[1]) self.assertFloatsEqual(sm[2, 10:12], self.objectPixelVal[1]) if False: import matplotlib matplotlib.use('Agg') import pylab as plt im1 = afwImage.ImageF(bb) hfoot1.insert(im1) im2 = afwImage.ImageF(bb) hfoot2.insert(im2) im3 = afwImage.ImageF(bb) hsum.insert(im3) plt.clf() plt.subplot(1, 3, 1) plt.imshow(im1.getArray(), interpolation='nearest', origin='lower') plt.subplot(1, 3, 2) plt.imshow(im2.getArray(), interpolation='nearest', origin='lower') plt.subplot(1, 3, 3) plt.imshow(im3.getArray(), interpolation='nearest', origin='lower') plt.savefig('merge.png')
def makeplots(butler, dataId, ps, sources=None, pids=None, minsize=0, maxpeaks=10): calexp = butler.get("calexp", **dataId) if sources is None: ss = butler.get('src', **dataId) else: ss = sources # print('Sources', ss) # print('Calexp', calexp) # print(dir(ss)) srcs = {} families = {} for src in ss: sid = src.getId() srcs[sid] = src parent = src.getParent() if parent == 0: continue if parent not in families: families[parent] = [] families[parent].append(src) # print 'Source', src # print ' ', dir(src) # print ' parent', src.getParent() # print ' footprint', src.getFootprint() print() lsstimg = calexp.getMaskedImage().getImage() img = lsstimg.getArray() schema = ss.getSchema() psfkey = schema.find("deblend_deblendedAsPsf").key nchildkey = schema.find("deblend_nChild").key toomanykey = schema.find("deblend_tooManyPeaks").key failedkey = schema.find("deblend_failed").key def getFlagString(src): ss = ['Nchild: %i' % src.get(nchildkey)] for key, s in [(psfkey, 'PSF'), (toomanykey, 'TooMany'), (failedkey, 'Failed')]: if src.get(key): ss.append(s) return ', '.join(ss) plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.9, hspace=0.2, wspace=0.3) sig1 = np.sqrt( np.median(calexp.getMaskedImage().getVariance().getArray().ravel())) pp = (img / np.sqrt(calexp.getMaskedImage().getVariance().getArray())).ravel() plt.clf() lo, hi = -4, 4 n, b, p = plt.hist(img.ravel() / sig1, 100, range=(lo, hi), histtype='step', color='b') plt.hist(pp, 100, range=(lo, hi), histtype='step', color='g') xx = np.linspace(lo, hi, 200) yy = 1. / (np.sqrt(2. * np.pi)) * np.exp(-0.5 * xx**2) yy *= sum(n) * (b[1] - b[0]) plt.plot(xx, yy, 'k-', alpha=0.5) plt.xlim(lo, hi) plt.title('image-wide sig1: %.1f' % sig1) ps.savefig() for ifam, (p, kids) in enumerate(families.items()): parent = srcs[p] pid = parent.getId() & 0xffff if len(pids) and pid not in pids: # print('Skipping pid', pid) continue if len(kids) < minsize: print('Skipping parent', pid, ': n kids', len(kids)) continue # if len(kids) < 5: # print 'Skipping family with', len(kids) # continue # print 'ifam', ifam # if ifam != 18: # print 'skipping' # continue print('Parent', parent) print('Kids', kids) print('Parent', parent.getId()) print('Kids', [k.getId() for k in kids]) pfoot = parent.getFootprint() bb = pfoot.getBBox() y0, y1, x0, x1 = bb.getMinY(), bb.getMaxY(), bb.getMinX(), bb.getMaxX() slc = slice(y0, y1 + 1), slice(x0, x1 + 1) ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=-10, vmax=40) mn, mx = ima['vmin'], ima['vmax'] if False: plt.clf() plt.imshow(img[slc], extent=bb_to_ext(bb), **ima) plt.title('Parent %i, %s' % (parent.getId(), getFlagString(parent))) ax = plt.axis() x, y = bb_to_xy(bb) plt.plot(x, y, 'r-', lw=2) for i, kid in enumerate(kids): kfoot = kid.getFootprint() kbb = kfoot.getBBox() kx, ky = bb_to_xy(kbb, margin=0.4) plt.plot(kx, ky, 'm-') for pk in pfoot.getPeaks(): plt.plot(pk.getIx(), pk.getIy(), 'r+', ms=10, mew=3) plt.axis(ax) ps.savefig() print('parent footprint:', pfoot) print('heavy?', pfoot.isHeavy()) plt.clf() pimg, h = foot_to_img(pfoot, lsstimg) plt.imshow(img_to_rgb(pimg.getArray(), mn, mx), extent=bb_to_ext(bb), **ima) tt = 'Parent %i' % parent.getId() if not h: tt += ', no HFoot' tt += ', ' + getFlagString(parent) plt.title(tt) ax = plt.axis() plt.plot([x0, x0, x1, x1, x0], [y0, y1, y1, y0, y0], 'r-', lw=2) for i, kid in enumerate(kids): kfoot = kid.getFootprint() kbb = kfoot.getBBox() kx, ky = bb_to_xy(kbb, margin=-0.1) plt.plot(kx, ky, 'm-', lw=1.5) for pk in pfoot.getPeaks(): plt.plot(pk.getIx(), pk.getIy(), 'r+', ms=10, mew=3) plt.axis(ax) ps.savefig() cols = int(np.ceil(np.sqrt(len(kids)))) rows = int(np.ceil(len(kids) / float(cols))) if False: plt.clf() for i, kid in enumerate(kids): plt.subplot(rows, cols, 1 + i) kfoot = kid.getFootprint() print('kfoot:', kfoot) print('heavy?', kfoot.isHeavy()) # print(dir(kid)) kbb = kfoot.getBBox() ky0, ky1, kx0, kx1 = kbb.getMinY(), kbb.getMaxY(), kbb.getMinX( ), kbb.getMaxX() kslc = slice(ky0, ky1 + 1), slice(kx0, kx1 + 1) plt.imshow(img[kslc], extent=bb_to_ext(kbb), **ima) plt.title('Child %i' % kid.getId()) plt.axis(ax) ps.savefig() plt.clf() for i, kid in enumerate(kids): plt.subplot(rows, cols, 1 + i) kfoot = kid.getFootprint() kbb = kfoot.getBBox() kimg, h = foot_to_img(kfoot, lsstimg) tt = getFlagString(kid) if not h: tt += ', no HFoot' plt.title('%s' % tt) if kimg is None: plt.axis(ax) continue plt.imshow(img_to_rgb(kimg.getArray(), mn, mx), extent=bb_to_ext(kbb), **ima) for pk in kfoot.getPeaks(): plt.plot(pk.getIx(), pk.getIy(), 'g+', ms=10, mew=3) plt.axis(ax) plt.suptitle('Child HeavyFootprints') ps.savefig() print() print('Re-running deblender...') psf = calexp.getPsf() psf_fwhm = psf.computeShape().getDeterminantRadius() * 2.35 deb = deblend( pfoot, calexp.getMaskedImage(), psf, psf_fwhm, verbose=True, maxNumberOfPeaks=maxpeaks, rampFluxAtEdge=True, clipStrayFluxFraction=0.01, ) print('Got', deb) def getDebFlagString(kid): ss = [] for k in [ 'skip', 'outOfBounds', 'tinyFootprint', 'noValidPixels', ('deblendedAsPsf', 'PSF'), 'psfFitFailed', 'psfFitBadDof', 'psfFitBigDecenter', 'psfFitWithDecenter', 'failedSymmetricTemplate', 'hasRampedTemplate', 'patched' ]: if len(k) == 2: k, s = k else: s = k if getattr(kid, k): ss.append(s) return ', '.join(ss) N = len(deb.peaks) cols = int(np.ceil(np.sqrt(N))) rows = int(np.ceil(N / float(cols))) for plotnum in range(4): plt.clf() for i, kid in enumerate(deb.peaks): # print 'child', kid # print ' flags:', getDebFlagString(kid) kfoot = None if plotnum == 0: kfoot = kid.getFluxPortion(strayFlux=False) supt = 'flux portion' elif plotnum == 1: kfoot = kid.getFluxPortion(strayFlux=True) supt = 'flux portion + stray' elif plotnum == 2: kfoot = afwDet.makeHeavyFootprint(kid.templateFootprint, kid.templateImage) supt = 'template' elif plotnum == 3: if kid.deblendedAsPsf: kfoot = afwDet.makeHeavyFootprint( kid.psfFootprint, kid.psfTemplate) kfoot.normalize() kfoot.clipToNonzero(kid.psfTemplate.getImage()) # print 'kfoot BB:', kfoot.getBBox() # print 'Img bb:', kid.psfTemplate.getImage().getBBox() # for sp in kfoot.getSpans(): # print ' span', sp else: kfoot = afwDet.makeHeavyFootprint( kid.templateFootprint, kid.templateImage) supt = 'psf template' kimg, h = foot_to_img(kfoot, None) tt = 'kid %i: %s' % (i, getDebFlagString(kid)) if not h: tt += ', no HFoot' plt.subplot(rows, cols, 1 + i) plt.title('%s' % tt, fontsize=8) if kimg is None: plt.axis(ax) continue kbb = kfoot.getBBox() plt.imshow(img_to_rgb(kimg.getArray(), mn, mx), extent=bb_to_ext(kbb), **ima) # plt.imshow(kimg.getArray(), extent=bb_to_ext(kbb), **ima) plt.axis(ax) plt.suptitle(supt) ps.savefig() for i, kid in enumerate(deb.peaks): if not kid.deblendedAsPsf: continue plt.clf() ima = dict(interpolation='nearest', origin='lower', cmap='gray') # vmin=0, vmax=kid.psfFitFlux) plt.subplot(2, 4, 1) # plt.title('fit psf 0') # plt.imshow(kid.psfFitDebugPsf0Img.getArray(), **ima) # plt.colorbar() # plt.title('valid pixels') # plt.imshow(kid.psfFitDebugValidPix, vmin=0, vmax=1, **ima) plt.title('weights') plt.imshow(kid.psfFitDebugWeight, vmin=0, **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.subplot(2, 4, 7) plt.title('valid pixels') plt.imshow(kid.psfFitDebugValidPix, vmin=0, vmax=1, **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.subplot(2, 4, 2) # plt.title('ramp weights') # plt.imshow(kid.psfFitDebugRampWeight, vmin=0, vmax=1, **ima) # plt.colorbar() sig = np.sqrt(kid.psfFitDebugVar.getArray()) data = kid.psfFitDebugStamp.getArray() model = kid.psfFitDebugPsfModel.getArray() chi = ((data - model) / sig) valid = kid.psfFitDebugValidPix plt.hist(np.clip((data / sig)[valid], -5, 5), 20, range=(-5, 5), histtype='step', color='m') plt.hist(np.clip((model / sig)[valid], -5, 5), 20, range=(-5, 5), histtype='step', color='r') plt.hist(np.clip(chi.ravel(), -5, 5), 20, range=(-5, 5), histtype='step', color='g') n, b, p = plt.hist(np.clip(chi[valid], -5, 5), 20, range=(-5, 5), histtype='step', color='b') xx = np.linspace(-5, 5, 200) yy = 1. / (np.sqrt(2. * np.pi)) * np.exp(-0.5 * xx**2) yy *= sum(n) * (b[1] - b[0]) plt.plot(xx, yy, 'k-', alpha=0.5) plt.xlim(-5, 5) print('Sum of ramp weights:', np.sum(kid.psfFitDebugRampWeight)) print('Quadrature sum of ramp weights:', np.sqrt(np.sum(kid.psfFitDebugRampWeight**2))) print('Number of valid pix:', np.sum(kid.psfFitDebugValidPix)) rw = kid.psfFitDebugRampWeight valid = kid.psfFitDebugValidPix # print 'valid values:', np.unique(valid) print('rw[valid]', np.sum(rw[valid])) print('rw range', rw.min(), rw.max()) # print 'rw', rw.shape, rw.dtype # print 'valid', valid.shape, valid.dtype # print 'rw[valid]:', rw[valid] myresid = np.sum(kid.psfFitDebugValidPix * kid.psfFitDebugRampWeight * ((kid.psfFitDebugStamp.getArray() - kid.psfFitDebugPsfModel.getArray()) / np.sqrt(kid.psfFitDebugVar.getArray()))**2) print('myresid:', myresid) plt.subplot(2, 4, 8) N = 20000 rwv = rw[valid] print('rwv', rwv) x = np.random.normal(size=(N, len(rwv))) ss = np.sum(rwv * x**2, axis=1) plt.hist(ss, 25) chi, dof = kid.psfFitBest plt.axvline(chi, color='r') mx = kid.psfFitDebugPsfModel.getArray().max() plt.subplot(2, 4, 3) # plt.title('fit psf') # plt.imshow(kid.psfFitDebugPsfImg.getArray(), **ima) # plt.colorbar() # plt.title('variance') # plt.imshow(kid.psfFitDebugVar.getArray(), vmin=0, **ima) # plt.colorbar() plt.title('model+noise') plt.imshow((kid.psfFitDebugPsfModel.getArray() + sig * np.random.normal(size=sig.shape)) * valid, vmin=0, vmax=mx, **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.subplot(2, 4, 4) plt.title('fit psf model') plt.imshow(kid.psfFitDebugPsfModel.getArray(), vmin=0, vmax=mx, **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.subplot(2, 4, 5) plt.title('fit psf image') plt.imshow(kid.psfFitDebugStamp.getArray(), vmin=0, vmax=mx, **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() chi = (kid.psfFitDebugValidPix * (kid.psfFitDebugStamp.getArray() - kid.psfFitDebugPsfModel.getArray()) / np.sqrt(kid.psfFitDebugVar.getArray())) plt.subplot(2, 4, 6) plt.title('fit psf chi') plt.imshow(-chi, vmin=-3, vmax=3, interpolation='nearest', origin='lower', cmap='RdBu') plt.xticks([]) plt.yticks([]) plt.colorbar() params = kid.psfFitParams (flux, sky, skyx, skyy) = params[:4] print('Model sum:', model.sum()) print('- sky', model.sum() - np.sum(valid) * sky) sig1 = np.median(sig) chi, dof = kid.psfFitBest plt.suptitle( 'PSF kid %i: flux %.1f, sky %.1f, sig1 %.1f' % (i, flux, sky, sig1)) # : chisq %g, dof %i' % (i, chi, dof)) ps.savefig()
def begin(self, exposure, sources, noiseImage=None, noiseMeanVar=None): # creates heavies, replaces all footprints with noise # We need the source table to be sorted by ID to do the parent lookups # (sources.find() below) if not sources.isSorted(): sources.sort() mi = exposure.getMaskedImage() im = mi.getImage() mask = mi.getMask() # Add temporary Mask planes for THISDET and OTHERDET self.removeplanes = [] bitmasks = [] for maskname in ['THISDET', 'OTHERDET']: try: # does it already exist? plane = mask.getMaskPlane(maskname) self.log.logdebug('Mask plane "%s" already existed' % maskname) except: # if not, add it; we should delete it when done. plane = mask.addMaskPlane(maskname) self.removeplanes.append(maskname) mask.clearMaskPlane(plane) bitmask = mask.getPlaneBitMask(maskname) bitmasks.append(bitmask) self.log.logdebug('Mask plane "%s": plane %i, bitmask %i = 0x%x' % (maskname, plane, bitmask, bitmask)) self.thisbitmask,self.otherbitmask = bitmasks del bitmasks # Start by creating HeavyFootprints for each source. # # The "getParent()" checks are here because top-level # sources (ie, those with no parents) are not supposed to # have HeavyFootprints, but child sources (ie, those that # have been deblended) should have HeavyFootprints # already. self.heavies = [] for source in sources: fp = source.getFootprint() hfp = afwDet.cast_HeavyFootprintF(fp) if source.getParent() and hfp is not None: # this source has been deblended; "fp" should # already be a HeavyFootprint (but may not be # if read from disk). # Swig downcasts it to Footprint, so we have to re-cast. self.heavies.append(hfp) else: # top-level source: copy pixels from the input # image. ### FIXME: the heavy footprint includes the mask ### and variance planes, which we shouldn't need ### (I don't think we ever want to modify them in ### the input image). Copying them around is ### wasteful. heavy = afwDet.makeHeavyFootprint(fp, mi) self.heavies.append(heavy) # We now create a noise HeavyFootprint for each top-level Source. # We'll put the noisy footprints in a map from id -> HeavyFootprint: self.heavyNoise = {} noisegen = self.getNoiseGenerator(exposure, noiseImage, noiseMeanVar) self.log.logdebug('Using noise generator: %s' % (str(noisegen))) for source in sources: if source.getParent(): continue fp = source.getFootprint() heavy = noisegen.getHeavyFootprint(fp) self.heavyNoise[source.getId()] = heavy # Also insert the noisy footprint into the image now. # Notice that we're just inserting it into "im", ie, # the Image, not the MaskedImage. heavy.insert(im) # Also set the OTHERDET bit afwDet.setMaskFromFootprint(mask, fp, self.otherbitmask)