def test_pixel_stdev(self): """Test that sources are detected on a simulated image with thresholdType='pixel_stdev', and that they have the right significance. """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() config.thresholdType = "pixel_stdev" config.reEstimateBackground = False # TempLocalBackground changes the peak value of the faintest peak, # so disable it for this test so that we can calculate an expected # answer without having to try to deal with backgrounds. config.doTempLocalBackground = False task = SourceDetectionTask(config=config, schema=schema) # Don't smooth, so that we can directly calculate the s/n from the exposure. result = task.detectFootprints(exposure, doSmooth=False) self.assertEqual(result.numPos, numX * numY) self.assertEqual(result.numNeg, 0) # Significance values for `pixel_stdev` should match image/sqrt(variance). for footprint in result.positive.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] stddev = np.sqrt(exposure.variance[point]) with self.subTest(str(point)): self.assertFloatsAlmostEqual(peak["significance"], value / stddev, rtol=1e-7, msg=str(point))
def test_stdev(self): """Test that sources are detected on a simulated image with thresholdType='stdev'. """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() # don't modify the image after detection. config.reEstimateBackground = False config.thresholdType = "stdev" task = SourceDetectionTask(config=config, schema=schema) self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=True) self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=False)
def test_significance(self): """Test that negative peaks have the right significance for thresholdType='stdev' for the non-convolved, non-local-background case. """ exposure, numX, numY = self._create_exposure() schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.thresholdPolarity = 'both' # don't modify the image after detection. config.reEstimateBackground = False config.doTempLocalBackground = False detection = SourceDetectionTask(config=config, schema=schema) result = detection.detectFootprints(exposure, doSmooth=False) bad = exposure.mask.getPlaneBitMask(config.statsMask) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP, sctrl) stddev = stats.getValue(afwMath.STDEVCLIP) # Don't bother checking positive footprints: those are tested more # thoroughly in test_detection.py. for footprint in result.negative.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] with self.subTest(str(point)): self.assertFloatsAlmostEqual( peak["significance"], -value / stddev, # S/N for negative peak rtol=1e-7, msg=str(point))
def test_detection_stdev(self): """Test detection and measurement on an exposure with negative sources for thresholdType="stdev". """ exposure, numX, numY = self._create_exposure() if display: disp = afwDisplay.Display(frame=1) disp.mtv(exposure, title=self._testMethodName + ": image with -ve sources") schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdPolarity = 'both' detection = SourceDetectionTask(config=config, schema=schema) algMetadata = dafBase.PropertyList() measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata) table = afwTable.SourceTable.make(schema) detections = detection.run(table, exposure) sources = detections.sources fpSets = detections.fpSets self.assertEqual(len(sources), numX * numY) self.assertEqual(fpSets.numPos, numX * numY / 2) self.assertEqual(fpSets.numNeg, numX * numY / 2) measurement.run(sources, exposure) nGoodCent = 0 nGoodShape = 0 for s in sources: cent = s.getCentroid() shape = s.getShape() if cent[0] == cent[0] and cent[1] == cent[1]: nGoodCent += 1 if (shape.getIxx() == shape.getIxx() and shape.getIyy() == shape.getIyy() and shape.getIxy() == shape.getIxy()): nGoodShape += 1 if display: xy = cent[0], cent[1] disp.dot('+', *xy) disp.dot(shape, *xy, ctype=afwDisplay.RED) self.assertEqual(nGoodCent, numX * numY) self.assertEqual(nGoodShape, numX * numY)
def testBasics(self): bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100), afwGeom.Extent2I(128, 127)) minCounts = 5000 maxCounts = 50000 starSigma = 1.5 numX = 5 numY = 5 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False task = SourceDetectionTask(config=config, schema=schema) for doSmooth in (False, True): taskSigma = 2.2 res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=taskSigma) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0) self.assertEqual(task.metadata.get("sigma"), taskSigma) self.assertEqual(task.metadata.get("doSmooth"), doSmooth) self.assertEqual(task.metadata.get("nGrow"), int(taskSigma * config.nSigmaToGrow + 0.5)) res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=None) taskSigma = task.metadata.get("sigma") self.assertTrue(abs(taskSigma - starSigma) < 0.1) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0)
def checkExposure(original, doTempLocalBackground, doTempWideBackground): config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdType = "pixel_stdev" config.doTempLocalBackground = doTempLocalBackground config.doTempWideBackground = doTempWideBackground schema = afwTable.SourceTable.makeMinimalSchema() task = SourceDetectionTask(config=config, schema=schema) exposure = original.clone() task.detectFootprints(exposure, sigma=3.21) self.assertFloatsEqual(exposure.image.array, original.image.array) # Mask is permitted to vary: DETECTED bit gets set self.assertFloatsEqual(exposure.variance.array, original.variance.array)
def test_deblend_task(self): # Set the random seed so that the noise field is unaffected np.random.seed(0) # Test that executing the deblend task works # In the future we can have more detailed tests, # but for now this at least ensures that the task isn't broken shape = (5, 31, 55) coords = [(15, 25), (10, 30), (17, 38)] amplitudes = [80, 60, 90] result = initData(shape, coords, amplitudes) targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result B, Ny, Nx = shape # Add some noise, otherwise the task will blow up due to # zero variance noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5) images += noise filters = "grizy" _images = afwImage.MultibandMaskedImage.fromArrays( filters, images.astype(np.float32), None, noise) coadds = [ afwImage.Exposure(img, dtype=img.image.array.dtype) for img in _images ] coadds = afwImage.MultibandExposure.fromExposures(filters, coadds) for b, coadd in enumerate(coadds): coadd.setPsf(psfs[b]) schema = SourceCatalog.Table.makeMinimalSchema() detectionTask = SourceDetectionTask(schema=schema) config = ScarletDeblendTask.ConfigClass() config.maxIter = 300 deblendTask = ScarletDeblendTask(schema=schema, config=config) table = SourceCatalog.Table.make(schema) detectionResult = detectionTask.run(table, coadds["r"]) catalog = detectionResult.sources self.assertEqual(len(catalog), 1) _, result = deblendTask.run(coadds, catalog)
def test_significance_stdev(self): """Check the non-smoothed, non-background updated peak significance values with thresholdType="stddev". """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() # don't modify the image after detection. config.reEstimateBackground = False config.doTempLocalBackground = False config.thresholdType = "stdev" task = SourceDetectionTask(config=config, schema=schema) result = self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=False) bad = exposure.mask.getPlaneBitMask(config.statsMask) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP, sctrl) stddev = stats.getValue(afwMath.STDEVCLIP) for footprint in result.positive.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] with self.subTest(str(point)): self.assertFloatsAlmostEqual(peak["significance"], value / stddev, rtol=1e-7, msg=str(point))
def test_deblend_task(self): # Set the random seed so that the noise field is unaffected np.random.seed(0) shape = (5, 100, 115) coords = [ # blend (15, 25), (10, 30), (17, 38), # isolated source (85, 90), ] amplitudes = [ # blend 80, 60, 90, # isolated source 20, ] result = initData(shape, coords, amplitudes) targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result B, Ny, Nx = shape # Add some noise, otherwise the task will blow up due to # zero variance noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5) images += noise filters = "grizy" _images = afwImage.MultibandMaskedImage.fromArrays( filters, images.astype(np.float32), None, noise) coadds = [ afwImage.Exposure(img, dtype=img.image.array.dtype) for img in _images ] coadds = afwImage.MultibandExposure.fromExposures(filters, coadds) for b, coadd in enumerate(coadds): coadd.setPsf(psfs[b]) schema = SourceCatalog.Table.makeMinimalSchema() detectionTask = SourceDetectionTask(schema=schema) # Adjust config options to test skipping parents config = ScarletDeblendTask.ConfigClass() config.maxIter = 100 config.maxFootprintArea = 1000 config.maxNumberOfPeaks = 4 deblendTask = ScarletDeblendTask(schema=schema, config=config) table = SourceCatalog.Table.make(schema) detectionResult = detectionTask.run(table, coadds["r"]) catalog = detectionResult.sources # Add a footprint that is too large src = catalog.addNew() halfLength = int(np.ceil(np.sqrt(config.maxFootprintArea) + 1)) ss = SpanSet.fromShape(halfLength, Stencil.BOX, offset=(50, 50)) bigfoot = Footprint(ss) bigfoot.addPeak(50, 50, 100) src.setFootprint(bigfoot) # Add a footprint with too many peaks src = catalog.addNew() ss = SpanSet.fromShape(10, Stencil.BOX, offset=(75, 20)) denseFoot = Footprint(ss) for n in range(config.maxNumberOfPeaks + 1): denseFoot.addPeak(70 + 2 * n, 15 + 2 * n, 10 * n) src.setFootprint(denseFoot) # Run the deblender result = deblendTask.run(coadds, catalog) # Make sure that the catalogs have the same sources in all bands, # and check that band-independent columns are equal bandIndependentColumns = [ "id", "parent", "deblend_nPeaks", "deblend_nChild", "deblend_peak_center_x", "deblend_peak_center_y", "deblend_runtime", "deblend_iterations", "deblend_logL", "deblend_spectrumInitFlag", "deblend_blendConvergenceFailedFlag", ] self.assertEqual(len(filters), len(result)) ref = result[filters[0]] for f in filters[1:]: for col in bandIndependentColumns: np.testing.assert_array_equal(result[f][col], ref[col]) # Check that other columns are consistent for f, _catalog in result.items(): parents = _catalog[_catalog["parent"] == 0] # Check that the number of deblended children is consistent self.assertEqual(np.sum(_catalog["deblend_nChild"]), len(_catalog) - len(parents)) for parent in parents: children = _catalog[_catalog["parent"] == parent.get("id")] # Check that nChild is set correctly self.assertEqual(len(children), parent.get("deblend_nChild")) # Check that parent columns are propagated to their children for parentCol, childCol in config.columnInheritance.items(): np.testing.assert_array_equal(parent.get(parentCol), children[childCol]) children = _catalog[_catalog["parent"] != 0] for child in children: fp = child.getFootprint() img = heavyFootprintToImage(fp) # Check that the flux at the center is correct. # Note: this only works in this test image because the # detected peak is in the same location as the scarlet peak. # If the peak is shifted, the flux value will be correct # but deblend_peak_center is not the correct location. px = child.get("deblend_peak_center_x") py = child.get("deblend_peak_center_y") flux = img.image[Point2I(px, py)] self.assertEqual(flux, child.get("deblend_peak_instFlux")) # Check that the peak positions match the catalog entry peaks = fp.getPeaks() self.assertEqual(px, peaks[0].getIx()) self.assertEqual(py, peaks[0].getIy()) # Check that all sources have the correct number of peaks for src in _catalog: fp = src.getFootprint() self.assertEqual(len(fp.peaks), src.get("deblend_nPeaks")) # Check that only the large foorprint was flagged as too big largeFootprint = np.zeros(len(_catalog), dtype=bool) largeFootprint[2] = True np.testing.assert_array_equal(largeFootprint, _catalog["deblend_parentTooBig"]) # Check that only the dense foorprint was flagged as too dense denseFootprint = np.zeros(len(_catalog), dtype=bool) denseFootprint[3] = True np.testing.assert_array_equal(denseFootprint, _catalog["deblend_tooManyPeaks"]) # Check that only the appropriate parents were skipped skipped = largeFootprint | denseFootprint np.testing.assert_array_equal(skipped, _catalog["deblend_skipped"])
def testBasics(self): bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100), afwGeom.Extent2I(128, 127)) minCounts = 2000 maxCounts = 20000 starSigma = 1.5 numX = 4 numY = 4 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) if display: ds9.mtv(exposure) schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdPolarity = 'both' detection = SourceDetectionTask(config=config, schema=schema) algMetadata = dafBase.PropertyList() measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata) table = afwTable.SourceTable.make(schema) detections = detection.makeSourceCatalog(table, exposure) sources = detections.sources fpSets = detections.fpSets self.assertEqual(len(sources), numX * numY) self.assertEqual(fpSets.numPos, numX * numY / 2) self.assertEqual(fpSets.numNeg, numX * numY / 2) measurement.run(sources, exposure) nGoodCent = 0 nGoodShape = 0 for s in sources: cent = s.getCentroid() shape = s.getShape() if cent[0] == cent[0] and cent[1] == cent[1]: nGoodCent += 1 if (shape.getIxx() == shape.getIxx() and shape.getIyy() == shape.getIyy() and shape.getIxy() == shape.getIxy()): nGoodShape += 1 if display: xy = cent[0] - exposure.getX0(), cent[1] - exposure.getY0() ds9.dot('+', *xy) ds9.dot(shape, *xy, ctype=ds9.RED) self.assertEqual(nGoodCent, numX * numY) self.assertEqual(nGoodShape, numX * numY)
def testIsScarletPrimaryFlag(self): """Test detect_isPrimary column when scarlet is used as the deblender """ # We need a multiband coadd for scarlet, # even though there is only one band coadds = afwImage.MultibandExposure.fromExposures(["test"], [self.exposure]) # Create a SkyMap with a tract that contains a portion of the image, # subdivided into 3x3 patches wcs = self.exposure.getWcs() tractBBox = Box2I(Point2I(100, 100), Extent2I(900, 900)) skyMap = MockSkyMap([tractBBox], wcs, 3) tractInfo = skyMap[0] patchInfo = tractInfo[0, 0] patchBBox = patchInfo.getInnerBBox() schema = SourceCatalog.Table.makeMinimalSchema() # Initialize the detection task detectionTask = SourceDetectionTask(schema=schema) # Initialize the fake source injection task skyConfig = SkyObjectsTask.ConfigClass() skySourcesTask = SkyObjectsTask(name="skySources", config=skyConfig) schema.addField("merge_peak_sky", type="Flag") # Initialize the deblender task scarletConfig = ScarletDeblendTask.ConfigClass() scarletConfig.maxIter = 20 scarletConfig.columnInheritance["merge_peak_sky"] = "merge_peak_sky" deblendTask = ScarletDeblendTask(schema=schema, config=scarletConfig) # We'll customize the configuration of measurement to just run the # minimal number of plugins to make setPrimaryFlags work. measureConfig = SingleFrameMeasurementTask.ConfigClass() measureConfig.plugins.names = ["base_SdssCentroid", "base_SkyCoord"] measureConfig.slots.psfFlux = None measureConfig.slots.apFlux = None measureConfig.slots.shape = None measureConfig.slots.modelFlux = None measureConfig.slots.calibFlux = None measureConfig.slots.gaussianFlux = None measureTask = SingleFrameMeasurementTask(config=measureConfig, schema=schema) primaryConfig = SetPrimaryFlagsTask.ConfigClass() setPrimaryTask = SetPrimaryFlagsTask(config=primaryConfig, schema=schema, name="setPrimaryFlags", isSingleFrame=False) table = SourceCatalog.Table.make(schema) # detect sources detectionResult = detectionTask.run(table, coadds["test"]) catalog = detectionResult.sources # add fake sources skySources = skySourcesTask.run(mask=self.exposure.mask, seed=0) for foot in skySources[:5]: src = catalog.addNew() src.setFootprint(foot) src.set("merge_peak_sky", True) # deblend result = deblendTask.run(coadds, catalog) # measure measureTask.run(result["test"], self.exposure) outputCat = result["test"] # Set the primary flags setPrimaryTask.run(outputCat, skyMap=skyMap, tractInfo=tractInfo, patchInfo=patchInfo) # There should be the same number of deblenedPrimary and # deblendedModelPrimary sources, # since they both have the same blended sources and only differ # over which model to use for the isolated sources. isPseudo = getPseudoSources(outputCat, primaryConfig.pseudoFilterList, schema, setPrimaryTask.log) self.assertEqual( np.sum(outputCat["detect_isDeblendedSource"] & ~isPseudo), np.sum(outputCat["detect_isDeblendedModelSource"])) # Check that the sources contained in a tract are all marked appropriately x = outputCat["slot_Centroid_x"] y = outputCat["slot_Centroid_y"] tractInner = tractBBox.contains(x, y) np.testing.assert_array_equal(outputCat["detect_isTractInner"], tractInner) # Check that the sources contained in a patch are all marked appropriately patchInner = patchBBox.contains(x, y) np.testing.assert_array_equal(outputCat["detect_isPatchInner"], patchInner) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["merge_peak_sky"])), 0) # Check that sky objects have not been deblended np.testing.assert_array_equal( isPseudo, isPseudo & (outputCat["deblend_nChild"] == 0))
algConfig.seeing = values algConfig.aperture.radii = [3, 6, 12, 24] algConfig.aperture.maxSincRadius = 0 measure_config.load( '/tigress/rea3/lsst/DM-8059/obs_subaru/config/apertures.py') deblend_config = SourceDeblendTask.ConfigClass() detect_config = SourceDetectionTask.ConfigClass() detect_config.isotropicGrow = True detect_config.doTempLocalBackground = False detect_config.thresholdValue = 5 detect_config.nSigmaToGrow = args.grow schema = afwTable.SourceTable.makeMinimalSchema() detectTask = SourceDetectionTask(schema, config=detect_config) deblendTask = SourceDeblendTask(schema, config=deblend_config) measureTask = SingleFrameMeasurementTask(schema, config=measure_config) peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) truthKey = schema.addField('truth_index', type=int, doc='Index to truth catalog') table = afwTable.SourceTable.make(schema) lsst_images = {} for filter_ in filter_names: lsst_images[filter_] = afwImage.ImageF('%s/image_%s.fits' %
def detect_and_deblend(*, exp, log): log = lsst.log.Log.getLogger("LSSTMEDSifier") thresh = 5.0 loglevel = 'INFO' # This schema holds all the measurements that will be run within the # stack It needs to be constructed before running anything and passed # to algorithms that make additional measurents. schema = afw_table.SourceTable.makeMinimalSchema() # Setup algorithms to run meas_config = SingleFrameMeasurementConfig() meas_config.plugins.names = [ "base_SdssCentroid", "base_PsfFlux", "base_SkyCoord", # "modelfit_ShapeletPsfApprox", "modelfit_DoubleShapeletPsfApprox", "modelfit_CModel", # "base_SdssShape", # "base_LocalBackground", ] # set these slots to none because we aren't running these algorithms meas_config.slots.apFlux = None meas_config.slots.gaussianFlux = None meas_config.slots.calibFlux = None meas_config.slots.modelFlux = None # goes with SdssShape above meas_config.slots.shape = None # fix odd issue where it things things are near the edge meas_config.plugins['base_SdssCentroid'].binmax = 1 meas_task = SingleFrameMeasurementTask( config=meas_config, schema=schema, ) # setup detection config detection_config = SourceDetectionConfig() detection_config.reEstimateBackground = False detection_config.thresholdValue = thresh detection_task = SourceDetectionTask(config=detection_config) detection_task.log.setLevel(getattr(lsst.log, loglevel)) deblend_config = SourceDeblendConfig() deblend_task = SourceDeblendTask(config=deblend_config, schema=schema) deblend_task.log.setLevel(getattr(lsst.log, loglevel)) # Detect objects table = afw_table.SourceTable.make(schema) result = detection_task.run(table, exp) sources = result.sources # run the deblender deblend_task.run(exp, sources) # Run on deblended images noise_replacer_config = NoiseReplacerConfig() footprints = { record.getId(): (record.getParent(), record.getFootprint()) for record in result.sources } # This constructor will replace all detected pixels with noise in the # image replacer = NoiseReplacer( noise_replacer_config, exposure=exp, footprints=footprints, ) nbad = 0 ntry = 0 kept_sources = [] for record in result.sources: # Skip parent objects where all children are inserted if record.get('deblend_nChild') != 0: continue ntry += 1 # This will insert a single source into the image replacer.insertSource(record.getId()) # Get the peak as before # peak = record.getFootprint().getPeaks()[0] # The bounding box will be for the parent object # bbox = record.getFootprint().getBBox() meas_task.callMeasure(record, exp) # Remove object replacer.removeSource(record.getId()) if record.getCentroidFlag(): nbad += 1 kept_sources.append(record) # Insert all objects back into image replacer.end() if ntry > 0: log.debug('nbad center: %d frac: %d' % (nbad, nbad / ntry)) nkeep = len(kept_sources) ntot = len(result.sources) log.debug('kept %d/%d non parents' % (nkeep, ntot)) return kept_sources