def test_significance(self): """Test that negative peaks have the right significance for thresholdType='stdev' for the non-convolved, non-local-background case. """ exposure, numX, numY = self._create_exposure() schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.thresholdPolarity = 'both' # don't modify the image after detection. config.reEstimateBackground = False config.doTempLocalBackground = False detection = SourceDetectionTask(config=config, schema=schema) result = detection.detectFootprints(exposure, doSmooth=False) bad = exposure.mask.getPlaneBitMask(config.statsMask) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP, sctrl) stddev = stats.getValue(afwMath.STDEVCLIP) # Don't bother checking positive footprints: those are tested more # thoroughly in test_detection.py. for footprint in result.negative.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] with self.subTest(str(point)): self.assertFloatsAlmostEqual( peak["significance"], -value / stddev, # S/N for negative peak rtol=1e-7, msg=str(point))
def test_pixel_stdev(self): """Test that sources are detected on a simulated image with thresholdType='pixel_stdev', and that they have the right significance. """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() config.thresholdType = "pixel_stdev" config.reEstimateBackground = False # TempLocalBackground changes the peak value of the faintest peak, # so disable it for this test so that we can calculate an expected # answer without having to try to deal with backgrounds. config.doTempLocalBackground = False task = SourceDetectionTask(config=config, schema=schema) # Don't smooth, so that we can directly calculate the s/n from the exposure. result = task.detectFootprints(exposure, doSmooth=False) self.assertEqual(result.numPos, numX * numY) self.assertEqual(result.numNeg, 0) # Significance values for `pixel_stdev` should match image/sqrt(variance). for footprint in result.positive.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] stddev = np.sqrt(exposure.variance[point]) with self.subTest(str(point)): self.assertFloatsAlmostEqual(peak["significance"], value / stddev, rtol=1e-7, msg=str(point))
def test_stdev(self): """Test that sources are detected on a simulated image with thresholdType='stdev'. """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() # don't modify the image after detection. config.reEstimateBackground = False config.thresholdType = "stdev" task = SourceDetectionTask(config=config, schema=schema) self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=True) self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=False)
def test_detection_stdev(self): """Test detection and measurement on an exposure with negative sources for thresholdType="stdev". """ exposure, numX, numY = self._create_exposure() if display: disp = afwDisplay.Display(frame=1) disp.mtv(exposure, title=self._testMethodName + ": image with -ve sources") schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdPolarity = 'both' detection = SourceDetectionTask(config=config, schema=schema) algMetadata = dafBase.PropertyList() measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata) table = afwTable.SourceTable.make(schema) detections = detection.run(table, exposure) sources = detections.sources fpSets = detections.fpSets self.assertEqual(len(sources), numX * numY) self.assertEqual(fpSets.numPos, numX * numY / 2) self.assertEqual(fpSets.numNeg, numX * numY / 2) measurement.run(sources, exposure) nGoodCent = 0 nGoodShape = 0 for s in sources: cent = s.getCentroid() shape = s.getShape() if cent[0] == cent[0] and cent[1] == cent[1]: nGoodCent += 1 if (shape.getIxx() == shape.getIxx() and shape.getIyy() == shape.getIyy() and shape.getIxy() == shape.getIxy()): nGoodShape += 1 if display: xy = cent[0], cent[1] disp.dot('+', *xy) disp.dot(shape, *xy, ctype=afwDisplay.RED) self.assertEqual(nGoodCent, numX * numY) self.assertEqual(nGoodShape, numX * numY)
def checkExposure(original, doTempLocalBackground, doTempWideBackground): config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdType = "pixel_stdev" config.doTempLocalBackground = doTempLocalBackground config.doTempWideBackground = doTempWideBackground schema = afwTable.SourceTable.makeMinimalSchema() task = SourceDetectionTask(config=config, schema=schema) exposure = original.clone() task.detectFootprints(exposure, sigma=3.21) self.assertFloatsEqual(exposure.image.array, original.image.array) # Mask is permitted to vary: DETECTED bit gets set self.assertFloatsEqual(exposure.variance.array, original.variance.array)
def testBasics(self): bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100), afwGeom.Extent2I(128, 127)) minCounts = 5000 maxCounts = 50000 starSigma = 1.5 numX = 5 numY = 5 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False task = SourceDetectionTask(config=config, schema=schema) for doSmooth in (False, True): taskSigma = 2.2 res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=taskSigma) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0) self.assertEqual(task.metadata.get("sigma"), taskSigma) self.assertEqual(task.metadata.get("doSmooth"), doSmooth) self.assertEqual(task.metadata.get("nGrow"), int(taskSigma * config.nSigmaToGrow + 0.5)) res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=None) taskSigma = task.metadata.get("sigma") self.assertTrue(abs(taskSigma - starSigma) < 0.1) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0)
def test_deblend_task(self): # Set the random seed so that the noise field is unaffected np.random.seed(0) # Test that executing the deblend task works # In the future we can have more detailed tests, # but for now this at least ensures that the task isn't broken shape = (5, 31, 55) coords = [(15, 25), (10, 30), (17, 38)] amplitudes = [80, 60, 90] result = initData(shape, coords, amplitudes) targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result B, Ny, Nx = shape # Add some noise, otherwise the task will blow up due to # zero variance noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5) images += noise filters = "grizy" _images = afwImage.MultibandMaskedImage.fromArrays( filters, images.astype(np.float32), None, noise) coadds = [ afwImage.Exposure(img, dtype=img.image.array.dtype) for img in _images ] coadds = afwImage.MultibandExposure.fromExposures(filters, coadds) for b, coadd in enumerate(coadds): coadd.setPsf(psfs[b]) schema = SourceCatalog.Table.makeMinimalSchema() detectionTask = SourceDetectionTask(schema=schema) config = ScarletDeblendTask.ConfigClass() config.maxIter = 300 deblendTask = ScarletDeblendTask(schema=schema, config=config) table = SourceCatalog.Table.make(schema) detectionResult = detectionTask.run(table, coadds["r"]) catalog = detectionResult.sources self.assertEqual(len(catalog), 1) _, result = deblendTask.run(coadds, catalog)
def testBasics(self): bbox = lsst.geom.Box2I(lsst.geom.Point2I(256, 100), lsst.geom.Extent2I(128, 127)) minCounts = 5000 maxCounts = 50000 starSigma = 1.5 numX = 5 numY = 5 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False task = SourceDetectionTask(config=config, schema=schema) for doSmooth in (False, True): taskSigma = 2.2 res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=taskSigma) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0) self.assertEqual(task.metadata.getScalar("sigma"), taskSigma) self.assertEqual(task.metadata.getScalar("doSmooth"), doSmooth) self.assertEqual(task.metadata.getScalar("nGrow"), int(taskSigma * config.nSigmaToGrow + 0.5)) res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=None) taskSigma = task.metadata.getScalar("sigma") self.assertLess(abs(taskSigma - starSigma), 0.1) self.assertEqual(res.numPos, numX * numY) self.assertEqual(res.numNeg, 0)
def test_significance_stdev(self): """Check the non-smoothed, non-background updated peak significance values with thresholdType="stddev". """ exposure, schema, numX, numY, starSigma = self._create_exposure() config = SourceDetectionTask.ConfigClass() # don't modify the image after detection. config.reEstimateBackground = False config.doTempLocalBackground = False config.thresholdType = "stdev" task = SourceDetectionTask(config=config, schema=schema) result = self._check_detectFootprints(exposure, numX, numY, starSigma, task, config, doSmooth=False) bad = exposure.mask.getPlaneBitMask(config.statsMask) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP, sctrl) stddev = stats.getValue(afwMath.STDEVCLIP) for footprint in result.positive.getFootprints(): for peak in footprint.peaks: point = lsst.geom.Point2I(peak.getIx(), peak.getIy()) value = exposure.image[point] with self.subTest(str(point)): self.assertFloatsAlmostEqual(peak["significance"], value / stddev, rtol=1e-7, msg=str(point))
def testIsScarletPrimaryFlag(self): """Test detect_isPrimary column when scarlet is used as the deblender """ # We need a multiband coadd for scarlet, # even though there is only one band coadds = afwImage.MultibandExposure.fromExposures(["test"], [self.exposure]) # Create a SkyMap with a tract that contains a portion of the image, # subdivided into 3x3 patches wcs = self.exposure.getWcs() tractBBox = Box2I(Point2I(100, 100), Extent2I(900, 900)) skyMap = MockSkyMap([tractBBox], wcs, 3) tractInfo = skyMap[0] patchInfo = tractInfo[0, 0] patchBBox = patchInfo.getInnerBBox() schema = SourceCatalog.Table.makeMinimalSchema() # Initialize the detection task detectionTask = SourceDetectionTask(schema=schema) # Initialize the fake source injection task skyConfig = SkyObjectsTask.ConfigClass() skySourcesTask = SkyObjectsTask(name="skySources", config=skyConfig) schema.addField("merge_peak_sky", type="Flag") # Initialize the deblender task scarletConfig = ScarletDeblendTask.ConfigClass() scarletConfig.maxIter = 20 scarletConfig.columnInheritance["merge_peak_sky"] = "merge_peak_sky" deblendTask = ScarletDeblendTask(schema=schema, config=scarletConfig) # We'll customize the configuration of measurement to just run the # minimal number of plugins to make setPrimaryFlags work. measureConfig = SingleFrameMeasurementTask.ConfigClass() measureConfig.plugins.names = ["base_SdssCentroid", "base_SkyCoord"] measureConfig.slots.psfFlux = None measureConfig.slots.apFlux = None measureConfig.slots.shape = None measureConfig.slots.modelFlux = None measureConfig.slots.calibFlux = None measureConfig.slots.gaussianFlux = None measureTask = SingleFrameMeasurementTask(config=measureConfig, schema=schema) primaryConfig = SetPrimaryFlagsTask.ConfigClass() setPrimaryTask = SetPrimaryFlagsTask(config=primaryConfig, schema=schema, name="setPrimaryFlags", isSingleFrame=False) table = SourceCatalog.Table.make(schema) # detect sources detectionResult = detectionTask.run(table, coadds["test"]) catalog = detectionResult.sources # add fake sources skySources = skySourcesTask.run(mask=self.exposure.mask, seed=0) for foot in skySources[:5]: src = catalog.addNew() src.setFootprint(foot) src.set("merge_peak_sky", True) # deblend result = deblendTask.run(coadds, catalog) # measure measureTask.run(result["test"], self.exposure) outputCat = result["test"] # Set the primary flags setPrimaryTask.run(outputCat, skyMap=skyMap, tractInfo=tractInfo, patchInfo=patchInfo) # There should be the same number of deblenedPrimary and # deblendedModelPrimary sources, # since they both have the same blended sources and only differ # over which model to use for the isolated sources. isPseudo = getPseudoSources(outputCat, primaryConfig.pseudoFilterList, schema, setPrimaryTask.log) self.assertEqual( np.sum(outputCat["detect_isDeblendedSource"] & ~isPseudo), np.sum(outputCat["detect_isDeblendedModelSource"])) # Check that the sources contained in a tract are all marked appropriately x = outputCat["slot_Centroid_x"] y = outputCat["slot_Centroid_y"] tractInner = tractBBox.contains(x, y) np.testing.assert_array_equal(outputCat["detect_isTractInner"], tractInner) # Check that the sources contained in a patch are all marked appropriately patchInner = patchBBox.contains(x, y) np.testing.assert_array_equal(outputCat["detect_isPatchInner"], patchInner) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["merge_peak_sky"])), 0) # Check that sky objects have not been deblended np.testing.assert_array_equal( isPseudo, isPseudo & (outputCat["deblend_nChild"] == 0))
subConfig[ algName].maxSincRadius = 0 # Disable sinc photometry because we're undersampled for subConfig in (measure_config.plugins, measure_config.undeblended): subConfig.names.add(algName2) values = [ii / 0.168 for ii in (0.65, 0.85, 1.1)] algConfig = subConfig[algName2] algConfig.seeing = values algConfig.aperture.radii = [3, 6, 12, 24] algConfig.aperture.maxSincRadius = 0 measure_config.load( '/tigress/rea3/lsst/DM-8059/obs_subaru/config/apertures.py') deblend_config = SourceDeblendTask.ConfigClass() detect_config = SourceDetectionTask.ConfigClass() detect_config.isotropicGrow = True detect_config.doTempLocalBackground = False detect_config.thresholdValue = 5 detect_config.nSigmaToGrow = args.grow schema = afwTable.SourceTable.makeMinimalSchema() detectTask = SourceDetectionTask(schema, config=detect_config) deblendTask = SourceDeblendTask(schema, config=deblend_config) measureTask = SingleFrameMeasurementTask(schema, config=measure_config) peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema() peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema) truthKey = schema.addField('truth_index', type=int,
def main(args): epoch = args.epoch hdus = pyfits.open("%s/ground/constant/epoch_catalog-%s-0.fits"%(args.type, args.epoch)) cat = hdus[1] exp = afwImage.ExposureF("%s/ground/constant/image-%s-0.fits"%(args.type, args.epoch)) psfDict = None if not os.path.isdir(args.psfs): if not os.path.exists(args.psfs): print "Psf location: %s does not exist."%args.psfs sys.exit(1) psfHdfs = pyfits.open(args.psfs) psfDict = {} for i in range(len(psfHdfs)): psf_number = psfHdfs[i].header.get("PSF_NO") if psf_number == None: psfDict[i+1] = i else: psfDict[psf_number] = i else: psfFormat = args.psfs + "/psfs_%d.fits" i_psf_number = getIndex("psf_number", cat) i_index = getIndex("index", cat) i_xmax = getIndex("xmax", cat) i_ymax = getIndex("ymax", cat) i_xmin = getIndex("xmin", cat) i_ymin = getIndex("ymin", cat) schema = afwTable.SourceTable.makeMinimalSchema() schema.addField("centroid_x", type=float) schema.addField("centroid_y", type=float) control = shape.HsmShapeBjControl() alg = shape.HsmShapeBjAlgorithm #control = measBase.SdssShapeControl() #alg = measBase.SdssShapeAlgorithm plugin = alg(control, "", schema) success = 0 failed = 0 count = 0 startTime = time.time() for item in cat.data: bbox = afwGeom.Box2I( afwGeom.Point2I(item[i_xmin], item[i_ymin]), afwGeom.Point2I(item[i_xmax], item[i_ymax]) ) subexp = afwImage.ExposureF(exp, bbox) #subarray = subexp.getMaskedImage().getImage().getArray() #subarray = numpy.repeat(numpy.repeat(subarray,2, axis=0), 2, axis=1) #subexp = afwImage.ExposureF(afwImage.MaskedImageF(afwImage.ImageF(subarray))) if psfDict: psf_number = item[i_psf_number] i = psfDict[psf_number] data = psfHdfs[i].data.astype(numpy.float64) else: psfFile = psfFormat % item[i_psf_number] psfImage = afwImage.ImageF(psfFile) data = psfImage.getArray().astype(numpy.float64) kernel = afwMath.FixedKernel(afwImage.ImageD(data)) psf = lsst.meas.algorithms.KernelPsf(kernel) subexp.setPsf(psf) subexp.setXY0(afwGeom.Point2I(0,0)) detection = SourceDetectionTask() table = afwTable.SourceTable.make(schema) detections = detection.run(table, subexp) detections.sources.defineCentroid("centroid") count = count+1 if len(detections.sources) == 0: print "ITEM %d, no sources, "%count, bbox continue record = detections.sources[0] record.setFootprint(detections.fpSets.positive.getFootprints()[0]) cen = record.getFootprint().getPeaks()[0].getCentroid() record.set('centroid_x', cen.getX()) record.set('centroid_y', cen.getY()) try: plugin.measure(record, subexp) print item[i_index], record.get("_e1"), record.get("_e2") success = success+1 except lsst.pex.exceptions.RuntimeError as e: print item[i_index], e.message failed = failed + 1 except Exception as e: print "FAILED: ", item[i_index], e.message failed = failed + 1 print "elapsed time = ", time.time()-startTime print "success = ", success, ", failed = ", failed
def run(self, inputDefects, camera): detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) if detectorId is None: raise RuntimeError("Cannot identify detector id.") detector = camera[detectorId] imageTypes = set() for inDefect in inputDefects: imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') imageTypes.add(imageType) # Determine common defect pixels separately for each input image type. splitDefects = list() for imageType in imageTypes: sumImage = afwImage.MaskedImageF(detector.getBBox()) count = 0 for inDefect in inputDefects: if imageType == inDefect.getMetadata().get( 'cpDefectGenImageType', 'UNKNOWN'): count += 1 for defect in inDefect: sumImage.image[defect.getBBox()] += 1.0 sumImage /= count nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) self.log.info( "Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) if self.config.combinationMode == 'AND': threshold = 1.0 elif self.config.combinationMode == 'OR': threshold = 0.0 elif self.config.combinationMode == 'FRACTION': threshold = self.config.combinationFraction else: raise RuntimeError( f"Got unsupported combinationMode {self.config.combinationMode}" ) indices = np.where(sumImage.getImage().getArray() > threshold) BADBIT = sumImage.getMask().getPlaneBitMask('BAD') sumImage.getMask().getArray()[indices] |= BADBIT self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) partialDefect = Defects.fromMask(sumImage, 'BAD') splitDefects.append(partialDefect) # Do final combination of separate image types finalImage = afwImage.MaskedImageF(detector.getBBox()) for inDefect in splitDefects: for defect in inDefect: finalImage.image[defect.getBBox()] += 1 finalImage /= len(splitDefects) nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) # This combination is the OR of all image types threshold = 0.0 indices = np.where(finalImage.getImage().getArray() > threshold) BADBIT = finalImage.getMask().getPlaneBitMask('BAD') finalImage.getMask().getArray()[indices] |= BADBIT self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) if self.config.edgesAsDefects: self.log.info("Masking edge pixels as defects.") # Do the same as IsrTask.maskEdges() box = detector.getBBox() subImage = finalImage[box] box.grow(-self.nPixBorder) SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) merged = Defects.fromMask(finalImage, 'BAD') merged.updateMetadata(camera=camera, detector=detector, filterName=None, setCalibId=True, setDate=True) return pipeBase.Struct(mergedDefects=merged, )
def testBasics(self): bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100), afwGeom.Extent2I(128, 127)) minCounts = 2000 maxCounts = 20000 starSigma = 1.5 numX = 4 numY = 4 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) if display: ds9.mtv(exposure) schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdPolarity = 'both' detection = SourceDetectionTask(config=config, schema=schema) algMetadata = dafBase.PropertyList() measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata) table = afwTable.SourceTable.make(schema) detections = detection.makeSourceCatalog(table, exposure) sources = detections.sources fpSets = detections.fpSets self.assertEqual(len(sources), numX * numY) self.assertEqual(fpSets.numPos, numX * numY / 2) self.assertEqual(fpSets.numNeg, numX * numY / 2) measurement.run(sources, exposure) nGoodCent = 0 nGoodShape = 0 for s in sources: cent = s.getCentroid() shape = s.getShape() if cent[0] == cent[0] and cent[1] == cent[1]: nGoodCent += 1 if (shape.getIxx() == shape.getIxx() and shape.getIyy() == shape.getIyy() and shape.getIxy() == shape.getIxy()): nGoodShape += 1 if display: xy = cent[0] - exposure.getX0(), cent[1] - exposure.getY0() ds9.dot('+', *xy) ds9.dot(shape, *xy, ctype=ds9.RED) self.assertEqual(nGoodCent, numX * numY) self.assertEqual(nGoodShape, numX * numY)
def testBasics(self): bbox = lsst.geom.Box2I(lsst.geom.Point2I(256, 100), lsst.geom.Extent2I(128, 127)) minCounts = 2000 maxCounts = 20000 starSigma = 1.5 numX = 4 numY = 4 coordList = self.makeCoordList( bbox=bbox, numX=numX, numY=numY, minCounts=minCounts, maxCounts=maxCounts, sigma=starSigma, ) kwid = 11 sky = 2000 addPoissonNoise = True exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=addPoissonNoise) if display: disp = afwDisplay.Display(frame=1) disp.mtv(exposure, title=self._testMethodName + ": image with -ve sources") schema = afwTable.SourceTable.makeMinimalSchema() config = SourceDetectionTask.ConfigClass() config.reEstimateBackground = False config.thresholdPolarity = 'both' detection = SourceDetectionTask(config=config, schema=schema) algMetadata = dafBase.PropertyList() measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata) table = afwTable.SourceTable.make(schema) detections = detection.makeSourceCatalog(table, exposure) sources = detections.sources fpSets = detections.fpSets self.assertEqual(len(sources), numX*numY) self.assertEqual(fpSets.numPos, numX*numY/2) self.assertEqual(fpSets.numNeg, numX*numY/2) measurement.run(sources, exposure) nGoodCent = 0 nGoodShape = 0 for s in sources: cent = s.getCentroid() shape = s.getShape() if cent[0] == cent[0] and cent[1] == cent[1]: nGoodCent += 1 if (shape.getIxx() == shape.getIxx() and shape.getIyy() == shape.getIyy() and shape.getIxy() == shape.getIxy()): nGoodShape += 1 if display: xy = cent[0], cent[1] disp.dot('+', *xy) disp.dot(shape, *xy, ctype=afwDisplay.RED) self.assertEqual(nGoodCent, numX*numY) self.assertEqual(nGoodShape, numX*numY)
def test_deblend_task(self): # Set the random seed so that the noise field is unaffected np.random.seed(0) shape = (5, 100, 115) coords = [ # blend (15, 25), (10, 30), (17, 38), # isolated source (85, 90), ] amplitudes = [ # blend 80, 60, 90, # isolated source 20, ] result = initData(shape, coords, amplitudes) targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result B, Ny, Nx = shape # Add some noise, otherwise the task will blow up due to # zero variance noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5) images += noise filters = "grizy" _images = afwImage.MultibandMaskedImage.fromArrays( filters, images.astype(np.float32), None, noise) coadds = [ afwImage.Exposure(img, dtype=img.image.array.dtype) for img in _images ] coadds = afwImage.MultibandExposure.fromExposures(filters, coadds) for b, coadd in enumerate(coadds): coadd.setPsf(psfs[b]) schema = SourceCatalog.Table.makeMinimalSchema() detectionTask = SourceDetectionTask(schema=schema) # Adjust config options to test skipping parents config = ScarletDeblendTask.ConfigClass() config.maxIter = 100 config.maxFootprintArea = 1000 config.maxNumberOfPeaks = 4 deblendTask = ScarletDeblendTask(schema=schema, config=config) table = SourceCatalog.Table.make(schema) detectionResult = detectionTask.run(table, coadds["r"]) catalog = detectionResult.sources # Add a footprint that is too large src = catalog.addNew() halfLength = int(np.ceil(np.sqrt(config.maxFootprintArea) + 1)) ss = SpanSet.fromShape(halfLength, Stencil.BOX, offset=(50, 50)) bigfoot = Footprint(ss) bigfoot.addPeak(50, 50, 100) src.setFootprint(bigfoot) # Add a footprint with too many peaks src = catalog.addNew() ss = SpanSet.fromShape(10, Stencil.BOX, offset=(75, 20)) denseFoot = Footprint(ss) for n in range(config.maxNumberOfPeaks + 1): denseFoot.addPeak(70 + 2 * n, 15 + 2 * n, 10 * n) src.setFootprint(denseFoot) # Run the deblender result = deblendTask.run(coadds, catalog) # Make sure that the catalogs have the same sources in all bands, # and check that band-independent columns are equal bandIndependentColumns = [ "id", "parent", "deblend_nPeaks", "deblend_nChild", "deblend_peak_center_x", "deblend_peak_center_y", "deblend_runtime", "deblend_iterations", "deblend_logL", "deblend_spectrumInitFlag", "deblend_blendConvergenceFailedFlag", ] self.assertEqual(len(filters), len(result)) ref = result[filters[0]] for f in filters[1:]: for col in bandIndependentColumns: np.testing.assert_array_equal(result[f][col], ref[col]) # Check that other columns are consistent for f, _catalog in result.items(): parents = _catalog[_catalog["parent"] == 0] # Check that the number of deblended children is consistent self.assertEqual(np.sum(_catalog["deblend_nChild"]), len(_catalog) - len(parents)) for parent in parents: children = _catalog[_catalog["parent"] == parent.get("id")] # Check that nChild is set correctly self.assertEqual(len(children), parent.get("deblend_nChild")) # Check that parent columns are propagated to their children for parentCol, childCol in config.columnInheritance.items(): np.testing.assert_array_equal(parent.get(parentCol), children[childCol]) children = _catalog[_catalog["parent"] != 0] for child in children: fp = child.getFootprint() img = heavyFootprintToImage(fp) # Check that the flux at the center is correct. # Note: this only works in this test image because the # detected peak is in the same location as the scarlet peak. # If the peak is shifted, the flux value will be correct # but deblend_peak_center is not the correct location. px = child.get("deblend_peak_center_x") py = child.get("deblend_peak_center_y") flux = img.image[Point2I(px, py)] self.assertEqual(flux, child.get("deblend_peak_instFlux")) # Check that the peak positions match the catalog entry peaks = fp.getPeaks() self.assertEqual(px, peaks[0].getIx()) self.assertEqual(py, peaks[0].getIy()) # Check that all sources have the correct number of peaks for src in _catalog: fp = src.getFootprint() self.assertEqual(len(fp.peaks), src.get("deblend_nPeaks")) # Check that only the large foorprint was flagged as too big largeFootprint = np.zeros(len(_catalog), dtype=bool) largeFootprint[2] = True np.testing.assert_array_equal(largeFootprint, _catalog["deblend_parentTooBig"]) # Check that only the dense foorprint was flagged as too dense denseFootprint = np.zeros(len(_catalog), dtype=bool) denseFootprint[3] = True np.testing.assert_array_equal(denseFootprint, _catalog["deblend_tooManyPeaks"]) # Check that only the appropriate parents were skipped skipped = largeFootprint | denseFootprint np.testing.assert_array_equal(skipped, _catalog["deblend_skipped"])
def detect_and_deblend(*, exp, log): log = lsst.log.Log.getLogger("LSSTMEDSifier") thresh = 5.0 loglevel = 'INFO' # This schema holds all the measurements that will be run within the # stack It needs to be constructed before running anything and passed # to algorithms that make additional measurents. schema = afw_table.SourceTable.makeMinimalSchema() # Setup algorithms to run meas_config = SingleFrameMeasurementConfig() meas_config.plugins.names = [ "base_SdssCentroid", "base_PsfFlux", "base_SkyCoord", # "modelfit_ShapeletPsfApprox", "modelfit_DoubleShapeletPsfApprox", "modelfit_CModel", # "base_SdssShape", # "base_LocalBackground", ] # set these slots to none because we aren't running these algorithms meas_config.slots.apFlux = None meas_config.slots.gaussianFlux = None meas_config.slots.calibFlux = None meas_config.slots.modelFlux = None # goes with SdssShape above meas_config.slots.shape = None # fix odd issue where it things things are near the edge meas_config.plugins['base_SdssCentroid'].binmax = 1 meas_task = SingleFrameMeasurementTask( config=meas_config, schema=schema, ) # setup detection config detection_config = SourceDetectionConfig() detection_config.reEstimateBackground = False detection_config.thresholdValue = thresh detection_task = SourceDetectionTask(config=detection_config) detection_task.log.setLevel(getattr(lsst.log, loglevel)) deblend_config = SourceDeblendConfig() deblend_task = SourceDeblendTask(config=deblend_config, schema=schema) deblend_task.log.setLevel(getattr(lsst.log, loglevel)) # Detect objects table = afw_table.SourceTable.make(schema) result = detection_task.run(table, exp) sources = result.sources # run the deblender deblend_task.run(exp, sources) # Run on deblended images noise_replacer_config = NoiseReplacerConfig() footprints = { record.getId(): (record.getParent(), record.getFootprint()) for record in result.sources } # This constructor will replace all detected pixels with noise in the # image replacer = NoiseReplacer( noise_replacer_config, exposure=exp, footprints=footprints, ) nbad = 0 ntry = 0 kept_sources = [] for record in result.sources: # Skip parent objects where all children are inserted if record.get('deblend_nChild') != 0: continue ntry += 1 # This will insert a single source into the image replacer.insertSource(record.getId()) # Get the peak as before # peak = record.getFootprint().getPeaks()[0] # The bounding box will be for the parent object # bbox = record.getFootprint().getBBox() meas_task.callMeasure(record, exp) # Remove object replacer.removeSource(record.getId()) if record.getCentroidFlag(): nbad += 1 kept_sources.append(record) # Insert all objects back into image replacer.end() if ntry > 0: log.debug('nbad center: %d frac: %d' % (nbad, nbad / ntry)) nkeep = len(kept_sources) ntot = len(result.sources) log.debug('kept %d/%d non parents' % (nkeep, ntot)) return kept_sources