def test_significance(self):
"""Test that negative peaks have the right significance for
thresholdType='stdev' for the non-convolved, non-local-background case.
"""
exposure, numX, numY = self._create_exposure()
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = 'both'
# don't modify the image after detection.
config.reEstimateBackground = False
config.doTempLocalBackground = False
detection = SourceDetectionTask(config=config, schema=schema)
result = detection.detectFootprints(exposure, doSmooth=False)
bad = exposure.mask.getPlaneBitMask(config.statsMask)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bad)
stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP,
sctrl)
stddev = stats.getValue(afwMath.STDEVCLIP)
# Don't bother checking positive footprints: those are tested more
# thoroughly in test_detection.py.
for footprint in result.negative.getFootprints():
for peak in footprint.peaks:
point = lsst.geom.Point2I(peak.getIx(), peak.getIy())
value = exposure.image[point]
with self.subTest(str(point)):
self.assertFloatsAlmostEqual(
peak["significance"],
-value / stddev, # S/N for negative peak
rtol=1e-7,
msg=str(point))
def test_pixel_stdev(self):
"""Test that sources are detected on a simulated image with
thresholdType='pixel_stdev', and that they have the right significance.
"""
exposure, schema, numX, numY, starSigma = self._create_exposure()
config = SourceDetectionTask.ConfigClass()
config.thresholdType = "pixel_stdev"
config.reEstimateBackground = False
# TempLocalBackground changes the peak value of the faintest peak,
# so disable it for this test so that we can calculate an expected
# answer without having to try to deal with backgrounds.
config.doTempLocalBackground = False
task = SourceDetectionTask(config=config, schema=schema)
# Don't smooth, so that we can directly calculate the s/n from the exposure.
result = task.detectFootprints(exposure, doSmooth=False)
self.assertEqual(result.numPos, numX * numY)
self.assertEqual(result.numNeg, 0)
# Significance values for `pixel_stdev` should match image/sqrt(variance).
for footprint in result.positive.getFootprints():
for peak in footprint.peaks:
point = lsst.geom.Point2I(peak.getIx(), peak.getIy())
value = exposure.image[point]
stddev = np.sqrt(exposure.variance[point])
with self.subTest(str(point)):
self.assertFloatsAlmostEqual(peak["significance"],
value / stddev,
rtol=1e-7,
msg=str(point))
def test_stdev(self):
"""Test that sources are detected on a simulated image with
thresholdType='stdev'.
"""
exposure, schema, numX, numY, starSigma = self._create_exposure()
config = SourceDetectionTask.ConfigClass()
# don't modify the image after detection.
config.reEstimateBackground = False
config.thresholdType = "stdev"
task = SourceDetectionTask(config=config, schema=schema)
self._check_detectFootprints(exposure,
numX,
numY,
starSigma,
task,
config,
doSmooth=True)
self._check_detectFootprints(exposure,
numX,
numY,
starSigma,
task,
config,
doSmooth=False)
def test_detection_stdev(self):
"""Test detection and measurement on an exposure with negative sources
for thresholdType="stdev".
"""
exposure, numX, numY = self._create_exposure()
if display:
disp = afwDisplay.Display(frame=1)
disp.mtv(exposure,
title=self._testMethodName + ": image with -ve sources")
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdPolarity = 'both'
detection = SourceDetectionTask(config=config, schema=schema)
algMetadata = dafBase.PropertyList()
measurement = SourceMeasurementTask(schema=schema,
algMetadata=algMetadata)
table = afwTable.SourceTable.make(schema)
detections = detection.run(table, exposure)
sources = detections.sources
fpSets = detections.fpSets
self.assertEqual(len(sources), numX * numY)
self.assertEqual(fpSets.numPos, numX * numY / 2)
self.assertEqual(fpSets.numNeg, numX * numY / 2)
measurement.run(sources, exposure)
nGoodCent = 0
nGoodShape = 0
for s in sources:
cent = s.getCentroid()
shape = s.getShape()
if cent[0] == cent[0] and cent[1] == cent[1]:
nGoodCent += 1
if (shape.getIxx() == shape.getIxx()
and shape.getIyy() == shape.getIyy()
and shape.getIxy() == shape.getIxy()):
nGoodShape += 1
if display:
xy = cent[0], cent[1]
disp.dot('+', *xy)
disp.dot(shape, *xy, ctype=afwDisplay.RED)
self.assertEqual(nGoodCent, numX * numY)
self.assertEqual(nGoodShape, numX * numY)
def checkExposure(original, doTempLocalBackground, doTempWideBackground):
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdType = "pixel_stdev"
config.doTempLocalBackground = doTempLocalBackground
config.doTempWideBackground = doTempWideBackground
schema = afwTable.SourceTable.makeMinimalSchema()
task = SourceDetectionTask(config=config, schema=schema)
exposure = original.clone()
task.detectFootprints(exposure, sigma=3.21)
self.assertFloatsEqual(exposure.image.array, original.image.array)
# Mask is permitted to vary: DETECTED bit gets set
self.assertFloatsEqual(exposure.variance.array, original.variance.array)
def testBasics(self):
bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100),
afwGeom.Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11
sky = 2000
addPoissonNoise = True
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=addPoissonNoise)
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
task = SourceDetectionTask(config=config, schema=schema)
for doSmooth in (False, True):
taskSigma = 2.2
res = task.detectFootprints(exposure,
doSmooth=doSmooth,
sigma=taskSigma)
self.assertEqual(res.numPos, numX * numY)
self.assertEqual(res.numNeg, 0)
self.assertEqual(task.metadata.get("sigma"), taskSigma)
self.assertEqual(task.metadata.get("doSmooth"), doSmooth)
self.assertEqual(task.metadata.get("nGrow"),
int(taskSigma * config.nSigmaToGrow + 0.5))
res = task.detectFootprints(exposure,
doSmooth=doSmooth,
sigma=None)
taskSigma = task.metadata.get("sigma")
self.assertTrue(abs(taskSigma - starSigma) < 0.1)
self.assertEqual(res.numPos, numX * numY)
self.assertEqual(res.numNeg, 0)
def checkExposure(original, doTempLocalBackground,
doTempWideBackground):
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdType = "pixel_stdev"
config.doTempLocalBackground = doTempLocalBackground
config.doTempWideBackground = doTempWideBackground
schema = afwTable.SourceTable.makeMinimalSchema()
task = SourceDetectionTask(config=config, schema=schema)
exposure = original.clone()
task.detectFootprints(exposure, sigma=3.21)
self.assertFloatsEqual(exposure.image.array, original.image.array)
# Mask is permitted to vary: DETECTED bit gets set
self.assertFloatsEqual(exposure.variance.array,
original.variance.array)
def test_deblend_task(self):
# Set the random seed so that the noise field is unaffected
np.random.seed(0)
# Test that executing the deblend task works
# In the future we can have more detailed tests,
# but for now this at least ensures that the task isn't broken
shape = (5, 31, 55)
coords = [(15, 25), (10, 30), (17, 38)]
amplitudes = [80, 60, 90]
result = initData(shape, coords, amplitudes)
targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result
B, Ny, Nx = shape
# Add some noise, otherwise the task will blow up due to
# zero variance
noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5)
images += noise
filters = "grizy"
_images = afwImage.MultibandMaskedImage.fromArrays(
filters, images.astype(np.float32), None, noise)
coadds = [
afwImage.Exposure(img, dtype=img.image.array.dtype)
for img in _images
]
coadds = afwImage.MultibandExposure.fromExposures(filters, coadds)
for b, coadd in enumerate(coadds):
coadd.setPsf(psfs[b])
schema = SourceCatalog.Table.makeMinimalSchema()
detectionTask = SourceDetectionTask(schema=schema)
config = ScarletDeblendTask.ConfigClass()
config.maxIter = 300
deblendTask = ScarletDeblendTask(schema=schema, config=config)
table = SourceCatalog.Table.make(schema)
detectionResult = detectionTask.run(table, coadds["r"])
catalog = detectionResult.sources
self.assertEqual(len(catalog), 1)
_, result = deblendTask.run(coadds, catalog)
def testBasics(self):
bbox = lsst.geom.Box2I(lsst.geom.Point2I(256, 100), lsst.geom.Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11
sky = 2000
addPoissonNoise = True
exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList,
addPoissonNoise=addPoissonNoise)
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
task = SourceDetectionTask(config=config, schema=schema)
for doSmooth in (False, True):
taskSigma = 2.2
res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=taskSigma)
self.assertEqual(res.numPos, numX * numY)
self.assertEqual(res.numNeg, 0)
self.assertEqual(task.metadata.getScalar("sigma"), taskSigma)
self.assertEqual(task.metadata.getScalar("doSmooth"), doSmooth)
self.assertEqual(task.metadata.getScalar("nGrow"), int(taskSigma * config.nSigmaToGrow + 0.5))
res = task.detectFootprints(exposure, doSmooth=doSmooth, sigma=None)
taskSigma = task.metadata.getScalar("sigma")
self.assertLess(abs(taskSigma - starSigma), 0.1)
self.assertEqual(res.numPos, numX * numY)
self.assertEqual(res.numNeg, 0)
def test_significance_stdev(self):
"""Check the non-smoothed, non-background updated peak significance
values with thresholdType="stddev".
"""
exposure, schema, numX, numY, starSigma = self._create_exposure()
config = SourceDetectionTask.ConfigClass()
# don't modify the image after detection.
config.reEstimateBackground = False
config.doTempLocalBackground = False
config.thresholdType = "stdev"
task = SourceDetectionTask(config=config, schema=schema)
result = self._check_detectFootprints(exposure,
numX,
numY,
starSigma,
task,
config,
doSmooth=False)
bad = exposure.mask.getPlaneBitMask(config.statsMask)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bad)
stats = afwMath.makeStatistics(exposure.maskedImage, afwMath.STDEVCLIP,
sctrl)
stddev = stats.getValue(afwMath.STDEVCLIP)
for footprint in result.positive.getFootprints():
for peak in footprint.peaks:
point = lsst.geom.Point2I(peak.getIx(), peak.getIy())
value = exposure.image[point]
with self.subTest(str(point)):
self.assertFloatsAlmostEqual(peak["significance"],
value / stddev,
rtol=1e-7,
msg=str(point))
def testIsScarletPrimaryFlag(self):
"""Test detect_isPrimary column when scarlet is used as the deblender
"""
# We need a multiband coadd for scarlet,
# even though there is only one band
coadds = afwImage.MultibandExposure.fromExposures(["test"],
[self.exposure])
# Create a SkyMap with a tract that contains a portion of the image,
# subdivided into 3x3 patches
wcs = self.exposure.getWcs()
tractBBox = Box2I(Point2I(100, 100), Extent2I(900, 900))
skyMap = MockSkyMap([tractBBox], wcs, 3)
tractInfo = skyMap[0]
patchInfo = tractInfo[0, 0]
patchBBox = patchInfo.getInnerBBox()
schema = SourceCatalog.Table.makeMinimalSchema()
# Initialize the detection task
detectionTask = SourceDetectionTask(schema=schema)
# Initialize the fake source injection task
skyConfig = SkyObjectsTask.ConfigClass()
skySourcesTask = SkyObjectsTask(name="skySources", config=skyConfig)
schema.addField("merge_peak_sky", type="Flag")
# Initialize the deblender task
scarletConfig = ScarletDeblendTask.ConfigClass()
scarletConfig.maxIter = 20
scarletConfig.columnInheritance["merge_peak_sky"] = "merge_peak_sky"
deblendTask = ScarletDeblendTask(schema=schema, config=scarletConfig)
# We'll customize the configuration of measurement to just run the
# minimal number of plugins to make setPrimaryFlags work.
measureConfig = SingleFrameMeasurementTask.ConfigClass()
measureConfig.plugins.names = ["base_SdssCentroid", "base_SkyCoord"]
measureConfig.slots.psfFlux = None
measureConfig.slots.apFlux = None
measureConfig.slots.shape = None
measureConfig.slots.modelFlux = None
measureConfig.slots.calibFlux = None
measureConfig.slots.gaussianFlux = None
measureTask = SingleFrameMeasurementTask(config=measureConfig,
schema=schema)
primaryConfig = SetPrimaryFlagsTask.ConfigClass()
setPrimaryTask = SetPrimaryFlagsTask(config=primaryConfig,
schema=schema,
name="setPrimaryFlags",
isSingleFrame=False)
table = SourceCatalog.Table.make(schema)
# detect sources
detectionResult = detectionTask.run(table, coadds["test"])
catalog = detectionResult.sources
# add fake sources
skySources = skySourcesTask.run(mask=self.exposure.mask, seed=0)
for foot in skySources[:5]:
src = catalog.addNew()
src.setFootprint(foot)
src.set("merge_peak_sky", True)
# deblend
result = deblendTask.run(coadds, catalog)
# measure
measureTask.run(result["test"], self.exposure)
outputCat = result["test"]
# Set the primary flags
setPrimaryTask.run(outputCat,
skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo)
# There should be the same number of deblenedPrimary and
# deblendedModelPrimary sources,
# since they both have the same blended sources and only differ
# over which model to use for the isolated sources.
isPseudo = getPseudoSources(outputCat, primaryConfig.pseudoFilterList,
schema, setPrimaryTask.log)
self.assertEqual(
np.sum(outputCat["detect_isDeblendedSource"] & ~isPseudo),
np.sum(outputCat["detect_isDeblendedModelSource"]))
# Check that the sources contained in a tract are all marked appropriately
x = outputCat["slot_Centroid_x"]
y = outputCat["slot_Centroid_y"]
tractInner = tractBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isTractInner"],
tractInner)
# Check that the sources contained in a patch are all marked appropriately
patchInner = patchBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isPatchInner"],
patchInner)
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["merge_peak_sky"])), 0)
# Check that sky objects have not been deblended
np.testing.assert_array_equal(
isPseudo, isPseudo & (outputCat["deblend_nChild"] == 0))
subConfig[
algName].maxSincRadius = 0 # Disable sinc photometry because we're undersampled
for subConfig in (measure_config.plugins, measure_config.undeblended):
subConfig.names.add(algName2)
values = [ii / 0.168 for ii in (0.65, 0.85, 1.1)]
algConfig = subConfig[algName2]
algConfig.seeing = values
algConfig.aperture.radii = [3, 6, 12, 24]
algConfig.aperture.maxSincRadius = 0
measure_config.load(
'/tigress/rea3/lsst/DM-8059/obs_subaru/config/apertures.py')
deblend_config = SourceDeblendTask.ConfigClass()
detect_config = SourceDetectionTask.ConfigClass()
detect_config.isotropicGrow = True
detect_config.doTempLocalBackground = False
detect_config.thresholdValue = 5
detect_config.nSigmaToGrow = args.grow
schema = afwTable.SourceTable.makeMinimalSchema()
detectTask = SourceDetectionTask(schema, config=detect_config)
deblendTask = SourceDeblendTask(schema, config=deblend_config)
measureTask = SingleFrameMeasurementTask(schema, config=measure_config)
peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
truthKey = schema.addField('truth_index',
type=int,
def main(args):
epoch = args.epoch
hdus = pyfits.open("%s/ground/constant/epoch_catalog-%s-0.fits"%(args.type, args.epoch))
cat = hdus[1]
exp = afwImage.ExposureF("%s/ground/constant/image-%s-0.fits"%(args.type, args.epoch))
psfDict = None
if not os.path.isdir(args.psfs):
if not os.path.exists(args.psfs):
print "Psf location: %s does not exist."%args.psfs
sys.exit(1)
psfHdfs = pyfits.open(args.psfs)
psfDict = {}
for i in range(len(psfHdfs)):
psf_number = psfHdfs[i].header.get("PSF_NO")
if psf_number == None:
psfDict[i+1] = i
else:
psfDict[psf_number] = i
else:
psfFormat = args.psfs + "/psfs_%d.fits"
i_psf_number = getIndex("psf_number", cat)
i_index = getIndex("index", cat)
i_xmax = getIndex("xmax", cat)
i_ymax = getIndex("ymax", cat)
i_xmin = getIndex("xmin", cat)
i_ymin = getIndex("ymin", cat)
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("centroid_x", type=float)
schema.addField("centroid_y", type=float)
control = shape.HsmShapeBjControl()
alg = shape.HsmShapeBjAlgorithm
#control = measBase.SdssShapeControl()
#alg = measBase.SdssShapeAlgorithm
plugin = alg(control, "", schema)
success = 0
failed = 0
count = 0
startTime = time.time()
for item in cat.data:
bbox = afwGeom.Box2I(
afwGeom.Point2I(item[i_xmin], item[i_ymin]),
afwGeom.Point2I(item[i_xmax], item[i_ymax])
)
subexp = afwImage.ExposureF(exp, bbox)
#subarray = subexp.getMaskedImage().getImage().getArray()
#subarray = numpy.repeat(numpy.repeat(subarray,2, axis=0), 2, axis=1)
#subexp = afwImage.ExposureF(afwImage.MaskedImageF(afwImage.ImageF(subarray)))
if psfDict:
psf_number = item[i_psf_number]
i = psfDict[psf_number]
data = psfHdfs[i].data.astype(numpy.float64)
else:
psfFile = psfFormat % item[i_psf_number]
psfImage = afwImage.ImageF(psfFile)
data = psfImage.getArray().astype(numpy.float64)
kernel = afwMath.FixedKernel(afwImage.ImageD(data))
psf = lsst.meas.algorithms.KernelPsf(kernel)
subexp.setPsf(psf)
subexp.setXY0(afwGeom.Point2I(0,0))
detection = SourceDetectionTask()
table = afwTable.SourceTable.make(schema)
detections = detection.run(table, subexp)
detections.sources.defineCentroid("centroid")
count = count+1
if len(detections.sources) == 0:
print "ITEM %d, no sources, "%count, bbox
continue
record = detections.sources[0]
record.setFootprint(detections.fpSets.positive.getFootprints()[0])
cen = record.getFootprint().getPeaks()[0].getCentroid()
record.set('centroid_x', cen.getX())
record.set('centroid_y', cen.getY())
try:
plugin.measure(record, subexp)
print item[i_index], record.get("_e1"), record.get("_e2")
success = success+1
except lsst.pex.exceptions.RuntimeError as e:
print item[i_index], e.message
failed = failed + 1
except Exception as e:
print "FAILED: ", item[i_index], e.message
failed = failed + 1
print "elapsed time = ", time.time()-startTime
print "success = ", success, ", failed = ", failed
def run(self, inputDefects, camera):
detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
if detectorId is None:
raise RuntimeError("Cannot identify detector id.")
detector = camera[detectorId]
imageTypes = set()
for inDefect in inputDefects:
imageType = inDefect.getMetadata().get('cpDefectGenImageType',
'UNKNOWN')
imageTypes.add(imageType)
# Determine common defect pixels separately for each input image type.
splitDefects = list()
for imageType in imageTypes:
sumImage = afwImage.MaskedImageF(detector.getBBox())
count = 0
for inDefect in inputDefects:
if imageType == inDefect.getMetadata().get(
'cpDefectGenImageType', 'UNKNOWN'):
count += 1
for defect in inDefect:
sumImage.image[defect.getBBox()] += 1.0
sumImage /= count
nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
self.log.info(
"Pre-merge %s pixels with non-zero detections for %s" %
(nDetected, imageType))
if self.config.combinationMode == 'AND':
threshold = 1.0
elif self.config.combinationMode == 'OR':
threshold = 0.0
elif self.config.combinationMode == 'FRACTION':
threshold = self.config.combinationFraction
else:
raise RuntimeError(
f"Got unsupported combinationMode {self.config.combinationMode}"
)
indices = np.where(sumImage.getImage().getArray() > threshold)
BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
sumImage.getMask().getArray()[indices] |= BADBIT
self.log.info("Post-merge %s pixels marked as defects for %s" %
(len(indices[0]), imageType))
partialDefect = Defects.fromMask(sumImage, 'BAD')
splitDefects.append(partialDefect)
# Do final combination of separate image types
finalImage = afwImage.MaskedImageF(detector.getBBox())
for inDefect in splitDefects:
for defect in inDefect:
finalImage.image[defect.getBBox()] += 1
finalImage /= len(splitDefects)
nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
self.log.info("Pre-final merge %s pixels with non-zero detections" %
(nDetected, ))
# This combination is the OR of all image types
threshold = 0.0
indices = np.where(finalImage.getImage().getArray() > threshold)
BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
finalImage.getMask().getArray()[indices] |= BADBIT
self.log.info("Post-final merge %s pixels marked as defects" %
(len(indices[0]), ))
if self.config.edgesAsDefects:
self.log.info("Masking edge pixels as defects.")
# Do the same as IsrTask.maskEdges()
box = detector.getBBox()
subImage = finalImage[box]
box.grow(-self.nPixBorder)
SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
merged = Defects.fromMask(finalImage, 'BAD')
merged.updateMetadata(camera=camera,
detector=detector,
filterName=None,
setCalibId=True,
setDate=True)
return pipeBase.Struct(mergedDefects=merged, )
def testBasics(self):
bbox = afwGeom.Box2I(afwGeom.Point2I(256, 100),
afwGeom.Extent2I(128, 127))
minCounts = 2000
maxCounts = 20000
starSigma = 1.5
numX = 4
numY = 4
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11
sky = 2000
addPoissonNoise = True
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=addPoissonNoise)
if display:
ds9.mtv(exposure)
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdPolarity = 'both'
detection = SourceDetectionTask(config=config, schema=schema)
algMetadata = dafBase.PropertyList()
measurement = SourceMeasurementTask(schema=schema,
algMetadata=algMetadata)
table = afwTable.SourceTable.make(schema)
detections = detection.makeSourceCatalog(table, exposure)
sources = detections.sources
fpSets = detections.fpSets
self.assertEqual(len(sources), numX * numY)
self.assertEqual(fpSets.numPos, numX * numY / 2)
self.assertEqual(fpSets.numNeg, numX * numY / 2)
measurement.run(sources, exposure)
nGoodCent = 0
nGoodShape = 0
for s in sources:
cent = s.getCentroid()
shape = s.getShape()
if cent[0] == cent[0] and cent[1] == cent[1]:
nGoodCent += 1
if (shape.getIxx() == shape.getIxx()
and shape.getIyy() == shape.getIyy()
and shape.getIxy() == shape.getIxy()):
nGoodShape += 1
if display:
xy = cent[0] - exposure.getX0(), cent[1] - exposure.getY0()
ds9.dot('+', *xy)
ds9.dot(shape, *xy, ctype=ds9.RED)
self.assertEqual(nGoodCent, numX * numY)
self.assertEqual(nGoodShape, numX * numY)
def testBasics(self):
bbox = lsst.geom.Box2I(lsst.geom.Point2I(256, 100), lsst.geom.Extent2I(128, 127))
minCounts = 2000
maxCounts = 20000
starSigma = 1.5
numX = 4
numY = 4
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11
sky = 2000
addPoissonNoise = True
exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList,
addPoissonNoise=addPoissonNoise)
if display:
disp = afwDisplay.Display(frame=1)
disp.mtv(exposure, title=self._testMethodName + ": image with -ve sources")
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdPolarity = 'both'
detection = SourceDetectionTask(config=config, schema=schema)
algMetadata = dafBase.PropertyList()
measurement = SourceMeasurementTask(schema=schema, algMetadata=algMetadata)
table = afwTable.SourceTable.make(schema)
detections = detection.makeSourceCatalog(table, exposure)
sources = detections.sources
fpSets = detections.fpSets
self.assertEqual(len(sources), numX*numY)
self.assertEqual(fpSets.numPos, numX*numY/2)
self.assertEqual(fpSets.numNeg, numX*numY/2)
measurement.run(sources, exposure)
nGoodCent = 0
nGoodShape = 0
for s in sources:
cent = s.getCentroid()
shape = s.getShape()
if cent[0] == cent[0] and cent[1] == cent[1]:
nGoodCent += 1
if (shape.getIxx() == shape.getIxx() and
shape.getIyy() == shape.getIyy() and
shape.getIxy() == shape.getIxy()):
nGoodShape += 1
if display:
xy = cent[0], cent[1]
disp.dot('+', *xy)
disp.dot(shape, *xy, ctype=afwDisplay.RED)
self.assertEqual(nGoodCent, numX*numY)
self.assertEqual(nGoodShape, numX*numY)
def test_deblend_task(self):
# Set the random seed so that the noise field is unaffected
np.random.seed(0)
shape = (5, 100, 115)
coords = [
# blend
(15, 25),
(10, 30),
(17, 38),
# isolated source
(85, 90),
]
amplitudes = [
# blend
80,
60,
90,
# isolated source
20,
]
result = initData(shape, coords, amplitudes)
targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result
B, Ny, Nx = shape
# Add some noise, otherwise the task will blow up due to
# zero variance
noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5)
images += noise
filters = "grizy"
_images = afwImage.MultibandMaskedImage.fromArrays(
filters, images.astype(np.float32), None, noise)
coadds = [
afwImage.Exposure(img, dtype=img.image.array.dtype)
for img in _images
]
coadds = afwImage.MultibandExposure.fromExposures(filters, coadds)
for b, coadd in enumerate(coadds):
coadd.setPsf(psfs[b])
schema = SourceCatalog.Table.makeMinimalSchema()
detectionTask = SourceDetectionTask(schema=schema)
# Adjust config options to test skipping parents
config = ScarletDeblendTask.ConfigClass()
config.maxIter = 100
config.maxFootprintArea = 1000
config.maxNumberOfPeaks = 4
deblendTask = ScarletDeblendTask(schema=schema, config=config)
table = SourceCatalog.Table.make(schema)
detectionResult = detectionTask.run(table, coadds["r"])
catalog = detectionResult.sources
# Add a footprint that is too large
src = catalog.addNew()
halfLength = int(np.ceil(np.sqrt(config.maxFootprintArea) + 1))
ss = SpanSet.fromShape(halfLength, Stencil.BOX, offset=(50, 50))
bigfoot = Footprint(ss)
bigfoot.addPeak(50, 50, 100)
src.setFootprint(bigfoot)
# Add a footprint with too many peaks
src = catalog.addNew()
ss = SpanSet.fromShape(10, Stencil.BOX, offset=(75, 20))
denseFoot = Footprint(ss)
for n in range(config.maxNumberOfPeaks + 1):
denseFoot.addPeak(70 + 2 * n, 15 + 2 * n, 10 * n)
src.setFootprint(denseFoot)
# Run the deblender
result = deblendTask.run(coadds, catalog)
# Make sure that the catalogs have the same sources in all bands,
# and check that band-independent columns are equal
bandIndependentColumns = [
"id",
"parent",
"deblend_nPeaks",
"deblend_nChild",
"deblend_peak_center_x",
"deblend_peak_center_y",
"deblend_runtime",
"deblend_iterations",
"deblend_logL",
"deblend_spectrumInitFlag",
"deblend_blendConvergenceFailedFlag",
]
self.assertEqual(len(filters), len(result))
ref = result[filters[0]]
for f in filters[1:]:
for col in bandIndependentColumns:
np.testing.assert_array_equal(result[f][col], ref[col])
# Check that other columns are consistent
for f, _catalog in result.items():
parents = _catalog[_catalog["parent"] == 0]
# Check that the number of deblended children is consistent
self.assertEqual(np.sum(_catalog["deblend_nChild"]),
len(_catalog) - len(parents))
for parent in parents:
children = _catalog[_catalog["parent"] == parent.get("id")]
# Check that nChild is set correctly
self.assertEqual(len(children), parent.get("deblend_nChild"))
# Check that parent columns are propagated to their children
for parentCol, childCol in config.columnInheritance.items():
np.testing.assert_array_equal(parent.get(parentCol),
children[childCol])
children = _catalog[_catalog["parent"] != 0]
for child in children:
fp = child.getFootprint()
img = heavyFootprintToImage(fp)
# Check that the flux at the center is correct.
# Note: this only works in this test image because the
# detected peak is in the same location as the scarlet peak.
# If the peak is shifted, the flux value will be correct
# but deblend_peak_center is not the correct location.
px = child.get("deblend_peak_center_x")
py = child.get("deblend_peak_center_y")
flux = img.image[Point2I(px, py)]
self.assertEqual(flux, child.get("deblend_peak_instFlux"))
# Check that the peak positions match the catalog entry
peaks = fp.getPeaks()
self.assertEqual(px, peaks[0].getIx())
self.assertEqual(py, peaks[0].getIy())
# Check that all sources have the correct number of peaks
for src in _catalog:
fp = src.getFootprint()
self.assertEqual(len(fp.peaks), src.get("deblend_nPeaks"))
# Check that only the large foorprint was flagged as too big
largeFootprint = np.zeros(len(_catalog), dtype=bool)
largeFootprint[2] = True
np.testing.assert_array_equal(largeFootprint,
_catalog["deblend_parentTooBig"])
# Check that only the dense foorprint was flagged as too dense
denseFootprint = np.zeros(len(_catalog), dtype=bool)
denseFootprint[3] = True
np.testing.assert_array_equal(denseFootprint,
_catalog["deblend_tooManyPeaks"])
# Check that only the appropriate parents were skipped
skipped = largeFootprint | denseFootprint
np.testing.assert_array_equal(skipped, _catalog["deblend_skipped"])
def detect_and_deblend(*, exp, log):
log = lsst.log.Log.getLogger("LSSTMEDSifier")
thresh = 5.0
loglevel = 'INFO'
# This schema holds all the measurements that will be run within the
# stack It needs to be constructed before running anything and passed
# to algorithms that make additional measurents.
schema = afw_table.SourceTable.makeMinimalSchema()
# Setup algorithms to run
meas_config = SingleFrameMeasurementConfig()
meas_config.plugins.names = [
"base_SdssCentroid",
"base_PsfFlux",
"base_SkyCoord",
# "modelfit_ShapeletPsfApprox",
"modelfit_DoubleShapeletPsfApprox",
"modelfit_CModel",
# "base_SdssShape",
# "base_LocalBackground",
]
# set these slots to none because we aren't running these algorithms
meas_config.slots.apFlux = None
meas_config.slots.gaussianFlux = None
meas_config.slots.calibFlux = None
meas_config.slots.modelFlux = None
# goes with SdssShape above
meas_config.slots.shape = None
# fix odd issue where it things things are near the edge
meas_config.plugins['base_SdssCentroid'].binmax = 1
meas_task = SingleFrameMeasurementTask(
config=meas_config,
schema=schema,
)
# setup detection config
detection_config = SourceDetectionConfig()
detection_config.reEstimateBackground = False
detection_config.thresholdValue = thresh
detection_task = SourceDetectionTask(config=detection_config)
detection_task.log.setLevel(getattr(lsst.log, loglevel))
deblend_config = SourceDeblendConfig()
deblend_task = SourceDeblendTask(config=deblend_config, schema=schema)
deblend_task.log.setLevel(getattr(lsst.log, loglevel))
# Detect objects
table = afw_table.SourceTable.make(schema)
result = detection_task.run(table, exp)
sources = result.sources
# run the deblender
deblend_task.run(exp, sources)
# Run on deblended images
noise_replacer_config = NoiseReplacerConfig()
footprints = {
record.getId(): (record.getParent(), record.getFootprint())
for record in result.sources
}
# This constructor will replace all detected pixels with noise in the
# image
replacer = NoiseReplacer(
noise_replacer_config,
exposure=exp,
footprints=footprints,
)
nbad = 0
ntry = 0
kept_sources = []
for record in result.sources:
# Skip parent objects where all children are inserted
if record.get('deblend_nChild') != 0:
continue
ntry += 1
# This will insert a single source into the image
replacer.insertSource(record.getId()) # Get the peak as before
# peak = record.getFootprint().getPeaks()[0]
# The bounding box will be for the parent object
# bbox = record.getFootprint().getBBox()
meas_task.callMeasure(record, exp)
# Remove object
replacer.removeSource(record.getId())
if record.getCentroidFlag():
nbad += 1
kept_sources.append(record)
# Insert all objects back into image
replacer.end()
if ntry > 0:
log.debug('nbad center: %d frac: %d' % (nbad, nbad / ntry))
nkeep = len(kept_sources)
ntot = len(result.sources)
log.debug('kept %d/%d non parents' % (nkeep, ntot))
return kept_sources