def test_detection_stdev(self):
"""Test detection and measurement on an exposure with negative sources
for thresholdType="stdev".
"""
exposure, numX, numY = self._create_exposure()
if display:
disp = afwDisplay.Display(frame=1)
disp.mtv(exposure,
title=self._testMethodName + ": image with -ve sources")
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdPolarity = 'both'
detection = SourceDetectionTask(config=config, schema=schema)
algMetadata = dafBase.PropertyList()
measurement = SourceMeasurementTask(schema=schema,
algMetadata=algMetadata)
table = afwTable.SourceTable.make(schema)
detections = detection.run(table, exposure)
sources = detections.sources
fpSets = detections.fpSets
self.assertEqual(len(sources), numX * numY)
self.assertEqual(fpSets.numPos, numX * numY / 2)
self.assertEqual(fpSets.numNeg, numX * numY / 2)
measurement.run(sources, exposure)
nGoodCent = 0
nGoodShape = 0
for s in sources:
cent = s.getCentroid()
shape = s.getShape()
if cent[0] == cent[0] and cent[1] == cent[1]:
nGoodCent += 1
if (shape.getIxx() == shape.getIxx()
and shape.getIyy() == shape.getIyy()
and shape.getIxy() == shape.getIxy()):
nGoodShape += 1
if display:
xy = cent[0], cent[1]
disp.dot('+', *xy)
disp.dot(shape, *xy, ctype=afwDisplay.RED)
self.assertEqual(nGoodCent, numX * numY)
self.assertEqual(nGoodShape, numX * numY)
def test_deblend_task(self):
# Set the random seed so that the noise field is unaffected
np.random.seed(0)
# Test that executing the deblend task works
# In the future we can have more detailed tests,
# but for now this at least ensures that the task isn't broken
shape = (5, 31, 55)
coords = [(15, 25), (10, 30), (17, 38)]
amplitudes = [80, 60, 90]
result = initData(shape, coords, amplitudes)
targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result
B, Ny, Nx = shape
# Add some noise, otherwise the task will blow up due to
# zero variance
noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5)
images += noise
filters = "grizy"
_images = afwImage.MultibandMaskedImage.fromArrays(
filters, images.astype(np.float32), None, noise)
coadds = [
afwImage.Exposure(img, dtype=img.image.array.dtype)
for img in _images
]
coadds = afwImage.MultibandExposure.fromExposures(filters, coadds)
for b, coadd in enumerate(coadds):
coadd.setPsf(psfs[b])
schema = SourceCatalog.Table.makeMinimalSchema()
detectionTask = SourceDetectionTask(schema=schema)
config = ScarletDeblendTask.ConfigClass()
config.maxIter = 300
deblendTask = ScarletDeblendTask(schema=schema, config=config)
table = SourceCatalog.Table.make(schema)
detectionResult = detectionTask.run(table, coadds["r"])
catalog = detectionResult.sources
self.assertEqual(len(catalog), 1)
_, result = deblendTask.run(coadds, catalog)
def test_deblend_task(self):
# Set the random seed so that the noise field is unaffected
np.random.seed(0)
shape = (5, 100, 115)
coords = [
# blend
(15, 25),
(10, 30),
(17, 38),
# isolated source
(85, 90),
]
amplitudes = [
# blend
80,
60,
90,
# isolated source
20,
]
result = initData(shape, coords, amplitudes)
targetPsfImage, psfImages, images, channels, seds, morphs, targetPsf, psfs = result
B, Ny, Nx = shape
# Add some noise, otherwise the task will blow up due to
# zero variance
noise = 10 * (np.random.rand(*images.shape).astype(np.float32) - .5)
images += noise
filters = "grizy"
_images = afwImage.MultibandMaskedImage.fromArrays(
filters, images.astype(np.float32), None, noise)
coadds = [
afwImage.Exposure(img, dtype=img.image.array.dtype)
for img in _images
]
coadds = afwImage.MultibandExposure.fromExposures(filters, coadds)
for b, coadd in enumerate(coadds):
coadd.setPsf(psfs[b])
schema = SourceCatalog.Table.makeMinimalSchema()
detectionTask = SourceDetectionTask(schema=schema)
# Adjust config options to test skipping parents
config = ScarletDeblendTask.ConfigClass()
config.maxIter = 100
config.maxFootprintArea = 1000
config.maxNumberOfPeaks = 4
deblendTask = ScarletDeblendTask(schema=schema, config=config)
table = SourceCatalog.Table.make(schema)
detectionResult = detectionTask.run(table, coadds["r"])
catalog = detectionResult.sources
# Add a footprint that is too large
src = catalog.addNew()
halfLength = int(np.ceil(np.sqrt(config.maxFootprintArea) + 1))
ss = SpanSet.fromShape(halfLength, Stencil.BOX, offset=(50, 50))
bigfoot = Footprint(ss)
bigfoot.addPeak(50, 50, 100)
src.setFootprint(bigfoot)
# Add a footprint with too many peaks
src = catalog.addNew()
ss = SpanSet.fromShape(10, Stencil.BOX, offset=(75, 20))
denseFoot = Footprint(ss)
for n in range(config.maxNumberOfPeaks + 1):
denseFoot.addPeak(70 + 2 * n, 15 + 2 * n, 10 * n)
src.setFootprint(denseFoot)
# Run the deblender
result = deblendTask.run(coadds, catalog)
# Make sure that the catalogs have the same sources in all bands,
# and check that band-independent columns are equal
bandIndependentColumns = [
"id",
"parent",
"deblend_nPeaks",
"deblend_nChild",
"deblend_peak_center_x",
"deblend_peak_center_y",
"deblend_runtime",
"deblend_iterations",
"deblend_logL",
"deblend_spectrumInitFlag",
"deblend_blendConvergenceFailedFlag",
]
self.assertEqual(len(filters), len(result))
ref = result[filters[0]]
for f in filters[1:]:
for col in bandIndependentColumns:
np.testing.assert_array_equal(result[f][col], ref[col])
# Check that other columns are consistent
for f, _catalog in result.items():
parents = _catalog[_catalog["parent"] == 0]
# Check that the number of deblended children is consistent
self.assertEqual(np.sum(_catalog["deblend_nChild"]),
len(_catalog) - len(parents))
for parent in parents:
children = _catalog[_catalog["parent"] == parent.get("id")]
# Check that nChild is set correctly
self.assertEqual(len(children), parent.get("deblend_nChild"))
# Check that parent columns are propagated to their children
for parentCol, childCol in config.columnInheritance.items():
np.testing.assert_array_equal(parent.get(parentCol),
children[childCol])
children = _catalog[_catalog["parent"] != 0]
for child in children:
fp = child.getFootprint()
img = heavyFootprintToImage(fp)
# Check that the flux at the center is correct.
# Note: this only works in this test image because the
# detected peak is in the same location as the scarlet peak.
# If the peak is shifted, the flux value will be correct
# but deblend_peak_center is not the correct location.
px = child.get("deblend_peak_center_x")
py = child.get("deblend_peak_center_y")
flux = img.image[Point2I(px, py)]
self.assertEqual(flux, child.get("deblend_peak_instFlux"))
# Check that the peak positions match the catalog entry
peaks = fp.getPeaks()
self.assertEqual(px, peaks[0].getIx())
self.assertEqual(py, peaks[0].getIy())
# Check that all sources have the correct number of peaks
for src in _catalog:
fp = src.getFootprint()
self.assertEqual(len(fp.peaks), src.get("deblend_nPeaks"))
# Check that only the large foorprint was flagged as too big
largeFootprint = np.zeros(len(_catalog), dtype=bool)
largeFootprint[2] = True
np.testing.assert_array_equal(largeFootprint,
_catalog["deblend_parentTooBig"])
# Check that only the dense foorprint was flagged as too dense
denseFootprint = np.zeros(len(_catalog), dtype=bool)
denseFootprint[3] = True
np.testing.assert_array_equal(denseFootprint,
_catalog["deblend_tooManyPeaks"])
# Check that only the appropriate parents were skipped
skipped = largeFootprint | denseFootprint
np.testing.assert_array_equal(skipped, _catalog["deblend_skipped"])
def testIsScarletPrimaryFlag(self):
"""Test detect_isPrimary column when scarlet is used as the deblender
"""
# We need a multiband coadd for scarlet,
# even though there is only one band
coadds = afwImage.MultibandExposure.fromExposures(["test"],
[self.exposure])
# Create a SkyMap with a tract that contains a portion of the image,
# subdivided into 3x3 patches
wcs = self.exposure.getWcs()
tractBBox = Box2I(Point2I(100, 100), Extent2I(900, 900))
skyMap = MockSkyMap([tractBBox], wcs, 3)
tractInfo = skyMap[0]
patchInfo = tractInfo[0, 0]
patchBBox = patchInfo.getInnerBBox()
schema = SourceCatalog.Table.makeMinimalSchema()
# Initialize the detection task
detectionTask = SourceDetectionTask(schema=schema)
# Initialize the fake source injection task
skyConfig = SkyObjectsTask.ConfigClass()
skySourcesTask = SkyObjectsTask(name="skySources", config=skyConfig)
schema.addField("merge_peak_sky", type="Flag")
# Initialize the deblender task
scarletConfig = ScarletDeblendTask.ConfigClass()
scarletConfig.maxIter = 20
scarletConfig.columnInheritance["merge_peak_sky"] = "merge_peak_sky"
deblendTask = ScarletDeblendTask(schema=schema, config=scarletConfig)
# We'll customize the configuration of measurement to just run the
# minimal number of plugins to make setPrimaryFlags work.
measureConfig = SingleFrameMeasurementTask.ConfigClass()
measureConfig.plugins.names = ["base_SdssCentroid", "base_SkyCoord"]
measureConfig.slots.psfFlux = None
measureConfig.slots.apFlux = None
measureConfig.slots.shape = None
measureConfig.slots.modelFlux = None
measureConfig.slots.calibFlux = None
measureConfig.slots.gaussianFlux = None
measureTask = SingleFrameMeasurementTask(config=measureConfig,
schema=schema)
primaryConfig = SetPrimaryFlagsTask.ConfigClass()
setPrimaryTask = SetPrimaryFlagsTask(config=primaryConfig,
schema=schema,
name="setPrimaryFlags",
isSingleFrame=False)
table = SourceCatalog.Table.make(schema)
# detect sources
detectionResult = detectionTask.run(table, coadds["test"])
catalog = detectionResult.sources
# add fake sources
skySources = skySourcesTask.run(mask=self.exposure.mask, seed=0)
for foot in skySources[:5]:
src = catalog.addNew()
src.setFootprint(foot)
src.set("merge_peak_sky", True)
# deblend
result = deblendTask.run(coadds, catalog)
# measure
measureTask.run(result["test"], self.exposure)
outputCat = result["test"]
# Set the primary flags
setPrimaryTask.run(outputCat,
skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo)
# There should be the same number of deblenedPrimary and
# deblendedModelPrimary sources,
# since they both have the same blended sources and only differ
# over which model to use for the isolated sources.
isPseudo = getPseudoSources(outputCat, primaryConfig.pseudoFilterList,
schema, setPrimaryTask.log)
self.assertEqual(
np.sum(outputCat["detect_isDeblendedSource"] & ~isPseudo),
np.sum(outputCat["detect_isDeblendedModelSource"]))
# Check that the sources contained in a tract are all marked appropriately
x = outputCat["slot_Centroid_x"]
y = outputCat["slot_Centroid_y"]
tractInner = tractBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isTractInner"],
tractInner)
# Check that the sources contained in a patch are all marked appropriately
patchInner = patchBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isPatchInner"],
patchInner)
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["merge_peak_sky"])), 0)
# Check that sky objects have not been deblended
np.testing.assert_array_equal(
isPseudo, isPseudo & (outputCat["deblend_nChild"] == 0))
def testBasics(self):
bbox = lsst.geom.Box2I(lsst.geom.Point2I(256, 100),
lsst.geom.Extent2I(128, 127))
minCounts = 2000
maxCounts = 20000
starSigma = 1.5
numX = 4
numY = 4
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11
sky = 2000
addPoissonNoise = True
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=addPoissonNoise)
if display:
disp = afwDisplay.Display(frame=1)
disp.mtv(exposure,
title=self._testMethodName + ": image with -ve sources")
schema = afwTable.SourceTable.makeMinimalSchema()
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
config.thresholdPolarity = 'both'
detection = SourceDetectionTask(config=config, schema=schema)
algMetadata = dafBase.PropertyList()
measurement = SourceMeasurementTask(schema=schema,
algMetadata=algMetadata)
table = afwTable.SourceTable.make(schema)
detections = detection.run(table, exposure)
sources = detections.sources
fpSets = detections.fpSets
self.assertEqual(len(sources), numX * numY)
self.assertEqual(fpSets.numPos, numX * numY / 2)
self.assertEqual(fpSets.numNeg, numX * numY / 2)
measurement.run(sources, exposure)
nGoodCent = 0
nGoodShape = 0
for s in sources:
cent = s.getCentroid()
shape = s.getShape()
if cent[0] == cent[0] and cent[1] == cent[1]:
nGoodCent += 1
if (shape.getIxx() == shape.getIxx()
and shape.getIyy() == shape.getIyy()
and shape.getIxy() == shape.getIxy()):
nGoodShape += 1
if display:
xy = cent[0], cent[1]
disp.dot('+', *xy)
disp.dot(shape, *xy, ctype=afwDisplay.RED)
self.assertEqual(nGoodCent, numX * numY)
self.assertEqual(nGoodShape, numX * numY)
def main(args):
epoch = args.epoch
hdus = pyfits.open("%s/ground/constant/epoch_catalog-%s-0.fits"%(args.type, args.epoch))
cat = hdus[1]
exp = afwImage.ExposureF("%s/ground/constant/image-%s-0.fits"%(args.type, args.epoch))
psfDict = None
if not os.path.isdir(args.psfs):
if not os.path.exists(args.psfs):
print "Psf location: %s does not exist."%args.psfs
sys.exit(1)
psfHdfs = pyfits.open(args.psfs)
psfDict = {}
for i in range(len(psfHdfs)):
psf_number = psfHdfs[i].header.get("PSF_NO")
if psf_number == None:
psfDict[i+1] = i
else:
psfDict[psf_number] = i
else:
psfFormat = args.psfs + "/psfs_%d.fits"
i_psf_number = getIndex("psf_number", cat)
i_index = getIndex("index", cat)
i_xmax = getIndex("xmax", cat)
i_ymax = getIndex("ymax", cat)
i_xmin = getIndex("xmin", cat)
i_ymin = getIndex("ymin", cat)
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("centroid_x", type=float)
schema.addField("centroid_y", type=float)
control = shape.HsmShapeBjControl()
alg = shape.HsmShapeBjAlgorithm
#control = measBase.SdssShapeControl()
#alg = measBase.SdssShapeAlgorithm
plugin = alg(control, "", schema)
success = 0
failed = 0
count = 0
startTime = time.time()
for item in cat.data:
bbox = afwGeom.Box2I(
afwGeom.Point2I(item[i_xmin], item[i_ymin]),
afwGeom.Point2I(item[i_xmax], item[i_ymax])
)
subexp = afwImage.ExposureF(exp, bbox)
#subarray = subexp.getMaskedImage().getImage().getArray()
#subarray = numpy.repeat(numpy.repeat(subarray,2, axis=0), 2, axis=1)
#subexp = afwImage.ExposureF(afwImage.MaskedImageF(afwImage.ImageF(subarray)))
if psfDict:
psf_number = item[i_psf_number]
i = psfDict[psf_number]
data = psfHdfs[i].data.astype(numpy.float64)
else:
psfFile = psfFormat % item[i_psf_number]
psfImage = afwImage.ImageF(psfFile)
data = psfImage.getArray().astype(numpy.float64)
kernel = afwMath.FixedKernel(afwImage.ImageD(data))
psf = lsst.meas.algorithms.KernelPsf(kernel)
subexp.setPsf(psf)
subexp.setXY0(afwGeom.Point2I(0,0))
detection = SourceDetectionTask()
table = afwTable.SourceTable.make(schema)
detections = detection.run(table, subexp)
detections.sources.defineCentroid("centroid")
count = count+1
if len(detections.sources) == 0:
print "ITEM %d, no sources, "%count, bbox
continue
record = detections.sources[0]
record.setFootprint(detections.fpSets.positive.getFootprints()[0])
cen = record.getFootprint().getPeaks()[0].getCentroid()
record.set('centroid_x', cen.getX())
record.set('centroid_y', cen.getY())
try:
plugin.measure(record, subexp)
print item[i_index], record.get("_e1"), record.get("_e2")
success = success+1
except lsst.pex.exceptions.RuntimeError as e:
print item[i_index], e.message
failed = failed + 1
except Exception as e:
print "FAILED: ", item[i_index], e.message
failed = failed + 1
print "elapsed time = ", time.time()-startTime
print "success = ", success, ", failed = ", failed
def detect_and_deblend(*, exp, log):
log = lsst.log.Log.getLogger("LSSTMEDSifier")
thresh = 5.0
loglevel = 'INFO'
# This schema holds all the measurements that will be run within the
# stack It needs to be constructed before running anything and passed
# to algorithms that make additional measurents.
schema = afw_table.SourceTable.makeMinimalSchema()
# Setup algorithms to run
meas_config = SingleFrameMeasurementConfig()
meas_config.plugins.names = [
"base_SdssCentroid",
"base_PsfFlux",
"base_SkyCoord",
# "modelfit_ShapeletPsfApprox",
"modelfit_DoubleShapeletPsfApprox",
"modelfit_CModel",
# "base_SdssShape",
# "base_LocalBackground",
]
# set these slots to none because we aren't running these algorithms
meas_config.slots.apFlux = None
meas_config.slots.gaussianFlux = None
meas_config.slots.calibFlux = None
meas_config.slots.modelFlux = None
# goes with SdssShape above
meas_config.slots.shape = None
# fix odd issue where it things things are near the edge
meas_config.plugins['base_SdssCentroid'].binmax = 1
meas_task = SingleFrameMeasurementTask(
config=meas_config,
schema=schema,
)
# setup detection config
detection_config = SourceDetectionConfig()
detection_config.reEstimateBackground = False
detection_config.thresholdValue = thresh
detection_task = SourceDetectionTask(config=detection_config)
detection_task.log.setLevel(getattr(lsst.log, loglevel))
deblend_config = SourceDeblendConfig()
deblend_task = SourceDeblendTask(config=deblend_config, schema=schema)
deblend_task.log.setLevel(getattr(lsst.log, loglevel))
# Detect objects
table = afw_table.SourceTable.make(schema)
result = detection_task.run(table, exp)
sources = result.sources
# run the deblender
deblend_task.run(exp, sources)
# Run on deblended images
noise_replacer_config = NoiseReplacerConfig()
footprints = {
record.getId(): (record.getParent(), record.getFootprint())
for record in result.sources
}
# This constructor will replace all detected pixels with noise in the
# image
replacer = NoiseReplacer(
noise_replacer_config,
exposure=exp,
footprints=footprints,
)
nbad = 0
ntry = 0
kept_sources = []
for record in result.sources:
# Skip parent objects where all children are inserted
if record.get('deblend_nChild') != 0:
continue
ntry += 1
# This will insert a single source into the image
replacer.insertSource(record.getId()) # Get the peak as before
# peak = record.getFootprint().getPeaks()[0]
# The bounding box will be for the parent object
# bbox = record.getFootprint().getBBox()
meas_task.callMeasure(record, exp)
# Remove object
replacer.removeSource(record.getId())
if record.getCentroidFlag():
nbad += 1
kept_sources.append(record)
# Insert all objects back into image
replacer.end()
if ntry > 0:
log.debug('nbad center: %d frac: %d' % (nbad, nbad / ntry))
nkeep = len(kept_sources)
ntot = len(result.sources)
log.debug('kept %d/%d non parents' % (nkeep, ntot))
return kept_sources
