def addCalibColumns(self, catalog, dataRef):
"""Add columns with local calibration evaluated at each centroid
for backwards compatibility with old repos.
This exists for the purpose of converting old src catalogs
(which don't have the expected local calib columns) to Source Tables.
Parameters
----------
catalog: `afwTable.SourceCatalog`
catalog to which calib columns will be added
dataRef: `lsst.daf.persistence.ButlerDataRef
for fetching the calibs from disk.
Returns
-------
newCat: `afwTable.SourceCatalog`
Source Catalog with requested local calib columns
"""
mapper = afwTable.SchemaMapper(catalog.schema)
measureConfig = SingleFrameMeasurementTask.ConfigClass()
measureConfig.doReplaceWithNoise = False
# Just need the WCS or the PhotoCalib attached to an exposue
exposure = dataRef.get('calexp_sub',
bbox=lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Point2I(0, 0)))
mapper = afwTable.SchemaMapper(catalog.schema)
mapper.addMinimalSchema(catalog.schema, True)
schema = mapper.getOutputSchema()
exposureIdInfo = dataRef.get("expIdInfo")
measureConfig.plugins.names = []
if self.config.doApplyExternalSkyWcs:
plugin = 'base_LocalWcs'
if plugin in schema:
raise RuntimeError(
f"{plugin} already in src catalog. Set doApplyExternalSkyWcs=False"
)
else:
measureConfig.plugins.names.add(plugin)
if self.config.doApplyExternalPhotoCalib:
plugin = 'base_LocalPhotoCalib'
if plugin in schema:
raise RuntimeError(
f"{plugin} already in src catalog. Set doApplyExternalPhotoCalib=False"
)
else:
measureConfig.plugins.names.add(plugin)
measurement = SingleFrameMeasurementTask(config=measureConfig,
schema=schema)
newCat = afwTable.SourceCatalog(schema)
newCat.extend(catalog, mapper=mapper)
measurement.run(measCat=newCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
return newCat
def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection and measurement Tasks
#
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False
detectionTask = SourceDetectionTask(config=config, schema=schema)
config = SingleFrameMeasurementTask.ConfigClass()
# Use the minimum set of plugins required.
config.plugins.names.clear()
for plugin in [
"base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux",
"base_PixelFlags"
]:
config.plugins.names.add(plugin)
config.plugins["base_CircularApertureFlux"].radii = [7.0]
# Use of the PSF flux is hardcoded in secondMomentStarSelector
config.slots.psfFlux = "base_CircularApertureFlux_7_0"
measureTask = SingleFrameMeasurementTask(schema, config=config)
#
# Create the measurePsf task
#
config = MeasurePsfTask.ConfigClass()
psfDeterminer = config.psfDeterminer.apply()
psfDeterminer.config.sizeCellX = 128
psfDeterminer.config.sizeCellY = 128
psfDeterminer.config.spatialOrder = 1
psfDeterminer.config.nEigenComponents = 3
measurePsfTask = MeasurePsfTask(config=config, schema=schema)
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
sources = detectionTask.run(tab, exposure, sigma=2).sources
measureTask.measure(exposure, sources)
result = measurePsfTask.run(exposure, sources)
psf = result.psf
cellSet = result.cellSet
if display: # display on ds9 (see also --debug argparse option)
frame = 1
ds9.mtv(exposure, frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, frame=frame)
if s.get("calib.psf.candidate"):
ds9.dot('x', *xy, ctype=ds9.YELLOW, frame=frame)
if s.get("calib.psf.used"):
ds9.dot('o', *xy, size=4, ctype=ds9.RED, frame=frame)
def setUp(self):
config = SingleFrameMeasurementTask.ConfigClass()
config.slots.apFlux = 'base_CircularApertureFlux_12_0'
self.schema = afwTable.SourceTable.makeMinimalSchema()
self.measureSources = SingleFrameMeasurementTask(self.schema,
config=config)
bbox = afwGeom.BoxI(afwGeom.PointI(0, 0),
afwGeom.ExtentI(self.width, self.height))
self.cellSet = afwMath.SpatialCellSet(bbox, 100)
self.footprintSet = afwDetection.FootprintSet(
self.mi, afwDetection.Threshold(self.detectThresh), "DETECTED")
self.catalog = self.measure(self.footprintSet, self.exposure)
for source in self.catalog:
try:
cand = measAlg.makePsfCandidate(source, self.exposure)
self.cellSet.insertCandidate(cand)
except Exception as e:
print(e)
continue
def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection task
#
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
#
# And the measurement Task
#
config = SingleFrameMeasurementTask.ConfigClass()
config.algorithms.names = ["base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux"]
config.algorithms["base_CircularApertureFlux"].radii = [1, 2, 4, 8, 16] # pixels
config.slots.instFlux = None
config.slots.modelFlux = None
config.slots.psfFlux = None
algMetadata = dafBase.PropertyList()
measureTask = SingleFrameMeasurementTask(schema, algMetadata=algMetadata, config=config)
radii = algMetadata.get("base_CircularApertureFlux_radii")
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
result = detectionTask.run(tab, exposure)
sources = result.sources
print("Found %d sources (%d +ve, %d -ve)" % (len(sources), result.fpSets.numPos, result.fpSets.numNeg))
measureTask.run(sources, exposure)
if display: # display on ds9 (see also --debug argparse option)
frame = 1
ds9.mtv(exposure, frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags.negative") else ds9.GREEN, frame=frame)
ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame)
for i in range(s.get("flux.aperture.nProfile")):
ds9.dot('o', *xy, size=radii[i], ctype=ds9.YELLOW, frame=frame)
def testInclude(self):
schema = afwTable.SourceTable.makeMinimalSchema()
# Create the detection task
config = SourceDetectionTask.ConfigClass()
config.reEstimateBackground = False # Turn off so that background does not change from orig
detectionTask = SourceDetectionTask(config=config, schema=schema)
# Create the deblender Task
debConfig = measDeb.SourceDeblendConfig()
debTask = measDeb.SourceDeblendTask(schema, config=debConfig)
# Create the measurement Task
config = SingleFrameMeasurementTask.ConfigClass()
measureTask = SingleFrameMeasurementTask(schema, config=config)
# Create the output table
tab = afwTable.SourceTable.make(schema)
# Process the data
result = detectionTask.run(tab, self.calexp)
sources = result.sources
# Run the deblender
debTask.run(self.calexp, sources)
# Run the measurement task: this where the replace-with-noise occurs
measureTask.run(sources, self.calexp)
plotOnFailure = False
if display:
plotOnFailure = True
# The relative differences ranged from 0.02 to ~2. This rtol is somewhat
# random, but will certainly catch the pathology if it occurs.
self.assertFloatsAlmostEqual(
self.calexpOrig.getMaskedImage().getImage().getArray(),
self.calexp.getMaskedImage().getImage().getArray(),
rtol=1E-3,
printFailures=False,
plotOnFailure=plotOnFailure)
def setUp(self):
config = SingleFrameMeasurementTask.ConfigClass()
config.slots.apFlux = 'base_CircularApertureFlux_12_0'
self.schema = afwTable.SourceTable.makeMinimalSchema()
self.measureSources = SingleFrameMeasurementTask(self.schema,
config=config)
width, height = 110, 301
self.mi = afwImage.MaskedImageF(afwGeom.ExtentI(width, height))
self.mi.set(0)
sd = 3 # standard deviation of image
self.mi.getVariance().set(sd * sd)
self.mi.getMask().addMaskPlane("DETECTED")
self.ksize = 31 # size of desired kernel
sigma1 = 1.75
sigma2 = 2 * sigma1
self.exposure = afwImage.makeExposure(self.mi)
self.exposure.setPsf(
measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5 * sigma1, 1,
0.1))
cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
cdMatrix.shape = (2, 2)
wcs = afwGeom.makeSkyWcs(crpix=afwGeom.PointD(0, 0),
crval=afwGeom.SpherePoint(
0.0, 0.0, afwGeom.degrees),
cdMatrix=cdMatrix)
self.exposure.setWcs(wcs)
#
# Make a kernel with the exactly correct basis functions. Useful for debugging
#
basisKernelList = []
for sigma in (sigma1, sigma2):
basisKernel = afwMath.AnalyticKernel(
self.ksize, self.ksize,
afwMath.GaussianFunction2D(sigma, sigma))
basisImage = afwImage.ImageD(basisKernel.getDimensions())
basisKernel.computeImage(basisImage, True)
basisImage /= np.sum(basisImage.getArray())
if sigma == sigma1:
basisImage0 = basisImage
else:
basisImage -= basisImage0
basisKernelList.append(afwMath.FixedKernel(basisImage))
order = 1 # 1 => up to linear
spFunc = afwMath.PolynomialFunction2D(order)
exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc)
exactKernel.setSpatialParameters([[1.0, 0, 0],
[0.0, 0.5 * 1e-2, 0.2e-2]])
rand = afwMath.Random() # make these tests repeatable by setting seed
addNoise = True
if addNoise:
im = self.mi.getImage()
afwMath.randomGaussianImage(im, rand) # N(0, 1)
im *= sd # N(0, sd^2)
del im
xarr, yarr = [], []
for x, y in [
(20, 20),
(60, 20),
(30, 35),
(50, 50),
(20, 90),
(70, 160),
(25, 265),
(75, 275),
(85, 30),
(50, 120),
(70, 80),
(60, 210),
(20, 210),
]:
xarr.append(x)
yarr.append(y)
for x, y in zip(xarr, yarr):
dx = rand.uniform() - 0.5 # random (centered) offsets
dy = rand.uniform() - 0.5
k = exactKernel.getSpatialFunction(1)(
x, y) # functional variation of Kernel ...
b = (k * sigma1**2 / ((1 - k) * sigma2**2)
) # ... converted double Gaussian's "b"
#flux = 80000 - 20*x - 10*(y/float(height))**2
flux = 80000 * (1 + 0.1 * (rand.uniform() - 0.5))
I0 = flux * (1 + b) / (2 * np.pi * (sigma1**2 + b * sigma2**2))
for iy in range(y - self.ksize // 2, y + self.ksize // 2 + 1):
if iy < 0 or iy >= self.mi.getHeight():
continue
for ix in range(x - self.ksize // 2, x + self.ksize // 2 + 1):
if ix < 0 or ix >= self.mi.getWidth():
continue
I = I0 * psfVal(ix, iy, x + dx, y + dy, sigma1, sigma2, b)
Isample = rand.poisson(I) if addNoise else I
self.mi.getImage().set(
ix, iy,
self.mi.getImage().get(ix, iy) + Isample)
self.mi.getVariance().set(
ix, iy,
self.mi.getVariance().get(ix, iy) + I)
bbox = afwGeom.BoxI(afwGeom.PointI(0, 0),
afwGeom.ExtentI(width, height))
self.cellSet = afwMath.SpatialCellSet(bbox, 100)
self.footprintSet = afwDetection.FootprintSet(
self.mi, afwDetection.Threshold(100), "DETECTED")
self.catalog = self.measure(self.footprintSet, self.exposure)
for source in self.catalog:
try:
cand = measAlg.makePsfCandidate(source, self.exposure)
self.cellSet.insertCandidate(cand)
except Exception as e:
print(e)
continue
def testIsScarletPrimaryFlag(self):
"""Test detect_isPrimary column when scarlet is used as the deblender
"""
# We need a multiband coadd for scarlet,
# even though there is only one band
coadds = afwImage.MultibandExposure.fromExposures(["test"],
[self.exposure])
# Create a SkyMap with a tract that contains a portion of the image,
# subdivided into 3x3 patches
wcs = self.exposure.getWcs()
tractBBox = Box2I(Point2I(100, 100), Extent2I(900, 900))
skyMap = MockSkyMap([tractBBox], wcs, 3)
tractInfo = skyMap[0]
patchInfo = tractInfo[0, 0]
patchBBox = patchInfo.getInnerBBox()
schema = SourceCatalog.Table.makeMinimalSchema()
# Initialize the detection task
detectionTask = SourceDetectionTask(schema=schema)
# Initialize the fake source injection task
skyConfig = SkyObjectsTask.ConfigClass()
skySourcesTask = SkyObjectsTask(name="skySources", config=skyConfig)
schema.addField("merge_peak_sky", type="Flag")
# Initialize the deblender task
scarletConfig = ScarletDeblendTask.ConfigClass()
scarletConfig.maxIter = 20
scarletConfig.columnInheritance["merge_peak_sky"] = "merge_peak_sky"
deblendTask = ScarletDeblendTask(schema=schema, config=scarletConfig)
# We'll customize the configuration of measurement to just run the
# minimal number of plugins to make setPrimaryFlags work.
measureConfig = SingleFrameMeasurementTask.ConfigClass()
measureConfig.plugins.names = ["base_SdssCentroid", "base_SkyCoord"]
measureConfig.slots.psfFlux = None
measureConfig.slots.apFlux = None
measureConfig.slots.shape = None
measureConfig.slots.modelFlux = None
measureConfig.slots.calibFlux = None
measureConfig.slots.gaussianFlux = None
measureTask = SingleFrameMeasurementTask(config=measureConfig,
schema=schema)
primaryConfig = SetPrimaryFlagsTask.ConfigClass()
setPrimaryTask = SetPrimaryFlagsTask(config=primaryConfig,
schema=schema,
name="setPrimaryFlags",
isSingleFrame=False)
table = SourceCatalog.Table.make(schema)
# detect sources
detectionResult = detectionTask.run(table, coadds["test"])
catalog = detectionResult.sources
# add fake sources
skySources = skySourcesTask.run(mask=self.exposure.mask, seed=0)
for foot in skySources[:5]:
src = catalog.addNew()
src.setFootprint(foot)
src.set("merge_peak_sky", True)
# deblend
result = deblendTask.run(coadds, catalog)
# measure
measureTask.run(result["test"], self.exposure)
outputCat = result["test"]
# Set the primary flags
setPrimaryTask.run(outputCat,
skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo)
# There should be the same number of deblenedPrimary and
# deblendedModelPrimary sources,
# since they both have the same blended sources and only differ
# over which model to use for the isolated sources.
isPseudo = getPseudoSources(outputCat, primaryConfig.pseudoFilterList,
schema, setPrimaryTask.log)
self.assertEqual(
np.sum(outputCat["detect_isDeblendedSource"] & ~isPseudo),
np.sum(outputCat["detect_isDeblendedModelSource"]))
# Check that the sources contained in a tract are all marked appropriately
x = outputCat["slot_Centroid_x"]
y = outputCat["slot_Centroid_y"]
tractInner = tractBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isTractInner"],
tractInner)
# Check that the sources contained in a patch are all marked appropriately
patchInner = patchBBox.contains(x, y)
np.testing.assert_array_equal(outputCat["detect_isPatchInner"],
patchInner)
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["merge_peak_sky"])), 0)
# Check that sky objects have not been deblended
np.testing.assert_array_equal(
isPseudo, isPseudo & (outputCat["deblend_nChild"] == 0))
def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection task
#
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
#
# And the measurement Task
#
config = SingleFrameMeasurementTask.ConfigClass()
config.algorithms.names = [
"base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux"
]
config.algorithms["base_CircularApertureFlux"].radii = [
1, 2, 4, 8, 12, 16
] # pixels
config.slots.gaussianFlux = None
config.slots.modelFlux = None
config.slots.psfFlux = None
algMetadata = dafBase.PropertyList()
measureTask = SingleFrameMeasurementTask(schema,
algMetadata=algMetadata,
config=config)
radii = algMetadata.getArray("BASE_CIRCULARAPERTUREFLUX_RADII")
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
result = detectionTask.run(tab, exposure)
sources = result.sources
print("Found %d sources (%d +ve, %d -ve)" %
(len(sources), result.fpSets.numPos, result.fpSets.numNeg))
measureTask.run(sources, exposure)
if display: # display image (see also --debug argparse option)
afwDisplay.setDefaultMaskTransparency(75)
frame = 1
disp = afwDisplay.Display(frame=frame)
disp.mtv(exposure)
with disp.Buffering():
for s in sources:
xy = s.getCentroid()
disp.dot('+',
*xy,
ctype=afwDisplay.CYAN
if s.get("flags_negative") else afwDisplay.GREEN)
disp.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
for radius in radii:
disp.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
def run(display=False):
exposure = loadData()
schema = afwTable.SourceTable.makeMinimalSchema()
#
# Create the detection task
#
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
#
# And the measurement Task
#
config = SingleFrameMeasurementTask.ConfigClass()
config.plugins.names.clear()
for plugin in [
"base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux",
"base_GaussianFlux"
]:
config.plugins.names.add(plugin)
config.slots.psfFlux = None
config.slots.apFlux = "base_CircularApertureFlux_3_0"
measureTask = SingleFrameMeasurementTask(schema, config=config)
#
# Print the schema the configuration produced
#
print schema
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# Process the data
#
result = detectionTask.run(tab, exposure)
sources = result.sources
print "Found %d sources (%d +ve, %d -ve)" % (
len(sources), result.fpSets.numPos, result.fpSets.numNeg)
measureTask.run(sources, exposure)
if display: # display on ds9 (see also --debug argparse option)
frame = 1
ds9.mtv(exposure, frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot(
'+',
*xy,
ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN,
frame=frame)
ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame)
ds9.dot(
'o',
*xy,
size=config.plugins["base_CircularApertureFlux"].radii[0],
ctype=ds9.YELLOW,
frame=frame)
