def makeSourceCat(self, distortedWcs):
"""Make a source catalog by reading the position reference stars and distorting the positions
"""
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=distortedWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
sourceCat.reserve(len(refCat))
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey) / 100)
return sourceCat
def doDetection(exp,
threshold=5.0,
thresholdType='pixel_stdev',
thresholdPolarity='positive',
doSmooth=True,
doMeasure=True,
asDF=False):
# Modeled from meas_algorithms/tests/testMeasure.py
schema = afwTable.SourceTable.makeMinimalSchema()
config = measAlg.SourceDetectionTask.ConfigClass()
config.thresholdPolarity = thresholdPolarity
config.reEstimateBackground = False
config.thresholdValue = threshold
config.thresholdType = thresholdType
detectionTask = measAlg.SourceDetectionTask(config=config, schema=schema)
detectionTask.log.setLevel(log_level)
# Do measurement too, so we can get x- and y-coord centroids
config = measBase.SingleFrameMeasurementTask.ConfigClass()
# Use the minimum set of plugins required.
config.plugins = [
"base_CircularApertureFlux",
"base_PixelFlags",
"base_SkyCoord",
"base_PsfFlux",
"base_GaussianCentroid",
"base_GaussianFlux",
"base_PeakLikelihoodFlux",
"base_PeakCentroid",
"base_SdssCentroid",
"base_SdssShape",
"base_NaiveCentroid",
#"ip_diffim_NaiveDipoleCentroid",
#"ip_diffim_NaiveDipoleFlux",
"ip_diffim_PsfDipoleFlux",
"ip_diffim_ClassificationDipole",
]
config.slots.centroid = "base_GaussianCentroid" #"ip_diffim_NaiveDipoleCentroid"
#config.plugins["base_CircularApertureFlux"].radii = [3.0, 7.0, 15.0, 25.0]
#config.slots.psfFlux = "base_CircularApertureFlux_7_0" # Use of the PSF flux is hardcoded in secondMomentStarSelector
config.slots.calibFlux = None
config.slots.modelFlux = None
config.slots.instFlux = None
config.slots.shape = "base_SdssShape"
config.doReplaceWithNoise = False
measureTask = measBase.SingleFrameMeasurementTask(schema, config=config)
measureTask.log.setLevel(log_level)
table = afwTable.SourceTable.make(schema)
sources = detectionTask.run(table, exp, doSmooth=doSmooth).sources
measureTask.measure(sources, exposure=exp)
if asDF:
sources = catalogToDF(
sources
) #pd.DataFrame({col: sources.columns[col] for col in sources.schema.getNames()})
return sources
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = self.makeSourceCat(self.tanWcs, sourceSchema=sourceSchema)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def testEMPlugin(self):
msConfig = self.makeConfig()
msConfig.plugins[self.algName].nGauss = 2
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema=schema,
config=msConfig)
exposure = afwImage.ExposureF(os.path.join(self.dataDir, "exp.fits"))
# this is a double gaussian with a .7/.3 ratio of inner to outer
# we expect ixx = iyy = sigma*sigma
psf = makePsf(67, 4.0, .7, sigma2=10.0, mult2=.3)
exposure.setPsf(psf)
source = runMeasure(task, schema, exposure)
# Be sure there were no failures
self.assertEqual(source.get(self.algName + "_flag"), False)
self.assertEqual(source.get(self.algName + "_flag_rangeError"), False)
self.assertEqual(source.get(self.algName + "_flag_maxIters"), False)
self.assertEqual(source.get(self.algName + "_flag_noPsf"), False)
self.msfKey = lsst.shapelet.MultiShapeletFunctionKey(
schema[self.algName], lsst.shapelet.HERMITE)
# check the two component result to be sure it is close to the input PSF
# we don't control the order of EmPsfApprox, so order by size.
msf = source.get(self.msfKey)
components = msf.getComponents()
self.assertEqual(len(components), 2)
comp0 = components[0]
comp1 = components[1]
flux0 = comp0.getCoefficients()[0]
flux1 = comp1.getCoefficients()[0]
if flux0 < flux1:
temp = comp1
comp1 = comp0
comp0 = temp
# We are not looking for really close matches in this unit test, which is why
# the tolerances are set rather large. Really just a check that we are getting
# some kind of reasonable value for the fit. A more quantitative test may be needed.
self.assertFloatsAlmostEqual(flux0 / flux1, 7.0 / 3.0, rtol=.05)
self.assertFloatsAlmostEqual(comp0.getEllipse().getCore().getIxx(),
16.0,
rtol=.05)
self.assertFloatsAlmostEqual(comp0.getEllipse().getCore().getIyy(),
16.0,
rtol=.05)
self.assertFloatsAlmostEqual(comp0.getEllipse().getCore().getIxy(),
0.0,
atol=.1)
self.assertFloatsAlmostEqual(comp1.getEllipse().getCore().getIxx(),
100.0,
rtol=.05)
self.assertFloatsAlmostEqual(comp1.getEllipse().getCore().getIyy(),
100.0,
rtol=.05)
self.assertFloatsAlmostEqual(comp1.getEllipse().getCore().getIxy(),
0.0,
atol=.1)
def testSingleFrameMeasurementTransform(self):
"""Test applying a transform task to the results of single frame measurement."""
schema = afwTable.SourceTable.makeMinimalSchema()
sfmConfig = measBase.SingleFrameMeasurementConfig(plugins=[PLUGIN_NAME])
# We don't use slots in this test
for key in sfmConfig.slots:
setattr(sfmConfig.slots, key, None)
sfmTask = measBase.SingleFrameMeasurementTask(schema, config=sfmConfig)
transformTask = TransformTask(measConfig=sfmConfig,
inputSchema=sfmTask.schema, outputDataset="src")
self._transformAndCheck(sfmConfig, sfmTask.schema, transformTask)
def testFootprintsMeasure(self):
"""Check that we can measure the objects in a detectionSet"""
xcentroid = [10.0, 14.0, 9.0]
ycentroid = [8.0, 11.5061728, 14.0]
flux = [51.0, 101.0, 20.0]
ds = afwDetection.FootprintSet(self.mi, afwDetection.Threshold(10), "DETECTED")
if display:
ds9.mtv(self.mi, frame=0)
ds9.mtv(self.mi.getVariance(), frame=1)
measureSourcesConfig = measBase.SingleFrameMeasurementConfig()
measureSourcesConfig.algorithms["base_CircularApertureFlux"].radii = [3.0]
measureSourcesConfig.algorithms.names = ["base_NaiveCentroid", "base_SdssShape", "base_PsfFlux",
"base_CircularApertureFlux"]
measureSourcesConfig.slots.centroid = "base_NaiveCentroid"
measureSourcesConfig.slots.psfFlux = "base_PsfFlux"
measureSourcesConfig.slots.apFlux = "base_CircularApertureFlux_3_0"
measureSourcesConfig.slots.modelFlux = None
measureSourcesConfig.slots.instFlux = None
measureSourcesConfig.slots.calibFlux = None
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=measureSourcesConfig)
measCat = afwTable.SourceCatalog(schema)
# now run the SFM task with the test plugin
sigma = 1e-10
psf = algorithms.DoubleGaussianPsf(11, 11, sigma) # i.e. a single pixel
self.exposure.setPsf(psf)
task.run(measCat, self.exposure)
for i, source in enumerate(measCat):
xc, yc = source.getX() - self.mi.getX0(), source.getY() - self.mi.getY0()
if display:
ds9.dot("+", xc, yc)
self.assertAlmostEqual(source.getX(), xcentroid[i], 6)
self.assertAlmostEqual(source.getY(), ycentroid[i], 6)
self.assertEqual(source.getApFlux(), flux[i])
# 29 pixels in 3pixel circular ap.
self.assertAlmostEqual(source.getApFluxErr(), math.sqrt(29), 6)
# We're using a delta-function PSF, so the psfFlux should be the pixel under the centroid,
# iff the object's centred in the pixel
if xc == int(xc) and yc == int(yc):
self.assertAlmostEqual(source.getPsfFlux(),
self.exposure.getMaskedImage().getImage().get(int(xc + 0.5),
int(yc + 0.5)))
self.assertAlmostEqual(source.getPsfFluxErr(),
self.exposure.getMaskedImage().getVariance().get(int(xc + 0.5),
int(yc + 0.5)))
def testMissingPsf(self):
msConfig = self.makeConfig()
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema=schema,
config=msConfig)
exposure = afwImage.ExposureF(os.path.join(self.dataDir, "exp.fits"))
# Strip the psf
exposure.setPsf(None)
source = runMeasure(task, schema, exposure)
self.assertEqual(source.get(self.algName + "_flag"), True)
self.assertEqual(source.get(self.algName + "_flag_rangeError"), False)
self.assertEqual(source.get(self.algName + "_flag_maxIters"), False)
self.assertEqual(source.get(self.algName + "_flag_noPsf"), True)
def setUp(self):
im = afwImage.ImageF(self.monetFile("small.fits"))
self.mi = afwImage.MaskedImageF(im, afwImage.Mask(im.getDimensions()),
afwImage.ImageF(im.getDimensions()))
self.ds = afwDetection.FootprintSet(self.mi,
afwDetection.Threshold(100))
if display:
ds9.mtv(self.mi.getImage())
ds9.erase()
for foot in self.ds.getFootprints():
bbox = foot.getBBox()
x0, y0 = bbox.getMinX(), bbox.getMinY()
x1, y1 = bbox.getMaxX(), bbox.getMaxY()
xc = (x0 + x1) / 2.0
yc = (y0 + y1) / 2.0
if display:
ds9.dot("+", xc, yc, ctype=ds9.BLUE)
if False:
x0 -= 0.5
y0 -= 0.5
x1 += 0.5
y1 += 0.5
ds9.line([(x0, y0), (x1, y0), (x1, y1), (x0, y1),
(x0, y0)],
ctype=ds9.RED)
msConfig = measBase.SingleFrameMeasurementConfig()
msConfig.algorithms.names = ["base_GaussianCentroid"]
msConfig.plugins["base_GaussianCentroid"].doFootprintCheck = False
msConfig.slots.centroid = "base_GaussianCentroid"
msConfig.slots.shape = None
msConfig.slots.apFlux = None
msConfig.slots.modelFlux = None
msConfig.slots.psfFlux = None
msConfig.slots.instFlux = None
msConfig.slots.calibFlux = None
schema = afwTable.SourceTable.makeMinimalSchema()
self.task = measBase.SingleFrameMeasurementTask(schema,
config=msConfig)
self.ssMeasured = afwTable.SourceCatalog(schema)
self.ssMeasured.table.defineCentroid("base_GaussianCentroid")
self.ssTruth = afwTable.SourceCatalog(schema)
self.ssTruth.table.defineCentroid("base_GaussianCentroid")
self.readTruth(self.monetFile("positions.dat-original"))
def testUnexpectedError(self):
msConfig = self.makeConfig()
msConfig.plugins[self.algName].nGauss = 2
# This generates an unexpected exception in Erin's code
msConfig.plugins[self.algName].nTries = 0
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema=schema,
config=msConfig)
exposure = afwImage.ExposureF(os.path.join(self.dataDir, "exp.fits"))
psf = makePsf(67, 4.0, .7, 12.0, .3)
exposure.setPsf(psf)
source = runMeasure(task, schema, exposure)
self.assertEqual(source.get(self.algName + "_flag"), True)
self.assertEqual(source.get(self.algName + "_flag_rangeError"), False)
self.assertEqual(source.get(self.algName + "_flag_maxIters"), False)
self.assertEqual(source.get(self.algName + "_flag_noPsf"), False)
def measureFree(exposure, center, msConfig):
"""Unforced measurement"""
schema = afwTable.SourceTable.makeMinimalSchema()
algMeta = PropertyList()
task = measBase.SingleFrameMeasurementTask(schema,
config=msConfig,
algMetadata=algMeta)
measCat = afwTable.SourceCatalog(schema)
source = measCat.addNew()
source.getTable().setMetadata(algMeta)
ss = afwDetection.FootprintSet(exposure.getMaskedImage(),
afwDetection.Threshold(0.1))
fp = ss.getFootprints()[0]
source.setFootprint(fp)
task.run(measCat, exposure)
return source
def testMaxIter(self):
msConfig = self.makeConfig()
msConfig.plugins[self.algName].nGauss = 2
msConfig.plugins[self.algName].tolerance = 1e-10
# we know the code can't fit this in one iteration
msConfig.plugins[self.algName].maxIters = 1
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema=schema,
config=msConfig)
exposure = afwImage.ExposureF(os.path.join(self.dataDir, "exp.fits"))
psf = makePsf(67, 4.0, .7, 12.0, .3)
exposure.setPsf(psf)
source = runMeasure(task, schema, exposure)
self.assertEqual(source.get(self.algName + "_flag"), True)
self.assertEqual(source.get(self.algName + "_flag_rangeError"), False)
self.assertEqual(source.get(self.algName + "_flag_maxIters"), True)
self.assertEqual(source.get(self.algName + "_flag_noPsf"), False)
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
distortedWcs = afwImage.DistortedTanWcs(self.tanWcs,
afwGeom.IdentityXYTransform())
self.exposure.setWcs(distortedWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=distortedWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceFluxKey = sourceSchema["slot_ApFlux_flux"].asKey()
sourceFluxSigmaKey = sourceSchema["slot_ApFlux_fluxSigma"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceFluxKey, refObj.get(refFluxRKey))
src.set(sourceFluxSigmaKey, refObj.get(refFluxRKey) / 100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometryUsed'):
count += 1
self.assertEqual(count, len(results.matches))
def testPsfFlux(self):
"""Test that fluxes are measured correctly."""
#
# Total flux in image
#
flux = afwMath.makeStatistics(self.exp.getMaskedImage(),
afwMath.SUM).getValue()
self.assertAlmostEqual(flux / self.instFlux, 1.0)
#
# Various algorithms
#
rad = 10.0
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("centroid_x", type=float)
schema.addField("centroid_y", type=float)
schema.addField("centroid_flag", type='Flag')
sfm_config = measBase.SingleFrameMeasurementConfig()
sfm_config.doReplaceWithNoise = False
sfm_config.plugins = ["base_CircularApertureFlux", "base_PsfFlux"]
sfm_config.slots.centroid = "centroid"
sfm_config.slots.shape = None
sfm_config.slots.psfFlux = None
sfm_config.slots.gaussianFlux = None
sfm_config.slots.apFlux = None
sfm_config.slots.modelFlux = None
sfm_config.slots.calibFlux = None
sfm_config.plugins["base_SdssShape"].maxShift = 10.0
sfm_config.plugins["base_CircularApertureFlux"].radii = [rad]
task = measBase.SingleFrameMeasurementTask(schema, config=sfm_config)
measCat = afwTable.SourceCatalog(schema)
source = measCat.addNew()
source.set("centroid_x", self.xc)
source.set("centroid_y", self.yc)
task.run(measCat, self.exp)
for algName in ["base_CircularApertureFlux_10_0", "base_PsfFlux"]:
instFlux = source.get(algName + "_instFlux")
flag = source.get(algName + "_flag")
self.assertEqual(flag, False)
self.assertAlmostEqual(
instFlux / self.instFlux, 1.0, 4,
"Measuring with %s: %g v. %g" %
(algName, instFlux, self.instFlux))
def mySetup(self):
msConfig = measBase.SingleFrameMeasurementConfig()
msConfig.algorithms.names = ["base_SdssCentroid"]
msConfig.slots.centroid = "base_SdssCentroid"
msConfig.slots.shape = None
msConfig.slots.apFlux = None
msConfig.slots.modelFlux = None
msConfig.slots.psfFlux = None
msConfig.slots.instFlux = None
msConfig.slots.calibFlux = None
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=msConfig)
measCat = afwTable.SourceCatalog(schema)
source = measCat.addNew()
fp = afwDetection.Footprint(self.exp.getBBox(afwImage.LOCAL))
fp.addPeak(50, 50, 1000.0)
source.setFootprint(fp)
# Then run the default SFM task. Results not checked
task.run(measCat, self.exp)
return source
def makeSourceCat(self, wcs, sourceSchema=None, doScatterCentroids=False):
"""Make a source catalog by reading the position reference stars using
the proviced WCS.
Optionally provide a schema for the source catalog (to allow
AstrometryTask in the test methods to update it with the
"calib_astrometry_used" flag). Otherwise, a minimal SourceTable
schema will be created.
Optionally, via doScatterCentroids, add some scatter to the centroids
assiged to the source catalog (otherwise they will be identical to
those of the reference catalog).
"""
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=wcs,
filterName="r")
refCat = loadRes.refCat
if sourceSchema is None:
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCat.resize(len(refCat))
scatterFactor = 1.0
if doScatterCentroids:
np.random.seed(12345)
scatterFactor = np.random.uniform(0.999, 1.001, len(sourceCat))
sourceCat["slot_Centroid_x"] = scatterFactor * refCat["centroid_x"]
sourceCat["slot_Centroid_y"] = scatterFactor * refCat["centroid_y"]
sourceCat["slot_ApFlux_instFlux"] = refCat["r_flux"]
sourceCat["slot_ApFlux_instFluxErr"] = refCat["r_flux"] / 100
# Deliberately add some outliers to check that the magnitude
# outlier rejection code is being run.
sourceCat["slot_ApFlux_instFlux"][0:4] *= 1000.0
return sourceCat
def testRejectBlends(self):
"""Test the PcaPsfDeterminerTask blend removal."""
"""
We give it a single blended source, asking it to remove blends,
and check that it barfs in the expected way.
"""
psfDeterminerClass = measAlg.psfDeterminerRegistry["pca"]
config = psfDeterminerClass.ConfigClass()
config.doRejectBlends = True
psfDeterminer = psfDeterminerClass(config=config)
schema = afwTable.SourceTable.makeMinimalSchema()
# Use The single frame measurement task to populate the schema with standard keys
measBase.SingleFrameMeasurementTask(schema)
catalog = afwTable.SourceCatalog(schema)
source = catalog.addNew()
# Make the source blended, with necessary information to calculate pca
spanShift = afwGeom.Point2I(54, 123)
spans = afwGeom.SpanSet.fromShape(6, offset=spanShift)
foot = afwDetection.Footprint(spans, self.exposure.getBBox())
foot.addPeak(45, 123, 6)
foot.addPeak(47, 126, 5)
source.setFootprint(foot)
centerKey = afwTable.Point2DKey(source.schema['slot_Centroid'])
shapeKey = afwTable.QuadrupoleKey(schema['slot_Shape'])
source.set(centerKey, afwGeom.Point2D(46, 124))
source.set(shapeKey, afwGeom.Quadrupole(1.1, 2.2, 1))
candidates = [measAlg.makePsfCandidate(source, self.exposure)]
metadata = dafBase.PropertyList()
with self.assertRaises(RuntimeError) as cm:
psfDeterminer.determinePsf(self.exposure, candidates, metadata)
self.assertEqual(str(cm.exception),
"All PSF candidates removed as blends")
def showPsfCandidates(exposure, psfCellSet, psf=None, display=None, normalize=True, showBadCandidates=True,
fitBasisComponents=False, variance=None, chi=None):
"""Display the PSF candidates.
If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs
(and residuals)
If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
If fitBasisComponents is true, also find the best linear combination of the PSF's components
(if they exist)
"""
if not display:
display = afwDisplay.Display()
if chi is None:
if variance is not None: # old name for chi
chi = variance
#
# Show us the ccandidates
#
mos = displayUtils.Mosaic()
#
candidateCenters = []
candidateCentersBad = []
candidateIndex = 0
for cell in psfCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
rchi2 = cand.getChi2()
if rchi2 > 1e100:
rchi2 = numpy.nan
if not showBadCandidates and cand.isBad():
continue
if psf:
im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x")
try:
im = cand.getMaskedImage() # copy of this object's image
xc, yc = cand.getXCenter(), cand.getYCenter()
margin = 0 if True else 5
w, h = im.getDimensions()
bbox = lsst.geom.BoxI(lsst.geom.PointI(margin, margin), im.getDimensions())
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue())
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim.getVariance().set(stdev**2)
bim.assign(im, bbox)
im = bim
xc += margin
yc += margin
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
except Exception:
continue
if not variance:
im_resid.append(im.Factory(im, True))
if True: # tweak up centroids
mi = im
psfIm = mi.getImage()
config = measBase.SingleFrameMeasurementTask.ConfigClass()
config.slots.centroid = "base_SdssCentroid"
schema = afwTable.SourceTable.makeMinimalSchema()
measureSources = measBase.SingleFrameMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
extra = 10 # enough margin to run the sdss centroider
miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra)
miBig[extra:-extra, extra:-extra, afwImage.LOCAL] = mi
miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra)
mi = miBig
del miBig
exp = afwImage.makeExposure(mi)
exp.setPsf(psf)
footprintSet = afwDet.FootprintSet(mi,
afwDet.Threshold(0.5*numpy.max(psfIm.getArray())),
"DETECTED")
footprintSet.makeSources(catalog)
if len(catalog) == 0:
raise RuntimeError("Failed to detect any objects")
measureSources.run(catalog, exp)
if len(catalog) == 1:
source = catalog[0]
else: # more than one source; find the once closest to (xc, yc)
dmin = None # an invalid value to catch logic errors
for i, s in enumerate(catalog):
d = numpy.hypot(xc - s.getX(), yc - s.getY())
if i == 0 or d < dmin:
source, dmin = s, d
xc, yc = source.getCentroid()
# residuals using spatial model
try:
subtractPsf(psf, im, xc, yc)
except Exception:
continue
resid = im
if variance:
resid = resid.getImage()
var = im.getVariance()
var = var.Factory(var, True)
numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt
resid /= var
im_resid.append(resid)
# Fit the PSF components directly to the data (i.e. ignoring the spatial model)
if fitBasisComponents:
im = cand.getMaskedImage()
im = im.Factory(im, True)
im.setXY0(cand.getMaskedImage().getXY0())
try:
noSpatialKernel = psf.getKernel()
except Exception:
noSpatialKernel = None
if noSpatialKernel:
candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter())
fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter)
params = fit[0]
kernels = afwMath.KernelList(fit[1])
outputKernel = afwMath.LinearCombinationKernel(kernels, params)
outImage = afwImage.ImageD(outputKernel.getDimensions())
outputKernel.computeImage(outImage, False)
im -= outImage.convertF()
resid = im
if margin > 0:
bim = im.Factory(w + 2*margin, h + 2*margin)
afwMath.randomGaussianImage(bim.getImage(), afwMath.Random())
bim *= stdev
bim.assign(resid, bbox)
resid = bim
if variance:
resid = resid.getImage()
resid /= var
im_resid.append(resid)
im = im_resid.makeMosaic()
else:
im = cand.getMaskedImage()
if normalize:
im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
objId = splitId(cand.getSource().getId(), True)["objId"]
if psf:
lab = "%d chi^2 %.1f" % (objId, rchi2)
ctype = afwDisplay.RED if cand.isBad() else afwDisplay.GREEN
else:
lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfInstFlux())
ctype = afwDisplay.GREEN
mos.append(im, lab, ctype)
if False and numpy.isnan(rchi2):
display.mtv(cand.getMaskedImage().getImage(), title="showPsfCandidates: candidate")
print("amp", cand.getAmplitude())
im = cand.getMaskedImage()
center = (candidateIndex, xc - im.getX0(), yc - im.getY0())
candidateIndex += 1
if cand.isBad():
candidateCentersBad.append(center)
else:
candidateCenters.append(center)
if variance:
title = "chi(Psf fit)"
else:
title = "Stars & residuals"
mosaicImage = mos.makeMosaic(display=display, title=title)
with display.Buffering():
for centers, color in ((candidateCenters, afwDisplay.GREEN), (candidateCentersBad, afwDisplay.RED)):
for cen in centers:
bbox = mos.getBBox(cen[0])
display.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), ctype=color)
return mosaicImage
def run(self, dataRef, selectDataList=[]):
"""Draw randoms for a given patch
"""
# first test if the forced-src file exists
# do not process if the patch doesn't exist
try:
dataRef.get(self.config.coaddName + "Coadd_forced_src")
except:
self.log.info("No forced_src file found for %s. Skipping..." % (dataRef.dataId))
return
# verbose
self.log.info("Processing %s" % (dataRef.dataId))
# create a seed that depends on patch id
# so it is consistent among filters
if self.config.seed == -1:
p = [int(d) for d in dataRef.dataId["patch"].split(",") ]
numpy.random.seed(seed=dataRef.dataId["tract"]*10000+p[0]*10+ p[1])
else:
numpy.random.seed(seed=self.config.seed)
# compute sky mean and sky std_dev for this patch
# in 2" diameter apertures (~12 pixels x 0.17"/pixel)
# import source list for getting sky objects
sources = dataRef.get(self.config.coaddName + "Coadd_meas")
if True:
sky_apertures = sources['base_CircularApertureFlux_12_0_flux'][sources['merge_peak_sky']]
select = numpy.isfinite(sky_apertures)
sky_mean = numpy.mean(sky_apertures[select])
sky_std = numpy.std(sky_apertures[select])
# NOTE: to get 5-sigma limiting magnitudes:
# print -2.5*numpy.log10(5.0*sky_std/coadd.getCalib().getFluxMag0()[0])
else:
sky_mean = 0.0
sky_std = 0.0
# get coadd, coadd info and coadd psf object
coadd = dataRef.get(self.config.coaddName + "Coadd_calexp")
psf = coadd.getPsf()
var = coadd.getMaskedImage().getVariance().getArray()
skyInfo = self.getSkyInfo(dataRef)
# wcs and reference point (wrt tract)
# See http://hsca.ipmu.jp/hscsphinx_test/scripts/print_coord.html
# for coordinate routines.
wcs = coadd.getWcs()
xy0 = coadd.getXY0()
# dimension in pixels
dim = coadd.getDimensions()
# define measurement algorithms
# mostly copied from /data1a/ana/hscPipe5/Linux64/meas_base/5.3-hsc/tests/testInputCount.py
measureSourcesConfig = measBase.SingleFrameMeasurementConfig()
measureSourcesConfig.plugins.names = ['base_PixelFlags', 'base_PeakCentroid', 'base_InputCount', 'base_SdssShape']
measureSourcesConfig.slots.centroid = "base_PeakCentroid"
measureSourcesConfig.slots.psfFlux = None
measureSourcesConfig.slots.apFlux = None
measureSourcesConfig.slots.modelFlux = None
measureSourcesConfig.slots.instFlux = None
measureSourcesConfig.slots.calibFlux = None
measureSourcesConfig.slots.shape = None
# it seems it is still necessary to manually add the
# bright-star mask flag by hand
measureSourcesConfig.plugins['base_PixelFlags'].masksFpCenter.append("BRIGHT_OBJECT")
measureSourcesConfig.plugins['base_PixelFlags'].masksFpAnywhere.append("BRIGHT_OBJECT")
measureSourcesConfig.validate()
# add PSF shape
# sdssShape_psf = self.schema.addField("shape_sdss_psf", type="MomentsD", doc="PSF xx from SDSS algorithm", units="pixel")
# shape_sdss_psf = self.schema.addField("shape_sdss_psf", type="MomentsD", doc="PSF yy from SDSS algorithm", units="pixel")
# shape_sdss_psf = self.schema.addField("shape_sdss_psf", type="MomentsD", doc="PSF xy from SDSS algorithm", units="pixel")
# additional columns
# random number to adjust sky density
adjust_density = self.schema.addField("adjust_density", type=float, doc="Random number between [0:1] to adjust sky density", units='')
# sky mean and variance for the entire patch
sky_mean_key = self.schema.addField("sky_mean", type=float, doc="Mean of sky value in 2\" diamter apertures", units='count')
sky_std_key = self.schema.addField("sky_std", type=float, doc="Standard deviation of sky value in 2\" diamter apertures", units='count')
# pixel variance at random point position
pix_variance = self.schema.addField("pix_variance", type=float, doc="Pixel variance at random point position", units="flx^2")
# add healpix map value (if healpix map is given)
if self.depthMap.map is not None:
depth_key = self.schema.addField("isFullDepthColor", type="Flag", doc="True if full depth and full colors at point position", units='')
# task and output catalog
task = measBase.SingleFrameMeasurementTask(self.schema, config=measureSourcesConfig)
table = afwTable.SourceTable.make(self.schema, self.makeIdFactory(dataRef))
catalog = afwTable.SourceCatalog(table)
if self.config.N == -1:
# to output a constant random
# number density, first compute
# the area in degree
pixel_area = coadd.getWcs().getPixelScale().asDegrees()**2
area = pixel_area * dim[0] * dim[1]
N = self.iround(area*self.config.Nden*60.0*60.0)
else:
# fixed number if random points
N = self.config.N
# verbose
self.log.info("Drawing %d random points" % (N))
# loop over N random points
for i in range(N):
# for i in range(100):
# draw one random point
x = numpy.random.random()*(dim[0]-1)
y = numpy.random.random()*(dim[1]-1)
# get coordinates
radec = wcs.pixelToSky(afwGeom.Point2D(x + xy0.getX(), y + xy0.getY()))
xy = wcs.skyToPixel(radec)
# new record in table
record = catalog.addNew()
record.setCoord(radec)
# get PSF moments and evaluate size
#size_psf = 1.0
#try:
# shape_sdss_psf_val = psf.computeShape(afwGeom.Point2D(xy))
#except:
# pass
#else:
# record.set(shape_sdss_psf, shape_sdss_psf_val)
# size_psf = shape_sdss_psf_val.getDeterminantRadius()
# object has no footprint
radius = 0
spanset1 = SpanSet.fromShape(radius, stencil=Stencil.CIRCLE, offset=afwGeom.Point2I(xy))
foot = Footprint(spanset1)
foot.addPeak(xy[0], xy[1], 0.0)
record.setFootprint(foot)
# draw a number between 0 and 1 to adjust sky density
record.set(adjust_density, numpy.random.random())
# add sky properties
record.set(sky_mean_key, sky_mean)
record.set(sky_std_key, sky_std)
# add local (pixel) variance
record.set(pix_variance, float(var[self.iround(y), self.iround(x)]))
# required for setPrimaryFlags
record.set(catalog.getCentroidKey(), afwGeom.Point2D(xy))
# add healpix map value
if self.depthMap.map is not None:
mapIndex = healpy.pixelfunc.ang2pix(self.depthMap.nside, numpy.pi/2.0 - radec[1].asRadians(), radec[0].asRadians(), nest=self.depthMap.nest)
record.setFlag(depth_key, self.depthMap.map[mapIndex])
# run measurements
task.run(catalog, coadd)
self.setPrimaryFlags.run(catalog, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo, includeDeblend=False)
# write catalog
if self.config.fileOutName == "":
if self.config.dirOutName == "" :
fileOutName = dataRef.get(self.config.coaddName + "Coadd_forced_src_filename")[0].replace('forced_src', 'ran')
self.log.info("WARNING: the output file will be written in {0:s}.".format(fileOutName))
else:
fileOutName = "{0}/{1}/{2}/{3}/ran-{1}-{2}-{3}.fits".format(self.config.dirOutName,dataRef.dataId["filter"],dataRef.dataId["tract"],dataRef.dataId["patch"])
else:
fileOutName = self.config.fileOutName
self.mkdir_p(os.path.dirname(fileOutName))
catalog.writeFits(fileOutName)
# to do. Define output name in init (not in paf) and
# allow parallel processing
# write sources
# if self.config.doWriteSources:
# dataRef.put(result.sources, self.dataPrefix + 'src')
return
def testUndeblendedMeasurement(self):
"""Check undeblended measurement and aperture correction"""
width, height = 100, 100 # Dimensions of image
x0, y0 = 1234, 5678 # Offset of image
radius = 3.0 # Aperture radius
xCenter, yCenter = width//2, height//2 # Position of first source; integer values, for convenience
xOffset, yOffset = 1, 1 # Offset from first source to second source
flux1, flux2 = 1000, 1 # Flux of sources
apCorrValue = 3.21 # Aperture correction value to apply
image = afwImage.MaskedImageF(afwGeom.ExtentI(width, height))
image.setXY0(x0, y0)
image.getVariance().set(1.0)
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("centroid_x", type=np.float64)
schema.addField("centroid_y", type=np.float64)
schema.addField("centroid_flag", type='Flag')
schema.getAliasMap().set("slot_Centroid", "centroid")
sfmConfig = measBase.SingleFrameMeasurementConfig()
algName = "base_CircularApertureFlux"
for subConfig in (sfmConfig.plugins, sfmConfig.undeblended):
subConfig.names = [algName]
subConfig[algName].radii = [radius]
subConfig[algName].maxSincRadius = 0 # Disable sinc photometry because we're undersampled
slots = sfmConfig.slots
slots.centroid = "centroid"
slots.shape = None
slots.psfShape = None
slots.apFlux = None
slots.modelFlux = None
slots.psfFlux = None
slots.instFlux = None
slots.calibFlux = None
fieldName = lsst.meas.base.CircularApertureFluxAlgorithm.makeFieldPrefix(algName, radius)
measBase.addApCorrName(fieldName)
apCorrConfig = measBase.ApplyApCorrConfig()
apCorrConfig.proxies = {"undeblended_" + fieldName: fieldName}
sfm = measBase.SingleFrameMeasurementTask(config=sfmConfig, schema=schema)
apCorr = measBase.ApplyApCorrTask(config=apCorrConfig, schema=schema)
cat = afwTable.SourceCatalog(schema)
parent = cat.addNew()
parent.set("centroid_x", x0 + xCenter)
parent.set("centroid_y", y0 + yCenter)
spanSetParent = afwGeom.SpanSet.fromShape(int(radius))
spanSetParent = spanSetParent.shiftedBy(x0 + xCenter, y0 + yCenter)
parent.setFootprint(afwDetection.Footprint(spanSetParent))
# First child is bright, dominating the blend
child1 = cat.addNew()
child1.set("centroid_x", parent.get("centroid_x"))
child1.set("centroid_y", parent.get("centroid_y"))
child1.setParent(parent.getId())
image.set(xCenter, yCenter, (flux1, 0, 0))
spanSetChild1 = afwGeom.SpanSet.fromShape(1)
spanSetChild1 = spanSetChild1.shiftedBy(x0 + xCenter, y0 + yCenter)
foot1 = afwDetection.Footprint(spanSetChild1)
child1.setFootprint(afwDetection.HeavyFootprintF(foot1, image))
# Second child is fainter, but we want to be able to measure it!
child2 = cat.addNew()
child2.set("centroid_x", parent.get("centroid_x") + xOffset)
child2.set("centroid_y", parent.get("centroid_y") + yOffset)
child2.setParent(parent.getId())
image.set(xCenter + xOffset, yCenter + yOffset, (flux2, 0, 0))
spanSetChild2 = afwGeom.SpanSet.fromShape(1)
tmpPoint = (x0 + xCenter + xOffset, y0 + yCenter + yOffset)
spanSetChild2 = spanSetChild2.shiftedBy(*tmpPoint)
foot2 = afwDetection.Footprint(spanSetChild2)
child2.setFootprint(afwDetection.HeavyFootprintF(foot2, image))
spans = foot1.spans.union(foot2.spans)
bbox = afwGeom.Box2I()
bbox.include(foot1.getBBox())
bbox.include(foot2.getBBox())
parent.setFootprint(afwDetection.Footprint(spans, bbox))
exposure = afwImage.makeExposure(image)
sfm.run(cat, exposure)
def checkSource(source, baseName, expectedFlux):
"""Check that we get the expected results"""
self.assertEqual(source.get(baseName + "_flux"), expectedFlux)
self.assertGreater(source.get(baseName + "_fluxSigma"), 0)
self.assertFalse(source.get(baseName + "_flag"))
# Deblended
checkSource(child1, fieldName, flux1)
checkSource(child2, fieldName, flux2)
# Undeblended
checkSource(child1, "undeblended_" + fieldName, flux1 + flux2)
checkSource(child2, "undeblended_" + fieldName, flux1 + flux2)
# Apply aperture correction
apCorrMap = afwImage.ApCorrMap()
apCorrMap[fieldName + "_flux"] = afwMath.ChebyshevBoundedField(
image.getBBox(),
apCorrValue*np.ones((1, 1), dtype=np.float64)
)
apCorrMap[fieldName + "_fluxSigma"] = afwMath.ChebyshevBoundedField(
image.getBBox(),
apCorrValue*np.zeros((1, 1), dtype=np.float64)
)
apCorr.run(cat, apCorrMap)
# Deblended
checkSource(child1, fieldName, flux1*apCorrValue)
checkSource(child2, fieldName, flux2*apCorrValue)
# Undeblended
checkSource(child1, "undeblended_" + fieldName, (flux1 + flux2)*apCorrValue)
checkSource(child2, "undeblended_" + fieldName, (flux1 + flux2)*apCorrValue)
self.assertIn(fieldName + "_apCorr", schema)
self.assertIn(fieldName + "_apCorrSigma", schema)
self.assertIn("undeblended_" + fieldName + "_apCorr", schema)
self.assertIn("undeblended_" + fieldName + "_apCorrSigma", schema)
def testInputCounts(self, showPlot=False):
# Generate a simulated coadd of four overlapping-but-offset CCDs.
# Populate it with three sources.
# Demonstrate that we can correctly recover the number of images which
# contribute to each source.
size = 20 # Size of images (pixels)
value = 100.0 # Source flux
ccdPositions = [
lsst.geom.Point2D(8, 0),
lsst.geom.Point2D(10, 10),
lsst.geom.Point2D(-8, -8),
lsst.geom.Point2D(-8, 8)
]
# Represent sources by a tuple of position and expected number of
# contributing CCDs (based on the size/positions given above).
Source = namedtuple("Source", ["pos", "count"])
sources = [
Source(pos=lsst.geom.Point2D(6, 6), count=2),
Source(pos=lsst.geom.Point2D(10, 10), count=3),
Source(pos=lsst.geom.Point2D(14, 14), count=1)
]
# These lines are used in the creation of WCS information
scale = 1.0e-5 * lsst.geom.degrees
cdMatrix = afwGeom.makeCdMatrix(scale=scale)
crval = lsst.geom.SpherePoint(0.0, 0.0, lsst.geom.degrees)
# Construct the info needed to set the exposure object
imageBox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Extent2I(size, size))
wcsRef = afwGeom.makeSkyWcs(crpix=lsst.geom.Point2D(0, 0),
crval=crval,
cdMatrix=cdMatrix)
# Create the exposure object, and set it up to be the output of a coadd
exp = afwImage.ExposureF(size, size)
exp.setWcs(wcsRef)
exp.getInfo().setCoaddInputs(
afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(),
afwTable.ExposureTable.makeMinimalSchema()))
# Set the fake CCDs that "went into" making this coadd, using the
# differing wcs objects created above.
ccds = exp.getInfo().getCoaddInputs().ccds
for pos in ccdPositions:
record = ccds.addNew()
record.setWcs(
afwGeom.makeSkyWcs(crpix=pos, crval=crval, cdMatrix=cdMatrix))
record.setBBox(imageBox)
record.setValidPolygon(afwGeom.Polygon(lsst.geom.Box2D(imageBox)))
# Configure a SingleFrameMeasurementTask to run InputCounts.
measureSourcesConfig = measBase.SingleFrameMeasurementConfig()
measureSourcesConfig.plugins.names = [
"base_PeakCentroid", "base_InputCount"
]
measureSourcesConfig.slots.centroid = "base_PeakCentroid"
measureSourcesConfig.slots.psfFlux = None
measureSourcesConfig.slots.apFlux = None
measureSourcesConfig.slots.modelFlux = None
measureSourcesConfig.slots.gaussianFlux = None
measureSourcesConfig.slots.calibFlux = None
measureSourcesConfig.slots.shape = None
measureSourcesConfig.validate()
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema,
config=measureSourcesConfig)
catalog = afwTable.SourceCatalog(schema)
# Add simulated sources to the measurement catalog.
for src in sources:
spans = afwGeom.SpanSet.fromShape(1)
spans = spans.shiftedBy(int(src.pos.getX()), int(src.pos.getY()))
foot = afwDetection.Footprint(spans)
peak = foot.getPeaks().addNew()
peak.setFx(src.pos[0])
peak.setFy(src.pos[1])
peak.setPeakValue(value)
catalog.addNew().setFootprint(foot)
task.run(catalog, exp)
for src, rec in zip(sources, catalog):
self.assertEqual(rec.get("base_InputCount_value"), src.count)
if display:
ccdVennDiagram(exp)
def setUp(self):
width, height = 100, 300
self.mi = afwImage.MaskedImageF(afwGeom.ExtentI(width, height))
self.mi.set(0)
self.mi.getVariance().set(10)
self.mi.getMask().addMaskPlane("DETECTED")
self.FWHM = 5
self.ksize = 25 # size of desired kernel
self.exposure = afwImage.makeExposure(self.mi)
psf = roundTripPsf(
2,
algorithms.DoubleGaussianPsf(
self.ksize, self.ksize,
self.FWHM / (2 * math.sqrt(2 * math.log(2))), 1, 0.1))
self.exposure.setPsf(psf)
for x, y in [
(20, 20),
#(30, 35), (50, 50),
(60, 20),
(60, 210),
(20, 210)
]:
flux = 10000 - 0 * x - 10 * y
sigma = 3 + 0.01 * (y - self.mi.getHeight() / 2)
psf = roundTripPsf(
3,
algorithms.DoubleGaussianPsf(self.ksize, self.ksize, sigma, 1,
0.1))
im = psf.computeImage().convertF()
im *= flux
x0y0 = afwGeom.PointI(x - self.ksize // 2, y - self.ksize // 2)
smi = self.mi.getImage().Factory(
self.mi.getImage(),
afwGeom.BoxI(x0y0, afwGeom.ExtentI(self.ksize)),
afwImage.LOCAL)
if False: # Test subtraction with non-centered psfs
im = afwMath.offsetImage(im, 0.5, 0.5)
smi += im
del psf
del im
del smi
roundTripPsf(
4,
algorithms.DoubleGaussianPsf(
self.ksize, self.ksize,
self.FWHM / (2 * math.sqrt(2 * math.log(2))), 1, 0.1))
self.cellSet = afwMath.SpatialCellSet(
afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.ExtentI(width, height)),
100)
ds = afwDetection.FootprintSet(self.mi, afwDetection.Threshold(10),
"DETECTED")
#
# Prepare to measure
#
schema = afwTable.SourceTable.makeMinimalSchema()
sfm_config = measBase.SingleFrameMeasurementConfig()
sfm_config.plugins = [
"base_SdssCentroid", "base_CircularApertureFlux", "base_PsfFlux",
"base_SdssShape", "base_GaussianFlux", "base_PixelFlags"
]
sfm_config.slots.centroid = "base_SdssCentroid"
sfm_config.slots.shape = "base_SdssShape"
sfm_config.slots.psfFlux = "base_PsfFlux"
sfm_config.slots.instFlux = None
sfm_config.slots.apFlux = "base_CircularApertureFlux_3_0"
sfm_config.slots.modelFlux = "base_GaussianFlux"
sfm_config.slots.calibFlux = None
sfm_config.plugins["base_SdssShape"].maxShift = 10.0
sfm_config.plugins["base_CircularApertureFlux"].radii = [3.0]
task = measBase.SingleFrameMeasurementTask(schema, config=sfm_config)
measCat = afwTable.SourceCatalog(schema)
# detect the sources and run with the measurement task
ds.makeSources(measCat)
task.run(measCat, self.exposure)
for source in measCat:
self.cellSet.insertCandidate(
algorithms.makePsfCandidate(source, self.exposure))
def testDetection(self):
"""Test object detection"""
#
# Fix defects
#
# Mask known bad pixels
#
measAlgorithmsDir = lsst.utils.getPackageDir('meas_algorithms')
badPixels = defects.policyToBadRegionList(os.path.join(measAlgorithmsDir,
"policy/BadPixels.paf"))
# did someone lie about the origin of the maskedImage? If so, adjust bad pixel list
if self.XY0.getX() != self.mi.getX0() or self.XY0.getY() != self.mi.getY0():
dx = self.XY0.getX() - self.mi.getX0()
dy = self.XY0.getY() - self.mi.getY0()
for bp in badPixels:
bp.shift(-dx, -dy)
algorithms.interpolateOverDefects(self.mi, self.psf, badPixels)
#
# Subtract background
#
bgGridSize = 64 # was 256 ... but that gives only one region and the spline breaks
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE)
bctrl.setNxSample(int(self.mi.getWidth()/bgGridSize) + 1)
bctrl.setNySample(int(self.mi.getHeight()/bgGridSize) + 1)
backobj = afwMath.makeBackground(self.mi.getImage(), bctrl)
self.mi.getImage()[:] -= backobj.getImageF()
#
# Remove CRs
#
crConfig = algorithms.FindCosmicRaysConfig()
algorithms.findCosmicRays(self.mi, self.psf, 0, pexConfig.makePolicy(crConfig))
#
# We do a pretty good job of interpolating, so don't propagagate the convolved CR/INTRP bits
# (we'll keep them for the original CR/INTRP pixels)
#
savedMask = self.mi.getMask().Factory(self.mi.getMask(), True)
saveBits = savedMask.getPlaneBitMask("CR") | \
savedMask.getPlaneBitMask("BAD") | \
savedMask.getPlaneBitMask("INTRP") # Bits to not convolve
savedMask &= saveBits
msk = self.mi.getMask()
msk &= ~saveBits # Clear the saved bits
del msk
#
# Smooth image
#
psf = algorithms.DoubleGaussianPsf(15, 15, self.FWHM/(2*math.sqrt(2*math.log(2))))
cnvImage = self.mi.Factory(self.mi.getBBox())
kernel = psf.getKernel()
afwMath.convolve(cnvImage, self.mi, kernel, afwMath.ConvolutionControl())
msk = cnvImage.getMask()
msk |= savedMask # restore the saved bits
del msk
threshold = afwDetection.Threshold(3, afwDetection.Threshold.STDEV)
#
# Only search the part of the frame that was PSF-smoothed
#
llc = lsst.geom.PointI(psf.getKernel().getWidth()//2, psf.getKernel().getHeight()//2)
urc = lsst.geom.PointI(cnvImage.getWidth() - llc[0] - 1, cnvImage.getHeight() - llc[1] - 1)
middle = cnvImage.Factory(cnvImage, lsst.geom.BoxI(llc, urc), afwImage.LOCAL)
ds = afwDetection.FootprintSet(middle, threshold, "DETECTED")
del middle
#
# Reinstate the saved (e.g. BAD) (and also the DETECTED | EDGE) bits in the unsmoothed image
#
savedMask[:] = cnvImage.getMask()
msk = self.mi.getMask()
msk |= savedMask
del msk
del savedMask
if display:
disp = afwDisplay.Display(frame=2)
disp.mtv(self.mi, title=self._testMethodName + ": image")
afwDisplay.Display(frame=3).mtv(cnvImage, title=self._testMethodName + ": cnvImage")
#
# Time to actually measure
#
schema = afwTable.SourceTable.makeMinimalSchema()
sfm_config = measBase.SingleFrameMeasurementConfig()
sfm_config.plugins = ["base_SdssCentroid", "base_CircularApertureFlux", "base_PsfFlux",
"base_SdssShape", "base_GaussianFlux",
"base_PixelFlags"]
sfm_config.slots.centroid = "base_SdssCentroid"
sfm_config.slots.shape = "base_SdssShape"
sfm_config.slots.psfFlux = "base_PsfFlux"
sfm_config.slots.gaussianFlux = None
sfm_config.slots.apFlux = "base_CircularApertureFlux_3_0"
sfm_config.slots.modelFlux = "base_GaussianFlux"
sfm_config.slots.calibFlux = None
sfm_config.plugins["base_SdssShape"].maxShift = 10.0
sfm_config.plugins["base_CircularApertureFlux"].radii = [3.0]
task = measBase.SingleFrameMeasurementTask(schema, config=sfm_config)
measCat = afwTable.SourceCatalog(schema)
# detect the sources and run with the measurement task
ds.makeSources(measCat)
self.exposure.setPsf(self.psf)
task.run(measCat, self.exposure)
self.assertGreater(len(measCat), 0)
for source in measCat:
if source.get("base_PixelFlags_flag_edge"):
continue
if display:
disp.dot("+", source.getX(), source.getY())
def testFootprintsMeasure(self):
"""Check that we can measure the objects in a detectionSet"""
xcentroid = [10.0, 14.0, 9.0]
ycentroid = [8.0, 11.5061728, 14.0]
flux = [51.0, 101.0, 20.0]
# sqrt of num pixels in aperture; note the second source is offset
# from the pixel grid.
fluxErr = [math.sqrt(29), math.sqrt(27), math.sqrt(29)]
footprints = afwDetection.FootprintSet(self.mi, afwDetection.Threshold(10), "DETECTED")
if display:
disp = afwDisplay.Display(frame=0)
disp.mtv(self.mi, title=self._testMethodName + ": image")
afwDisplay.Display(frame=1).mtv(self.mi.getVariance(), title=self._testMethodName + ": variance")
measureSourcesConfig = measBase.SingleFrameMeasurementConfig()
measureSourcesConfig.algorithms["base_CircularApertureFlux"].radii = [3.0]
# Numerical tests below assumes that we are not using sinc fluxes.
measureSourcesConfig.algorithms["base_CircularApertureFlux"].maxSincRadius = 0.0
measureSourcesConfig.algorithms.names = ["base_NaiveCentroid", "base_SdssShape", "base_PsfFlux",
"base_CircularApertureFlux"]
measureSourcesConfig.slots.centroid = "base_NaiveCentroid"
measureSourcesConfig.slots.psfFlux = "base_PsfFlux"
measureSourcesConfig.slots.apFlux = "base_CircularApertureFlux_3_0"
measureSourcesConfig.slots.modelFlux = None
measureSourcesConfig.slots.gaussianFlux = None
measureSourcesConfig.slots.calibFlux = None
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=measureSourcesConfig)
measCat = afwTable.SourceCatalog(schema)
footprints.makeSources(measCat)
# now run the SFM task with the test plugin
sigma = 1e-10
psf = algorithms.DoubleGaussianPsf(11, 11, sigma) # i.e. a single pixel
self.exposure.setPsf(psf)
task.run(measCat, self.exposure)
self.assertEqual(len(measCat), len(flux))
for i, source in enumerate(measCat):
xc, yc = source.getX(), source.getY()
if display:
disp.dot("+", xc, yc)
self.assertAlmostEqual(source.getX(), xcentroid[i], 6)
self.assertAlmostEqual(source.getY(), ycentroid[i], 6)
self.assertEqual(source.getApInstFlux(), flux[i])
self.assertAlmostEqual(source.getApInstFluxErr(), fluxErr[i], 6)
# We're using a delta-function PSF, so the psfFlux should be the
# pixel under the centroid, iff the object's centred in the pixel
if xc == int(xc) and yc == int(yc):
self.assertAlmostEqual(source.getPsfInstFlux(),
self.exposure.getMaskedImage().getImage()[int(xc + 0.5),
int(yc + 0.5)])
self.assertAlmostEqual(source.getPsfInstFluxErr(),
self.exposure.getMaskedImage().getVariance()[int(xc + 0.5),
int(yc + 0.5)])
def setUp(self):
size = 128 # size of image (pixels)
center = afwGeom.Point2D(size // 2, size // 2) # object center
width = 2 # PSF width
flux = 10.0 # Flux of object
variance = 1.0 # Mean variance value
varianceStd = 0.1 # Standard deviation of the variance value
# Set a seed for predictable randomness
np.random.seed(300)
# Create a random image to be used as variance plane
variancePlane = np.random.normal(variance, varianceStd,
size * size).reshape(size, size)
# Initial setup of an image
exp = afwImage.ExposureF(size, size)
image = exp.getMaskedImage().getImage()
mask = exp.getMaskedImage().getMask()
var = exp.getMaskedImage().getVariance()
image.set(0.0)
mask.set(0)
var.getArray()[:, :] = variancePlane
# Put down a PSF
psfSize = int(6 * width + 1) # Size of PSF image; must be odd
psf = afwDetection.GaussianPsf(psfSize, psfSize, width)
exp.setPsf(psf)
psfImage = psf.computeImage(center).convertF()
psfImage *= flux
image.Factory(image,
psfImage.getBBox(afwImage.PARENT)).__iadd__(psfImage)
var.Factory(var, psfImage.getBBox(afwImage.PARENT)).__iadd__(psfImage)
# Put in some bad pixels to ensure they're ignored
for i in range(-5, 6):
bad = size // 2 + i * width
var.getArray()[bad, :] = float("nan")
mask.getArray()[bad, :] = mask.getPlaneBitMask("BAD")
var.getArray()[:, bad] = float("nan")
mask.getArray()[:, bad] = mask.getPlaneBitMask("BAD")
# Put in some unmasked bad pixels outside the expected aperture, to ensure the aperture is working
var.getArray()[0, 0] = float("nan")
var.getArray()[0, -1] = float("nan")
var.getArray()[-1, 0] = float("nan")
var.getArray()[-1, -1] = float("nan")
if display:
import lsst.afw.display as afwDisplay
afwDisplay.getDisplay(1).mtv(image)
afwDisplay.getDisplay(2).mtv(mask)
afwDisplay.getDisplay(3).mtv(var)
config = measBase.SingleFrameMeasurementConfig()
config.plugins.names = [
"base_NaiveCentroid", "base_SdssShape", "base_Variance"
]
config.slots.centroid = "base_NaiveCentroid"
config.slots.psfFlux = None
config.slots.apFlux = None
config.slots.modelFlux = None
config.slots.instFlux = None
config.slots.calibFlux = None
config.slots.shape = "base_SdssShape"
config.slots.psfShape = None
config.plugins["base_Variance"].mask = ["BAD", "SAT"]
config.validate()
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
spans = afwGeom.SpanSet.fromShape(int(width))
spans = spans.shiftedBy(int(center.getX()), int(center.getY()))
foot = afwDetection.Footprint(spans)
peak = foot.getPeaks().addNew()
peak.setIx(int(center.getX()))
peak.setIy(int(center.getY()))
peak.setFx(center.getX())
peak.setFy(center.getY())
peak.setPeakValue(flux)
source = catalog.addNew()
source.setFootprint(foot)
self.variance = variance
self.varianceStd = varianceStd
self.mask = mask
self.catalog = catalog
self.exp = exp
self.task = task
self.source = source
def setUp(self):
self.schema = afwTable.SourceTable.makeMinimalSchema()
config = measBase.SingleFrameMeasurementConfig()
config.algorithms.names = [
"base_PixelFlags",
"base_SdssCentroid",
"base_GaussianFlux",
"base_SdssShape",
"base_CircularApertureFlux",
"base_PsfFlux",
]
config.algorithms["base_CircularApertureFlux"].radii = [3.0]
config.slots.centroid = "base_SdssCentroid"
config.slots.psfFlux = "base_PsfFlux"
config.slots.apFlux = "base_CircularApertureFlux_3_0"
config.slots.modelFlux = None
config.slots.instFlux = None
config.slots.calibFlux = None
config.slots.shape = "base_SdssShape"
self.measureTask = measBase.SingleFrameMeasurementTask(self.schema,
config=config)
width, height = 110, 301
self.mi = afwImage.MaskedImageF(afwGeom.ExtentI(width, height))
self.mi.set(0)
sd = 3 # standard deviation of image
self.mi.getVariance().set(sd * sd)
self.mi.getMask().addMaskPlane("DETECTED")
self.FWHM = 5
self.ksize = 31 # size of desired kernel
sigma1 = 1.75
sigma2 = 2 * sigma1
self.exposure = afwImage.makeExposure(self.mi)
self.exposure.setPsf(
measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5 * sigma1, 1,
0.1))
self.exposure.setDetector(DetectorWrapper().detector)
#
# Make a kernel with the exactly correct basis functions. Useful for debugging
#
basisKernelList = []
for sigma in (sigma1, sigma2):
basisKernel = afwMath.AnalyticKernel(
self.ksize, self.ksize,
afwMath.GaussianFunction2D(sigma, sigma))
basisImage = afwImage.ImageD(basisKernel.getDimensions())
basisKernel.computeImage(basisImage, True)
basisImage /= np.sum(basisImage.getArray())
if sigma == sigma1:
basisImage0 = basisImage
else:
basisImage -= basisImage0
basisKernelList.append(afwMath.FixedKernel(basisImage))
order = 1 # 1 => up to linear
spFunc = afwMath.PolynomialFunction2D(order)
exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc)
exactKernel.setSpatialParameters([[1.0, 0, 0],
[0.0, 0.5 * 1e-2, 0.2e-2]])
self.exactPsf = measAlg.PcaPsf(exactKernel)
rand = afwMath.Random() # make these tests repeatable by setting seed
addNoise = True
if addNoise:
im = self.mi.getImage()
afwMath.randomGaussianImage(im, rand) # N(0, 1)
im *= sd # N(0, sd^2)
del im
xarr, yarr = [], []
for x, y in [
(20, 20),
(60, 20),
(30, 35),
(50, 50),
(20, 90),
(70, 160),
(25, 265),
(75, 275),
(85, 30),
(50, 120),
(70, 80),
(60, 210),
(20, 210),
]:
xarr.append(x)
yarr.append(y)
for x, y in zip(xarr, yarr):
dx = rand.uniform() - 0.5 # random (centered) offsets
dy = rand.uniform() - 0.5
k = exactKernel.getSpatialFunction(1)(
x, y) # functional variation of Kernel ...
b = (k * sigma1**2 / ((1 - k) * sigma2**2)
) # ... converted double Gaussian's "b"
# flux = 80000 - 20*x - 10*(y/float(height))**2
flux = 80000 * (1 + 0.1 * (rand.uniform() - 0.5))
I0 = flux * (1 + b) / (2 * np.pi * (sigma1**2 + b * sigma2**2))
for iy in range(y - self.ksize // 2, y + self.ksize // 2 + 1):
if iy < 0 or iy >= self.mi.getHeight():
continue
for ix in range(x - self.ksize // 2, x + self.ksize // 2 + 1):
if ix < 0 or ix >= self.mi.getWidth():
continue
intensity = I0 * psfVal(ix, iy, x + dx, y + dy, sigma1,
sigma2, b)
Isample = rand.poisson(
intensity) if addNoise else intensity
self.mi.getImage().set(
ix, iy,
self.mi.getImage().get(ix, iy) + Isample)
self.mi.getVariance().set(
ix, iy,
self.mi.getVariance().get(ix, iy) + intensity)
#
bbox = afwGeom.BoxI(afwGeom.PointI(0, 0),
afwGeom.ExtentI(width, height))
self.cellSet = afwMath.SpatialCellSet(bbox, 100)
self.footprintSet = afwDetection.FootprintSet(
self.mi, afwDetection.Threshold(100), "DETECTED")
self.catalog = self.measure(self.footprintSet, self.exposure)
for source in self.catalog:
try:
cand = measAlg.makePsfCandidate(source, self.exposure)
self.cellSet.insertCandidate(cand)
except Exception as e:
print(e)
continue
def setUp(self):
self.x0, self.y0 = 0, 0
self.nx, self.ny = 512, 512 #2048, 4096
self.sky = 100.0
self.nObj = 100
# make a detector with distortion
self.detector = DetectorWrapper(
bbox = afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(self.nx, self.ny)),
orientation = cameraGeom.Orientation(afwGeom.Point2D(255.0, 255.0)),
radialDistortion = 0.925,
).detector
# make a detector with no distortion
self.flatDetector = DetectorWrapper(
bbox = afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(self.nx, self.ny)),
orientation = cameraGeom.Orientation(afwGeom.Point2D(255.0, 255.0)),
radialDistortion = 0.0,
).detector
# detection policies
detConfig = measAlg.SourceDetectionConfig()
# Cannot use default background approximation order (6) for such a small image.
detConfig.background.approxOrderX = 4
# measurement policies
measConfig = measBase.SingleFrameMeasurementConfig()
measConfig.algorithms.names = [
"base_SdssCentroid",
"base_SdssShape",
"base_GaussianFlux",
"base_PsfFlux",
]
measConfig.slots.centroid = "base_SdssCentroid"
measConfig.slots.shape = "base_SdssShape"
measConfig.slots.psfFlux = "base_PsfFlux"
measConfig.slots.apFlux = None
measConfig.slots.modelFlux = None
measConfig.slots.instFlux = None
measConfig.slots.calibFlux = None
self.schema = afwTable.SourceTable.makeMinimalSchema()
detConfig.validate()
measConfig.validate()
self.detTask = measAlg.SourceDetectionTask(config=detConfig, schema=self.schema)
self.measTask = measBase.SingleFrameMeasurementTask(config=measConfig, schema=self.schema)
# psf star selector
starSelectorConfig = measAlg.SecondMomentStarSelectorTask.ConfigClass()
starSelectorConfig.fluxLim = 5000.0
starSelectorConfig.histSize = 32
starSelectorConfig.clumpNSigma = 1.0
starSelectorConfig.badFlags = []
self.starSelector = measAlg.SecondMomentStarSelectorTask(
config=starSelectorConfig, schema=self.schema
)
# psf determiner
psfDeterminerFactory = measAlg.psfDeterminerRegistry["pca"]
psfDeterminerConfig = psfDeterminerFactory.ConfigClass()
width, height = self.nx, self.ny
nEigenComponents = 3
psfDeterminerConfig.sizeCellX = width//3
psfDeterminerConfig.sizeCellY = height//3
psfDeterminerConfig.nEigenComponents = nEigenComponents
psfDeterminerConfig.spatialOrder = 1
psfDeterminerConfig.kernelSizeMin = 31
psfDeterminerConfig.nStarPerCell = 0
psfDeterminerConfig.nStarPerCellSpatialFit = 0 # unlimited
self.psfDeterminer = psfDeterminerFactory(psfDeterminerConfig)
def makeSourceSchema(self):
schema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(schema=schema) # expand the schema
return schema
