def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = self.makeSourceCat(self.tanWcs, sourceSchema=sourceSchema)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.matcher.sourceSelector.config.minSnr = 0
def testMagnitudeOutlierRejection(self):
"""Test rejection of magnitude outliers.
This test only tests the outlier rejection, and not any other
part of the matching or astrometry fitter.
"""
config = AstrometryTask.ConfigClass()
config.doMagnitudeOutlierRejection = True
config.magnitudeOutlierRejectionNSigma = 4.0
solver = AstrometryTask(config=config, refObjLoader=None)
nTest = 100
refSchema = lsst.afw.table.SimpleTable.makeMinimalSchema()
refSchema.addField('refFlux', 'F')
refCat = lsst.afw.table.SimpleCatalog(refSchema)
refCat.resize(nTest)
srcSchema = lsst.afw.table.SourceTable.makeMinimalSchema()
srcSchema.addField('srcFlux', 'F')
srcCat = lsst.afw.table.SourceCatalog(srcSchema)
srcCat.resize(nTest)
np.random.seed(12345)
refMag = np.full(nTest, 20.0)
srcMag = np.random.normal(size=nTest, loc=0.0, scale=1.0)
# Determine the sigma of the random sample
zp = np.median(refMag[:-4] - srcMag[:-4])
sigma = scipy.stats.median_abs_deviation(srcMag[:-4], scale='normal')
# Deliberately alter some magnitudes to be outliers.
srcMag[-3] = (config.magnitudeOutlierRejectionNSigma + 0.1) * sigma + (
20.0 - zp)
srcMag[-4] = -(config.magnitudeOutlierRejectionNSigma +
0.1) * sigma + (20.0 - zp)
refCat['refFlux'] = (refMag * units.ABmag).to_value(units.nJy)
srcCat['srcFlux'] = 10.0**(srcMag / (-2.5))
# Deliberately poison some reference fluxes.
refCat['refFlux'][-1] = np.inf
refCat['refFlux'][-2] = np.nan
matchesIn = []
for ref, src in zip(refCat, srcCat):
matchesIn.append(
lsst.afw.table.ReferenceMatch(first=ref,
second=src,
distance=0.0))
matchesOut = solver._removeMagnitudeOutliers('srcFlux', 'refFlux',
matchesIn)
# We should lose the 4 outliers we created.
self.assertEqual(len(matchesOut), len(matchesIn) - 4)
def run():
exposure, srcCat = loadData()
schema = srcCat.getSchema()
#
# Create the astrometry task
#
config = AstrometryTask.ConfigClass()
config.refObjLoader.filterMap = {"_unknown_": "r"}
config.matcher.sourceFluxType = "Psf" # sample catalog does not contain aperture flux
aTask = AstrometryTask(config=config)
#
# And the photometry Task
#
config = PhotoCalTask.ConfigClass()
config.applyColorTerms = False # we don't have any available, so this suppresses a warning
pTask = PhotoCalTask(config=config, schema=schema)
#
# The tasks may have added extra elements to the schema (e.g. AstrometryTask's centroidKey to
# handle distortion; photometryTask's config.outputField). If this is so, we need to add
# these columns to the Source table.
#
# We wouldn't need to do this if we created the schema prior to measuring the exposure,
# but in this case we read the sources from disk
#
if schema != srcCat.getSchema(): # the tasks added fields
print("Adding columns to the source catalogue")
cat = afwTable.SourceCatalog(schema)
cat.table.defineCentroid(srcCat.table.getCentroidDefinition())
cat.table.definePsfFlux(srcCat.table.getPsfFluxDefinition())
scm = afwTable.SchemaMapper(srcCat.getSchema(), schema)
for schEl in srcCat.getSchema():
scm.addMapping(schEl.getKey(), True)
cat.extend(srcCat, True, scm) # copy srcCat to cat, adding new columns
srcCat = cat
del cat
#
# Process the data
#
matches = aTask.run(exposure, srcCat).matches
result = pTask.run(exposure, matches)
calib = result.calib
fm0, fm0Err = calib.getFluxMag0()
print("Used %d calibration sources out of %d matches" %
(len(result.matches), len(matches)))
delta = result.arrays.refMag - result.arrays.srcMag
q25, q75 = np.percentile(delta, [25, 75])
print("RMS error is %.3fmmsg (robust %.3f, Calib says %.3f)" %
(np.std(delta), 0.741 *
(q75 - q25), 2.5 / np.log(10) * fm0Err / fm0))
def run():
exposure, srcCat = loadData()
schema = srcCat.getSchema()
#
# Create the astrometry task
#
config = AstrometryTask.ConfigClass()
config.refObjLoader.filterMap = {"_unknown_": "r"}
config.matcher.sourceFluxType = "Psf" # sample catalog does not contain aperture flux
aTask = AstrometryTask(config=config)
#
# And the photometry Task
#
config = PhotoCalTask.ConfigClass()
config.applyColorTerms = False # we don't have any available, so this suppresses a warning
pTask = PhotoCalTask(config=config, schema=schema)
#
# The tasks may have added extra elements to the schema (e.g. AstrometryTask's centroidKey to
# handle distortion; photometryTask's config.outputField). If this is so, we need to add
# these columns to the Source table.
#
# We wouldn't need to do this if we created the schema prior to measuring the exposure,
# but in this case we read the sources from disk
#
if schema != srcCat.getSchema(): # the tasks added fields
print("Adding columns to the source catalogue")
cat = afwTable.SourceCatalog(schema)
cat.table.defineCentroid(srcCat.table.getCentroidDefinition())
cat.table.definePsfFlux(srcCat.table.getPsfFluxDefinition())
scm = afwTable.SchemaMapper(srcCat.getSchema(), schema)
for schEl in srcCat.getSchema():
scm.addMapping(schEl.getKey(), True)
cat.extend(srcCat, True, scm) # copy srcCat to cat, adding new columns
srcCat = cat; del cat
#
# Process the data
#
matches = aTask.run(exposure, srcCat).matches
result = pTask.run(exposure, matches)
calib = result.calib
fm0, fm0Err = calib.getFluxMag0()
print("Used %d calibration sources out of %d matches" % (len(result.matches), len(matches)))
delta = result.arrays.refMag - result.arrays.srcMag
q25, q75 = np.percentile(delta, [25, 75])
print("RMS error is %.3fmmsg (robust %.3f, Calib says %.3f)" % (np.std(delta), 0.741*(q75 - q25),
2.5/np.log(10)*fm0Err/fm0))
def testMaxMeanDistance(self):
"""If the astrometric fit does not satisfy the maxMeanDistanceArcsec
threshold, ensure task raises an lsst.pipe.base.TaskError.
"""
self.exposure.setWcs(self.tanWcs)
config = AstrometryTask.ConfigClass()
config.maxMeanDistanceArcsec = 0.0 # To ensure a "deemed" WCS failure
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
sourceCat = self.makeSourceCat(self.tanWcs, doScatterCentroids=True)
with self.assertRaisesRegex(pipeBase.TaskError,
"Fatal astrometry failure detected"):
solver.run(sourceCat=sourceCat, exposure=self.exposure)
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.sourceSelector.config.minSnr = 0
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
distortedWcs = afwImage.DistortedTanWcs(self.tanWcs,
afwGeom.IdentityXYTransform())
self.exposure.setWcs(distortedWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=distortedWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceFluxKey = sourceSchema["slot_ApFlux_flux"].asKey()
sourceFluxSigmaKey = sourceSchema["slot_ApFlux_fluxSigma"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceFluxKey, refObj.get(refFluxRKey))
src.set(sourceFluxSigmaKey, refObj.get(refFluxRKey) / 100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometryUsed'):
count += 1
self.assertEqual(count, len(results.matches))
def runAstrometry(self, butler, exp, icSrc):
refObjLoaderConfig = LoadIndexedReferenceObjectsTask.ConfigClass()
refObjLoaderConfig.ref_dataset_name = 'gaia_dr2_20191105'
refObjLoaderConfig.pixelMargin = 1000
refObjLoader = LoadIndexedReferenceObjectsTask(
butler=butler, config=refObjLoaderConfig)
astromConfig = AstrometryTask.ConfigClass()
astromConfig.wcsFitter.retarget(FitAffineWcsTask)
astromConfig.referenceSelector.doMagLimit = True
magLimit = MagnitudeLimit()
magLimit.minimum = 1
magLimit.maximum = 15
astromConfig.referenceSelector.magLimit = magLimit
astromConfig.referenceSelector.magLimit.fluxField = "phot_g_mean_flux"
astromConfig.matcher.maxRotationDeg = 5.99
astromConfig.matcher.maxOffsetPix = 3000
astromConfig.sourceSelector['matcher'].minSnr = 10
solver = AstrometryTask(config=astromConfig, refObjLoader=refObjLoader)
# TODO: Change this to doing this the proper way
referenceFilterName = self.config.referenceFilterOverride
referenceFilterLabel = afwImage.FilterLabel(
physical=referenceFilterName, band=referenceFilterName)
originalFilterLabel = exp.getFilterLabel(
) # there's a better way of doing this with the task I think
exp.setFilterLabel(referenceFilterLabel)
try:
astromResult = solver.run(sourceCat=icSrc, exposure=exp)
exp.setFilterLabel(originalFilterLabel)
except (RuntimeError, TaskError):
self.log.warn("Solver failed to run completely")
exp.setFilterLabel(originalFilterLabel)
return None
scatter = astromResult.scatterOnSky.asArcseconds()
if scatter < 1:
return astromResult
else:
self.log.warn("Failed to find an acceptable match")
return None
def doTest(self, pixelsToTanPixels, order=3):
"""Test using pixelsToTanPixels to distort the source positions
"""
distortedWcs = afwGeom.makeModifiedWcs(pixelTransform=pixelsToTanPixels, wcs=self.tanWcs,
modifyActualPixels=False)
self.exposure.setWcs(distortedWcs)
sourceCat = self.makeSourceCat(distortedWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = order
config.wcsFitter.numRejIter = 0
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
fitWcs = self.exposure.getWcs()
self.assertRaises(Exception, self.assertWcsAlmostEqualOverBBox, fitWcs, distortedWcs)
self.assertWcsAlmostEqualOverBBox(distortedWcs, fitWcs, self.bbox,
maxDiffSky=0.01*afwGeom.arcseconds, maxDiffPix=0.02)
srcCoordKey = afwTable.CoordKey(sourceCat.schema["coord"])
refCoordKey = afwTable.CoordKey(results.refCat.schema["coord"])
refCentroidKey = afwTable.Point2DKey(results.refCat.schema["centroid"])
maxAngSep = afwGeom.Angle(0)
maxPixSep = 0
for refObj, src, d in results.matches:
refCoord = refObj.get(refCoordKey)
refPixPos = refObj.get(refCentroidKey)
srcCoord = src.get(srcCoordKey)
srcPixPos = src.getCentroid()
angSep = refCoord.separation(srcCoord)
maxAngSep = max(maxAngSep, angSep)
pixSep = math.hypot(*(srcPixPos-refPixPos))
maxPixSep = max(maxPixSep, pixSep)
print("max angular separation = %0.4f arcsec" % (maxAngSep.asArcseconds(),))
print("max pixel separation = %0.3f" % (maxPixSep,))
self.assertLess(maxAngSep.asArcseconds(), 0.0026)
self.assertLess(maxPixSep, 0.015)
# try again, but without fitting the WCS
config.forceKnownWcs = True
solverNoFit = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsNoFit = solverNoFit.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertIsNone(resultsNoFit.scatterOnSky)
# fitting should result in matches that are at least as good
# (strictly speaking fitting might result in a larger match list with
# some outliers, but in practice this test passes)
meanFitDist = np.mean([match.distance for match in results.matches])
meanNoFitDist = np.mean([match.distance for match in resultsNoFit.matches])
self.assertLessEqual(meanFitDist, meanNoFitDist)
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox, wcs=self.tanWcs, filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader, schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCat.reserve(len(refCat))
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey)/100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def doTest(self, pixelsToTanPixels, order=3):
"""Test using pixelsToTanPixels to distort the source positions
"""
distortedWcs = afwGeom.makeModifiedWcs(pixelTransform=pixelsToTanPixels, wcs=self.tanWcs,
modifyActualPixels=False)
self.exposure.setWcs(distortedWcs)
sourceCat = self.makeSourceCat(distortedWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = order
config.wcsFitter.numRejIter = 0
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
fitWcs = self.exposure.getWcs()
self.assertRaises(Exception, self.assertWcsAlmostEqualOverBBox, fitWcs, distortedWcs)
self.assertWcsAlmostEqualOverBBox(distortedWcs, fitWcs, self.bbox,
maxDiffSky=0.01*lsst.geom.arcseconds, maxDiffPix=0.02)
srcCoordKey = afwTable.CoordKey(sourceCat.schema["coord"])
refCoordKey = afwTable.CoordKey(results.refCat.schema["coord"])
refCentroidKey = afwTable.Point2DKey(results.refCat.schema["centroid"])
maxAngSep = 0*lsst.geom.radians
maxPixSep = 0
for refObj, src, d in results.matches:
refCoord = refObj.get(refCoordKey)
refPixPos = refObj.get(refCentroidKey)
srcCoord = src.get(srcCoordKey)
srcPixPos = src.getCentroid()
angSep = refCoord.separation(srcCoord)
maxAngSep = max(maxAngSep, angSep)
pixSep = math.hypot(*(srcPixPos-refPixPos))
maxPixSep = max(maxPixSep, pixSep)
print("max angular separation = %0.4f arcsec" % (maxAngSep.asArcseconds(),))
print("max pixel separation = %0.3f" % (maxPixSep,))
self.assertLess(maxAngSep.asArcseconds(), 0.0038)
self.assertLess(maxPixSep, 0.021)
# try again, invoking the reference selector
config.referenceSelector.doUnresolved = True
config.referenceSelector.unresolved.name = 'resolved'
solverRefSelect = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsRefSelect = solverRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsRefSelect.matches), len(results.matches))
# try again, but without fitting the WCS, no reference selector
config.referenceSelector.doUnresolved = False
config.forceKnownWcs = True
solverNoFit = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsNoFit = solverNoFit.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertIsNone(resultsNoFit.scatterOnSky)
# fitting should result in matches that are at least as good
# (strictly speaking fitting might result in a larger match list with
# some outliers, but in practice this test passes)
meanFitDist = np.mean([match.distance for match in results.matches])
meanNoFitDist = np.mean([match.distance for match in resultsNoFit.matches])
self.assertLessEqual(meanFitDist, meanNoFitDist)
# try once again, without fitting the WCS, with the reference selector
# (this goes through a different code path)
config.referenceSelector.doUnresolved = True
solverNoFitRefSelect = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
resultsNoFitRefSelect = solverNoFitRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsNoFitRefSelect.matches), len(resultsNoFit.matches))
class joinMatchListWithCatalogTestCase(unittest.TestCase):
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.matcher.sourceSelector.config.minSnr = 0
def tearDown(self):
del self.srcSet
del self.bbox
del self.exposure
del self.astrom
def getAstrometrySolution(self):
return self.astrom.solve(exposure=self.exposure, sourceCat=self.srcSet)
def testJoin(self):
res = self.getAstrometrySolution()
matches = res.matches
matchmeta = res.matchMeta
normalized = packMatches(matches)
normalized.table.setMetadata(matchmeta)
matches2 = self.astrom.refObjLoader.joinMatchListWithCatalog(normalized, self.srcSet)
self.assertEqual(len(matches2), len(matches))
for i in range(len(matches)):
self.assertEqual(matches2[i].second.table, matches[i].second.table)
self.assertEqual(matches2[i].second.getId(), matches[i].second.getId())
self.assertEqual(matches2[i].second, matches[i].second) # no deep copying, so we can compare ptrs
self.assertEqual(matches2[i].first.getId(), matches[i].first.getId())
self.assertEqual(matches2[i].first.getRa().asDegrees(), matches[i].first.getRa().asDegrees())
self.assertEqual(matches2[i].first.getDec().asDegrees(), matches[i].first.getDec().asDegrees())
self.assertEqual(matches2[i].first.get("i_flux"), matches[i].first.get("i_flux"))
def testJoinAllFluxes(self):
"""Test that we can read all the fluxes back from an a.n.d catalogue"""
res = self.getAstrometrySolution()
matches = res.matches
matchmeta = res.matchMeta
normalized = packMatches(matches)
normalized.table.setMetadata(matchmeta)
matches2 = self.astrom.refObjLoader.joinMatchListWithCatalog(normalized, self.srcSet)
self.assertGreater(len(matches2), 0)
ref = matches2[0][0]
names = ref.getSchema().getNames()
for b in ("u", "g", "r", "i", "z"):
self.assertIn("%s_flux" % (b,), names)
self.assertIn("%s_fluxSigma" % (b,), names)
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = int(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure,
sensorRef,
templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
# sigma of PSF of template image before warping
templateSigma = templateExposure.getPsf().computeShape(
).getDeterminantRadius()
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel(
).getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight,
scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn(
"Src product does not exist; running detection, measurement, selection"
)
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(
makeKernelBasisList(
self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(
exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
mc = afwTable.MatchControl()
mc.findOnlyClosest = False
matches = afwTable.matchRaDec(templateSources,
selectSources,
1.0 * afwGeom.arcseconds, mc)
else:
raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
+
"but template sources not available. Cannot match science "
+
"sources with template sources. Run process* on data from "
+ "which templates are built.")
kernelSources = self.sourceSelector.selectStars(
exposure, selectSources, matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [
k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize
]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources),
len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources,
templateExposure,
selectSources)
warpedExp = self.register.warpExposure(templateExposure,
wcsResults.wcs,
exposure.getWcs(),
exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable(
).getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable(
).getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [
m.first.get(refCoordKey) for m in wcsResults.matches
]
residuals = [
m.first.get(refCoordKey).getOffsetFrom(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [
m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
colors = numpy.array([
-2.5 * numpy.log10(srcToMatch[x].get("g")) +
2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
if x in srcToMatch.keys()
])
dlong = numpy.array([
r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
dlat = numpy.array([
r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
idx1 = numpy.where(
colors < self.sourceSelector.config.grMin)
idx2 = numpy.where(
(colors >= self.sourceSelector.config.grMin)
& (colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(
colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma * \
(numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma * \
(numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma * \
(numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian",
numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian",
numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian",
numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian",
numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian",
numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian",
numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
# Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure,
self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(
exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
# already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel)
subtractedExposure = decorrResult.correctedExposure
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" %
(len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if not self.config.doDipoleFitting:
self.measurement.run(diaSources, subtractedExposure)
else:
if self.config.doSubtract:
self.measurement.run(diaSources, subtractedExposure,
exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure,
exposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs(
).pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"),
diaSources, matchRadPixel)
srcMatchDict = dict([(srcMatch.second.getId(),
srcMatch.first.getId())
for srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" %
(len(srcMatchDict), len(diaSources)))
else:
self.log.warn(
"Src product does not exist; cannot match with diaSources"
)
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure,
sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn(
"No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info(
"Matched %d / %d diaSources to reference catalog" %
(len(refMatches), len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(),
refMatch.first.getId())
for refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if sid in srcMatchDict:
diaSource.set("srcMatchId", srcMatchDict[sid])
if sid in refMatchDict:
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources,
self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cand)
# Get basis list to build control sample kernels
basisList = kernelCandList[0].getKernel(
KernelCandidateF.ORIG).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids,
diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources,
self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources,
diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def run():
exposure, srcCat = loadData()
schema = srcCat.getSchema()
#
# Create the reference catalog loader
#
refCatDir = os.path.join(lsst.utils.getPackageDir('meas_astrom'), 'tests',
'data', 'sdssrefcat')
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
#
# Create the astrometry task
#
config = AstrometryTask.ConfigClass()
config.matcher.sourceFluxType = "Psf" # sample catalog does not contain aperture flux
config.matcher.minSnr = 0 # disable S/N test because sample catalog does not contain flux sigma
aTask = AstrometryTask(config=config, refObjLoader=refObjLoader)
#
# And the photometry Task
#
config = PhotoCalTask.ConfigClass()
config.applyColorTerms = False # we don't have any available, so this suppresses a warning
# The associated data has been prepared on the basis that we use PsfFlux to perform photometry.
config.fluxField = "base_PsfFlux_flux"
pTask = PhotoCalTask(config=config, schema=schema)
#
# The tasks may have added extra elements to the schema (e.g. AstrometryTask's centroidKey to
# handle distortion; photometryTask's config.outputField). If this is so, we need to add
# these columns to the Source table.
#
# We wouldn't need to do this if we created the schema prior to measuring the exposure,
# but in this case we read the sources from disk
#
if schema != srcCat.getSchema(): # the tasks added fields
print("Adding columns to the source catalogue")
cat = afwTable.SourceCatalog(schema)
cat.table.defineCentroid(srcCat.table.getCentroidDefinition())
cat.table.definePsfFlux(srcCat.table.getPsfFluxDefinition())
scm = afwTable.SchemaMapper(srcCat.getSchema(), schema)
for schEl in srcCat.getSchema():
scm.addMapping(schEl.getKey(), True)
cat.extend(srcCat, True, scm) # copy srcCat to cat, adding new columns
srcCat = cat
del cat
#
# Process the data
#
matches = aTask.run(exposure, srcCat).matches
result = pTask.run(exposure, matches)
calib = result.calib
fm0, fm0Err = calib.getFluxMag0()
print("Used %d calibration sources out of %d matches" %
(len(result.matches), len(matches)))
delta = result.arrays.refMag - result.arrays.srcMag
q25, q75 = np.percentile(delta, [25, 75])
print("RMS error is %.3fmmsg (robust %.3f, Calib says %.3f)" %
(np.std(delta), 0.741 *
(q75 - q25), 2.5 / np.log(10) * fm0Err / fm0))
def runDataRef(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = int(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
if self.config.subtract.name == 'zogy':
subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
doWarping=True,
spatiallyVarying=self.config.doSpatiallyVarying,
doPreConvolve=self.config.doPreConvolve)
subtractedExposure = subtractRes.subtractedExposure
elif self.config.subtract.name == 'al':
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
# sigma of PSF of template image before warping
templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
preConvPsf = None
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
targetFwhmPix=templateSigma*FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
mc = afwTable.MatchControl()
mc.findOnlyClosest = False
matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
mc)
else:
raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
"but template sources not available. Cannot match science "
"sources with template sources. Run process* on data from "
"which templates are built.")
kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
matches=matches).sourceCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(exposure, selectSources,
matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory
)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
exposure.getWcs(), exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
2.5*numpy.log10(srcToMatch[x].get("r"))
for x in sids if x in srcToMatch.keys()])
dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
(colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma*(
(numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma*(
(numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma*(
(numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
# Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(exposure, sensorRef,
templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above),
# thus it may have already been decorrelated. Thus, we do not decorrelate if
# doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
preConvKernel = None
if preConvPsf is not None:
preConvKernel = preConvPsf.getLocalKernel()
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel,
spatiallyVarying=self.config.doSpatiallyVarying,
preConvKernel=preConvKernel)
subtractedExposure = decorrResult.correctedExposure
# END (if subtractAlgorithm == 'AL')
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
newDipoleFitting = self.config.doDipoleFitting
self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
if not newDipoleFitting:
# Just fit dipole in diffim
self.measurement.run(diaSources, subtractedExposure)
else:
# Use (matched) template and science image (if avail.) to constrain dipole fitting
if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
self.measurement.run(diaSources, subtractedExposure, exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure, exposure)
if self.config.doForcedMeasurement:
# Run forced psf photometry on the PVI at the diaSource locations.
# Copy the measured flux and error into the diaSource.
forcedSources = self.forcedMeasurement.generateMeasCat(
exposure, diaSources, subtractedExposure.getWcs())
self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
"ip_diffim_forced_PsfFlux_instFlux", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
"ip_diffim_forced_PsfFlux_instFluxErr", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
"ip_diffim_forced_PsfFlux_area", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
"ip_diffim_forced_PsfFlux_flag", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
"ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
"ip_diffim_forced_PsfFlux_flag_edge", True)
for diaSource, forcedSource in zip(diaSources, forcedSources):
diaSource.assign(forcedSource, mapper)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec/exposure.getWcs().pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if sid in srcMatchDict:
diaSource.set("srcMatchId", srcMatchDict[sid])
if sid in refMatchDict:
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cand)
# Get basis list to build control sample kernels
basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
controlCandList = (
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList))
kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
class JoinMatchListWithCatalogTestCase(unittest.TestCase):
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.sourceSelector.config.minSnr = 0
def tearDown(self):
del self.srcSet
del self.bbox
del self.exposure
del self.astrom
def getAstrometrySolution(self):
return self.astrom.solve(exposure=self.exposure, sourceCat=self.srcSet)
def testJoin(self):
res = self.getAstrometrySolution()
matches = res.matches
matchmeta = res.matchMeta
normalized = packMatches(matches)
normalized.table.setMetadata(matchmeta)
matches2 = self.astrom.refObjLoader.joinMatchListWithCatalog(normalized, self.srcSet)
self.assertEqual(len(matches2), len(matches))
for i in range(len(matches)):
self.assertEqual(matches2[i].second.table, matches[i].second.table)
self.assertEqual(matches2[i].second.getId(), matches[i].second.getId())
self.assertEqual(matches2[i].second, matches[i].second) # no deep copying, so we can compare ptrs
self.assertEqual(matches2[i].first.getId(), matches[i].first.getId())
self.assertEqual(matches2[i].first.getRa().asDegrees(), matches[i].first.getRa().asDegrees())
self.assertEqual(matches2[i].first.getDec().asDegrees(), matches[i].first.getDec().asDegrees())
self.assertEqual(matches2[i].first.get("i_flux"), matches[i].first.get("i_flux"))
def testJoinAllFluxes(self):
"""Test that we can read all the fluxes from a reference catalog"""
res = self.getAstrometrySolution()
matches = res.matches
matchmeta = res.matchMeta
normalized = packMatches(matches)
normalized.table.setMetadata(matchmeta)
matches2 = self.astrom.refObjLoader.joinMatchListWithCatalog(normalized, self.srcSet)
self.assertGreater(len(matches2), 0)
ref = matches2[0][0]
refSchema = ref.getSchema()
for b in ("u", "g", "r", "i", "z"):
self.assertIn("%s_flux" % (b,), refSchema)
self.assertIn("%s_fluxErr" % (b,), refSchema)