def setup_module(module):
lsst.utils.tests.init()
if os.path.exists(DATAREPO_ROOT):
print("Deleting existing repo: %r" % (DATAREPO_ROOT,))
shutil.rmtree(DATAREPO_ROOT)
if not haveMeasBase:
raise unittest.SkipTest("meas_base could not be imported")
# Create a task that creates simulated images and builds a coadd from them
mocksTask = lsst.pipe.tasks.mocks.MockCoaddTask()
# Create an instance of DetectCoaddSourcesTask to measure on the coadd.
# There's no noise in these images, so we set a direct-value threshold,
# and the background weighting (when using Approximate) to False
detectConfig = DetectCoaddSourcesTask.ConfigClass()
# Images have no noise, so we can't use the default DynamicDetectionTask
detectConfig.detection.retarget(lsst.meas.algorithms.SourceDetectionTask)
detectConfig.detection.thresholdType = "value"
detectConfig.detection.thresholdValue = 0.01
detectConfig.detection.background.weighting = False
detectTask = DetectCoaddSourcesTask(config=detectConfig)
butler = lsst.pipe.tasks.mocks.makeDataRepo(DATAREPO_ROOT)
mocksTask.buildAllInputs(butler)
addMaskPlanes(butler)
mocksTask.buildCoadd(butler)
mocksTask.buildMockCoadd(butler)
detectTask.writeSchemas(butler)
# Now run the seperate multiband tasks on the Coadd to make the reference
# catalog for the forced photometry tests.
runTaskOnPatches(butler, detectTask, mocksTask)
mergeDetConfig = MergeDetectionsTask.ConfigClass()
mergeDetConfig.priorityList = ['r', ]
mergeDetTask = MergeDetectionsTask(config=mergeDetConfig, butler=butler)
mergeDetTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeDetTask, mocksTask)
measMergedConfig = MeasureMergedCoaddSourcesTask.ConfigClass()
measMergedConfig.measurement.slots.shape = "base_SdssShape"
measMergedConfig.measurement.plugins['base_PixelFlags'].masksFpAnywhere = []
measMergedConfig.propagateFlags.flags = {} # Disable flag propagation: no flags to propagate
measMergedConfig.doMatchSources = False # We don't have a reference catalog available
measMergedTask = MeasureMergedCoaddSourcesTask(config=measMergedConfig, butler=butler)
measMergedTask.writeSchemas(butler)
runTaskOnPatches(butler, measMergedTask, mocksTask)
mergeMeasConfig = MergeMeasurementsTask.ConfigClass()
mergeMeasConfig.priorityList = ['r', ]
mergeMeasTask = MergeMeasurementsTask(config=mergeMeasConfig, butler=butler)
mergeMeasTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeMeasTask, mocksTask)
runForcedPhotCoaddTask(butler, mocksTask)
runForcedPhotCcdTask(butler)
def setup_module(module):
lsst.utils.tests.init()
if os.path.exists(DATAREPO_ROOT):
print("Deleting existing repo: %r" % (DATAREPO_ROOT,))
shutil.rmtree(DATAREPO_ROOT)
if not haveMeasBase:
raise unittest.SkipTest("meas_base could not be imported")
# Create a task that creates simulated images and builds a coadd from them
mocksTask = lsst.pipe.tasks.mocks.MockCoaddTask()
# Create an instance of DetectCoaddSourcesTask to measure on the coadd.
# There's no noise in these images, so we set a direct-value threshold,
# and the background weighting (when using Approximate) to False
detectConfig = DetectCoaddSourcesTask.ConfigClass()
# Images have no noise, so we can't use the default DynamicDetectionTask
detectConfig.detection.retarget(lsst.meas.algorithms.SourceDetectionTask)
detectConfig.detection.thresholdType = "value"
detectConfig.detection.thresholdValue = 0.01
detectConfig.detection.background.weighting = False
detectTask = DetectCoaddSourcesTask(config=detectConfig)
butler = lsst.pipe.tasks.mocks.makeDataRepo(DATAREPO_ROOT)
mocksTask.buildAllInputs(butler)
addMaskPlanes(butler)
mocksTask.buildCoadd(butler)
mocksTask.buildMockCoadd(butler)
detectTask.writeSchemas(butler)
# Now run the seperate multiband tasks on the Coadd to make the reference
# catalog for the forced photometry tests.
runTaskOnPatches(butler, detectTask, mocksTask)
mergeDetConfig = MergeDetectionsTask.ConfigClass()
mergeDetConfig.priorityList = ['r', ]
mergeDetTask = MergeDetectionsTask(config=mergeDetConfig, butler=butler)
mergeDetTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeDetTask, mocksTask)
deblendSourcesConfig = DeblendCoaddSourcesTask.ConfigClass()
deblendSourcesTask = DeblendCoaddSourcesTask(config=deblendSourcesConfig, butler=butler)
deblendSourcesTask.writeSchemas(butler)
runTaskOnPatchList(butler, deblendSourcesTask, mocksTask)
measMergedConfig = MeasureMergedCoaddSourcesTask.ConfigClass()
measMergedConfig.measurement.slots.shape = "base_SdssShape"
measMergedConfig.measurement.plugins['base_PixelFlags'].masksFpAnywhere = []
measMergedConfig.propagateFlags.flags = {} # Disable flag propagation: no flags to propagate
measMergedConfig.doMatchSources = False # We don't have a reference catalog available
measMergedTask = MeasureMergedCoaddSourcesTask(config=measMergedConfig, butler=butler)
measMergedTask.writeSchemas(butler)
runTaskOnPatches(butler, measMergedTask, mocksTask)
mergeMeasConfig = MergeMeasurementsTask.ConfigClass()
mergeMeasConfig.priorityList = ['r', ]
mergeMeasTask = MergeMeasurementsTask(config=mergeMeasConfig, butler=butler)
mergeMeasTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeMeasTask, mocksTask)
runForcedPhotCoaddTask(butler, mocksTask)
runForcedPhotCcdTask(butler)
def setUpClass(cls):
"""Create 200MB of test data."""
# Start by assuming nothing failed
cls.failed = False
if os.path.exists(DATAREPO_ROOT):
print(f"Deleting existing repo: {DATAREPO_ROOT}")
# Do not ignore errors since failure to clean up is indicative
shutil.rmtree(DATAREPO_ROOT)
# Create a task that creates simulated images and builds a coadd from them
mocksTask = lsst.pipe.tasks.mocks.MockCoaddTask()
# Create an instance of DetectCoaddSourcesTask to measure on the coadd.
# There's no noise in these images, so we set a direct-value threshold,
# and the background weighting (when using Approximate) to False
detectConfig = DetectCoaddSourcesTask.ConfigClass()
# Images have no noise, so we can't use the default DynamicDetectionTask
detectConfig.detection.retarget(
lsst.meas.algorithms.SourceDetectionTask)
detectConfig.detection.thresholdType = "value"
detectConfig.detection.thresholdValue = 0.01
detectConfig.detection.background.weighting = False
detectTask = DetectCoaddSourcesTask(config=detectConfig)
butler = lsst.pipe.tasks.mocks.makeDataRepo(DATAREPO_ROOT)
mocksTask.buildAllInputs(butler)
addMaskPlanes(butler)
mocksTask.buildCoadd(butler)
mocksTask.buildMockCoadd(butler)
detectTask.writeSchemas(butler)
# Now run the seperate multiband tasks on the Coadd to make the reference
# catalog for the forced photometry tests.
runTaskOnPatches(butler, detectTask, mocksTask)
mergeDetConfig = MergeDetectionsTask.ConfigClass()
mergeDetConfig.priorityList = [
'r',
]
mergeDetTask = MergeDetectionsTask(config=mergeDetConfig,
butler=butler)
mergeDetTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeDetTask, mocksTask)
deblendSourcesConfig = DeblendCoaddSourcesTask.ConfigClass()
deblendSourcesTask = DeblendCoaddSourcesTask(
config=deblendSourcesConfig, butler=butler)
deblendSourcesTask.writeSchemas(butler)
runTaskOnPatchList(butler, deblendSourcesTask, mocksTask)
measMergedConfig = MeasureMergedCoaddSourcesTask.ConfigClass()
measMergedConfig.measurement.slots.shape = "base_SdssShape"
measMergedConfig.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = []
measMergedConfig.propagateFlags.flags = {
} # Disable flag propagation: no flags to propagate
measMergedConfig.doMatchSources = False # We don't have a reference catalog available
measMergedTask = MeasureMergedCoaddSourcesTask(config=measMergedConfig,
butler=butler)
measMergedTask.writeSchemas(butler)
runTaskOnPatches(butler, measMergedTask, mocksTask)
mergeMeasConfig = MergeMeasurementsTask.ConfigClass()
mergeMeasConfig.priorityList = [
'r',
]
mergeMeasTask = MergeMeasurementsTask(config=mergeMeasConfig,
butler=butler)
mergeMeasTask.writeSchemas(butler)
runTaskOnPatchList(butler, mergeMeasTask, mocksTask)
runForcedPhotCoaddTask(butler, mocksTask)
runForcedPhotCcdTask(butler)
class CoaddsTestCase(lsst.utils.tests.TestCase):
def setupClass(self):
"""Delete the data repo if it exists"""
def setUp(self):
# Create a task that creates simulated images and builds a coadd from them
self.mocksTask = lsst.pipe.tasks.mocks.MockCoaddTask()
# Create an instance of DetectCoaddSourcesTask to measure on the coadd.
# There's no noise in these images, so we set a direct-value threshold,
# and the background weighting (when using Approximate) to False
detectConfig = DetectCoaddSourcesTask.ConfigClass()
detectConfig.detection.thresholdType = "value"
detectConfig.detection.thresholdValue = 0.01
detectConfig.detection.background.weighting = False
self.detectTask = DetectCoaddSourcesTask(config=detectConfig)
if REUSE_DATAREPO and os.path.exists(
os.path.join(DATAREPO_ROOT, "_mapper")):
self.butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
else:
self.butler = lsst.pipe.tasks.mocks.makeDataRepo(DATAREPO_ROOT)
self.mocksTask.buildAllInputs(self.butler)
self.addMaskPlanes()
self.checkMaskPlanesExist()
self.mocksTask.buildCoadd(self.butler)
self.mocksTask.buildMockCoadd(self.butler)
self.detectTask.writeSchemas(self.butler)
# Now run the seperate multiband tasks on the Coadd to make the reference
# catalog for the forced photometry tests.
self.runTaskOnPatches(self.detectTask)
mergeDetConfig = MergeDetectionsTask.ConfigClass()
mergeDetConfig.priorityList = [
'r',
]
mergeDetTask = MergeDetectionsTask(config=mergeDetConfig,
butler=self.butler)
mergeDetTask.writeSchemas(self.butler)
self.runTaskOnPatchList(mergeDetTask)
measMergedConfig = MeasureMergedCoaddSourcesTask.ConfigClass()
measMergedConfig.measurement.slots.shape = "base_SdssShape"
measMergedConfig.measurement.doApplyApCorr = "no"
measMergedConfig.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = []
measMergedConfig.propagateFlags.flags = {
} # Disable flag propagation: no flags to propagate
measMergedConfig.doMatchSources = False # We don't have a reference catalog available
measMergedTask = MeasureMergedCoaddSourcesTask(
config=measMergedConfig, butler=self.butler)
measMergedTask.writeSchemas(self.butler)
self.runTaskOnPatches(measMergedTask)
mergeMeasConfig = MergeMeasurementsTask.ConfigClass()
mergeMeasConfig.priorityList = [
'r',
]
mergeMeasTask = MergeMeasurementsTask(config=mergeMeasConfig,
butler=self.butler)
mergeMeasTask.writeSchemas(self.butler)
self.runTaskOnPatchList(mergeMeasTask)
def tearDown(self):
if not REUSE_DATAREPO:
shutil.rmtree(DATAREPO_ROOT)
del self.mocksTask
del self.detectTask
del self.butler
def getCalexpIds(self, butler):
if len(butler.repository._parents) is not 1:
raise RuntimeError(
"This test assumes one input repository, can not continue with %s"
% butler.repository._parents)
return butler.repository._parents[0]._mapper.index['calexp']['visit'][
None]
def addMaskPlanes(self):
butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
# Get the dataId for each calexp in the repository
calexpDataIds = self.getCalexpIds(butler)
# Loop over each of the calexp and add the CROSSTALK and NOT_DEBLENDED mask planes
for Id in calexpDataIds:
image = butler.get('calexp', Id)
mask = image.getMaskedImage().getMask()
mask.addMaskPlane("CROSSTALK")
mask.addMaskPlane("NOT_DEBLENDED")
butler.put(image, 'calexp', dataId=Id)
def checkMaskPlanesExist(self):
butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
# Get the dataId for each calexp in the repository
calexpDataIds = self.getCalexpIds(butler)
# Loop over each Id and verify the mask planes were added
for ID in calexpDataIds:
image = butler.get('calexp', ID)
mask = image.getMaskedImage().getMask()
self.assert_('CROSSTALK' in mask.getMaskPlaneDict().keys())
self.assert_('NOT_DEBLENDED' in mask.getMaskPlaneDict().keys())
def runTaskOnPatches(self, task, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
for dataRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
task.run(dataRef)
def runTaskOnPatchList(self, task, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
for dataRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
task.run([dataRef])
def runTaskOnCcds(self, task, tract=0):
catalog = self.butler.get("observations", tract=tract, immediate=True)
visitKey = catalog.getSchema().find("visit").key
ccdKey = catalog.getSchema().find("ccd").key
for record in catalog:
dataRef = self.butler.dataRef("forced_src",
tract=tract,
visit=record.getI(visitKey),
ccd=record.getI(ccdKey))
task.run(dataRef)
def getObsDict(self, tract=0):
catalog = self.butler.get("observations", tract=tract, immediate=True)
visitKey = catalog.getSchema().find("visit").key
ccdKey = catalog.getSchema().find("ccd").key
obsDict = {}
for record in catalog:
visit = record.getI(visitKey)
ccd = record.getI(ccdKey)
obsDict.setdefault(visit, {})[ccd] = record
return obsDict
def comparePsfs(self, a, b):
if a is None and b is None:
return
ak = lsst.meas.algorithms.KernelPsf.swigConvert(a).getKernel()
bk = lsst.meas.algorithms.KernelPsf.swigConvert(b).getKernel()
self.assertEqual(type(ak), type(bk))
self.assertEqual(ak.getDimensions(), bk.getDimensions())
self.assertEqual(ak.getNKernelParameters(), ak.getNKernelParameters())
self.assertEqual(ak.getNSpatialParameters(),
ak.getNSpatialParameters())
for aFuncParams, bFuncParams in zip(ak.getSpatialParameters(),
bk.getSpatialParameters()):
for aParam, bParam in zip(aFuncParams, bFuncParams):
self.assertEqual(aParam, bParam)
@unittest.skip("Remove test until DM-5174 is complete")
def testMasksRemoved(self):
image = self.butler.get(self.mocksTask.config.coaddName + 'Coadd_mock',\
{'filter':'r','tract':0, 'patch':'0,0'})
keys = image.getMaskedImage().getMask().getMaskPlaneDict().keys()
self.assert_('CROSSTALK' not in keys)
self.assert_('NOT_DEBLENDED' not in keys)
def testTempExpInputs(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
for visit, obsVisitDict in self.getObsDict(tract).iteritems():
foundOneTempExp = False
for patchRef in self.mocksTask.iterPatchRefs(
self.butler, tractInfo):
try:
tempExp = patchRef.get(self.mocksTask.config.coaddName +
"Coadd_tempExp",
visit=visit,
immediate=True)
foundOneTempExp = True
except:
continue
self.assertEqual(tractInfo.getWcs(), tempExp.getWcs())
coaddInputs = tempExp.getInfo().getCoaddInputs()
self.assertEqual(len(coaddInputs.visits), 1)
visitRecord = coaddInputs.visits[0]
self.assertEqual(visitRecord.getWcs(), tempExp.getWcs())
self.assertEqual(visitRecord.getBBox(), tempExp.getBBox())
self.assert_(len(coaddInputs.ccds) > 0)
ccdKey = coaddInputs.ccds.getSchema().find("ccd").key
for ccdRecord in coaddInputs.ccds:
ccd = ccdRecord.getI(ccdKey)
obsRecord = obsVisitDict[ccd]
self.assertEqual(obsRecord.getId(), ccdRecord.getId())
self.assertEqual(obsRecord.getWcs(), ccdRecord.getWcs())
self.assertEqual(obsRecord.getBBox(), ccdRecord.getBBox())
self.comparePsfs(obsRecord.getPsf(), ccdRecord.getPsf())
self.assert_(foundOneTempExp)
def testCoaddInputs(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
obsCatalog = self.butler.get("observations",
tract=tract,
immediate=True)
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd",
immediate=True)
self.assertEqual(tractInfo.getWcs(), coaddExp.getWcs())
coaddInputs = coaddExp.getInfo().getCoaddInputs()
try:
ccdVisitKey = coaddInputs.ccds.getSchema().find("visit").key
except:
print patchRef.dataId
print coaddInputs.ccds.getSchema()
raise
for ccdRecord in coaddInputs.ccds:
obsRecord = obsCatalog.find(ccdRecord.getId())
self.assertEqual(obsRecord.getId(), ccdRecord.getId())
self.assertEqual(obsRecord.getWcs(), ccdRecord.getWcs())
self.assertEqual(obsRecord.getBBox(), ccdRecord.getBBox())
self.comparePsfs(obsRecord.getPsf(), ccdRecord.getPsf())
self.assert_(
coaddInputs.visits.find(ccdRecord.getL(ccdVisitKey))
is not None)
for visitRecord in coaddInputs.visits:
nCcds = len([
ccdRecord for ccdRecord in coaddInputs.ccds
if ccdRecord.getL(ccdVisitKey) == visitRecord.getId()
])
self.assert_(nCcds >= 1)
self.assert_(nCcds <= 2)
def testPsfInstallation(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd",
immediate=True)
ccdCat = coaddExp.getInfo().getCoaddInputs().ccds
savedPsf = lsst.meas.algorithms.CoaddPsf.swigConvert(
coaddExp.getPsf())
newPsf = lsst.meas.algorithms.CoaddPsf(ccdCat, coaddExp.getWcs())
self.assertEqual(savedPsf.getComponentCount(), len(ccdCat))
self.assertEqual(newPsf.getComponentCount(), len(ccdCat))
for n, record in enumerate(ccdCat):
self.assert_(
lsst.afw.table.io.comparePersistablePtrs(
savedPsf.getPsf(n), record.getPsf()))
self.assert_(
lsst.afw.table.io.comparePersistablePtrs(
newPsf.getPsf(n), record.getPsf()))
self.assert_(
lsst.afw.table.io.comparePersistablePtrs(
savedPsf.getWcs(n), record.getWcs()))
self.assert_(
lsst.afw.table.io.comparePersistablePtrs(
newPsf.getWcs(n), record.getWcs()))
self.assertEqual(savedPsf.getBBox(n), record.getBBox())
self.assertEqual(newPsf.getBBox(n), record.getBBox())
def testCoaddPsf(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName +
"Coadd_skyMap",
immediate=True)
tractInfo = skyMap[tract]
# Start by finding objects that never appeared on the edge of an image
simSrcCat = self.butler.get("simsrc", tract=tract, immediate=True)
simSrcSchema = simSrcCat.getSchema()
objectIdKey = simSrcSchema.find("objectId").key
centroidInBBoxKey = simSrcSchema.find("centroidInBBox").key
partialOverlapKey = simSrcSchema.find("partialOverlap").key
simSrcByObject = {}
for simSrcRecord in simSrcCat:
simSrcByObject.setdefault(simSrcRecord.getL(objectIdKey),
[]).append(simSrcRecord)
pureObjectIds = set(
) # set will contain objects that never appear on edges
for objectId, simSrcRecords in simSrcByObject.iteritems():
inAnyImages = False
for simSrcRecord in simSrcRecords:
if simSrcRecord.getFlag(centroidInBBoxKey):
if simSrcRecord.getFlag(partialOverlapKey):
break
inAnyImages = True
else: # only get here if we didn't break
if inAnyImages:
pureObjectIds.add(objectId)
truthCatalog = self.butler.get("truth", tract=tract, immediate=True)
truthCatalog.sort()
nTested = 0
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd",
immediate=True)
coaddWcs = coaddExp.getWcs()
coaddPsf = coaddExp.getPsf()
coaddBBox = lsst.afw.geom.Box2D(coaddExp.getBBox())
for objectId in pureObjectIds:
truthRecord = truthCatalog.find(objectId)
position = coaddWcs.skyToPixel(truthRecord.getCoord())
if not coaddBBox.contains(position):
continue
try:
psfImage = coaddPsf.computeImage(position)
except:
continue
psfImageBBox = psfImage.getBBox()
if not coaddExp.getBBox().contains(psfImageBBox):
continue
starImage = lsst.afw.image.ImageF(
coaddExp.getMaskedImage().getImage(),
psfImageBBox).convertD()
starImage /= starImage.getArray().sum()
psfImage /= psfImage.getArray().sum()
residuals = lsst.afw.image.ImageD(starImage, True)
residuals -= psfImage
self.assertClose(starImage.getArray(),
psfImage.getArray(),
rtol=1E-3,
atol=1E-2)
nTested += 1
if nTested == 0:
print(
"WARNING: CoaddPsf test inconclusive (this can occur randomly, but very rarely; "
"first try running the test again)")
def testForcedPhotCoaddTask(self):
config = lsst.meas.base.ForcedPhotCoaddConfig()
config.references.filter = 'r'
config.measurement.doApplyApCorr = "no"
task = lsst.meas.base.ForcedPhotCoaddTask(config=config,
butler=self.butler)
task.writeSchemas(self.butler)
self.runTaskOnPatches(task)
def testForcedPhotCcdTask(self):
config = lsst.meas.base.ForcedPhotCcdConfig()
config.references.filter = 'r'
config.measurement.doApplyApCorr = "no"
task = lsst.meas.base.ForcedPhotCcdTask(config=config,
butler=self.butler)
self.runTaskOnCcds(task)
class CoaddsTestCase(lsst.utils.tests.TestCase):
def setupClass(self):
"""Delete the data repo if it exists"""
def setUp(self):
# Create a task that creates simulated images and builds a coadd from them
self.mocksTask = lsst.pipe.tasks.mocks.MockCoaddTask()
# Create an instance of DetectCoaddSourcesTask to measure on the coadd.
# There's no noise in these images, so we set a direct-value threshold,
# and the background weighting (when using Approximate) to False
detectConfig = DetectCoaddSourcesTask.ConfigClass()
detectConfig.detection.thresholdType = "value"
detectConfig.detection.thresholdValue = 0.01
detectConfig.detection.background.weighting = False
self.detectTask = DetectCoaddSourcesTask(config=detectConfig)
if REUSE_DATAREPO and os.path.exists(os.path.join(DATAREPO_ROOT, "_mapper")):
self.butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
else:
self.butler = lsst.pipe.tasks.mocks.makeDataRepo(DATAREPO_ROOT)
self.mocksTask.buildAllInputs(self.butler)
self.addMaskPlanes()
self.checkMaskPlanesExist()
self.mocksTask.buildCoadd(self.butler)
self.mocksTask.buildMockCoadd(self.butler)
self.detectTask.writeSchemas(self.butler)
# Now run the seperate multiband tasks on the Coadd to make the reference
# catalog for the forced photometry tests.
self.runTaskOnPatches(self.detectTask)
mergeDetConfig = MergeDetectionsTask.ConfigClass()
mergeDetConfig.priorityList = ['r', ]
mergeDetTask = MergeDetectionsTask(config=mergeDetConfig, butler=self.butler)
mergeDetTask.writeSchemas(self.butler)
self.runTaskOnPatchList(mergeDetTask)
measMergedConfig = MeasureMergedCoaddSourcesTask.ConfigClass()
measMergedConfig.measurement.slots.shape = "base_SdssShape"
measMergedConfig.measurement.doApplyApCorr = "no"
measMergedConfig.measurement.plugins['base_PixelFlags'].masksFpAnywhere = []
measMergedConfig.propagateFlags.flags = {} # Disable flag propagation: no flags to propagate
measMergedTask = MeasureMergedCoaddSourcesTask(config=measMergedConfig, butler=self.butler)
measMergedTask.writeSchemas(self.butler)
self.runTaskOnPatches(measMergedTask)
mergeMeasConfig = MergeMeasurementsTask.ConfigClass()
mergeMeasConfig.priorityList = ['r', ]
mergeMeasTask = MergeMeasurementsTask(config=mergeMeasConfig, butler=self.butler)
mergeMeasTask.writeSchemas(self.butler)
self.runTaskOnPatchList(mergeMeasTask)
def tearDown(self):
if not REUSE_DATAREPO:
shutil.rmtree(DATAREPO_ROOT)
del self.mocksTask
del self.detectTask
del self.butler
def getCalexpIds(self, butler):
if len(butler.repository._parents) is not 1:
raise RuntimeError("This test assumes one input repository, can not continue with %s"
% butler.repository._parents)
return butler.repository._parents[0]._mapper.index['calexp']['visit'][None]
def addMaskPlanes(self):
butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
# Get the dataId for each calexp in the repository
calexpDataIds = self.getCalexpIds(butler)
# Loop over each of the calexp and add the CROSSTALK and NOT_DEBLENDED mask planes
for Id in calexpDataIds:
image = butler.get('calexp', Id)
mask = image.getMaskedImage().getMask()
mask.addMaskPlane("CROSSTALK")
mask.addMaskPlane("NOT_DEBLENDED")
butler.put(image, 'calexp', dataId=Id)
def checkMaskPlanesExist(self):
butler = lsst.daf.persistence.Butler(DATAREPO_ROOT)
# Get the dataId for each calexp in the repository
calexpDataIds = self.getCalexpIds(butler)
# Loop over each Id and verify the mask planes were added
for ID in calexpDataIds:
image = butler.get('calexp', ID)
mask = image.getMaskedImage().getMask()
self.assert_('CROSSTALK' in mask.getMaskPlaneDict().keys())
self.assert_('NOT_DEBLENDED' in mask.getMaskPlaneDict().keys())
def runTaskOnPatches(self, task, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
for dataRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
task.run(dataRef)
def runTaskOnPatchList(self, task, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
for dataRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
task.run([dataRef])
def runTaskOnCcds(self, task, tract=0):
catalog = self.butler.get("observations", tract=tract, immediate=True)
visitKey = catalog.getSchema().find("visit").key
ccdKey = catalog.getSchema().find("ccd").key
for record in catalog:
dataRef = self.butler.dataRef("forced_src", tract=tract, visit=record.getI(visitKey),
ccd=record.getI(ccdKey))
task.run(dataRef)
def getObsDict(self, tract=0):
catalog = self.butler.get("observations", tract=tract, immediate=True)
visitKey = catalog.getSchema().find("visit").key
ccdKey = catalog.getSchema().find("ccd").key
obsDict = {}
for record in catalog:
visit = record.getI(visitKey)
ccd = record.getI(ccdKey)
obsDict.setdefault(visit, {})[ccd] = record
return obsDict
def comparePsfs(self, a, b):
if a is None and b is None:
return
ak = lsst.meas.algorithms.KernelPsf.swigConvert(a).getKernel()
bk = lsst.meas.algorithms.KernelPsf.swigConvert(b).getKernel()
self.assertEqual(type(ak), type(bk))
self.assertEqual(ak.getDimensions(), bk.getDimensions())
self.assertEqual(ak.getNKernelParameters(), ak.getNKernelParameters())
self.assertEqual(ak.getNSpatialParameters(), ak.getNSpatialParameters())
for aFuncParams, bFuncParams in zip(ak.getSpatialParameters(), bk.getSpatialParameters()):
for aParam, bParam in zip(aFuncParams, bFuncParams):
self.assertEqual(aParam, bParam)
@unittest.skip("Remove test until DM-5174 is complete")
def testMasksRemoved(self):
image = self.butler.get(self.mocksTask.config.coaddName + 'Coadd_mock',\
{'filter':'r','tract':0, 'patch':'0,0'})
keys = image.getMaskedImage().getMask().getMaskPlaneDict().keys()
self.assert_('CROSSTALK' not in keys)
self.assert_('NOT_DEBLENDED' not in keys)
def testTempExpInputs(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
for visit, obsVisitDict in self.getObsDict(tract).iteritems():
foundOneTempExp = False
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
try:
tempExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd_tempExp", visit=visit,
immediate=True)
foundOneTempExp = True
except:
continue
self.assertEqual(tractInfo.getWcs(), tempExp.getWcs())
coaddInputs = tempExp.getInfo().getCoaddInputs()
self.assertEqual(len(coaddInputs.visits), 1)
visitRecord = coaddInputs.visits[0]
self.assertEqual(visitRecord.getWcs(), tempExp.getWcs())
self.assertEqual(visitRecord.getBBox(), tempExp.getBBox())
self.assert_(len(coaddInputs.ccds) > 0)
ccdKey = coaddInputs.ccds.getSchema().find("ccd").key
for ccdRecord in coaddInputs.ccds:
ccd = ccdRecord.getI(ccdKey)
obsRecord = obsVisitDict[ccd]
self.assertEqual(obsRecord.getId(), ccdRecord.getId())
self.assertEqual(obsRecord.getWcs(), ccdRecord.getWcs())
self.assertEqual(obsRecord.getBBox(), ccdRecord.getBBox())
self.comparePsfs(obsRecord.getPsf(), ccdRecord.getPsf())
self.assert_(foundOneTempExp)
def testCoaddInputs(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
obsCatalog = self.butler.get("observations", tract=tract, immediate=True)
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd", immediate=True)
self.assertEqual(tractInfo.getWcs(), coaddExp.getWcs())
coaddInputs = coaddExp.getInfo().getCoaddInputs()
try:
ccdVisitKey = coaddInputs.ccds.getSchema().find("visit").key
except:
print patchRef.dataId
print coaddInputs.ccds.getSchema()
raise
for ccdRecord in coaddInputs.ccds:
obsRecord = obsCatalog.find(ccdRecord.getId())
self.assertEqual(obsRecord.getId(), ccdRecord.getId())
self.assertEqual(obsRecord.getWcs(), ccdRecord.getWcs())
self.assertEqual(obsRecord.getBBox(), ccdRecord.getBBox())
self.comparePsfs(obsRecord.getPsf(), ccdRecord.getPsf())
self.assert_(coaddInputs.visits.find(ccdRecord.getL(ccdVisitKey)) is not None)
for visitRecord in coaddInputs.visits:
nCcds = len([ccdRecord for ccdRecord in coaddInputs.ccds
if ccdRecord.getL(ccdVisitKey) == visitRecord.getId()])
self.assert_(nCcds >= 1)
self.assert_(nCcds <= 2)
def testPsfInstallation(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd", immediate=True)
ccdCat = coaddExp.getInfo().getCoaddInputs().ccds
savedPsf = lsst.meas.algorithms.CoaddPsf.swigConvert(coaddExp.getPsf())
newPsf = lsst.meas.algorithms.CoaddPsf(ccdCat, coaddExp.getWcs())
self.assertEqual(savedPsf.getComponentCount(), len(ccdCat))
self.assertEqual(newPsf.getComponentCount(), len(ccdCat))
for n, record in enumerate(ccdCat):
self.assert_(lsst.afw.table.io.comparePersistablePtrs(savedPsf.getPsf(n), record.getPsf()))
self.assert_(lsst.afw.table.io.comparePersistablePtrs(newPsf.getPsf(n), record.getPsf()))
self.assert_(lsst.afw.table.io.comparePersistablePtrs(savedPsf.getWcs(n), record.getWcs()))
self.assert_(lsst.afw.table.io.comparePersistablePtrs(newPsf.getWcs(n), record.getWcs()))
self.assertEqual(savedPsf.getBBox(n), record.getBBox())
self.assertEqual(newPsf.getBBox(n), record.getBBox())
def testCoaddPsf(self, tract=0):
skyMap = self.butler.get(self.mocksTask.config.coaddName + "Coadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
# Start by finding objects that never appeared on the edge of an image
simSrcCat = self.butler.get("simsrc", tract=tract, immediate=True)
simSrcSchema = simSrcCat.getSchema()
objectIdKey = simSrcSchema.find("objectId").key
centroidInBBoxKey = simSrcSchema.find("centroidInBBox").key
partialOverlapKey = simSrcSchema.find("partialOverlap").key
simSrcByObject = {}
for simSrcRecord in simSrcCat:
simSrcByObject.setdefault(simSrcRecord.getL(objectIdKey), []).append(simSrcRecord)
pureObjectIds = set() # set will contain objects that never appear on edges
for objectId, simSrcRecords in simSrcByObject.iteritems():
inAnyImages = False
for simSrcRecord in simSrcRecords:
if simSrcRecord.getFlag(centroidInBBoxKey):
if simSrcRecord.getFlag(partialOverlapKey):
break
inAnyImages = True
else: # only get here if we didn't break
if inAnyImages:
pureObjectIds.add(objectId)
truthCatalog = self.butler.get("truth", tract=tract, immediate=True)
truthCatalog.sort()
nTested = 0
for patchRef in self.mocksTask.iterPatchRefs(self.butler, tractInfo):
coaddExp = patchRef.get(self.mocksTask.config.coaddName + "Coadd", immediate=True)
coaddWcs = coaddExp.getWcs()
coaddPsf = coaddExp.getPsf()
coaddBBox = lsst.afw.geom.Box2D(coaddExp.getBBox())
for objectId in pureObjectIds:
truthRecord = truthCatalog.find(objectId)
position = coaddWcs.skyToPixel(truthRecord.getCoord())
if not coaddBBox.contains(position):
continue
try:
psfImage = coaddPsf.computeImage(position)
except:
continue
psfImageBBox = psfImage.getBBox()
if not coaddExp.getBBox().contains(psfImageBBox):
continue
starImage = lsst.afw.image.ImageF(coaddExp.getMaskedImage().getImage(),
psfImageBBox).convertD()
starImage /= starImage.getArray().sum()
psfImage /= psfImage.getArray().sum()
residuals = lsst.afw.image.ImageD(starImage, True)
residuals -= psfImage
self.assertClose(starImage.getArray(), psfImage.getArray(), rtol=1E-3, atol=1E-2)
nTested += 1
if nTested == 0:
print ("WARNING: CoaddPsf test inconclusive (this can occur randomly, but very rarely; "
"first try running the test again)")
def testForcedPhotCoaddTask(self):
config = lsst.meas.base.ForcedPhotCoaddConfig()
config.references.filter = 'r'
config.measurement.doApplyApCorr = "no"
task = lsst.meas.base.ForcedPhotCoaddTask(config=config, butler=self.butler)
task.writeSchemas(self.butler)
self.runTaskOnPatches(task)
def testForcedPhotCcdTask(self):
config = lsst.meas.base.ForcedPhotCcdConfig()
config.references.filter = 'r'
config.measurement.doApplyApCorr = "no"
task = lsst.meas.base.ForcedPhotCcdTask(config=config, butler=self.butler)
self.runTaskOnCcds(task)
