def testMockDiaPipelineTask(self):
config = MockDiaPipelineTask.ConfigClass()
config.apdb.db_url = "testing_only"
task = MockDiaPipelineTask(config=config)
pipelineTests.assertValidInitOutput(task)
result = task.run(pandas.DataFrame(), pandas.DataFrame(),
afwImage.ExposureF(), afwImage.ExposureF(),
afwImage.ExposureF(), 42, 'k')
pipelineTests.assertValidOutput(task, result)
self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable",
self.visitId)
self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp",
self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"diaSourceTable": self.visitId,
"solarSystemObjectTable": self.visitId,
"diffIm": self.visitId,
"exposure": self.visitId,
"template": self.visitId,
"apdbMarker": self.visitId,
"associatedDiaSources": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockDetectAndMeasureTask(self):
task = MockDetectAndMeasureTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(
science=afwImage.ExposureF(),
matchedTemplate=afwImage.ExposureF(),
difference=afwImage.ExposureF(),
selectSources=afwTable.SourceCatalog(),
)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_matchedExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceTempExp",
self.visitId)
self.butler.put(afwTable.SourceCatalog(), "src", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"science": self.visitId,
"matchedTemplate": self.visitId,
"difference": self.visitId,
"selectSources": self.visitId,
"diaSources": self.visitId,
"subtractedMeasuredExposure": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockGetTemplateTask(self):
task = MockGetTemplateTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(
coaddExposures=[afwImage.ExposureF()],
bbox=lsst.geom.Box2I(),
wcs=None, # Should not be allowed, but too hard to fake a SkyWcs
dataIds=[],
)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig())
self.butler.put(skymap,
lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
self.skymapId)
self.butler.put(afwImage.ExposureF(), "goodSeeingCoadd", self.patchId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.skymapVisitId, {
"bbox": self.visitId,
"wcs": self.visitId,
"skyMap": self.skymapId,
"coaddExposures": [self.patchId],
"template": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMissingData(self):
"""Test the run method with no data.
"""
result = self.task.run(None, None)
testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertIsNone(meas)
def testAssertValidOutputPass(self):
task = VisitTask()
inA = butlerTests.MetricsExample(data=[1, 2, 3])
inB = butlerTests.MetricsExample(data=[4, 0, 1])
result = task.run(inA, inB)
# should not throw
assertValidOutput(task, result)
def testMockIsr(self):
# Testing MockIsrTask is tricky because the real ISR has an unstable
# interface with dozens of potential inputs, too many to pass through
# runTestQuantum. I don't see a good way to test the inputs;
# fortunately, this is unlikely to matter for the overall goal of
# testing ap_verify's interaction with the AP pipeline.
task = MockIsrTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwImage.ExposureF())
pipelineTests.assertValidOutput(task, result)
def testValidEmpty(self):
"""Test the run method with a valid but zero result.
"""
metricComplete = self.makeTask(self.magCut, self.magCut + 5)
result = metricComplete.run(self.fakeCat, self.band)
testUtils.assertValidOutput(metricComplete, result)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_pipe.apFakesCompleteness"))
self.assertEqual(meas.quantity, 0 * u.dimensionless_unscaled)
def testAssertValidOutputMultiple(self):
task = VisitTask()
def run(a, b):
return Struct(outA=[a], outB=b)
task.run = run
inA = butlerTests.MetricsExample(data=[1, 2, 3])
inB = butlerTests.MetricsExample(data=[4, 0, 1])
result = task.run(inA, inB)
with self.assertRaises(AssertionError):
assertValidOutput(task, result)
def testAssertValidOutputSingle(self):
task = PatchTask()
def run(a, b):
return Struct(out=b)
task.run = run
inA = butlerTests.MetricsExample(data=[1, 2, 3])
inB = butlerTests.MetricsExample(data=[4, 0, 1])
result = task.run([inA], inB)
with self.assertRaises(AssertionError):
assertValidOutput(task, result)
def testMockCharacterizeImageTask(self):
task = MockCharacterizeImageTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwImage.ExposureF())
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"exposure": self.exposureId,
"characterized": self.visitId,
"sourceCat": self.visitId,
"backgroundModel": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testValid(self):
"""Test the run method.
"""
result = self.task.run(self.fakeCat, self.band)
testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_pipe.apFakesCompleteness"))
# Work around for Mac failing this test.
self.assertAlmostEqual(
meas.quantity.value,
((self.expectedAllMatched / self.targetSources) *
u.dimensionless_unscaled).value,
places=2)
def testMockTransformDiaSourceCatalogTask(self):
task = MockTransformDiaSourceCatalogTask(
initInputs=afwTable.SourceCatalog())
pipelineTests.assertValidInitOutput(task)
result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k',
42)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc",
self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp",
self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"diaSourceCat": self.visitId,
"diffIm": self.visitId,
"diaSourceTable": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockAlardLuptonSubtractTask(self):
task = MockAlardLuptonSubtractTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwImage.ExposureF(), afwImage.ExposureF(),
afwTable.SourceCatalog())
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwTable.SourceCatalog(), "src", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"template": self.visitId,
"science": self.visitId,
"sources": self.visitId,
"difference": self.visitId,
"matchedTemplate": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def _testRun(self, doPackageAlerts=False, doSolarSystemAssociation=False):
"""Test the normal workflow of each ap_pipe step.
"""
config = self._makeDefaultConfig(
doPackageAlerts=doPackageAlerts,
doSolarSystemAssociation=doSolarSystemAssociation)
task = DiaPipelineTask(config=config)
# Set DataFrame index testing to always return False. Mocks return
# true for this check otherwise.
task.testDataFrameIndex = lambda x: False
diffIm = Mock(spec=afwImage.ExposureF)
exposure = Mock(spec=afwImage.ExposureF)
template = Mock(spec=afwImage.ExposureF)
diaSrc = MagicMock(spec=pd.DataFrame())
ssObjects = MagicMock(spec=pd.DataFrame())
ccdExposureIdBits = 32
# Each of these subtasks should be called once during diaPipe
# execution. We use mocks here to check they are being executed
# appropriately.
subtasksToMock = [
"diaCatalogLoader",
"diaCalculation",
"diaForcedSource",
]
if doPackageAlerts:
subtasksToMock.append("alertPackager")
else:
self.assertFalse(hasattr(task, "alertPackager"))
if not doSolarSystemAssociation:
self.assertFalse(hasattr(task, "solarSystemAssociator"))
def concatMock(data):
return MagicMock(spec=pd.DataFrame)
# Mock out the run() methods of these two Tasks to ensure they
# return data in the correct form.
@lsst.utils.timer.timeMethod
def solarSystemAssociator_run(self, unAssocDiaSources,
solarSystemObjectTable, diffIm):
return lsst.pipe.base.Struct(
nTotalSsObjects=42,
nAssociatedSsObjects=30,
ssoAssocDiaSources=MagicMock(spec=pd.DataFrame()),
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))
@lsst.utils.timer.timeMethod
def associator_run(self, table, diaObjects):
return lsst.pipe.base.Struct(
nUpdatedDiaObjects=2,
nUnassociatedDiaObjects=3,
matchedDiaSources=MagicMock(spec=pd.DataFrame()),
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))
# apdb isn't a subtask, but still needs to be mocked out for correct
# execution in the test environment.
with patch.multiple(
task, **{task: DEFAULT
for task in subtasksToMock + ["apdb"]}):
with patch('lsst.ap.association.diaPipe.pd.concat', new=concatMock), \
patch('lsst.ap.association.association.AssociationTask.run', new=associator_run), \
patch('lsst.ap.association.ssoAssociation.SolarSystemAssociationTask.run',
new=solarSystemAssociator_run):
result = task.run(diaSrc, ssObjects, diffIm, exposure,
template, ccdExposureIdBits, "g")
for subtaskName in subtasksToMock:
getattr(task, subtaskName).run.assert_called_once()
assertValidOutput(task, result)
self.assertEqual(result.apdbMarker.db_url, "sqlite://")
meta = task.getFullMetadata()
# Check that the expected metadata has been set.
self.assertEqual(meta["diaPipe.numUpdatedDiaObjects"], 2)
self.assertEqual(meta["diaPipe.numUnassociatedDiaObjects"], 3)
# and that associators ran once or not at all.
self.assertEqual(
len(
meta.getArray(
"diaPipe:associator.associator_runEndUtc")), 1)
if doSolarSystemAssociation:
self.assertEqual(
len(
meta.getArray("diaPipe:solarSystemAssociator."
"solarSystemAssociator_runEndUtc")),
1)
else:
self.assertNotIn("diaPipe:solarSystemAssociator", meta)