def testMockDiaPipelineTask(self): config = MockDiaPipelineTask.ConfigClass() config.apdb.db_url = "testing_only" task = MockDiaPipelineTask(config=config) pipelineTests.assertValidInitOutput(task) result = task.run(pandas.DataFrame(), pandas.DataFrame(), afwImage.ExposureF(), afwImage.ExposureF(), afwImage.ExposureF(), 42, 'k') pipelineTests.assertValidOutput(task, result) self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId) self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "diaSourceTable": self.visitId, "solarSystemObjectTable": self.visitId, "diffIm": self.visitId, "exposure": self.visitId, "template": self.visitId, "apdbMarker": self.visitId, "associatedDiaSources": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockDetectAndMeasureTask(self): task = MockDetectAndMeasureTask() pipelineTests.assertValidInitOutput(task) result = task.run( science=afwImage.ExposureF(), matchedTemplate=afwImage.ExposureF(), difference=afwImage.ExposureF(), selectSources=afwTable.SourceCatalog(), ) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_matchedExp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceTempExp", self.visitId) self.butler.put(afwTable.SourceCatalog(), "src", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "science": self.visitId, "matchedTemplate": self.visitId, "difference": self.visitId, "selectSources": self.visitId, "diaSources": self.visitId, "subtractedMeasuredExposure": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockGetTemplateTask(self): task = MockGetTemplateTask() pipelineTests.assertValidInitOutput(task) result = task.run( coaddExposures=[afwImage.ExposureF()], bbox=lsst.geom.Box2I(), wcs=None, # Should not be allowed, but too hard to fake a SkyWcs dataIds=[], ) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig()) self.butler.put(skymap, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, self.skymapId) self.butler.put(afwImage.ExposureF(), "goodSeeingCoadd", self.patchId) quantum = pipelineTests.makeQuantum( task, self.butler, self.skymapVisitId, { "bbox": self.visitId, "wcs": self.visitId, "skyMap": self.skymapId, "coaddExposures": [self.patchId], "template": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMissingData(self): """Test the run method with no data. """ result = self.task.run(None, None) testUtils.assertValidOutput(self.task, result) meas = result.measurement self.assertIsNone(meas)
def testAssertValidOutputPass(self): task = VisitTask() inA = butlerTests.MetricsExample(data=[1, 2, 3]) inB = butlerTests.MetricsExample(data=[4, 0, 1]) result = task.run(inA, inB) # should not throw assertValidOutput(task, result)
def testMockIsr(self): # Testing MockIsrTask is tricky because the real ISR has an unstable # interface with dozens of potential inputs, too many to pass through # runTestQuantum. I don't see a good way to test the inputs; # fortunately, this is unlikely to matter for the overall goal of # testing ap_verify's interaction with the AP pipeline. task = MockIsrTask() pipelineTests.assertValidInitOutput(task) result = task.run(afwImage.ExposureF()) pipelineTests.assertValidOutput(task, result)
def testValidEmpty(self): """Test the run method with a valid but zero result. """ metricComplete = self.makeTask(self.magCut, self.magCut + 5) result = metricComplete.run(self.fakeCat, self.band) testUtils.assertValidOutput(metricComplete, result) meas = result.measurement self.assertEqual(meas.metric_name, Name(metric="ap_pipe.apFakesCompleteness")) self.assertEqual(meas.quantity, 0 * u.dimensionless_unscaled)
def testAssertValidOutputMultiple(self): task = VisitTask() def run(a, b): return Struct(outA=[a], outB=b) task.run = run inA = butlerTests.MetricsExample(data=[1, 2, 3]) inB = butlerTests.MetricsExample(data=[4, 0, 1]) result = task.run(inA, inB) with self.assertRaises(AssertionError): assertValidOutput(task, result)
def testAssertValidOutputSingle(self): task = PatchTask() def run(a, b): return Struct(out=b) task.run = run inA = butlerTests.MetricsExample(data=[1, 2, 3]) inB = butlerTests.MetricsExample(data=[4, 0, 1]) result = task.run([inA], inB) with self.assertRaises(AssertionError): assertValidOutput(task, result)
def testMockCharacterizeImageTask(self): task = MockCharacterizeImageTask() pipelineTests.assertValidInitOutput(task) result = task.run(afwImage.ExposureF()) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "exposure": self.exposureId, "characterized": self.visitId, "sourceCat": self.visitId, "backgroundModel": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testValid(self): """Test the run method. """ result = self.task.run(self.fakeCat, self.band) testUtils.assertValidOutput(self.task, result) meas = result.measurement self.assertEqual(meas.metric_name, Name(metric="ap_pipe.apFakesCompleteness")) # Work around for Mac failing this test. self.assertAlmostEqual( meas.quantity.value, ((self.expectedAllMatched / self.targetSources) * u.dimensionless_unscaled).value, places=2)
def testMockTransformDiaSourceCatalogTask(self): task = MockTransformDiaSourceCatalogTask( initInputs=afwTable.SourceCatalog()) pipelineTests.assertValidInitOutput(task) result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42) pipelineTests.assertValidOutput(task, result) self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "diaSourceCat": self.visitId, "diffIm": self.visitId, "diaSourceTable": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockAlardLuptonSubtractTask(self): task = MockAlardLuptonSubtractTask() pipelineTests.assertValidInitOutput(task) result = task.run(afwImage.ExposureF(), afwImage.ExposureF(), afwTable.SourceCatalog()) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwTable.SourceCatalog(), "src", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "template": self.visitId, "science": self.visitId, "sources": self.visitId, "difference": self.visitId, "matchedTemplate": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def _testRun(self, doPackageAlerts=False, doSolarSystemAssociation=False): """Test the normal workflow of each ap_pipe step. """ config = self._makeDefaultConfig( doPackageAlerts=doPackageAlerts, doSolarSystemAssociation=doSolarSystemAssociation) task = DiaPipelineTask(config=config) # Set DataFrame index testing to always return False. Mocks return # true for this check otherwise. task.testDataFrameIndex = lambda x: False diffIm = Mock(spec=afwImage.ExposureF) exposure = Mock(spec=afwImage.ExposureF) template = Mock(spec=afwImage.ExposureF) diaSrc = MagicMock(spec=pd.DataFrame()) ssObjects = MagicMock(spec=pd.DataFrame()) ccdExposureIdBits = 32 # Each of these subtasks should be called once during diaPipe # execution. We use mocks here to check they are being executed # appropriately. subtasksToMock = [ "diaCatalogLoader", "diaCalculation", "diaForcedSource", ] if doPackageAlerts: subtasksToMock.append("alertPackager") else: self.assertFalse(hasattr(task, "alertPackager")) if not doSolarSystemAssociation: self.assertFalse(hasattr(task, "solarSystemAssociator")) def concatMock(data): return MagicMock(spec=pd.DataFrame) # Mock out the run() methods of these two Tasks to ensure they # return data in the correct form. @lsst.utils.timer.timeMethod def solarSystemAssociator_run(self, unAssocDiaSources, solarSystemObjectTable, diffIm): return lsst.pipe.base.Struct( nTotalSsObjects=42, nAssociatedSsObjects=30, ssoAssocDiaSources=MagicMock(spec=pd.DataFrame()), unAssocDiaSources=MagicMock(spec=pd.DataFrame())) @lsst.utils.timer.timeMethod def associator_run(self, table, diaObjects): return lsst.pipe.base.Struct( nUpdatedDiaObjects=2, nUnassociatedDiaObjects=3, matchedDiaSources=MagicMock(spec=pd.DataFrame()), unAssocDiaSources=MagicMock(spec=pd.DataFrame())) # apdb isn't a subtask, but still needs to be mocked out for correct # execution in the test environment. with patch.multiple( task, **{task: DEFAULT for task in subtasksToMock + ["apdb"]}): with patch('lsst.ap.association.diaPipe.pd.concat', new=concatMock), \ patch('lsst.ap.association.association.AssociationTask.run', new=associator_run), \ patch('lsst.ap.association.ssoAssociation.SolarSystemAssociationTask.run', new=solarSystemAssociator_run): result = task.run(diaSrc, ssObjects, diffIm, exposure, template, ccdExposureIdBits, "g") for subtaskName in subtasksToMock: getattr(task, subtaskName).run.assert_called_once() assertValidOutput(task, result) self.assertEqual(result.apdbMarker.db_url, "sqlite://") meta = task.getFullMetadata() # Check that the expected metadata has been set. self.assertEqual(meta["diaPipe.numUpdatedDiaObjects"], 2) self.assertEqual(meta["diaPipe.numUnassociatedDiaObjects"], 3) # and that associators ran once or not at all. self.assertEqual( len( meta.getArray( "diaPipe:associator.associator_runEndUtc")), 1) if doSolarSystemAssociation: self.assertEqual( len( meta.getArray("diaPipe:solarSystemAssociator." "solarSystemAssociator_runEndUtc")), 1) else: self.assertNotIn("diaPipe:solarSystemAssociator", meta)