def testMockCalibrateTask(self): task = MockCalibrateTask() pipelineTests.assertValidInitOutput(task) # Even the real CalibrateTask won't pass assertValidOutput, because for # some reason the outputs are injected in runQuantum rather than run. self.butler.put(afwImage.ExposureF(), "icExp", self.visitId) self.butler.put(afwMath.BackgroundList(), "icExpBackground", self.visitId) self.butler.put(afwTable.SourceCatalog(), "icSrc", self.visitId) self.butler.put(afwTable.SimpleCatalog(), "gaia_dr2_20200414", self.htmId) self.butler.put(afwTable.SimpleCatalog(), "ps1_pv3_3pi_20170110", self.htmId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "exposure": self.visitId, "background": self.visitId, "icSourceCat": self.visitId, "astromRefCat": [self.htmId], "photoRefCat": [self.htmId], "outputExposure": self.visitId, "outputCat": self.visitId, "outputBackground": self.visitId, "matches": self.visitId, "matchesDenormalized": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockGetTemplateTask(self): task = MockGetTemplateTask() pipelineTests.assertValidInitOutput(task) result = task.run( coaddExposures=[afwImage.ExposureF()], bbox=lsst.geom.Box2I(), wcs=None, # Should not be allowed, but too hard to fake a SkyWcs dataIds=[], ) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig()) self.butler.put(skymap, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, self.skymapId) self.butler.put(afwImage.ExposureF(), "goodSeeingCoadd", self.patchId) quantum = pipelineTests.makeQuantum( task, self.butler, self.skymapVisitId, { "bbox": self.visitId, "wcs": self.visitId, "skyMap": self.skymapId, "coaddExposures": [self.patchId], "template": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockDetectAndMeasureTask(self): task = MockDetectAndMeasureTask() pipelineTests.assertValidInitOutput(task) result = task.run( science=afwImage.ExposureF(), matchedTemplate=afwImage.ExposureF(), difference=afwImage.ExposureF(), selectSources=afwTable.SourceCatalog(), ) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_matchedExp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceTempExp", self.visitId) self.butler.put(afwTable.SourceCatalog(), "src", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "science": self.visitId, "matchedTemplate": self.visitId, "difference": self.visitId, "selectSources": self.visitId, "diaSources": self.visitId, "subtractedMeasuredExposure": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockDiaPipelineTask(self): config = MockDiaPipelineTask.ConfigClass() config.apdb.db_url = "testing_only" task = MockDiaPipelineTask(config=config) pipelineTests.assertValidInitOutput(task) result = task.run(pandas.DataFrame(), pandas.DataFrame(), afwImage.ExposureF(), afwImage.ExposureF(), afwImage.ExposureF(), 42, 'k') pipelineTests.assertValidOutput(task, result) self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId) self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "diaSourceTable": self.visitId, "solarSystemObjectTable": self.visitId, "diffIm": self.visitId, "exposure": self.visitId, "template": self.visitId, "apdbMarker": self.visitId, "associatedDiaSources": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testRunTestQuantumPatchWithRun(self): task = PatchTask() dataId = {"skymap": "sky", "tract": 42} data = self._makePatchTestData(dataId) quantum = makeQuantum( task, self.butler, dataId, { "a": [dataset[0] for dataset in data["PatchA"]], "b": dataId, "out": [dataset[0] for dataset in data["PatchA"]], }, ) runTestQuantum(task, self.butler, quantum, mockRun=False) # Can we use runTestQuantum to verify that task.run got called with # correct inputs/outputs? inB = data["PatchB"][0][1] for dataset in data["PatchA"]: patchId = dataset[0] self.assertTrue(self.butler.datasetExists("PatchOut", patchId)) self.assertEqual( self.butler.get("PatchOut", patchId), butlerTests.MetricsExample(data=(dataset[1] + inB)))
def testRunTestQuantumVisitWithRun(self): task = VisitTask() dataId = {"instrument": "notACam", "visit": 102} data = self._makeVisitTestData(dataId) quantum = makeQuantum( task, self.butler, dataId, {key: dataId for key in {"a", "b", "outA", "outB"}}) runTestQuantum(task, self.butler, quantum, mockRun=False) # Can we use runTestQuantum to verify that task.run got called with # correct inputs/outputs? self.assertTrue(self.butler.datasetExists("VisitOutA", dataId)) self.assertEqual( self.butler.get("VisitOutA", dataId), butlerTests.MetricsExample(data=(data["VisitA"] + data["VisitB"])), ) self.assertTrue(self.butler.datasetExists("VisitOutB", dataId)) self.assertEqual( self.butler.get("VisitOutB", dataId), butlerTests.MetricsExample(data=(data["VisitA"] * max(data["VisitB"]))), )
def testMockCharacterizeImageTask(self): task = MockCharacterizeImageTask() pipelineTests.assertValidInitOutput(task) result = task.run(afwImage.ExposureF()) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "exposure": self.exposureId, "characterized": self.visitId, "sourceCat": self.visitId, "backgroundModel": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def _checkDoRefcats(self, doAstrometry, doPhotoCal, ids): """Test whether run is called with the correct arguments. In the case of `CalibrateTask`, the inputs should not depend on the task configuration. Parameters ---------- doAstrometry, doPhotoCal : `bool` Values of the config flags of the same name. ids : `dict` [`str`] A mapping from the input dataset type to the data ID of the dataset to process. """ config = CalibrateConfig() config.doWriteMatches = False # no real output to write config.doAstrometry = doAstrometry config.doPhotoCal = doPhotoCal config.connections.photoRefCat = "cal_ref_cat" config.connections.astromRefCat = "cal_ref_cat" task = CalibrateTask(config=config) quantumId = ids["exposure"] quantum = testUtils.makeQuantum(task, self.butler, quantumId, ids) run = testUtils.runTestQuantum(task, self.butler, quantum) run.assert_called_once() self.assertEqual(run.call_args[0], ()) # Some arguments unprintable because we don't have a full environment # So just check which ones were passed in self.assertEqual( run.call_args[1].keys(), {"exposure", "exposureIdInfo", "background", "icSourceCat"})
def testRunQuantumNone(self): class NoneTask(DummyTask): def run(self, *args, **kwargs): return Struct(measurement=None) config = NoneTask.ConfigClass() config.connections.package = "verify" config.connections.metric = "DummyApdb" task = NoneTask(config=config) butler, quantum, input = self._prepareQuantum(task) with unittest.mock.patch.object( lsst.pipe.base.ButlerQuantumContext, "put") as put: testUtils.runTestQuantum(task, butler, quantum, mockRun=False) # Should not attempt to write nonexistent data put.assert_not_called()
def testRunQuantum(self): """Test the run quantum method with a gen3 butler. """ root = tempfile.mkdtemp() dimensions = {"instrument": ["notACam"], "skymap": ["skyMap"], "tract": [0, 42], } testRepo = butlerTests.makeTestRepo(root, dimensions) fakesTask = CreateRandomApFakesTask() connections = fakesTask.config.ConnectionsClass( config=fakesTask.config) butlerTests.addDatasetType( testRepo, connections.skyMap.name, connections.skyMap.dimensions, connections.skyMap.storageClass) butlerTests.addDatasetType( testRepo, connections.fakeCat.name, connections.fakeCat.dimensions, connections.fakeCat.storageClass) dataId = {"skymap": "skyMap", "tract": 0} butler = butlerTests.makeTestCollection(testRepo) butler.put(self.simpleMap, "skyMap", {"skymap": "skyMap"}) quantum = testUtils.makeQuantum( fakesTask, butler, dataId, {key: dataId for key in {"skyMap", "fakeCat"}}) run = testUtils.runTestQuantum(fakesTask, butler, quantum, True) # Actual input dataset omitted for simplicity run.assert_called_once_with(tractId=dataId["tract"], skyMap=self.simpleMap) shutil.rmtree(root, ignore_errors=True)
def testRunTestQuantumPatchOptionalInput(self): config = PatchConfig() config.doUseB = False task = PatchTask(config=config) dataId = {"skymap": "sky", "tract": 42} data = self._makePatchTestData(dataId) quantum = makeQuantum( task, self.butler, dataId, { # Use lists, not sets, to ensure order agrees with test # assertion. "a": [dataset[0] for dataset in data["PatchA"]], "out": [dataset[0] for dataset in data["PatchA"]], }, ) run = runTestQuantum(task, self.butler, quantum, mockRun=True) # Can we use the mock to verify that task.run got called with the # correct inputs? run.assert_called_once_with(a=[ butlerTests.MetricsExample(data=dataset[1]) for dataset in data["PatchA"] ])
def testRunQuantumException(self): class ExceptionalTask(DummyTask): def run(self, *args, **kwargs): raise MetricComputationError() config = ExceptionalTask.ConfigClass() config.connections.package = "verify" config.connections.metric = "DummyApdb" task = ExceptionalTask(config=config) butler, quantum, input = self._prepareQuantum(task) with unittest.mock.patch.object( lsst.pipe.base.ButlerQuantumContext, "put") as put: testUtils.runTestQuantum(task, butler, quantum, mockRun=False) # Should not propagate MetricComputationError # Should not attempt to write data that was never returned put.assert_not_called()
def testMockTransformDiaSourceCatalogTask(self): task = MockTransformDiaSourceCatalogTask( initInputs=afwTable.SourceCatalog()) pipelineTests.assertValidInitOutput(task) result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42) pipelineTests.assertValidOutput(task, result) self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId) self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "diaSourceCat": self.visitId, "diffIm": self.visitId, "diaSourceTable": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testRunQuantum(self): butler, quantum, input = self._prepareQuantum(self.task) run = testUtils.runTestQuantum(self.task, butler, quantum) # Did output data ID get passed to DummyTask.run? expectedId = lsst.daf.butler.DataCoordinate.standardize( {"instrument": self.CAMERA_ID}, universe=butler.registry.dimensions) run.assert_called_once_with(dbInfo=[input], outputDataId=expectedId)
def testMockAlardLuptonSubtractTask(self): task = MockAlardLuptonSubtractTask() pipelineTests.assertValidInitOutput(task) result = task.run(afwImage.ExposureF(), afwImage.ExposureF(), afwTable.SourceCatalog()) pipelineTests.assertValidOutput(task, result) self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId) self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) self.butler.put(afwTable.SourceCatalog(), "src", self.visitId) quantum = pipelineTests.makeQuantum( task, self.butler, self.visitId, { "template": self.visitId, "science": self.visitId, "sources": self.visitId, "difference": self.visitId, "matchedTemplate": self.visitId, }) pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testSkypixHandling(self): task = SkyPixTask() dataId = {"htm7": 157227} # connection declares skypix, but Butler uses htm7 data = butlerTests.MetricsExample(data=[1, 2, 3]) self.butler.put(data, "PixA", dataId) quantum = makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "out"}}) run = runTestQuantum(task, self.butler, quantum, mockRun=True) # PixA dataset should have been retrieved by runTestQuantum run.assert_called_once_with(a=data)
def testRunTestQuantumVisitMockRun(self): task = VisitTask() dataId = butlerTests.expandUniqueId(self.butler, {"visit": 102}) data = self._makeVisitTestData(dataId) quantum = makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "b", "outA", "outB"}}) run = runTestQuantum(task, self.butler, quantum, mockRun=True) # Can we use the mock to verify that task.run got called with the # correct inputs? run.assert_called_once_with(a=butlerTests.MetricsExample(data=data["VisitA"]), b=butlerTests.MetricsExample(data=data["VisitB"]))
def testRunTestQuantumPatchMockRun(self): task = PatchTask() dataId = butlerTests.expandUniqueId(self.butler, {"tract": 42}) data = self._makePatchTestData(dataId) quantum = makeQuantum(task, self.butler, dataId, { # Use lists, not sets, to ensure order agrees with test assertion "a": [dataset[0] for dataset in data["PatchA"]], "b": dataId, "out": [dataset[0] for dataset in data["PatchA"]], }) run = runTestQuantum(task, self.butler, quantum, mockRun=True) # Can we use the mock to verify that task.run got called with the # correct inputs? run.assert_called_once_with( a=[butlerTests.MetricsExample(data=dataset[1]) for dataset in data["PatchA"]], b=butlerTests.MetricsExample(data=data["PatchB"][0][1]) )
def testRunQuantum(self): """Test the run quantum method with a gen3 butler. """ root = tempfile.mkdtemp() dimensions = { "instrument": ["notACam"], "skymap": ["deepCoadd_skyMap"], "tract": [0, 42], "visit": [1234, 4321], "detector": [25, 26] } testRepo = butlerTests.makeTestRepo(root, dimensions) matchTask = MatchApFakesTask() connections = matchTask.config.ConnectionsClass( config=matchTask.config) dataId = { "instrument": "notACam", "skymap": "deepCoadd_skyMap", "tract": 0, "visit": 1234, "detector": 25 } butlerTests.addDatasetType(testRepo, connections.fakeCat.name, connections.fakeCat.dimensions, connections.fakeCat.storageClass) butlerTests.addDatasetType(testRepo, connections.diffIm.name, connections.diffIm.dimensions, connections.diffIm.storageClass) butlerTests.addDatasetType( testRepo, connections.associatedDiaSources.name, connections.associatedDiaSources.dimensions, connections.associatedDiaSources.storageClass) butlerTests.addDatasetType(testRepo, connections.matchedDiaSources.name, connections.matchedDiaSources.dimensions, connections.matchedDiaSources.storageClass) butler = butlerTests.makeTestCollection(testRepo) butler.put(self.fakeCat, connections.fakeCat.name, { "tract": dataId["tract"], "skymap": dataId["skymap"] }) butler.put( self.exposure, connections.diffIm.name, { "instrument": dataId["instrument"], "visit": dataId["visit"], "detector": dataId["detector"] }) butler.put( self.sourceCat, connections.associatedDiaSources.name, { "instrument": dataId["instrument"], "visit": dataId["visit"], "detector": dataId["detector"] }) quantum = testUtils.makeQuantum( matchTask, butler, dataId, { key: dataId for key in { "fakeCat", "diffIm", "associatedDiaSources", "matchedDiaSources" } }) run = testUtils.runTestQuantum(matchTask, butler, quantum) # Actual input dataset omitted for simplicity run.assert_called_once() shutil.rmtree(root, ignore_errors=True)