def testMakeQuantumCorruptedDataId(self):
task = VisitTask()
dataId = {"instrument": "notACam", "visit": 102}
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
# fourth argument should be a mapping keyed by component name
makeQuantum(task, self.butler, dataId, dataId)
def testMakeQuantumCorruptedDataId(self):
task = VisitTask()
dataId = butlerTests.expandUniqueId(self.butler, {"visit": 102})
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
# fourth argument should be a mapping keyed by component name
makeQuantum(task, self.butler, dataId, dataId)
def testMakeQuantumMissingDataId(self):
task = VisitTask()
dataId = butlerTests.expandUniqueId(self.butler, {"visit": 102})
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "outA", "outB"}})
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "b", "outB"}})
def testMakeQuantumNoSuchDatatype(self):
config = VisitConfig()
config.connections.a = "Visit"
task = VisitTask(config=config)
dataId = butlerTests.expandUniqueId(self.butler, {"visit": 102})
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "b", "outA", "outB"}})
def testMakeQuantumExtraMultiple(self):
task = PatchTask()
dataId = butlerTests.expandUniqueId(self.butler, {"tract": 42})
self._makePatchTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId, {
"a": [dict(dataId, patch=patch) for patch in {0, 1}],
"b": [dataId],
"out": [dict(dataId, patch=patch) for patch in {0, 1}],
})
def testMakeQuantumNoSuchDatatype(self):
config = VisitConfig()
config.connections.a = "Visit"
task = VisitTask(config=config)
dataId = {"instrument": "notACam", "visit": 102}
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId,
{key: dataId
for key in {"a", "b", "outA", "outB"}})
def testMakeQuantumMissingDataId(self):
task = VisitTask()
dataId = {"instrument": "notACam", "visit": 102}
self._makeVisitTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId,
{key: dataId
for key in {"a", "outA", "outB"}})
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataId,
{key: dataId
for key in {"a", "b", "outB"}})
def testRunQuantum(self):
"""Test the run quantum method with a gen3 butler.
"""
root = tempfile.mkdtemp()
dimensions = {"instrument": ["notACam"],
"skymap": ["skyMap"],
"tract": [0, 42],
}
testRepo = butlerTests.makeTestRepo(root, dimensions)
fakesTask = CreateRandomApFakesTask()
connections = fakesTask.config.ConnectionsClass(
config=fakesTask.config)
butlerTests.addDatasetType(
testRepo,
connections.skyMap.name,
connections.skyMap.dimensions,
connections.skyMap.storageClass)
butlerTests.addDatasetType(
testRepo,
connections.fakeCat.name,
connections.fakeCat.dimensions,
connections.fakeCat.storageClass)
dataId = {"skymap": "skyMap", "tract": 0}
butler = butlerTests.makeTestCollection(testRepo)
butler.put(self.simpleMap, "skyMap", {"skymap": "skyMap"})
quantum = testUtils.makeQuantum(
fakesTask, butler, dataId,
{key: dataId for key in {"skyMap", "fakeCat"}})
run = testUtils.runTestQuantum(fakesTask, butler, quantum, True)
# Actual input dataset omitted for simplicity
run.assert_called_once_with(tractId=dataId["tract"], skyMap=self.simpleMap)
shutil.rmtree(root, ignore_errors=True)
def _checkDoRefcats(self, doAstrometry, doPhotoCal, ids):
"""Test whether run is called with the correct arguments.
In the case of `CalibrateTask`, the inputs should not depend on the
task configuration.
Parameters
----------
doAstrometry, doPhotoCal : `bool`
Values of the config flags of the same name.
ids : `dict` [`str`]
A mapping from the input dataset type to the data ID of the
dataset to process.
"""
config = CalibrateConfig()
config.doWriteMatches = False # no real output to write
config.doAstrometry = doAstrometry
config.doPhotoCal = doPhotoCal
config.connections.photoRefCat = "cal_ref_cat"
config.connections.astromRefCat = "cal_ref_cat"
task = CalibrateTask(config=config)
quantumId = ids["exposure"]
quantum = testUtils.makeQuantum(task, self.butler, quantumId, ids)
run = testUtils.runTestQuantum(task, self.butler, quantum)
run.assert_called_once()
self.assertEqual(run.call_args[0], ())
# Some arguments unprintable because we don't have a full environment
# So just check which ones were passed in
self.assertEqual(
run.call_args[1].keys(),
{"exposure", "exposureIdInfo", "background", "icSourceCat"})
def testRunTestQuantumPatchWithRun(self):
task = PatchTask()
dataId = {"skymap": "sky", "tract": 42}
data = self._makePatchTestData(dataId)
quantum = makeQuantum(
task,
self.butler,
dataId,
{
"a": [dataset[0] for dataset in data["PatchA"]],
"b": dataId,
"out": [dataset[0] for dataset in data["PatchA"]],
},
)
runTestQuantum(task, self.butler, quantum, mockRun=False)
# Can we use runTestQuantum to verify that task.run got called with
# correct inputs/outputs?
inB = data["PatchB"][0][1]
for dataset in data["PatchA"]:
patchId = dataset[0]
self.assertTrue(self.butler.datasetExists("PatchOut", patchId))
self.assertEqual(
self.butler.get("PatchOut", patchId),
butlerTests.MetricsExample(data=(dataset[1] + inB)))
def testMockGetTemplateTask(self):
task = MockGetTemplateTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(
coaddExposures=[afwImage.ExposureF()],
bbox=lsst.geom.Box2I(),
wcs=None, # Should not be allowed, but too hard to fake a SkyWcs
dataIds=[],
)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig())
self.butler.put(skymap,
lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
self.skymapId)
self.butler.put(afwImage.ExposureF(), "goodSeeingCoadd", self.patchId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.skymapVisitId, {
"bbox": self.visitId,
"wcs": self.visitId,
"skyMap": self.skymapId,
"coaddExposures": [self.patchId],
"template": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testRunTestQuantumVisitWithRun(self):
task = VisitTask()
dataId = {"instrument": "notACam", "visit": 102}
data = self._makeVisitTestData(dataId)
quantum = makeQuantum(
task, self.butler, dataId,
{key: dataId
for key in {"a", "b", "outA", "outB"}})
runTestQuantum(task, self.butler, quantum, mockRun=False)
# Can we use runTestQuantum to verify that task.run got called with
# correct inputs/outputs?
self.assertTrue(self.butler.datasetExists("VisitOutA", dataId))
self.assertEqual(
self.butler.get("VisitOutA", dataId),
butlerTests.MetricsExample(data=(data["VisitA"] + data["VisitB"])),
)
self.assertTrue(self.butler.datasetExists("VisitOutB", dataId))
self.assertEqual(
self.butler.get("VisitOutB", dataId),
butlerTests.MetricsExample(data=(data["VisitA"] *
max(data["VisitB"]))),
)
def testMockDetectAndMeasureTask(self):
task = MockDetectAndMeasureTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(
science=afwImage.ExposureF(),
matchedTemplate=afwImage.ExposureF(),
difference=afwImage.ExposureF(),
selectSources=afwTable.SourceCatalog(),
)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_matchedExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceTempExp",
self.visitId)
self.butler.put(afwTable.SourceCatalog(), "src", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"science": self.visitId,
"matchedTemplate": self.visitId,
"difference": self.visitId,
"selectSources": self.visitId,
"diaSources": self.visitId,
"subtractedMeasuredExposure": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testRunTestQuantumPatchOptionalInput(self):
config = PatchConfig()
config.doUseB = False
task = PatchTask(config=config)
dataId = {"skymap": "sky", "tract": 42}
data = self._makePatchTestData(dataId)
quantum = makeQuantum(
task,
self.butler,
dataId,
{
# Use lists, not sets, to ensure order agrees with test
# assertion.
"a": [dataset[0] for dataset in data["PatchA"]],
"out": [dataset[0] for dataset in data["PatchA"]],
},
)
run = runTestQuantum(task, self.butler, quantum, mockRun=True)
# Can we use the mock to verify that task.run got called with the
# correct inputs?
run.assert_called_once_with(a=[
butlerTests.MetricsExample(data=dataset[1])
for dataset in data["PatchA"]
])
def testMockDiaPipelineTask(self):
config = MockDiaPipelineTask.ConfigClass()
config.apdb.db_url = "testing_only"
task = MockDiaPipelineTask(config=config)
pipelineTests.assertValidInitOutput(task)
result = task.run(pandas.DataFrame(), pandas.DataFrame(),
afwImage.ExposureF(), afwImage.ExposureF(),
afwImage.ExposureF(), 42, 'k')
pipelineTests.assertValidOutput(task, result)
self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable",
self.visitId)
self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp",
self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"diaSourceTable": self.visitId,
"solarSystemObjectTable": self.visitId,
"diffIm": self.visitId,
"exposure": self.visitId,
"template": self.visitId,
"apdbMarker": self.visitId,
"associatedDiaSources": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockCalibrateTask(self):
task = MockCalibrateTask()
pipelineTests.assertValidInitOutput(task)
# Even the real CalibrateTask won't pass assertValidOutput, because for
# some reason the outputs are injected in runQuantum rather than run.
self.butler.put(afwImage.ExposureF(), "icExp", self.visitId)
self.butler.put(afwMath.BackgroundList(), "icExpBackground",
self.visitId)
self.butler.put(afwTable.SourceCatalog(), "icSrc", self.visitId)
self.butler.put(afwTable.SimpleCatalog(), "gaia_dr2_20200414",
self.htmId)
self.butler.put(afwTable.SimpleCatalog(), "ps1_pv3_3pi_20170110",
self.htmId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"exposure": self.visitId,
"background": self.visitId,
"icSourceCat": self.visitId,
"astromRefCat": [self.htmId],
"photoRefCat": [self.htmId],
"outputExposure": self.visitId,
"outputCat": self.visitId,
"outputBackground": self.visitId,
"matches": self.visitId,
"matchesDenormalized": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMakeQuantumExtraMultiple(self):
task = PatchTask()
dataId = {"skymap": "sky", "tract": 42}
self._makePatchTestData(dataId)
with self.assertRaises(ValueError):
makeQuantum(
task,
self.butler,
dataId,
{
"a": [dict(dataId, patch=patch) for patch in {0, 1}],
"b": [dataId],
"out": [dict(dataId, patch=patch) for patch in {0, 1}],
},
)
def testMakeQuantumInvalidDimension(self):
config = VisitConfig()
config.connections.a = "PatchA"
task = VisitTask(config=config)
dataIdV = butlerTests.expandUniqueId(self.butler, {"visit": 102})
dataIdP = butlerTests.expandUniqueId(self.butler, {"patch": 0})
inA = [1, 2, 3]
inB = [4, 0, 1]
self.butler.put(butlerTests.MetricsExample(data=inA), "VisitA", dataIdV)
self.butler.put(butlerTests.MetricsExample(data=inA), "PatchA", dataIdP)
self.butler.put(butlerTests.MetricsExample(data=inB), "VisitB", dataIdV)
with self.assertRaises(ValueError):
makeQuantum(task, self.butler, dataIdV, {
"a": dataIdP,
"b": dataIdV,
"outA": dataIdV,
"outB": dataIdV,
})
def testSkypixHandling(self):
task = SkyPixTask()
dataId = {"htm7": 157227} # connection declares skypix, but Butler uses htm7
data = butlerTests.MetricsExample(data=[1, 2, 3])
self.butler.put(data, "PixA", dataId)
quantum = makeQuantum(task, self.butler, dataId, {key: dataId for key in {"a", "out"}})
run = runTestQuantum(task, self.butler, quantum, mockRun=True)
# PixA dataset should have been retrieved by runTestQuantum
run.assert_called_once_with(a=data)
def testRunTestQuantumVisitMockRun(self):
task = VisitTask()
dataId = butlerTests.expandUniqueId(self.butler, {"visit": 102})
data = self._makeVisitTestData(dataId)
quantum = makeQuantum(task, self.butler, dataId,
{key: dataId for key in {"a", "b", "outA", "outB"}})
run = runTestQuantum(task, self.butler, quantum, mockRun=True)
# Can we use the mock to verify that task.run got called with the
# correct inputs?
run.assert_called_once_with(a=butlerTests.MetricsExample(data=data["VisitA"]),
b=butlerTests.MetricsExample(data=data["VisitB"]))
def testMockCharacterizeImageTask(self):
task = MockCharacterizeImageTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwImage.ExposureF())
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"exposure": self.exposureId,
"characterized": self.visitId,
"sourceCat": self.visitId,
"backgroundModel": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testMockTransformDiaSourceCatalogTask(self):
task = MockTransformDiaSourceCatalogTask(
initInputs=afwTable.SourceCatalog())
pipelineTests.assertValidInitOutput(task)
result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k',
42)
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc",
self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp",
self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"diaSourceCat": self.visitId,
"diffIm": self.visitId,
"diaSourceTable": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def _prepareQuantum(self, task):
inputId = {
"instrument": self.CAMERA_ID,
"visit": self.VISIT_ID,
"detector": self.CHIP_ID,
}
butler = butlerTests.makeTestCollection(self.repo)
# task.config not persistable if it refers to a local class
# We don't actually use the persisted config, so just make a new one
info = task.ConfigClass()
butler.put(info, "apdb_marker", inputId)
quantum = testUtils.makeQuantum(task, butler, inputId, {
"dbInfo": [inputId],
"measurement": inputId
})
return (butler, quantum, info)
def testMockAlardLuptonSubtractTask(self):
task = MockAlardLuptonSubtractTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwImage.ExposureF(), afwImage.ExposureF(),
afwTable.SourceCatalog())
pipelineTests.assertValidOutput(task, result)
self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp",
self.visitId)
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwTable.SourceCatalog(), "src", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId, {
"template": self.visitId,
"science": self.visitId,
"sources": self.visitId,
"difference": self.visitId,
"matchedTemplate": self.visitId,
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
def testRunTestQuantumPatchMockRun(self):
task = PatchTask()
dataId = butlerTests.expandUniqueId(self.butler, {"tract": 42})
data = self._makePatchTestData(dataId)
quantum = makeQuantum(task, self.butler, dataId, {
# Use lists, not sets, to ensure order agrees with test assertion
"a": [dataset[0] for dataset in data["PatchA"]],
"b": dataId,
"out": [dataset[0] for dataset in data["PatchA"]],
})
run = runTestQuantum(task, self.butler, quantum, mockRun=True)
# Can we use the mock to verify that task.run got called with the
# correct inputs?
run.assert_called_once_with(
a=[butlerTests.MetricsExample(data=dataset[1]) for dataset in data["PatchA"]],
b=butlerTests.MetricsExample(data=data["PatchB"][0][1])
)
def testRunQuantum(self):
"""Test the run quantum method with a gen3 butler.
"""
root = tempfile.mkdtemp()
dimensions = {
"instrument": ["notACam"],
"skymap": ["deepCoadd_skyMap"],
"tract": [0, 42],
"visit": [1234, 4321],
"detector": [25, 26]
}
testRepo = butlerTests.makeTestRepo(root, dimensions)
matchTask = MatchApFakesTask()
connections = matchTask.config.ConnectionsClass(
config=matchTask.config)
dataId = {
"instrument": "notACam",
"skymap": "deepCoadd_skyMap",
"tract": 0,
"visit": 1234,
"detector": 25
}
butlerTests.addDatasetType(testRepo, connections.fakeCat.name,
connections.fakeCat.dimensions,
connections.fakeCat.storageClass)
butlerTests.addDatasetType(testRepo, connections.diffIm.name,
connections.diffIm.dimensions,
connections.diffIm.storageClass)
butlerTests.addDatasetType(
testRepo, connections.associatedDiaSources.name,
connections.associatedDiaSources.dimensions,
connections.associatedDiaSources.storageClass)
butlerTests.addDatasetType(testRepo,
connections.matchedDiaSources.name,
connections.matchedDiaSources.dimensions,
connections.matchedDiaSources.storageClass)
butler = butlerTests.makeTestCollection(testRepo)
butler.put(self.fakeCat, connections.fakeCat.name, {
"tract": dataId["tract"],
"skymap": dataId["skymap"]
})
butler.put(
self.exposure, connections.diffIm.name, {
"instrument": dataId["instrument"],
"visit": dataId["visit"],
"detector": dataId["detector"]
})
butler.put(
self.sourceCat, connections.associatedDiaSources.name, {
"instrument": dataId["instrument"],
"visit": dataId["visit"],
"detector": dataId["detector"]
})
quantum = testUtils.makeQuantum(
matchTask, butler, dataId, {
key: dataId
for key in {
"fakeCat", "diffIm", "associatedDiaSources",
"matchedDiaSources"
}
})
run = testUtils.runTestQuantum(matchTask, butler, quantum)
# Actual input dataset omitted for simplicity
run.assert_called_once()
shutil.rmtree(root, ignore_errors=True)
def testMakeQuantumInvalidDimension(self):
config = VisitConfig()
config.connections.a = "PatchA"
task = VisitTask(config=config)
dataIdV = {"instrument": "notACam", "visit": 102}
dataIdVExtra = {"instrument": "notACam", "visit": 102, "detector": 42}
dataIdP = {"skymap": "sky", "tract": 42, "patch": 0}
inA = [1, 2, 3]
inB = [4, 0, 1]
self.butler.put(butlerTests.MetricsExample(data=inA), "VisitA",
dataIdV)
self.butler.put(butlerTests.MetricsExample(data=inA), "PatchA",
dataIdP)
self.butler.put(butlerTests.MetricsExample(data=inB), "VisitB",
dataIdV)
# dataIdV is correct everywhere, dataIdP should error
with self.assertRaises(ValueError):
makeQuantum(
task,
self.butler,
dataIdV,
{
"a": dataIdP,
"b": dataIdV,
"outA": dataIdV,
"outB": dataIdV,
},
)
with self.assertRaises(ValueError):
makeQuantum(
task,
self.butler,
dataIdP,
{
"a": dataIdV,
"b": dataIdV,
"outA": dataIdV,
"outB": dataIdV,
},
)
# should not accept small changes, either
with self.assertRaises(ValueError):
makeQuantum(
task,
self.butler,
dataIdV,
{
"a": dataIdV,
"b": dataIdV,
"outA": dataIdVExtra,
"outB": dataIdV,
},
)
with self.assertRaises(ValueError):
makeQuantum(
task,
self.butler,
dataIdVExtra,
{
"a": dataIdV,
"b": dataIdV,
"outA": dataIdV,
"outB": dataIdV,
},
)
