def setUpClass(cls):
"""
Generate donutCatalog needed for task.
"""
moduleDir = getModulePath()
cls.testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(cls.testDataDir,
"pipelineConfigs")
cls.repoDir = os.path.join(cls.testDataDir, "gen3TestRepo")
cls.runName = "run1"
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
instrument = "lsst.obs.lsst.LsstCam"
cls.cameraName = "LSSTCam"
pipelineYaml = os.path.join(testPipelineConfigDir,
"testBasePipeline.yaml")
pipeCmd = writePipetaskCmd(cls.repoDir,
cls.runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
runProgram(pipeCmd)
def testWritePipetaskCmd(self):
repoName = "testRepo"
instrument = "lsst.obs.lsst.LsstCam"
collections = "refcats"
runName = "run2"
# Test writing with task name
taskName = "lsst.ts.wep.testTask"
testCmdTask = self._writePipetaskCmd(repoName,
instrument,
collections,
runName,
taskName=taskName)
pipeOutTask = writePipetaskCmd(repoName,
runName,
instrument,
collections,
taskName=taskName)
self.assertEqual(testCmdTask, pipeOutTask)
# Test writing with pipeline
pipelineYamlFile = "testPipeOut.yaml"
testCmdYaml = self._writePipetaskCmd(repoName,
instrument,
collections,
runName,
pipelineName=pipelineYamlFile)
pipeOutYaml = writePipetaskCmd(repoName,
runName,
instrument,
collections,
pipelineYaml=pipelineYamlFile)
self.assertEqual(testCmdYaml, pipeOutYaml)
assertMsg = "At least one of taskName or pipelineYaml must not be None"
with self.assertRaises(ValueError) as context:
writePipetaskCmd(repoName, runName, instrument, collections)
self.assertTrue(assertMsg in str(context.exception))
def testPipelineOnePairOnly(self):
pipeCmd = writePipetaskCmd(
self.repoDir,
self.testRunName,
self.instrument,
self.collections,
pipelineYaml=self.pipelineYaml,
)
pipeCmd += f" -d 'exposure IN ({self.visitNum}) and detector IN (191, 192)'"
runProgram(pipeCmd)
# Get Butler with updated registry
self.butler = dafButler.Butler(self.repoDir)
donutExtra = self.butler.get("donutStampsExtra",
dataId=self.dataIdExtra,
collections=[self.testRunName])
donutIntra = self.butler.get("donutStampsIntra",
dataId=self.dataIdIntra,
collections=[self.testRunName])
zernAvg = self.butler.get(
"zernikeEstimateAvg",
dataId=self.dataIdExtra,
collections=[self.testRunName],
)
zernRaw = self.butler.get(
"zernikeEstimateRaw",
dataId=self.dataIdExtra,
collections=[self.testRunName],
)
self.assertEqual(len(donutExtra), 2)
self.assertEqual(len(donutExtra), len(donutIntra))
self.assertEqual(np.shape(zernAvg), (19, ))
self.assertEqual(np.shape(zernRaw), (2, 19))
self.badDataId = copy(self.dataIdExtra)
self.badDataId["detector"] = 195
with self.assertRaises(LookupError):
self.butler.get(
"donutStampsExtra",
dataId=self.badDataId,
collections=[self.testRunName],
)
def setUpClass(cls):
"""
Run the pipeline only once since it takes a
couple minutes with the ISR.
"""
moduleDir = getModulePath()
testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs")
cls.repoDir = "/repo/main"
# Create a temporary test directory
# under /repo/main/u/$USER
# to ensure write access is granted
user = os.getlogin()
tempDir = os.path.join(cls.repoDir, "u", user)
cls.testDir = tempfile.TemporaryDirectory(dir=tempDir)
testDirName = os.path.split(cls.testDir.name)[1] # temp dir name
cls.runName = os.path.join("u", user, testDirName)
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
# Point to the collections with
# the raw images and calibrations
collections = "LATISS/raw/all,LATISS/calib"
instrument = "lsst.obs.lsst.Latiss"
cls.cameraName = "LATISS"
pipelineYaml = os.path.join(testPipelineConfigDir,
"testLatissPipeline.yaml")
pipeCmd = writePipetaskCmd(cls.repoDir,
cls.runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
pipeCmd += " -d 'exposure IN (2021090800487, 2021090800488) AND visit_system=0'"
runProgram(pipeCmd)
def setUpClass(cls):
"""
Run the pipeline only once since it takes a
couple minutes with the ISR.
"""
moduleDir = getModulePath()
testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs")
cls.repoDir = os.path.join(testDataDir, "gen3TestRepo")
cls.runName = "run2"
# The visit number for the test data
cls.visitNum = 4021123106000
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
# Point to the collections for the reference catalogs,
# the raw images and the camera model in the calib directory
# that comes from `butler write-curated-calibrations`.
cls.collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
cls.instrument = "lsst.obs.lsst.LsstCam"
cls.cameraName = "LSSTCam"
cls.pipelineYaml = os.path.join(testPipelineConfigDir,
"testCwfsPipeline.yaml")
pipeCmd = writePipetaskCmd(
cls.repoDir,
cls.runName,
cls.instrument,
cls.collections,
pipelineYaml=cls.pipelineYaml,
)
pipeCmd += f" -d 'exposure IN ({cls.visitNum})'"
runProgram(pipeCmd)
def testPipeline(self):
"""
Test that the task runs in a pipeline. Also functions as a test of
runQuantum function.
"""
# Run pipeline command
runName = "run1"
instrument = "lsst.obs.lsst.LsstCam"
collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
exposureId = 4021123106001 # Exposure ID for test extra-focal image
testPipelineConfigDir = os.path.join(self.testDataDir,
"pipelineConfigs")
pipelineYaml = os.path.join(testPipelineConfigDir,
"testDonutCatWcsPipeline.yaml")
pipetaskCmd = writePipetaskCmd(self.repoDir,
runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
# Update task configuration to match pointing information
pipetaskCmd += f" -d 'exposure IN ({exposureId})'"
# Check that run doesn't already exist due to previous improper cleanup
collectionsList = list(self.registry.queryCollections())
if runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName)
runProgram(cleanUpCmd)
# Run pipeline task
runProgram(pipetaskCmd)
# Test instrument matches
pipelineButler = dafButler.Butler(self.repoDir)
donutCatDf_S11 = pipelineButler.get(
"donutCatalog",
dataId={
"instrument": "LSSTCam",
"detector": 94,
"visit": exposureId
},
collections=[f"{runName}"],
)
donutCatDf_S10 = pipelineButler.get(
"donutCatalog",
dataId={
"instrument": "LSSTCam",
"detector": 93,
"visit": exposureId
},
collections=[f"{runName}"],
)
# Check 4 sources in each detector
self.assertEqual(len(donutCatDf_S11), 4)
self.assertEqual(len(donutCatDf_S10), 4)
# Check outputs are correct
outputDf = pd.concat([donutCatDf_S11, donutCatDf_S10])
self.assertEqual(len(outputDf), 8)
self.assertCountEqual(
outputDf.columns,
[
"coord_ra",
"coord_dec",
"centroid_x",
"centroid_y",
"source_flux",
],
)
self.assertCountEqual(
[
3806.7636478057957,
2806.982895217227,
607.3861483168994,
707.3972344551466,
614.607342274194,
714.6336433247832,
3815.2649173460436,
2815.0561553920156,
],
outputDf["centroid_x"],
)
self.assertCountEqual(
[
3196.070534224157,
2195.666002294077,
394.8907003737886,
394.9087004171349,
396.2407036464963,
396.22270360324296,
3196.1965343932648,
2196.188002312585,
],
outputDf["centroid_y"],
)
fluxTruth = np.ones(8)
fluxTruth[:6] = 3630780.5477010026
fluxTruth[6:] = 363078.0547701003
self.assertCountEqual(outputDf["source_flux"], fluxTruth)
# Clean up
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName)
runProgram(cleanUpCmd)
