def setUpClass(cls):
"""
Generate donutCatalog needed for task.
"""
moduleDir = getModulePath()
cls.testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(cls.testDataDir,
"pipelineConfigs")
cls.repoDir = os.path.join(cls.testDataDir, "gen3TestRepo")
cls.runName = "run1"
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
instrument = "lsst.obs.lsst.LsstCam"
cls.cameraName = "LSSTCam"
pipelineYaml = os.path.join(testPipelineConfigDir,
"testBasePipeline.yaml")
pipeCmd = writePipetaskCmd(cls.repoDir,
cls.runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
runProgram(pipeCmd)
def setUp(self):
self.config = EstimateZernikesCwfsTaskConfig()
self.task = EstimateZernikesCwfsTask(config=self.config)
self.butler = dafButler.Butler(self.repoDir)
self.registry = self.butler.registry
self.dataIdExtra = {
"instrument": "LSSTCam",
"detector": 191,
"exposure": self.visitNum,
"visit": self.visitNum,
}
self.dataIdIntra = {
"instrument": "LSSTCam",
"detector": 192,
"exposure": self.visitNum,
"visit": self.visitNum,
}
self.testRunName = "testTaskRun"
self.collectionsList = list(self.registry.queryCollections())
if self.testRunName in self.collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, self.testRunName)
runProgram(cleanUpCmd)
def tearDown(self):
# Get Butler with updated registry
self.butler = dafButler.Butler(self.repoDir)
self.registry = self.butler.registry
self.collectionsList = list(self.registry.queryCollections())
if self.testRunName in self.collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, self.testRunName)
runProgram(cleanUpCmd)
def _genFakeFlat(self, fakeFlatDir, detector):
currWorkDir = os.getcwd()
os.chdir(fakeFlatDir)
command = "makeGainImages.py"
argstring = "--detector_list %s" % detector
runProgram(command, argstring=argstring)
os.chdir(currWorkDir)
def _genFakeFlat(self, fakeFlatDir, detector):
currWorkDir = os.getcwd()
os.chdir(fakeFlatDir)
command = "makeGainImages.py"
argstring = "--detector_list %s" % detector
runProgram(command, argstring=argstring)
os.chdir(currWorkDir)
def ingestImages(self, imgFiles):
"""Ingest the image files.
Parameters
----------
imgFiles : str
Image files.
"""
command = "ingestImages.py"
argstring = "%s %s" % (self.destDir, imgFiles)
runProgram(command, argstring=argstring)
def ingestImages(self, imgFiles):
"""Ingest the image files.
Parameters
----------
imgFiles : str
Image files.
"""
command = "ingestImages.py"
argstring = "%s %s" % (self.destDir, imgFiles)
runProgram(command, argstring=argstring)
def ingestCalibs(self, calibFiles):
"""Ingest the calibration files.
Parameters
----------
calibFiles : str
Calibration files.
"""
command = "ingestCalibs.py"
argstring = "%s %s --validity 99999 --output %s" % (
self.destDir, calibFiles, self.destDir)
runProgram(command, argstring=argstring)
def ingestCalibs(self, calibFiles):
"""Ingest the calibration files.
Parameters
----------
calibFiles : str
Calibration files.
"""
command = "ingestCalibs.py"
argstring = "%s %s --validity 99999 --output %s" % (
self.destDir, calibFiles, self.destDir)
runProgram(command, argstring=argstring)
def testPipelineOnePairOnly(self):
pipeCmd = writePipetaskCmd(
self.repoDir,
self.testRunName,
self.instrument,
self.collections,
pipelineYaml=self.pipelineYaml,
)
pipeCmd += f" -d 'exposure IN ({self.visitNum}) and detector IN (191, 192)'"
runProgram(pipeCmd)
# Get Butler with updated registry
self.butler = dafButler.Butler(self.repoDir)
donutExtra = self.butler.get("donutStampsExtra",
dataId=self.dataIdExtra,
collections=[self.testRunName])
donutIntra = self.butler.get("donutStampsIntra",
dataId=self.dataIdIntra,
collections=[self.testRunName])
zernAvg = self.butler.get(
"zernikeEstimateAvg",
dataId=self.dataIdExtra,
collections=[self.testRunName],
)
zernRaw = self.butler.get(
"zernikeEstimateRaw",
dataId=self.dataIdExtra,
collections=[self.testRunName],
)
self.assertEqual(len(donutExtra), 2)
self.assertEqual(len(donutExtra), len(donutIntra))
self.assertEqual(np.shape(zernAvg), (19, ))
self.assertEqual(np.shape(zernRaw), (2, 19))
self.badDataId = copy(self.dataIdExtra)
self.badDataId["detector"] = 195
with self.assertRaises(LookupError):
self.butler.get(
"donutStampsExtra",
dataId=self.badDataId,
collections=[self.testRunName],
)
def _ingestImagesByButler(self, imgFiles, configFile=None):
"""Ingest the images by butler.
Parameters
----------
imgFiles : str
Image files.
configFile : str, optional
Config override file(s). (the default is None.)
"""
command = "ingestImages.py"
argstring = "%s %s" % (self.destDir, imgFiles)
if configFile is not None:
argstring += " --configfile %s" % configFile
runProgram(command, argstring=argstring)
def setUpClass(cls):
"""
Run the pipeline only once since it takes a
couple minutes with the ISR.
"""
moduleDir = getModulePath()
testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs")
cls.repoDir = "/repo/main"
# Create a temporary test directory
# under /repo/main/u/$USER
# to ensure write access is granted
user = os.getlogin()
tempDir = os.path.join(cls.repoDir, "u", user)
cls.testDir = tempfile.TemporaryDirectory(dir=tempDir)
testDirName = os.path.split(cls.testDir.name)[1] # temp dir name
cls.runName = os.path.join("u", user, testDirName)
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
# Point to the collections with
# the raw images and calibrations
collections = "LATISS/raw/all,LATISS/calib"
instrument = "lsst.obs.lsst.Latiss"
cls.cameraName = "LATISS"
pipelineYaml = os.path.join(testPipelineConfigDir,
"testLatissPipeline.yaml")
pipeCmd = writePipetaskCmd(cls.repoDir,
cls.runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
pipeCmd += " -d 'exposure IN (2021090800487, 2021090800488) AND visit_system=0'"
runProgram(pipeCmd)
def doISR(self, inputDir, rerunName="run1"):
"""Do the ISR.
ISR: Instrument signature removal.
Parameters
----------
inputDir : str
Input data directory.
rerunName : str, optional
Rerun name. (the default is "run1".)
"""
command = "runIsr.py"
argstring = "%s --id --rerun=%s" % (inputDir, rerunName)
if (self.isrConfigFilePath is not None):
argstring += " --configfile %s" % self.isrConfigFilePath
runProgram(command, argstring=argstring)
def doISR(self, inputDir, rerunName="run1"):
"""Do the ISR.
ISR: Instrument signature removal.
Parameters
----------
inputDir : str
Input data directory.
rerunName : str, optional
Rerun name. (the default is "run1".)
"""
# Do the ISR
command = "runIsr.py"
argstring = "%s --id --rerun=%s" % (inputDir, rerunName)
if self.isrConfigFilePath is not None:
argstring += " --configfile %s --no-versions" % self.isrConfigFilePath
runProgram(command, argstring=argstring)
def setUpClass(cls):
"""
Run the pipeline only once since it takes a
couple minutes with the ISR.
"""
moduleDir = getModulePath()
testDataDir = os.path.join(moduleDir, "tests", "testData")
testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs")
cls.repoDir = os.path.join(testDataDir, "gen3TestRepo")
cls.runName = "run2"
# The visit number for the test data
cls.visitNum = 4021123106000
# Check that run doesn't already exist due to previous improper cleanup
butler = dafButler.Butler(cls.repoDir)
registry = butler.registry
collectionsList = list(registry.queryCollections())
if cls.runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
# Point to the collections for the reference catalogs,
# the raw images and the camera model in the calib directory
# that comes from `butler write-curated-calibrations`.
cls.collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
cls.instrument = "lsst.obs.lsst.LsstCam"
cls.cameraName = "LSSTCam"
cls.pipelineYaml = os.path.join(testPipelineConfigDir,
"testCwfsPipeline.yaml")
pipeCmd = writePipetaskCmd(
cls.repoDir,
cls.runName,
cls.instrument,
cls.collections,
pipelineYaml=cls.pipelineYaml,
)
pipeCmd += f" -d 'exposure IN ({cls.visitNum})'"
runProgram(pipeCmd)
def repackageComCamImgFromPhoSim(self):
"""Repackage the ComCam amplifier images from PhoSim to the single 16
extension MEFs for processing.
ComCam: commissioning camera.
MEF: multi-extension frames.
"""
# Make a temp directory
tmpDirPath = os.path.join(self.outputImgDir, "tmp")
self._makeDir(tmpDirPath)
for imgType in (self.PISTON_INTRA_DIR_NAME,
self.PISTON_EXTRA_DIR_NAME):
# Repackage the images to that temp directory
command = "phosim_repackager.py"
phosimAmgImgDir = os.path.join(self.outputImgDir, imgType)
argstring = "%s --out_dir=%s" % (phosimAmgImgDir, tmpDirPath)
runProgram(command, argstring=argstring)
# Remove the image data in the original directory
argString = "-rf %s/lsst_a_*.fits.gz" % phosimAmgImgDir
runProgram("rm", argstring=argString)
# Put the repackaged data into the image directory
argstring = "%s/*.fits %s" % (tmpDirPath, phosimAmgImgDir)
runProgram("mv", argstring=argstring)
# Remove the temp directory
shutil.rmtree(tmpDirPath)
def _repackageComCamImages(self, isEimg=False):
"""Repackage the ComCam images from PhoSim for processing.
Parameters
----------
isEimg : bool, optional
Is eimage or not. (the default is False.)
"""
# Make a temporary directory
tmpDirPath = os.path.join(self.outputImgDir, "tmp")
self._makeDir(tmpDirPath)
intraFocalDirName = self.getIntraFocalDirName()
extraFocalDirName = self.getExtraFocalDirName()
for imgType in (intraFocalDirName, extraFocalDirName):
# Repackage the images to the temporary directory
command = "phosim_repackager.py"
phosimImgDir = os.path.join(self.outputImgDir, imgType)
argstring = "%s --out_dir=%s" % (phosimImgDir, tmpDirPath)
if (isEimg):
argstring += " --eimage"
runProgram(command, argstring=argstring)
# Remove the image data in the original directory
argString = "-rf %s/*.fits*" % phosimImgDir
runProgram("rm", argstring=argString)
# Put the repackaged data into the image directory
argstring = "%s/*.fits %s" % (tmpDirPath, phosimImgDir)
runProgram("mv", argstring=argstring)
# Remove the temporary directory
shutil.rmtree(tmpDirPath)
def testPipeline(self):
"""
Test that the task runs in a pipeline. Also functions as a test of
runQuantum function.
"""
# Run pipeline command
runName = "run1"
instrument = "lsst.obs.lsst.LsstCam"
collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all"
exposureId = 4021123106001 # Exposure ID for test extra-focal image
testPipelineConfigDir = os.path.join(self.testDataDir,
"pipelineConfigs")
pipelineYaml = os.path.join(testPipelineConfigDir,
"testDonutCatWcsPipeline.yaml")
pipetaskCmd = writePipetaskCmd(self.repoDir,
runName,
instrument,
collections,
pipelineYaml=pipelineYaml)
# Update task configuration to match pointing information
pipetaskCmd += f" -d 'exposure IN ({exposureId})'"
# Check that run doesn't already exist due to previous improper cleanup
collectionsList = list(self.registry.queryCollections())
if runName in collectionsList:
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName)
runProgram(cleanUpCmd)
# Run pipeline task
runProgram(pipetaskCmd)
# Test instrument matches
pipelineButler = dafButler.Butler(self.repoDir)
donutCatDf_S11 = pipelineButler.get(
"donutCatalog",
dataId={
"instrument": "LSSTCam",
"detector": 94,
"visit": exposureId
},
collections=[f"{runName}"],
)
donutCatDf_S10 = pipelineButler.get(
"donutCatalog",
dataId={
"instrument": "LSSTCam",
"detector": 93,
"visit": exposureId
},
collections=[f"{runName}"],
)
# Check 4 sources in each detector
self.assertEqual(len(donutCatDf_S11), 4)
self.assertEqual(len(donutCatDf_S10), 4)
# Check outputs are correct
outputDf = pd.concat([donutCatDf_S11, donutCatDf_S10])
self.assertEqual(len(outputDf), 8)
self.assertCountEqual(
outputDf.columns,
[
"coord_ra",
"coord_dec",
"centroid_x",
"centroid_y",
"source_flux",
],
)
self.assertCountEqual(
[
3806.7636478057957,
2806.982895217227,
607.3861483168994,
707.3972344551466,
614.607342274194,
714.6336433247832,
3815.2649173460436,
2815.0561553920156,
],
outputDf["centroid_x"],
)
self.assertCountEqual(
[
3196.070534224157,
2195.666002294077,
394.8907003737886,
394.9087004171349,
396.2407036464963,
396.22270360324296,
3196.1965343932648,
2196.188002312585,
],
outputDf["centroid_y"],
)
fluxTruth = np.ones(8)
fluxTruth[:6] = 3630780.5477010026
fluxTruth[6:] = 363078.0547701003
self.assertCountEqual(outputDf["source_flux"], fluxTruth)
# Clean up
cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName)
runProgram(cleanUpCmd)
def tearDownClass(cls):
cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName)
runProgram(cleanUpCmd)
def _makeFakeFlat(self, detector):
command = "makeGainImages.py"
argstring = "--detector_list %s" % detector
runProgram(command, argstring=argstring)
def _makeFakeFlat(self, detector):
command = "makeGainImages.py"
argstring = "--detector_list %s" % detector
runProgram(command, argstring=argstring)
