def setUpClass(cls): """ Generate donutCatalog needed for task. """ moduleDir = getModulePath() cls.testDataDir = os.path.join(moduleDir, "tests", "testData") testPipelineConfigDir = os.path.join(cls.testDataDir, "pipelineConfigs") cls.repoDir = os.path.join(cls.testDataDir, "gen3TestRepo") cls.runName = "run1" # Check that run doesn't already exist due to previous improper cleanup butler = dafButler.Butler(cls.repoDir) registry = butler.registry collectionsList = list(registry.queryCollections()) if cls.runName in collectionsList: cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName) runProgram(cleanUpCmd) collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all" instrument = "lsst.obs.lsst.LsstCam" cls.cameraName = "LSSTCam" pipelineYaml = os.path.join(testPipelineConfigDir, "testBasePipeline.yaml") pipeCmd = writePipetaskCmd(cls.repoDir, cls.runName, instrument, collections, pipelineYaml=pipelineYaml) runProgram(pipeCmd)
def setUp(self): self.config = EstimateZernikesCwfsTaskConfig() self.task = EstimateZernikesCwfsTask(config=self.config) self.butler = dafButler.Butler(self.repoDir) self.registry = self.butler.registry self.dataIdExtra = { "instrument": "LSSTCam", "detector": 191, "exposure": self.visitNum, "visit": self.visitNum, } self.dataIdIntra = { "instrument": "LSSTCam", "detector": 192, "exposure": self.visitNum, "visit": self.visitNum, } self.testRunName = "testTaskRun" self.collectionsList = list(self.registry.queryCollections()) if self.testRunName in self.collectionsList: cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, self.testRunName) runProgram(cleanUpCmd)
def tearDown(self): # Get Butler with updated registry self.butler = dafButler.Butler(self.repoDir) self.registry = self.butler.registry self.collectionsList = list(self.registry.queryCollections()) if self.testRunName in self.collectionsList: cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, self.testRunName) runProgram(cleanUpCmd)
def _genFakeFlat(self, fakeFlatDir, detector): currWorkDir = os.getcwd() os.chdir(fakeFlatDir) command = "makeGainImages.py" argstring = "--detector_list %s" % detector runProgram(command, argstring=argstring) os.chdir(currWorkDir)
def ingestImages(self, imgFiles): """Ingest the image files. Parameters ---------- imgFiles : str Image files. """ command = "ingestImages.py" argstring = "%s %s" % (self.destDir, imgFiles) runProgram(command, argstring=argstring)
def ingestCalibs(self, calibFiles): """Ingest the calibration files. Parameters ---------- calibFiles : str Calibration files. """ command = "ingestCalibs.py" argstring = "%s %s --validity 99999 --output %s" % ( self.destDir, calibFiles, self.destDir) runProgram(command, argstring=argstring)
def testPipelineOnePairOnly(self): pipeCmd = writePipetaskCmd( self.repoDir, self.testRunName, self.instrument, self.collections, pipelineYaml=self.pipelineYaml, ) pipeCmd += f" -d 'exposure IN ({self.visitNum}) and detector IN (191, 192)'" runProgram(pipeCmd) # Get Butler with updated registry self.butler = dafButler.Butler(self.repoDir) donutExtra = self.butler.get("donutStampsExtra", dataId=self.dataIdExtra, collections=[self.testRunName]) donutIntra = self.butler.get("donutStampsIntra", dataId=self.dataIdIntra, collections=[self.testRunName]) zernAvg = self.butler.get( "zernikeEstimateAvg", dataId=self.dataIdExtra, collections=[self.testRunName], ) zernRaw = self.butler.get( "zernikeEstimateRaw", dataId=self.dataIdExtra, collections=[self.testRunName], ) self.assertEqual(len(donutExtra), 2) self.assertEqual(len(donutExtra), len(donutIntra)) self.assertEqual(np.shape(zernAvg), (19, )) self.assertEqual(np.shape(zernRaw), (2, 19)) self.badDataId = copy(self.dataIdExtra) self.badDataId["detector"] = 195 with self.assertRaises(LookupError): self.butler.get( "donutStampsExtra", dataId=self.badDataId, collections=[self.testRunName], )
def _ingestImagesByButler(self, imgFiles, configFile=None): """Ingest the images by butler. Parameters ---------- imgFiles : str Image files. configFile : str, optional Config override file(s). (the default is None.) """ command = "ingestImages.py" argstring = "%s %s" % (self.destDir, imgFiles) if configFile is not None: argstring += " --configfile %s" % configFile runProgram(command, argstring=argstring)
def setUpClass(cls): """ Run the pipeline only once since it takes a couple minutes with the ISR. """ moduleDir = getModulePath() testDataDir = os.path.join(moduleDir, "tests", "testData") testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs") cls.repoDir = "/repo/main" # Create a temporary test directory # under /repo/main/u/$USER # to ensure write access is granted user = os.getlogin() tempDir = os.path.join(cls.repoDir, "u", user) cls.testDir = tempfile.TemporaryDirectory(dir=tempDir) testDirName = os.path.split(cls.testDir.name)[1] # temp dir name cls.runName = os.path.join("u", user, testDirName) # Check that run doesn't already exist due to previous improper cleanup butler = dafButler.Butler(cls.repoDir) registry = butler.registry collectionsList = list(registry.queryCollections()) if cls.runName in collectionsList: cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName) runProgram(cleanUpCmd) # Point to the collections with # the raw images and calibrations collections = "LATISS/raw/all,LATISS/calib" instrument = "lsst.obs.lsst.Latiss" cls.cameraName = "LATISS" pipelineYaml = os.path.join(testPipelineConfigDir, "testLatissPipeline.yaml") pipeCmd = writePipetaskCmd(cls.repoDir, cls.runName, instrument, collections, pipelineYaml=pipelineYaml) pipeCmd += " -d 'exposure IN (2021090800487, 2021090800488) AND visit_system=0'" runProgram(pipeCmd)
def doISR(self, inputDir, rerunName="run1"): """Do the ISR. ISR: Instrument signature removal. Parameters ---------- inputDir : str Input data directory. rerunName : str, optional Rerun name. (the default is "run1".) """ command = "runIsr.py" argstring = "%s --id --rerun=%s" % (inputDir, rerunName) if (self.isrConfigFilePath is not None): argstring += " --configfile %s" % self.isrConfigFilePath runProgram(command, argstring=argstring)
def doISR(self, inputDir, rerunName="run1"): """Do the ISR. ISR: Instrument signature removal. Parameters ---------- inputDir : str Input data directory. rerunName : str, optional Rerun name. (the default is "run1".) """ # Do the ISR command = "runIsr.py" argstring = "%s --id --rerun=%s" % (inputDir, rerunName) if self.isrConfigFilePath is not None: argstring += " --configfile %s --no-versions" % self.isrConfigFilePath runProgram(command, argstring=argstring)
def setUpClass(cls): """ Run the pipeline only once since it takes a couple minutes with the ISR. """ moduleDir = getModulePath() testDataDir = os.path.join(moduleDir, "tests", "testData") testPipelineConfigDir = os.path.join(testDataDir, "pipelineConfigs") cls.repoDir = os.path.join(testDataDir, "gen3TestRepo") cls.runName = "run2" # The visit number for the test data cls.visitNum = 4021123106000 # Check that run doesn't already exist due to previous improper cleanup butler = dafButler.Butler(cls.repoDir) registry = butler.registry collectionsList = list(registry.queryCollections()) if cls.runName in collectionsList: cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName) runProgram(cleanUpCmd) # Point to the collections for the reference catalogs, # the raw images and the camera model in the calib directory # that comes from `butler write-curated-calibrations`. cls.collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all" cls.instrument = "lsst.obs.lsst.LsstCam" cls.cameraName = "LSSTCam" cls.pipelineYaml = os.path.join(testPipelineConfigDir, "testCwfsPipeline.yaml") pipeCmd = writePipetaskCmd( cls.repoDir, cls.runName, cls.instrument, cls.collections, pipelineYaml=cls.pipelineYaml, ) pipeCmd += f" -d 'exposure IN ({cls.visitNum})'" runProgram(pipeCmd)
def repackageComCamImgFromPhoSim(self): """Repackage the ComCam amplifier images from PhoSim to the single 16 extension MEFs for processing. ComCam: commissioning camera. MEF: multi-extension frames. """ # Make a temp directory tmpDirPath = os.path.join(self.outputImgDir, "tmp") self._makeDir(tmpDirPath) for imgType in (self.PISTON_INTRA_DIR_NAME, self.PISTON_EXTRA_DIR_NAME): # Repackage the images to that temp directory command = "phosim_repackager.py" phosimAmgImgDir = os.path.join(self.outputImgDir, imgType) argstring = "%s --out_dir=%s" % (phosimAmgImgDir, tmpDirPath) runProgram(command, argstring=argstring) # Remove the image data in the original directory argString = "-rf %s/lsst_a_*.fits.gz" % phosimAmgImgDir runProgram("rm", argstring=argString) # Put the repackaged data into the image directory argstring = "%s/*.fits %s" % (tmpDirPath, phosimAmgImgDir) runProgram("mv", argstring=argstring) # Remove the temp directory shutil.rmtree(tmpDirPath)
def _repackageComCamImages(self, isEimg=False): """Repackage the ComCam images from PhoSim for processing. Parameters ---------- isEimg : bool, optional Is eimage or not. (the default is False.) """ # Make a temporary directory tmpDirPath = os.path.join(self.outputImgDir, "tmp") self._makeDir(tmpDirPath) intraFocalDirName = self.getIntraFocalDirName() extraFocalDirName = self.getExtraFocalDirName() for imgType in (intraFocalDirName, extraFocalDirName): # Repackage the images to the temporary directory command = "phosim_repackager.py" phosimImgDir = os.path.join(self.outputImgDir, imgType) argstring = "%s --out_dir=%s" % (phosimImgDir, tmpDirPath) if (isEimg): argstring += " --eimage" runProgram(command, argstring=argstring) # Remove the image data in the original directory argString = "-rf %s/*.fits*" % phosimImgDir runProgram("rm", argstring=argString) # Put the repackaged data into the image directory argstring = "%s/*.fits %s" % (tmpDirPath, phosimImgDir) runProgram("mv", argstring=argstring) # Remove the temporary directory shutil.rmtree(tmpDirPath)
def testPipeline(self): """ Test that the task runs in a pipeline. Also functions as a test of runQuantum function. """ # Run pipeline command runName = "run1" instrument = "lsst.obs.lsst.LsstCam" collections = "refcats/gen2,LSSTCam/calib,LSSTCam/raw/all" exposureId = 4021123106001 # Exposure ID for test extra-focal image testPipelineConfigDir = os.path.join(self.testDataDir, "pipelineConfigs") pipelineYaml = os.path.join(testPipelineConfigDir, "testDonutCatWcsPipeline.yaml") pipetaskCmd = writePipetaskCmd(self.repoDir, runName, instrument, collections, pipelineYaml=pipelineYaml) # Update task configuration to match pointing information pipetaskCmd += f" -d 'exposure IN ({exposureId})'" # Check that run doesn't already exist due to previous improper cleanup collectionsList = list(self.registry.queryCollections()) if runName in collectionsList: cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName) runProgram(cleanUpCmd) # Run pipeline task runProgram(pipetaskCmd) # Test instrument matches pipelineButler = dafButler.Butler(self.repoDir) donutCatDf_S11 = pipelineButler.get( "donutCatalog", dataId={ "instrument": "LSSTCam", "detector": 94, "visit": exposureId }, collections=[f"{runName}"], ) donutCatDf_S10 = pipelineButler.get( "donutCatalog", dataId={ "instrument": "LSSTCam", "detector": 93, "visit": exposureId }, collections=[f"{runName}"], ) # Check 4 sources in each detector self.assertEqual(len(donutCatDf_S11), 4) self.assertEqual(len(donutCatDf_S10), 4) # Check outputs are correct outputDf = pd.concat([donutCatDf_S11, donutCatDf_S10]) self.assertEqual(len(outputDf), 8) self.assertCountEqual( outputDf.columns, [ "coord_ra", "coord_dec", "centroid_x", "centroid_y", "source_flux", ], ) self.assertCountEqual( [ 3806.7636478057957, 2806.982895217227, 607.3861483168994, 707.3972344551466, 614.607342274194, 714.6336433247832, 3815.2649173460436, 2815.0561553920156, ], outputDf["centroid_x"], ) self.assertCountEqual( [ 3196.070534224157, 2195.666002294077, 394.8907003737886, 394.9087004171349, 396.2407036464963, 396.22270360324296, 3196.1965343932648, 2196.188002312585, ], outputDf["centroid_y"], ) fluxTruth = np.ones(8) fluxTruth[:6] = 3630780.5477010026 fluxTruth[6:] = 363078.0547701003 self.assertCountEqual(outputDf["source_flux"], fluxTruth) # Clean up cleanUpCmd = writeCleanUpRepoCmd(self.repoDir, runName) runProgram(cleanUpCmd)
def tearDownClass(cls): cleanUpCmd = writeCleanUpRepoCmd(cls.repoDir, cls.runName) runProgram(cleanUpCmd)
def _makeFakeFlat(self, detector): command = "makeGainImages.py" argstring = "--detector_list %s" % detector runProgram(command, argstring=argstring)