def processCcd(REPO,visit,ccdnum,**kw): from lsst.pipe.tasks.processCcd import ProcessCcdTask command = "{REPO} --id visit={EXPNUM} ccdnum={CCDNUM} --output {output} --configfile {configfile}" args = command.format(REPO=REPO,EXPNUM=visit,CCDNUM=ccdnum,**kw).split() print "Running:\n%s\n" % " ".join(args) ProcessCcdTask.parseAndRun(args=args) return
def setUpClass(cls): """Runs ProcessCcdTask so the test* methods can inspect the results.""" try: cls.datadir = getPackageDir("testdata_decam") except pexExcept.NotFoundError: message = "testdata_decam not setup. Skipping." warnings.warn(message) raise unittest.SkipTest(message) cls.outPath = tempfile.mkdtemp() if OutputName is None else OutputName cls.dataId = {'visit': 229388, 'ccdnum': 1} argsList = [ os.path.join(cls.datadir, "rawData"), "--output", cls.outPath, "--id" ] argsList += ["%s=%s" % (key, val) for key, val in cls.dataId.items()] argsList += ["--calib", os.path.join(cls.datadir, "rawData/cpCalib")] argsList += [ "--config", "calibrate.doPhotoCal=False", "calibrate.doAstrometry=False" ] argsList.append('--doraise') disableImplicitThreading() # avoid contention with other processes fullResult = ProcessCcdTask.parseAndRun(args=argsList, doReturnResults=True) cls.butler = fullResult.parsedCmd.butler cls.config = fullResult.parsedCmd.config
def py_pccd_process_by_id(repo: RepoInfo, id: str, stdout=parsl.AUTO_LOGNAME): argslist = repo.cli_as_list() print("BENC ARGS LIST 1: {}".format(argslist)) if id: argslist += ["--id", id] else: argslist += ["--id"] print("BENC ARGS LIST 2: {}".format(argslist)) from lsst.pipe.tasks.processCcd import ProcessCcdTask try: ProcessCcdTask.parseAndRun(args=argslist) except SystemExit as e: raise ValueError("SystemExit {} from ProcessCcdTask".format(e.code))
def testInterface(self): obsTestDir = lsst.utils.getPackageDir('obs_test') inputDir = os.path.join(obsTestDir, "data", "input") # Configure a ProcessCcd task such that it will return a minimal # number of measurements plus our test plugin. cfg = ProcessCcdConfig() cfg.calibrate.measurement.plugins.names = ["base_SdssCentroid", "base_SkyCoord", PLUGIN_NAME] cfg.calibrate.measurement.slots.shape = None cfg.calibrate.measurement.slots.psfFlux = None cfg.calibrate.measurement.slots.apFlux = None cfg.calibrate.measurement.slots.gaussianFlux = None cfg.calibrate.measurement.slots.modelFlux = None cfg.calibrate.measurement.slots.calibFlux = None # no reference catalog, so... cfg.calibrate.doAstrometry = False cfg.calibrate.doPhotoCal = False # disable aperture correction because we aren't measuring aperture flux cfg.calibrate.doApCorr = False # Extendedness requires modelFlux, disabled above. cfg.calibrate.catalogCalculation.plugins.names.discard("base_ClassificationExtendedness") # Process the test data with ProcessCcd then perform a transform. with tempDirectory() as tempDir: measResult = ProcessCcdTask.parseAndRun(args=[inputDir, "--output", tempDir, "--id", "visit=1"], config=cfg, doReturnResults=True) trArgs = [tempDir, "--output", tempDir, "--id", "visit=1", "-c", "inputConfigType=processCcd_config"] trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True) # It should be possible to reprocess the data through a new transform task with exactly # the same configuration without throwing. This check is useful since we are # constructing the task on the fly, which could conceivably cause problems with # configuration/metadata persistence. trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True) measSrcs = measResult.resultList[0].result.calibRes.sourceCat trSrcs = trResult.resultList[0].result # The length of the measured and transformed catalogs should be the same. self.assertEqual(len(measSrcs), len(trSrcs)) # Each source should have been measured & transformed appropriately. for measSrc, trSrc in zip(measSrcs, trSrcs): # The TrivialMeasurement should be transformed as defined above. self.assertEqual(trSrc[PLUGIN_NAME], measSrc[PLUGIN_NAME]) self.assertEqual(trSrc[PLUGIN_NAME + "_transform"], -1.0 * measSrc[PLUGIN_NAME]) # The SdssCentroid should be transformed to celestial coordinates. # Checking that the full transformation has been done correctly is # out of scope for this test case; we just ensure that there's # plausible position in the transformed record. trCoord = afwTable.CoordKey(trSrcs.schema["base_SdssCentroid"]).get(trSrc) self.assertAlmostEqual(measSrc.getCoord().getLongitude(), trCoord.getLongitude()) self.assertAlmostEqual(measSrc.getCoord().getLatitude(), trCoord.getLatitude())
def testFakeProcessing(self): # Set the random seed for predictability np.random.seed(500) # Set ouput path and create a dataId outPath = tempfile.mkdtemp() if OutputName is None \ else "{}-ProcessCcd".format(OutputName) dataId = dict(visit=1) dataIdStrList = ["{}={}".format(*item) for item in dataId.items()] mask = None maskPlaneName = "FAKE" try: # Set the configurations for running the fake object piepline processCcdConfig = getObsTestConfig(ProcessCcdTask) processCcdConfig.calibrate.doInsertFakes = True processCcdConfig.calibrate.insertFakes.retarget( FakeSourcesTestTask) # Run ProcessCcd pCcdResult = ProcessCcdTask.parseAndRun(args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, config=processCcdConfig) # Check the Catalog contains properly measured fake sources sourceCat = pCcdResult.resultList[0].result.calibRes.sourceCat self.checkSourceCatalog(sourceCat) exposure = pCcdResult.resultList[0].result.calibRes.exposure mask = exposure.getMaskedImage().getMask() maskBit = 2**mask.getMaskPlaneDict()[maskPlaneName] fakeMask = np.bitwise_and(mask.getArray(), maskBit) self.assertGreater(fakeMask.sum(), 0) # Clean up temporary directories if needed finally: if mask: # Remove FAKE so as not to contaminate later tests lsst.afw.image.Mask[lsst.afw.image.MaskPixel].removeMaskPlane( maskPlaneName) if OutputName is None: shutil.rmtree(outPath) else: message = "testFakeProcessing.py's output data saved to {}" print(message.format(OutputName))
def testFakeProcessing(self): # Set the random seed for predictability np.random.seed(500) # Set ouput path and create a dataId outPath = tempfile.mkdtemp() if OutputName is None \ else "{}-ProcessCcd".format(OutputName) dataId = dict(visit=1) dataIdStrList = ["{}={}".format(*item) for item in dataId.items()] mask = None maskPlaneName = "FAKE" try: # Set the configurations for running the fake object piepline processCcdConfig = getObsTestConfig(ProcessCcdTask) processCcdConfig.calibrate.doInsertFakes = True processCcdConfig.calibrate.insertFakes.retarget(FakeSourcesTestTask) # Run ProcessCcd pCcdResult = ProcessCcdTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, config=processCcdConfig) # Check the Catalog contains properly measured fake sources sourceCat = pCcdResult.resultList[0].result.calibRes.sourceCat self.checkSourceCatalog(sourceCat) exposure = pCcdResult.resultList[0].result.calibRes.exposure mask = exposure.getMaskedImage().getMask() maskBit = 2**mask.getMaskPlaneDict()[maskPlaneName] fakeMask = np.bitwise_and(mask.getArray(), maskBit) self.assertGreater(fakeMask.sum(), 0) # Clean up temporary directories if needed finally: if mask: # Remove FAKE so as not to contaminate later tests lsst.afw.image.Mask[lsst.afw.image.MaskPixel].removeMaskPlane(maskPlaneName) if OutputName is None: shutil.rmtree(outPath) else: message = "testFakeProcessing.py's output data saved to {}" print(message.format(OutputName))
def setUp(self): try: self.datadir = getPackageDir("testdata_decam") except pexExcept.NotFoundError: message = "testdata_decam not setup. Skipping." warnings.warn(message) raise unittest.SkipTest(message) self.outPath = tempfile.mkdtemp() if OutputName is None else OutputName self.dataId = {'visit': 229388, 'ccdnum': 1} argsList = [os.path.join(self.datadir, "rawData"), "--output", self.outPath, "--id"] argsList += ["%s=%s" % (key, val) for key, val in self.dataId.iteritems()] argsList += ["--config", "calibrate.doPhotoCal=False", "calibrate.doAstrometry=False", # Temporary until DM-4232 is fixed. "isr.assembleCcd.setGain=False"] fullResult = ProcessCcdTask.parseAndRun(args=argsList, doReturnResults=True) self.butler = fullResult.parsedCmd.butler self.config = fullResult.parsedCmd.config
def testProcessCcd(self): """test ProcessCcdTask via parseAndRun (simulating the command line) This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-ProcessCcd".format(OutputName) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] fullResult = ProcessCcdTask.parseAndRun(args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "-c", "charImage.doWriteExposure=True", "--id" ] + dataIdStrList, doReturnResults=True) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=np.float64) bgStdDev = bg0Arr.std(dtype=np.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate( (butler.get("calexp", dataId), result.exposure)): self.assertEqual( exposure.getBBox(), geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = np.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=np.float64) imStdDev = imageArr.std(dtype=np.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=np.float64) varStdDev = varArr.std(dtype=np.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertEqual(len(icSrc), 28) self.assertEqual( len(src), 185 + 100) # 185 real sources plus 100 sky sources expectedPlaces = 7 # Tolerance for numerical comparisons for name, var, val in [ ("bgMean", bgMean, 191.48635852060525), ("bgStdDev", bgStdDev, 0.2399466881603354), ("numGoodPix", numGoodPix, 1966820), ("imMean", imMean, 1.1237668985230562), ("imStdDev", imStdDev, 85.81296241298496), ("varMean", varMean, 131.24003624152013), ("varStdDev", varStdDev, 55.98012493452948), ("psfIxx", psfIxx, 2.769679536557131), ("psfIyy", psfIyy, 2.2013649766299324), ("psfIxy", psfIxy, 0.14797939531970852) ]: self.assertAlmostEqual(var, val, places=expectedPlaces, msg=name) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def testProcessCcd(self): """test ProcessCcdTask This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp() if OutputName is None else OutputName try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.iteritems() ] fullResult = ProcessCcdTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--id" ] + dataIdStrList, doReturnResults=True, ) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=numpy.float64) bgStdDev = bg0Arr.std(dtype=numpy.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate(( butler.get("calexp", dataId), result.exposure, )): self.assertEqual( exposure.getBBox(), afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = numpy.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=numpy.float64) imStdDev = imageArr.std(dtype=numpy.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=numpy.float64) varStdDev = varArr.std(dtype=numpy.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertAlmostEqual(bgMean, 191.51202961532354, places=7) self.assertAlmostEqual(bgStdDev, 0.28333327656694213, places=7) self.assertEqual(len(icSrc), 28) self.assertEqual(len(src), 185) self.assertEqual(numGoodPix, 1965540) self.assertAlmostEqual(imMean, 0.99578990765845121, places=7) self.assertAlmostEqual(imStdDev, 95.645409033639211, places=7) self.assertAlmostEqual(varMean, 131.16293718847217, places=7) self.assertAlmostEqual(varStdDev, 64.806576059889963, places=7) self.assertAlmostEqual(psfIxx, 2.8540775242108163, places=7) self.assertAlmostEqual(psfIyy, 2.17386182921239, places=7) self.assertAlmostEqual(psfIxy, 0.14400008637208778, places=7) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def testProcessCcd(self): """test ProcessCcdTask via parseAndRun (simulating the command line) This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-ProcessCcd".format(OutputName) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] fullResult = ProcessCcdTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "-c", "charImage.doWriteExposure=True", "--id"] + dataIdStrList, doReturnResults=True ) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=np.float64) bgStdDev = bg0Arr.std(dtype=np.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate((butler.get("calexp", dataId), result.exposure)): self.assertEqual(exposure.getBBox(), afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = np.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=np.float64) imStdDev = imageArr.std(dtype=np.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=np.float64) varStdDev = varArr.std(dtype=np.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertEqual(len(icSrc), 28) self.assertEqual(len(src), 186) expectedPlaces = 7 # Tolerance for numerical comparisons for name, var, val in [ ("bgMean", bgMean, 191.4866883539376), ("bgStdDev", bgStdDev, 0.24004985195839124), ("numGoodPix", numGoodPix, 1965472), ("imMean", imMean, 0.99616767174645282), ("imStdDev", imStdDev, 95.64615285227903), ("varMean", varMean, 131.14644188960244), ("varStdDev", varStdDev, 64.806980097108436), ("psfIxx", psfIxx, 2.8540502512317234), ("psfIyy", psfIyy, 2.17386622227748), ("psfIxy", psfIxy, 0.14397638586604858) ]: self.assertAlmostEqual(var, val, places=expectedPlaces, msg=name) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def testProcessCcd(self): """test ProcessCcdTask via parseAndRun (simulating the command line) This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-ProcessCcd".format(OutputName) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] fullResult = ProcessCcdTask.parseAndRun(args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "-c", "charImage.doWriteExposure=True", "--id" ] + dataIdStrList, doReturnResults=True) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=np.float64) bgStdDev = bg0Arr.std(dtype=np.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate( (butler.get("calexp", dataId), result.exposure)): self.assertEqual( exposure.getBBox(), geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = np.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=np.float64) imStdDev = imageArr.std(dtype=np.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=np.float64) varStdDev = varArr.std(dtype=np.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertEqual(len(icSrc), 28) self.assertEqual( len(src), 184 + 100) # 184 real sources plus 100 sky sources # NOTE: These values are purely empirical, and need to be # updated to reflect major algorithmic changes. # If this test fails after an algorithmic change due to # small numeric changes here, check on slack at # #dm-science-pipelines as to whether the changes are # reasonable, and then replace the failing values by # running the test to determine the updated values. expectedPlaces = 7 # Tolerance for numerical comparisons for name, var, val in [ ("bgMean", bgMean, 191.48623786891795), ("bgStdDev", bgStdDev, 0.23994185672586282), ("numGoodPix", numGoodPix, 1966606), ("imMean", imMean, 1.1242456954648634), ("imStdDev", imStdDev, 85.8129750182329), ("varMean", varMean, 131.24003624152013), ("varStdDev", varStdDev, 55.98012493452948), ("psfIxx", psfIxx, 2.843329671276296), ("psfIyy", psfIyy, 2.2249941554078156), ("psfIxy", psfIxy, 0.16073332780683286) ]: self.assertAlmostEqual(var, val, places=expectedPlaces, msg=name) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def testProcessCcd(self): """test ProcessCcdTask This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp() if OutputName is None else OutputName try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.iteritems()] fullResult = ProcessCcdTask.parseAndRun( args = [InputDir, "--output", outPath, "--id"] + dataIdStrList, doReturnResults = True, ) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result calexpBackground = butler.get("calexpBackground", dataId) bg0Arr = calexpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=numpy.float64) bgStdDev = bg0Arr.std(dtype=numpy.float64) icSrc = butler.get("icSrc", dataId) sources = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate(( butler.get("calexp", dataId), result.exposure, )): self.assertEqual(exposure.getBBox(), afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = numpy.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=numpy.float64) imStdDev = imageArr.std(dtype=numpy.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=numpy.float64) varStdDev = varArr.std(dtype=numpy.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(sources) =", len(sources)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertAlmostEqual(bgMean, 327.61695151065607, places=7) self.assertAlmostEqual(bgStdDev, 0.40081096398879795, places=7) self.assertEqual(len(icSrc), 28) self.assertEqual(len(sources), 169) self.assertEqual(numGoodPix, 1953514) self.assertAlmostEqual(imMean, 1.7045581411166495, places=7) self.assertAlmostEqual(imStdDev, 163.66290334994014, places=7) self.assertAlmostEqual(varMean, 212.5416436210289, places=7) self.assertAlmostEqual(varStdDev, 121.25084470503148, places=7) self.assertAlmostEqual(psfIxx, 2.790601924907123, places=7) self.assertAlmostEqual(psfIyy, 2.1989110429762513, places=7) self.assertAlmostEqual(psfIxy, 0.16599586021207952, places=7) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
#!/usr/bin/env python # # LSST Data Management System # Copyright 2008, 2009, 2010 LSST Corporation. # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # from lsst.pipe.tasks.processCcd import ProcessCcdTask ProcessCcdTask.parseAndRun()
def testProcessCcd(self): """test ProcessCcdTask via parseAndRun (simulating the command line) This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-ProcessCcd".format(OutputName) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] fullResult = ProcessCcdTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "-c", "charImage.doWriteExposure=True", "--id"] + dataIdStrList, doReturnResults=True ) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=np.float64) bgStdDev = bg0Arr.std(dtype=np.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate((butler.get("calexp", dataId), result.exposure)): self.assertEqual(exposure.getBBox(), afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = np.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=np.float64) imStdDev = imageArr.std(dtype=np.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=np.float64) varStdDev = varArr.std(dtype=np.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertEqual(len(icSrc), 28) self.assertEqual(len(src), 185) expectedPlaces = 7 # Tolerance for numerical comparisons for var, val in [ (bgMean, 191.51453611409124), (bgStdDev, 0.22438381414455047), (numGoodPix, 1965508), (imMean, 0.99592485493752636), (imStdDev, 95.64609939459902), (varMean, 131.16293718847217), (varStdDev, 64.806576059889963), (psfIxx, 2.8540480723051846), (psfIyy, 2.173868563513369), (psfIxy, 0.14397457739362085) ]: self.assertAlmostEqual(var, val, places=expectedPlaces) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def testProcessCcd(self): """test ProcessCcdTask This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time without breaking this test. """ outPath = tempfile.mkdtemp() if OutputName is None else OutputName try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.iteritems()] fullResult = ProcessCcdTask.parseAndRun( args = [InputDir, "--output", outPath, "--clobber-config", "--id"] + dataIdStrList, doReturnResults = True, ) butler = fullResult.parsedCmd.butler self.assertEqual(len(fullResult.resultList), 1) runResult = fullResult.resultList[0] fullDataId = runResult.dataRef.dataId self.assertEqual(len(fullDataId), 2) self.assertEqual(fullDataId["visit"], dataId["visit"]) self.assertEqual(fullDataId["filter"], "g") result = runResult.result icExpBackground = butler.get("icExpBackground", dataId, immediate=True) bg0Arr = icExpBackground.getImage().getArray() bgMean = bg0Arr.mean(dtype=numpy.float64) bgStdDev = bg0Arr.std(dtype=numpy.float64) icSrc = butler.get("icSrc", dataId) src = butler.get("src", dataId) # the following makes pyflakes linter happy and the code more robust oldImMean = None oldImStdDev = None oldVarMean = None oldVarStdDev = None oldPsfIxx = None oldPsfIyy = None oldPsfIxy = None for i, exposure in enumerate(( butler.get("calexp", dataId), result.exposure, )): self.assertEqual(exposure.getBBox(), afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000))) maskedImage = exposure.getMaskedImage() maskArr = maskedImage.getMask().getArray() numGoodPix = numpy.sum(maskArr == 0) imageArr = maskedImage.getImage().getArray() imMean = imageArr.mean(dtype=numpy.float64) imStdDev = imageArr.std(dtype=numpy.float64) varArr = maskedImage.getVariance().getArray() varMean = varArr.mean(dtype=numpy.float64) varStdDev = varArr.std(dtype=numpy.float64) psfShape = exposure.getPsf().computeShape() psfIxx = psfShape.getIxx() psfIyy = psfShape.getIyy() psfIxy = psfShape.getIxy() if i == 0: print("\nMeasured results:") print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev)) print("len(icSrc) :", len(icSrc)) print("len(src) :", len(src)) print("numGoodPix =", numGoodPix) print("image mean = %r, stdDev = %r" % (imMean, imStdDev)) print("variance mean = %r, stdDev = %r" % (varMean, varStdDev)) print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy)) self.assertAlmostEqual(bgMean, 191.51202961532354, places=7) self.assertAlmostEqual(bgStdDev, 0.28333327656694213, places=7) self.assertEqual(len(icSrc), 28) self.assertEqual(len(src), 185) self.assertEqual(numGoodPix, 1965540) self.assertAlmostEqual(imMean, 0.99578990765845121, places=7) self.assertAlmostEqual(imStdDev, 95.645409033639211, places=7) self.assertAlmostEqual(varMean, 131.16293718847217, places=7) self.assertAlmostEqual(varStdDev, 64.806576059889963, places=7) self.assertAlmostEqual(psfIxx, 2.8540775242108163, places=7) self.assertAlmostEqual(psfIyy, 2.17386182921239, places=7) self.assertAlmostEqual(psfIxy, 0.14400008637208778, places=7) else: self.assertEqual(imMean, oldImMean) self.assertEqual(imStdDev, oldImStdDev) self.assertEqual(varMean, oldVarMean) self.assertEqual(varStdDev, oldVarStdDev) self.assertEqual(psfIxx, oldPsfIxx) self.assertEqual(psfIyy, oldPsfIyy) self.assertEqual(psfIxy, oldPsfIxy) oldImMean = imMean oldImStdDev = imStdDev oldVarMean = varMean oldVarStdDev = varStdDev oldPsfIxx = psfIxx oldPsfIyy = psfIyy oldPsfIxy = psfIxy finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def __init__(self, **kwargs): """Construct a ProcessEimageTask """ ProcessCcdTask.__init__(self, **kwargs) numpy.random.seed(self.config.rngSeed)