def test_readIsrData_withTrans(self): """Test that all necessary calibration frames are retrieved. """ self.config.doAttachTransmissionCurve = True self.task = IsrTask(config=self.config) results = self.task.readIsrData(self.dataRef, self.inputExp) self.validateIsrData(results)
def validateIsrResults(self): """results should be a struct with components that are not None if included in the configuration file. Returns ------- results : `pipeBase.Struct` Results struct generated from the current ISR configuration. """ self.task = IsrTask(config=self.config) results = self.task.run( self.inputExp, camera=self.camera, bias=self.dataRef.get("bias"), dark=self.dataRef.get("dark"), flat=self.dataRef.get("flat"), bfKernel=self.dataRef.get("bfKernel"), defects=self.dataRef.get("defects"), fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234), opticsTransmission=self.dataRef.get("transmission_"), filterTransmission=self.dataRef.get("transmission_"), sensorTransmission=self.dataRef.get("transmission_"), atmosphereTransmission=self.dataRef.get("transmission_")) self.assertIsInstance(results, Struct) self.assertIsInstance(results.exposure, afwImage.Exposure) return results
def setUp(self): self.config = IsrTaskConfig() self.config.qa = IsrQaConfig() self.task = IsrTask(config=self.config) self.dataRef = isrMock.DataRefMock() self.camera = isrMock.IsrMock().getCamera() self.inputExp = isrMock.TrimmedRawMock().run() self.amp = self.inputExp.getDetector()[0] self.mi = self.inputExp.getMaskedImage()
def test_runDataRef(self): """Expect a dataRef to be handled correctly. """ self.config.doLinearize = False self.config.doWrite = False self.task = IsrTask(config=self.config) results = self.task.runDataRef(self.dataRef) self.assertIsInstance(results, Struct) self.assertIsInstance(results.exposure, afwImage.Exposure)
def runIsr(self, doCrosstalk): """Run DecamCpIsrTask with or without crosstalk correction """ dataRef = self.butler.dataRef('raw', dataId=self.dataId) rawExposure = self.butler.get('raw', dataId=self.dataId) camera = dataRef.get('camera') config = getObsDecamConfig(IsrTask) config.doCrosstalk = doCrosstalk decamCpIsrTask = IsrTask(config=config) isrData = decamCpIsrTask.readIsrData(dataRef, rawExposure) isrResult = decamCpIsrTask.run(rawExposure, camera=camera, **isrData.getDict()) return isrResult
def setUp(self): self.config = IsrTaskConfig() self.config.qa = IsrQaConfig() self.task = IsrTask(config=self.config) self.mockConfig = isrMock.IsrMockConfig() self.mockConfig.isTrimmed = False self.doGenerateImage = True self.dataRef = isrMock.DataRefMock(config=self.mockConfig) self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() self.inputExp = isrMock.RawMock(config=self.mockConfig).run() self.amp = self.inputExp.getDetector()[0] self.mi = self.inputExp.getMaskedImage()
def testSetPolygonIntersect(self): # Create a detector detector = DetectorWrapper().detector numPolygonPoints = 50 # Create an exposure with bounding box defined by detector exposure = afwImage.ExposureF(detector.getBBox()) exposure.setDetector(detector) pixelSizeMm = exposure.getDetector().getPixelSize()[0] pixX0 = exposure.getX0() pixY0 = exposure.getY0() pixX1 = pixX0 + exposure.getWidth() - 1 pixY1 = pixY0 + exposure.getHeight() - 1 fpCenter = exposure.getDetector().getCenter(FOCAL_PLANE) fpCenterX = fpCenter[0] fpCenterY = fpCenter[1] pixCenter = exposure.getDetector().getCenter(PIXELS) # Create an instance of IsrTask task = IsrTask() # Make a polygon that encompases entire ccd (radius of 2*max of width/height) fpRadius = 2.0*max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints) # Set the polygon that is the intersection of fpPolygon and ccd task.setValidPolygonIntersect(exposure, fpPolygon) # Since the ccd is fully contained in the fpPolygon, the intersection should be the ccdPolygon itself ccdPolygonPix = afwGeom.Polygon(exposure.getDetector().getCorners(PIXELS)) self.assertEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix) # Make a polygon that is entirely within, but smaller than, the ccd # (radius of 0.2*min of width/height) fpRadius = 0.2*min(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints) # Set the polygon that is the intersection of fpPolygon and ccd task.setValidPolygonIntersect(exposure, fpPolygon) # all vertices of polygon should be contained within the ccd for x in exposure.getInfo().getValidPolygon(): self.assertTrue(ccdPolygonPix.contains(afwGeom.Point2D(x))) # intersection is smaller than the ccd self.assertNotEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix) # make a simple square polygon that partly intersects the ccd, centered at ccd center fpPolygonSize = max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeSquarePolygon(fpCenterX, fpCenterY, fpPolygonSize) task.setValidPolygonIntersect(exposure, fpPolygon) # Check that the polygon contains the central pixel (offset by one to actually be "contained") pixCenterPlusOne = afwGeom.Point2D(pixCenter[0] + 1, pixCenter[1] + 1) self.assertTrue(exposure.getInfo().getValidPolygon().contains(afwGeom.Point2D(pixCenterPlusOne))) # Check that the polygon contains the upper right ccd edge self.assertTrue(exposure.getInfo().getValidPolygon().contains(afwGeom.Point2D(pixX1, pixY1)))
def testSetPolygonIntersect(self): # Create a detector detector = DetectorWrapper().detector numPolygonPoints = 50 # Create an exposure with bounding box defined by detector exposure = afwImage.ExposureF(detector.getBBox()) exposure.setDetector(detector) pixelSizeMm = exposure.getDetector().getPixelSize()[0] pixX0 = exposure.getX0() pixY0 = exposure.getY0() pixX1 = pixX0 + exposure.getWidth() - 1 pixY1 = pixY0 + exposure.getHeight() - 1 fpCenter = exposure.getDetector().getCenter(FOCAL_PLANE) fpCenterX = fpCenter[0] fpCenterY = fpCenter[1] pixCenter = exposure.getDetector().getCenter(PIXELS) # Create an instance of IsrTask task = IsrTask() # Make a polygon that encompases entire ccd (radius of 2*max of width/height) fpRadius = 2.0*max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints) # Set the polygon that is the intersection of fpPolygon and ccd task.setValidPolygonIntersect(exposure, fpPolygon) # Since the ccd is fully contained in the fpPolygon, the intersection should be the ccdPolygon itself ccdPolygonPix = afwGeom.Polygon(exposure.getDetector().getCorners(PIXELS)) self.assertEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix) # Make a polygon that is entirely within, but smaller than, the ccd # (radius of 0.2*min of width/height) fpRadius = 0.2*min(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints) # Set the polygon that is the intersection of fpPolygon and ccd task.setValidPolygonIntersect(exposure, fpPolygon) # all vertices of polygon should be contained within the ccd for x in exposure.getInfo().getValidPolygon(): self.assertTrue(ccdPolygonPix.contains(lsst.geom.Point2D(x))) # intersection is smaller than the ccd self.assertNotEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix) # make a simple square polygon that partly intersects the ccd, centered at ccd center fpPolygonSize = max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm) fpPolygon = makeSquarePolygon(fpCenterX, fpCenterY, fpPolygonSize) task.setValidPolygonIntersect(exposure, fpPolygon) # Check that the polygon contains the central pixel (offset by one to actually be "contained") pixCenterPlusOne = lsst.geom.Point2D(pixCenter[0] + 1, pixCenter[1] + 1) self.assertTrue(exposure.getInfo().getValidPolygon().contains(lsst.geom.Point2D(pixCenterPlusOne))) # Check that the polygon contains the upper right ccd edge self.assertTrue(exposure.getInfo().getValidPolygon().contains(lsst.geom.Point2D(pixX1, pixY1)))
def setUp(self): """Constructs a CCD with two amplifiers and prepares for ISR""" np.random.seed(12345) baseValue = 100.0 gain = 1.0 readNoise = 123456789.0 saturation = 987654321.0 height = 234 imageSize = Extent2I(123, height) overscanSize = Extent2I(16, height) self.sigma = 1.234 # Set up the various regions overscan1 = Box2I(Point2I(0, 0), overscanSize) image1 = Box2I(Point2I(overscanSize[0], 0), imageSize) image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize) overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize) leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height)) rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height)) target1 = Box2I(Point2I(0, 0), imageSize) target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize) # Set the pixels exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height))) yy = np.arange(0, height, 1, dtype=np.float32) leftImage = ExposureF(exposure, leftBox) leftImage.image.array[:] = baseValue + yy[:, np.newaxis] rightImage = ExposureF(exposure, rightBox) rightImage.image.array[:] = baseValue - yy[:, np.newaxis] leftOverscan = ExposureF(exposure, overscan1) leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) rightOverscan = ExposureF(exposure, overscan2) rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) exposure.mask.array[:] = 0.0 exposure.variance.array[:] = np.nan # Construct the detectors amps = AmpInfoCatalog(AmpInfoTable.makeMinimalSchema()) makeAmplifier(amps, "left", target1, image1, overscan1, gain, readNoise, saturation) makeAmplifier(amps, "right", target2, image2, overscan2, gain, readNoise, saturation) ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height)) ccd = Detector("detector", 1, SCIENCE, "det1", ccdBox, amps, Orientation(), Extent2D(1.0, 1.0), {}) exposure.setDetector(ccd) header = PropertyList() header.add("EXPTIME", 0.0) exposure.getInfo().setVisitInfo(VisitInfo(header)) self.exposure = exposure self.config = IsrTask.ConfigClass() # Disable everything we don't care about self.config.doBias = False self.config.doDark = False self.config.doFlat = False self.config.doFringe = False self.config.doDefect = False self.config.doAddDistortionModel = False self.config.doWrite = False self.config.expectWcs = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doBrighterFatter = False self.config.doAttachTransmissionCurve = False # Set the things that match our test setup self.config.overscanFitType = "CHEB" self.config.overscanOrder = 1 self.config.doEmpiricalReadNoise = True self.task = IsrTask(config=self.config)
def summarizeVisit(butler, *, exp=None, extendedSummary=False, **kwargs): from astroquery.simbad import Simbad import astropy.units as u from astropy.time import Time from astropy.coordinates import SkyCoord, EarthLocation, AltAz def _airMassFromrRawMd(md): auxTelLocation = EarthLocation(lat=-30.244639 * u.deg, lon=-70.749417 * u.deg, height=2663 * u.m) time = Time(md['DATE-OBS']) skyLocation = SkyCoord(md['RASTART'], md['DECSTART'], unit=u.deg) altAz = AltAz(obstime=time, location=auxTelLocation) observationAltAz = skyLocation.transform_to(altAz) return observationAltAz.secz.value items = ["OBJECT", "expTime", "FILTER", "imageType"] obj, expTime, filterCompound, imageType = butler.queryMetadata( 'raw', items, **kwargs)[0] filt, grating = filterCompound.split('~') rawMd = butler.get('raw_md', **kwargs) airmass = _airMassFromrRawMd(rawMd) print(f"Object name: {obj}") print(f"expTime: {expTime}s") print(f"imageType: {imageType}") print(f"Filter: {filt}") print(f"Grating: {grating}") print(f"Airmass: {airmass:.3f}") if imageType not in ['BIAS', 'FLAT', 'DARK']: simbadObj = Simbad.query_object(obj) if simbadObj is None: print(f"Failed to find {obj} in Simbad.") else: assert (len(simbadObj.as_array()) == 1) raStr = simbadObj[0]['RA'] decStr = simbadObj[0]['DEC'] skyLocation = SkyCoord(raStr, decStr, unit=(u.hourangle, u.degree), frame='icrs') raRad, decRad = skyLocation.ra.rad, skyLocation.dec.rad print(f"obj RA (str): {raStr}") print(f"obj DEC (str): {decStr}") print(f"obj RA (rad): {raRad:5f}") print(f"obj DEC (rad): {decRad:5f}") print(f"obj RA (deg): {raRad*180/math.pi:5f}") print(f"obj DEC (deg): {decRad*180/math.pi:5f}") if exp is not None: # calc source coords from exp wcs ra = geom.Angle(raRad) dec = geom.Angle(decRad) targetLocation = geom.SpherePoint(ra, dec) pixCoord = exp.getWcs().skyToPixel(targetLocation) print(exp.getWcs()) print(f"Source location: {pixCoord} using exp provided") else: # try to find one, but not too hard datasetTypes = ['calexp', 'quickLookExp', 'postISRCCD'] for datasetType in datasetTypes: wcs = None typeUsed = None try: wcs = butler.get(datasetType + '_wcs', **kwargs) typeUsed = datasetType break except butlerExcept.NoResults: pass if wcs is not None: ra = geom.Angle(raRad) dec = geom.Angle(decRad) targetLocation = geom.SpherePoint(ra, dec) pixCoord = wcs.skyToPixel(targetLocation) print(wcs) print(f"Source location: {pixCoord} using {typeUsed}") if extendedSummary: print('\n--- Extended Summary ---') ranIsr = False if exp is None: print("Running isr to compute image stats...") # catch all the ISR chat # logRedirection1 = LogRedirect(1, open(os.devnull, 'w')) # logRedirection2 = LogRedirect(2, open(os.devnull, 'w')) # import ipdb as pdb; pdb.set_trace() from lsst.ip.isr.isrTask import IsrTask isrConfig = IsrTask.ConfigClass() isrConfig.doLinearize = False isrConfig.doBias = False isrConfig.doFlat = False isrConfig.doDark = False isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doWrite = False isrTask = IsrTask(config=isrConfig) dataRef = butler.dataRef('raw', **kwargs) exp = isrTask.runDataRef(dataRef).exposure wcs = exp.getWcs() ranIsr = True # logRedirection1.finish() # end re-direct # logRedirection2.finish() # end re-direct print(wcs) if simbadObj and ranIsr: ra = geom.Angle(raRad) dec = geom.Angle(decRad) targetLocation = geom.SpherePoint(ra, dec) pixCoord = wcs.skyToPixel(targetLocation) print( f"Source location: {pixCoord} using postISR just-reconstructed wcs" ) print( f'\nImage stats from {"just-constructed" if ranIsr else "provided"} exp:\n' ) print(f'Image mean: {np.mean(exp.image.array):.2f}') print(f'Image median: {np.median(exp.image.array):.2f}') print(f'Image min: {np.min(exp.image.array):.2f}') print(f'Image max: {np.max(exp.image.array):.2f}') # TODO: quartiles/percentiles here # number of masked pixels, saturated pixels print() print(f'BAD pixels: {countPixels(exp.maskedImage, "BAD")}') print(f'SAT pixels: {countPixels(exp.maskedImage, "SAT")}') print(f'CR pixels: {countPixels(exp.maskedImage, "CR")}') print(f'INTRP pixels: {countPixels(exp.maskedImage, "INTRP")}') print(f'DETECTED pixels: {countPixels(exp.maskedImage, "DETECTED")}') # detector = exp.getDetector() visitInfo = exp.getInfo().getVisitInfo() rotAngle = visitInfo.getBoresightRotAngle() boresight = visitInfo.getBoresightRaDec() md = butler.get('raw_md', **kwargs) print("\n From VisitInfo:") print(f"boresight: {boresight}") print(f"rotAngle: {rotAngle}") print(f" → {rotAngle.asDegrees():.4f} deg") print("\n From raw_md:") print(f"ROTPA: {md['ROTPA']} deg") print(f" → {(md['ROTPA']*math.pi/180):.6f} rad")
def setUp(self): """Constructs a CCD with two amplifiers and prepares for ISR""" np.random.seed(12345) baseValue = 100.0 gain = 1.0 readNoise = 123456789.0 saturation = 987654321.0 height = 234 imageSize = Extent2I(123, height) overscanSize = Extent2I(16, height) self.sigma = 1.234 # Set up the various regions overscan1 = Box2I(Point2I(0, 0), overscanSize) image1 = Box2I(Point2I(overscanSize[0], 0), imageSize) image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize) overscan2 = Box2I(Point2I(overscanSize[0] + 2 * imageSize[0], 0), overscanSize) leftBox = Box2I( overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height)) rightBox = Box2I( image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height)) target1 = Box2I(Point2I(0, 0), imageSize) target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize) # Set the pixels exposure = ExposureF( Box2I(Point2I(0, 0), Extent2I(imageSize[0] * 2 + overscanSize[0] * 2, height))) yy = np.arange(0, height, 1, dtype=np.float32) leftImage = ExposureF(exposure, leftBox) leftImage.image.array[:] = baseValue + yy[:, np.newaxis] rightImage = ExposureF(exposure, rightBox) rightImage.image.array[:] = baseValue - yy[:, np.newaxis] leftOverscan = ExposureF(exposure, overscan1) leftOverscan.image.array += np.random.normal( 0.0, self.sigma, leftOverscan.image.array.shape) rightOverscan = ExposureF(exposure, overscan2) rightOverscan.image.array += np.random.normal( 0.0, self.sigma, leftOverscan.image.array.shape) exposure.mask.array[:] = 0.0 exposure.variance.array[:] = np.nan # Construct the detectors amp1 = makeAmplifier("left", target1, image1, overscan1, gain, readNoise, saturation) amp2 = makeAmplifier("right", target2, image2, overscan2, gain, readNoise, saturation) ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height)) camBuilder = cameraGeom.Camera.Builder("fakeCam") detBuilder = camBuilder.add("detector", 1) detBuilder.setSerial("det1") detBuilder.setBBox(ccdBox) detBuilder.setPixelSize(Extent2D(1.0, 1.0)) detBuilder.setOrientation(cameraGeom.Orientation()) detBuilder.append(amp1) detBuilder.append(amp2) cam = camBuilder.finish() exposure.setDetector(cam.get('detector')) header = PropertyList() header.add("EXPTIME", 0.0) exposure.getInfo().setVisitInfo(VisitInfo(header)) self.exposure = exposure self.config = IsrTask.ConfigClass() # Disable everything we don't care about self.config.doBias = False self.config.doDark = False self.config.doFlat = False self.config.doFringe = False self.config.doDefect = False self.config.doWrite = False self.config.expectWcs = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doBrighterFatter = False self.config.doAttachTransmissionCurve = False self.config.doAssembleCcd = False self.config.doNanMasking = False self.config.doInterpolate = False self.config.maskNegativeVariance = False # This runs on mocks. # Set the things that match our test setup self.config.overscan.fitType = "CHEB" self.config.overscan.order = 1 self.config.doEmpiricalReadNoise = True self.task = IsrTask(config=self.config)
class EmpiricalVarianceTestCast(lsst.utils.tests.TestCase): def setUp(self): """Constructs a CCD with two amplifiers and prepares for ISR""" np.random.seed(12345) baseValue = 100.0 gain = 1.0 readNoise = 123456789.0 saturation = 987654321.0 height = 234 imageSize = Extent2I(123, height) overscanSize = Extent2I(16, height) self.sigma = 1.234 # Set up the various regions overscan1 = Box2I(Point2I(0, 0), overscanSize) image1 = Box2I(Point2I(overscanSize[0], 0), imageSize) image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize) overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize) leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height)) rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height)) target1 = Box2I(Point2I(0, 0), imageSize) target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize) # Set the pixels exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height))) yy = np.arange(0, height, 1, dtype=np.float32) leftImage = ExposureF(exposure, leftBox) leftImage.image.array[:] = baseValue + yy[:, np.newaxis] rightImage = ExposureF(exposure, rightBox) rightImage.image.array[:] = baseValue - yy[:, np.newaxis] leftOverscan = ExposureF(exposure, overscan1) leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) rightOverscan = ExposureF(exposure, overscan2) rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) exposure.mask.array[:] = 0.0 exposure.variance.array[:] = np.nan # Construct the detectors amp1 = makeAmplifier("left", target1, image1, overscan1, gain, readNoise, saturation) amp2 = makeAmplifier("right", target2, image2, overscan2, gain, readNoise, saturation) ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height)) camBuilder = cameraGeom.Camera.Builder("fakeCam") detBuilder = camBuilder.add("detector", 1) detBuilder.setSerial("det1") detBuilder.setBBox(ccdBox) detBuilder.setPixelSize(Extent2D(1.0, 1.0)) detBuilder.setOrientation(cameraGeom.Orientation()) detBuilder.append(amp1) detBuilder.append(amp2) cam = camBuilder.finish() exposure.setDetector(cam.get('detector')) header = PropertyList() header.add("EXPTIME", 0.0) exposure.getInfo().setVisitInfo(VisitInfo(header)) self.exposure = exposure self.config = IsrTask.ConfigClass() # Disable everything we don't care about self.config.doBias = False self.config.doDark = False self.config.doFlat = False self.config.doFringe = False self.config.doDefect = False self.config.doAddDistortionModel = False self.config.doWrite = False self.config.expectWcs = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doBrighterFatter = False self.config.doAttachTransmissionCurve = False self.config.doAssembleCcd = False self.config.doNanMasking = False self.config.doInterpolate = False # Set the things that match our test setup self.config.overscanFitType = "CHEB" self.config.overscanOrder = 1 self.config.doEmpiricalReadNoise = True self.task = IsrTask(config=self.config) def tearDown(self): del self.exposure def testEmpiricalVariance(self): results = self.task.run(self.exposure) postIsr = results.exposure self.assertFloatsEqual(postIsr.mask.array, 0) # Image is not exactly zero because the noise in the overscan (required to be able to set # the empirical variance) leads to a slight misestimate in the polynomial fit. self.assertFloatsAlmostEqual(np.median(postIsr.image.array), 0.0, atol=5.0e-2) self.assertFloatsAlmostEqual(np.nanmedian(postIsr.variance.array), self.sigma**2, rtol=5.0e-2)
class IsrTaskTestCases(lsst.utils.tests.TestCase): """Test IsrTask methods with trimmed raw data. """ def setUp(self): self.config = IsrTaskConfig() self.config.qa = IsrQaConfig() self.task = IsrTask(config=self.config) self.dataRef = isrMock.DataRefMock() self.camera = isrMock.IsrMock().getCamera() self.inputExp = isrMock.TrimmedRawMock().run() self.amp = self.inputExp.getDetector()[0] self.mi = self.inputExp.getMaskedImage() def validateIsrData(self, results): """results should be a struct with components that are not None if included in the configuration file. """ self.assertIsInstance(results, Struct) if self.config.doBias is True: self.assertIsNotNone(results.bias) if self.config.doDark is True: self.assertIsNotNone(results.dark) if self.config.doFlat is True: self.assertIsNotNone(results.flat) if self.config.doFringe is True: self.assertIsNotNone(results.fringes) if self.config.doDefect is True: self.assertIsNotNone(results.defects) if self.config.doBrighterFatter is True: self.assertIsNotNone(results.bfKernel) if self.config.doAttachTransmissionCurve is True: self.assertIsNotNone(results.opticsTransmission) self.assertIsNotNone(results.filterTransmission) self.assertIsNotNone(results.sensorTransmission) self.assertIsNotNone(results.atmosphereTransmission) def test_readIsrData_noTrans(self): """Test that all necessary calibration frames are retrieved. """ self.config.doAttachTransmissionCurve = False self.task = IsrTask(config=self.config) results = self.task.readIsrData(self.dataRef, self.inputExp) self.validateIsrData(results) def test_readIsrData_withTrans(self): """Test that all necessary calibration frames are retrieved. """ self.config.doAttachTransmissionCurve = True self.task = IsrTask(config=self.config) results = self.task.readIsrData(self.dataRef, self.inputExp) self.validateIsrData(results) def test_ensureExposure(self): """Test that an exposure has a usable instance class. """ self.assertIsInstance( self.task.ensureExposure(self.inputExp, self.camera, 0), afwImage.Exposure) def test_convertItoF(self): """Test conversion from integer to floating point pixels. """ result = self.task.convertIntToFloat(self.inputExp) self.assertEqual(result.getImage().getArray().dtype, np.dtype("float32")) self.assertEqual(result, self.inputExp) def test_updateVariance(self): """Expect The variance image should have a larger median value after this operation. """ statBefore = computeImageMedianAndStd( self.inputExp.variance[self.amp.getBBox()]) self.task.updateVariance(self.inputExp, self.amp) statAfter = computeImageMedianAndStd( self.inputExp.variance[self.amp.getBBox()]) self.assertGreater(statAfter[0], statBefore[0]) self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2) self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2) def test_darkCorrection(self): """Expect the median image value should decrease after this operation. """ darkIm = isrMock.DarkMock().run() statBefore = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.task.darkCorrection(self.inputExp, darkIm) statAfter = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.assertLess(statAfter[0], statBefore[0]) self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) def test_darkCorrection_noVisitInfo(self): """Expect the median image value should decrease after this operation. """ darkIm = isrMock.DarkMock().run() darkIm.getInfo().setVisitInfo(None) statBefore = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.task.darkCorrection(self.inputExp, darkIm) statAfter = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.assertLess(statAfter[0], statBefore[0]) self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2) self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2) def test_flatCorrection(self): """Expect the image median should increase (divide by < 1). """ flatIm = isrMock.FlatMock().run() statBefore = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.task.flatCorrection(self.inputExp, flatIm) statAfter = computeImageMedianAndStd( self.inputExp.image[self.amp.getBBox()]) self.assertGreater(statAfter[1], statBefore[1]) self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2) self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2) def test_saturationDetection(self): """Expect the saturation level detection/masking to scale with threshold. """ ampB = self.amp.rebuild() ampB.setSaturation(9000.0) self.task.saturationDetection(self.inputExp, ampB.finish()) countBefore = countMaskedPixels(self.mi, "SAT") ampB.setSaturation(8250.0) self.task.saturationDetection(self.inputExp, ampB.finish()) countAfter = countMaskedPixels(self.mi, "SAT") self.assertLessEqual(countBefore, countAfter) self.assertEqual(countBefore, 43) self.assertEqual(countAfter, 136) def test_measureBackground(self): """Expect the background measurement runs successfully and to save metadata values. """ self.config.qa.flatness.meshX = 20 self.config.qa.flatness.meshY = 20 self.task.measureBackground(self.inputExp, self.config.qa) self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL')) def test_flatContext(self): """Expect the flat context manager runs successfully (applying both flat and dark within the context), and results in the same image data after completion. """ darkExp = isrMock.DarkMock().run() flatExp = isrMock.FlatMock().run() mi = self.inputExp.getMaskedImage().clone() with self.task.flatContext(self.inputExp, flatExp, darkExp): contextStat = computeImageMedianAndStd( self.inputExp.getMaskedImage().getImage()) self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2) self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase): """Test IsrTask methods using untrimmed raw data. """ def setUp(self): self.config = IsrTaskConfig() self.config.qa = IsrQaConfig() self.task = IsrTask(config=self.config) self.mockConfig = isrMock.IsrMockConfig() self.mockConfig.isTrimmed = False self.doGenerateImage = True self.dataRef = isrMock.DataRefMock(config=self.mockConfig) self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() self.inputExp = isrMock.RawMock(config=self.mockConfig).run() self.amp = self.inputExp.getDetector()[0] self.mi = self.inputExp.getMaskedImage() def batchSetConfiguration(self, value): """Set the configuration state to a consistent value. Disable options we do not need as well. Parameters ---------- value : `bool` Value to switch common ISR configuration options to. """ self.config.qa.flatness.meshX = 20 self.config.qa.flatness.meshY = 20 self.config.doWrite = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doConvertIntToFloat = value self.config.doSaturation = value self.config.doSuspect = value self.config.doSetBadRegions = value self.config.doOverscan = value self.config.doBias = value self.config.doVariance = value self.config.doWidenSaturationTrails = value self.config.doBrighterFatter = value self.config.doDefect = value self.config.doSaturationInterpolation = value self.config.doDark = value self.config.doStrayLight = value self.config.doFlat = value self.config.doFringe = value self.config.doMeasureBackground = value self.config.doVignette = value self.config.doAttachTransmissionCurve = value self.config.doUseOpticsTransmission = value self.config.doUseFilterTransmission = value self.config.doUseSensorTransmission = value self.config.doUseAtmosphereTransmission = value self.config.qa.saveStats = value self.config.qa.doThumbnailOss = value self.config.qa.doThumbnailFlattened = value self.config.doApplyGains = not value self.config.doCameraSpecificMasking = value def validateIsrResults(self): """results should be a struct with components that are not None if included in the configuration file. Returns ------- results : `pipeBase.Struct` Results struct generated from the current ISR configuration. """ self.task = IsrTask(config=self.config) results = self.task.run( self.inputExp, camera=self.camera, bias=self.dataRef.get("bias"), dark=self.dataRef.get("dark"), flat=self.dataRef.get("flat"), bfKernel=self.dataRef.get("bfKernel"), defects=self.dataRef.get("defects"), fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234), opticsTransmission=self.dataRef.get("transmission_"), filterTransmission=self.dataRef.get("transmission_"), sensorTransmission=self.dataRef.get("transmission_"), atmosphereTransmission=self.dataRef.get("transmission_")) self.assertIsInstance(results, Struct) self.assertIsInstance(results.exposure, afwImage.Exposure) return results def test_overscanCorrection(self): """Expect that this should reduce the image variance with a full fit. The default fitType of MEDIAN will reduce the median value. This needs to operate on a RawMock() to have overscan data to use. The output types may be different when fitType != MEDIAN. """ statBefore = computeImageMedianAndStd( self.inputExp.image[self.amp.getRawDataBBox()]) oscanResults = self.task.overscanCorrection(self.inputExp, self.amp) self.assertIsInstance(oscanResults, Struct) self.assertIsInstance(oscanResults.imageFit, float) self.assertIsInstance(oscanResults.overscanFit, float) self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF) statAfter = computeImageMedianAndStd( self.inputExp.image[self.amp.getRawDataBBox()]) self.assertLess(statAfter[0], statBefore[0]) def test_overscanCorrectionMedianPerRow(self): """Expect that this should reduce the image variance with a full fit. fitType of MEDIAN_PER_ROW will reduce the median value. This needs to operate on a RawMock() to have overscan data to use. The output types may be different when fitType != MEDIAN_PER_ROW. """ self.config.overscan.fitType = 'MEDIAN_PER_ROW' statBefore = computeImageMedianAndStd( self.inputExp.image[self.amp.getRawDataBBox()]) oscanResults = self.task.overscanCorrection(self.inputExp, self.amp) self.assertIsInstance(oscanResults, Struct) self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF) self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF) self.assertIsInstance(oscanResults.overscanImage, afwImage.MaskedImageF) statAfter = computeImageMedianAndStd( self.inputExp.image[self.amp.getRawDataBBox()]) self.assertLess(statAfter[0], statBefore[0]) def test_runDataRef(self): """Expect a dataRef to be handled correctly. """ self.config.doLinearize = False self.config.doWrite = False self.task = IsrTask(config=self.config) results = self.task.runDataRef(self.dataRef) self.assertIsInstance(results, Struct) self.assertIsInstance(results.exposure, afwImage.Exposure) def test_run_allTrue(self): """Expect successful run with expected outputs when all non-exclusive configuration options are on. Output results should be tested more precisely by the individual function tests. """ self.batchSetConfiguration(True) self.validateIsrResults() def test_run_allFalse(self): """Expect successful run with expected outputs when all non-exclusive configuration options are off. Output results should be tested more precisely by the individual function tests. """ self.batchSetConfiguration(False) self.validateIsrResults() def test_failCases(self): """Expect failure with crosstalk enabled. Output results should be tested more precisely by the individual function tests. """ self.batchSetConfiguration(True) # This breaks it self.config.doCrosstalk = True with self.assertRaises(RuntimeError): self.validateIsrResults() def test_maskingCase_negativeVariance(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.doSaturation = False self.config.doWidenSaturationTrails = False self.config.doSaturationInterpolation = False self.config.doSuspect = False self.config.doSetBadRegions = False self.config.doDefect = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = True self.config.doInterpolate = False results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800) def test_maskingCase_noMasking(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.doSaturation = False self.config.doWidenSaturationTrails = False self.config.doSaturationInterpolation = False self.config.doSuspect = False self.config.doSetBadRegions = False self.config.doDefect = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False self.config.doInterpolate = False results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) def test_maskingCase_satMasking(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = False self.config.doSuspect = False self.config.doSetBadRegions = False self.config.doDefect = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) def test_maskingCase_satMaskingAndInterp(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = True self.config.doSuspect = False self.config.doSetBadRegions = False self.config.doDefect = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) def test_maskingCase_throughEdge(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = True self.config.numEdgeSuspect = 5 self.config.doSuspect = True self.config.doSetBadRegions = False self.config.doDefect = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0) def test_maskingCase_throughDefects(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = True self.config.numEdgeSuspect = 5 self.config.doSuspect = True self.config.doDefect = True self.config.doSetBadRegions = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) def test_maskingCase_throughDefectsAmpEdges(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = True self.config.numEdgeSuspect = 5 self.config.doSuspect = True self.config.doDefect = True self.config.edgeMaskLevel = 'AMP' self.config.doSetBadRegions = False self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000) def test_maskingCase_throughBad(self): """Test masking cases of configuration parameters. """ self.batchSetConfiguration(True) self.config.overscanFitType = "POLY" self.config.overscanOrder = 1 self.config.saturation = 20000.0 self.config.doSaturation = True self.config.doWidenSaturationTrails = True self.config.doSaturationInterpolation = True self.config.doSuspect = True self.config.doDefect = True self.config.doSetBadRegions = True self.config.doBrighterFatter = False self.config.maskNegativeVariance = False # These are mock images. results = self.validateIsrResults() self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000) self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0) self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
Popen.wait(ingest) print("finished ingesting spots images") spots_butler = Butler(SPOTS_REPO_DIR) visits = [] my_metaData = spots_butler.queryMetadata('raw', ['visit', 'dateObs']) for item in my_metaData: visits.append(item[0]) byamp_results = [] byamp_corrected_results = [] for visit in visits: print("Getting exposure # %d"%visit) sys.stdout.flush() exposure = spots_butler.get('raw', dataId={'visit': visit, 'detector': DETECTOR}) # Perform the instrument signature removal (mainly assembling the CCD) isrTask = IsrTask(config=isrConfig) exposure_isr = isrTask.run(exposure).exposure # For now, we're applying the gain manually ccd = exposure_isr.getDetector() for do_bf_corr in [False, True]: exposure_copy=exposure_isr.clone() for amp in ccd: if spots_bbox.overlaps(amp.getBBox()): gain = gain_data[amp.getName()] img = exposure_copy.image sim = img.Factory(img, amp.getBBox()) sim *= gain print(amp.getName(), gain, amp.getBBox()) sys.stdout.flush() if do_bf_corr: brighterFatterCorrection(exposure_copy[amp.getBBox()],bf_kernel.kernel[amp.getName()],20,10,False)