def test_readIsrData_withTrans(self):
"""Test that all necessary calibration frames are retrieved.
"""
self.config.doAttachTransmissionCurve = True
self.task = IsrTask(config=self.config)
results = self.task.readIsrData(self.dataRef, self.inputExp)
self.validateIsrData(results)
def validateIsrResults(self):
"""results should be a struct with components that are
not None if included in the configuration file.
Returns
-------
results : `pipeBase.Struct`
Results struct generated from the current ISR configuration.
"""
self.task = IsrTask(config=self.config)
results = self.task.run(
self.inputExp,
camera=self.camera,
bias=self.dataRef.get("bias"),
dark=self.dataRef.get("dark"),
flat=self.dataRef.get("flat"),
bfKernel=self.dataRef.get("bfKernel"),
defects=self.dataRef.get("defects"),
fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
opticsTransmission=self.dataRef.get("transmission_"),
filterTransmission=self.dataRef.get("transmission_"),
sensorTransmission=self.dataRef.get("transmission_"),
atmosphereTransmission=self.dataRef.get("transmission_"))
self.assertIsInstance(results, Struct)
self.assertIsInstance(results.exposure, afwImage.Exposure)
return results
def setUp(self):
self.config = IsrTaskConfig()
self.config.qa = IsrQaConfig()
self.task = IsrTask(config=self.config)
self.dataRef = isrMock.DataRefMock()
self.camera = isrMock.IsrMock().getCamera()
self.inputExp = isrMock.TrimmedRawMock().run()
self.amp = self.inputExp.getDetector()[0]
self.mi = self.inputExp.getMaskedImage()
def test_runDataRef(self):
"""Expect a dataRef to be handled correctly.
"""
self.config.doLinearize = False
self.config.doWrite = False
self.task = IsrTask(config=self.config)
results = self.task.runDataRef(self.dataRef)
self.assertIsInstance(results, Struct)
self.assertIsInstance(results.exposure, afwImage.Exposure)
def runIsr(self, doCrosstalk):
"""Run DecamCpIsrTask with or without crosstalk correction
"""
dataRef = self.butler.dataRef('raw', dataId=self.dataId)
rawExposure = self.butler.get('raw', dataId=self.dataId)
camera = dataRef.get('camera')
config = getObsDecamConfig(IsrTask)
config.doCrosstalk = doCrosstalk
decamCpIsrTask = IsrTask(config=config)
isrData = decamCpIsrTask.readIsrData(dataRef, rawExposure)
isrResult = decamCpIsrTask.run(rawExposure, camera=camera, **isrData.getDict())
return isrResult
def setUp(self):
self.config = IsrTaskConfig()
self.config.qa = IsrQaConfig()
self.task = IsrTask(config=self.config)
self.mockConfig = isrMock.IsrMockConfig()
self.mockConfig.isTrimmed = False
self.doGenerateImage = True
self.dataRef = isrMock.DataRefMock(config=self.mockConfig)
self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
self.amp = self.inputExp.getDetector()[0]
self.mi = self.inputExp.getMaskedImage()
def testSetPolygonIntersect(self):
# Create a detector
detector = DetectorWrapper().detector
numPolygonPoints = 50
# Create an exposure with bounding box defined by detector
exposure = afwImage.ExposureF(detector.getBBox())
exposure.setDetector(detector)
pixelSizeMm = exposure.getDetector().getPixelSize()[0]
pixX0 = exposure.getX0()
pixY0 = exposure.getY0()
pixX1 = pixX0 + exposure.getWidth() - 1
pixY1 = pixY0 + exposure.getHeight() - 1
fpCenter = exposure.getDetector().getCenter(FOCAL_PLANE)
fpCenterX = fpCenter[0]
fpCenterY = fpCenter[1]
pixCenter = exposure.getDetector().getCenter(PIXELS)
# Create an instance of IsrTask
task = IsrTask()
# Make a polygon that encompases entire ccd (radius of 2*max of width/height)
fpRadius = 2.0*max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints)
# Set the polygon that is the intersection of fpPolygon and ccd
task.setValidPolygonIntersect(exposure, fpPolygon)
# Since the ccd is fully contained in the fpPolygon, the intersection should be the ccdPolygon itself
ccdPolygonPix = afwGeom.Polygon(exposure.getDetector().getCorners(PIXELS))
self.assertEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix)
# Make a polygon that is entirely within, but smaller than, the ccd
# (radius of 0.2*min of width/height)
fpRadius = 0.2*min(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints)
# Set the polygon that is the intersection of fpPolygon and ccd
task.setValidPolygonIntersect(exposure, fpPolygon)
# all vertices of polygon should be contained within the ccd
for x in exposure.getInfo().getValidPolygon():
self.assertTrue(ccdPolygonPix.contains(afwGeom.Point2D(x)))
# intersection is smaller than the ccd
self.assertNotEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix)
# make a simple square polygon that partly intersects the ccd, centered at ccd center
fpPolygonSize = max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeSquarePolygon(fpCenterX, fpCenterY, fpPolygonSize)
task.setValidPolygonIntersect(exposure, fpPolygon)
# Check that the polygon contains the central pixel (offset by one to actually be "contained")
pixCenterPlusOne = afwGeom.Point2D(pixCenter[0] + 1, pixCenter[1] + 1)
self.assertTrue(exposure.getInfo().getValidPolygon().contains(afwGeom.Point2D(pixCenterPlusOne)))
# Check that the polygon contains the upper right ccd edge
self.assertTrue(exposure.getInfo().getValidPolygon().contains(afwGeom.Point2D(pixX1, pixY1)))
def testSetPolygonIntersect(self):
# Create a detector
detector = DetectorWrapper().detector
numPolygonPoints = 50
# Create an exposure with bounding box defined by detector
exposure = afwImage.ExposureF(detector.getBBox())
exposure.setDetector(detector)
pixelSizeMm = exposure.getDetector().getPixelSize()[0]
pixX0 = exposure.getX0()
pixY0 = exposure.getY0()
pixX1 = pixX0 + exposure.getWidth() - 1
pixY1 = pixY0 + exposure.getHeight() - 1
fpCenter = exposure.getDetector().getCenter(FOCAL_PLANE)
fpCenterX = fpCenter[0]
fpCenterY = fpCenter[1]
pixCenter = exposure.getDetector().getCenter(PIXELS)
# Create an instance of IsrTask
task = IsrTask()
# Make a polygon that encompases entire ccd (radius of 2*max of width/height)
fpRadius = 2.0*max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints)
# Set the polygon that is the intersection of fpPolygon and ccd
task.setValidPolygonIntersect(exposure, fpPolygon)
# Since the ccd is fully contained in the fpPolygon, the intersection should be the ccdPolygon itself
ccdPolygonPix = afwGeom.Polygon(exposure.getDetector().getCorners(PIXELS))
self.assertEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix)
# Make a polygon that is entirely within, but smaller than, the ccd
# (radius of 0.2*min of width/height)
fpRadius = 0.2*min(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeCircularPolygon(fpCenterX, fpCenterY, fpRadius, numPolygonPoints)
# Set the polygon that is the intersection of fpPolygon and ccd
task.setValidPolygonIntersect(exposure, fpPolygon)
# all vertices of polygon should be contained within the ccd
for x in exposure.getInfo().getValidPolygon():
self.assertTrue(ccdPolygonPix.contains(lsst.geom.Point2D(x)))
# intersection is smaller than the ccd
self.assertNotEqual(exposure.getInfo().getValidPolygon(), ccdPolygonPix)
# make a simple square polygon that partly intersects the ccd, centered at ccd center
fpPolygonSize = max(exposure.getWidth()*pixelSizeMm, exposure.getHeight()*pixelSizeMm)
fpPolygon = makeSquarePolygon(fpCenterX, fpCenterY, fpPolygonSize)
task.setValidPolygonIntersect(exposure, fpPolygon)
# Check that the polygon contains the central pixel (offset by one to actually be "contained")
pixCenterPlusOne = lsst.geom.Point2D(pixCenter[0] + 1, pixCenter[1] + 1)
self.assertTrue(exposure.getInfo().getValidPolygon().contains(lsst.geom.Point2D(pixCenterPlusOne)))
# Check that the polygon contains the upper right ccd edge
self.assertTrue(exposure.getInfo().getValidPolygon().contains(lsst.geom.Point2D(pixX1, pixY1)))
def setUp(self):
"""Constructs a CCD with two amplifiers and prepares for ISR"""
np.random.seed(12345)
baseValue = 100.0
gain = 1.0
readNoise = 123456789.0
saturation = 987654321.0
height = 234
imageSize = Extent2I(123, height)
overscanSize = Extent2I(16, height)
self.sigma = 1.234
# Set up the various regions
overscan1 = Box2I(Point2I(0, 0), overscanSize)
image1 = Box2I(Point2I(overscanSize[0], 0), imageSize)
image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize)
overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize)
leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height))
rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height))
target1 = Box2I(Point2I(0, 0), imageSize)
target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize)
# Set the pixels
exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height)))
yy = np.arange(0, height, 1, dtype=np.float32)
leftImage = ExposureF(exposure, leftBox)
leftImage.image.array[:] = baseValue + yy[:, np.newaxis]
rightImage = ExposureF(exposure, rightBox)
rightImage.image.array[:] = baseValue - yy[:, np.newaxis]
leftOverscan = ExposureF(exposure, overscan1)
leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
rightOverscan = ExposureF(exposure, overscan2)
rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
exposure.mask.array[:] = 0.0
exposure.variance.array[:] = np.nan
# Construct the detectors
amps = AmpInfoCatalog(AmpInfoTable.makeMinimalSchema())
makeAmplifier(amps, "left", target1, image1, overscan1, gain, readNoise, saturation)
makeAmplifier(amps, "right", target2, image2, overscan2, gain, readNoise, saturation)
ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height))
ccd = Detector("detector", 1, SCIENCE, "det1", ccdBox, amps, Orientation(), Extent2D(1.0, 1.0), {})
exposure.setDetector(ccd)
header = PropertyList()
header.add("EXPTIME", 0.0)
exposure.getInfo().setVisitInfo(VisitInfo(header))
self.exposure = exposure
self.config = IsrTask.ConfigClass()
# Disable everything we don't care about
self.config.doBias = False
self.config.doDark = False
self.config.doFlat = False
self.config.doFringe = False
self.config.doDefect = False
self.config.doAddDistortionModel = False
self.config.doWrite = False
self.config.expectWcs = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doBrighterFatter = False
self.config.doAttachTransmissionCurve = False
# Set the things that match our test setup
self.config.overscanFitType = "CHEB"
self.config.overscanOrder = 1
self.config.doEmpiricalReadNoise = True
self.task = IsrTask(config=self.config)
def summarizeVisit(butler, *, exp=None, extendedSummary=False, **kwargs):
from astroquery.simbad import Simbad
import astropy.units as u
from astropy.time import Time
from astropy.coordinates import SkyCoord, EarthLocation, AltAz
def _airMassFromrRawMd(md):
auxTelLocation = EarthLocation(lat=-30.244639 * u.deg,
lon=-70.749417 * u.deg,
height=2663 * u.m)
time = Time(md['DATE-OBS'])
skyLocation = SkyCoord(md['RASTART'], md['DECSTART'], unit=u.deg)
altAz = AltAz(obstime=time, location=auxTelLocation)
observationAltAz = skyLocation.transform_to(altAz)
return observationAltAz.secz.value
items = ["OBJECT", "expTime", "FILTER", "imageType"]
obj, expTime, filterCompound, imageType = butler.queryMetadata(
'raw', items, **kwargs)[0]
filt, grating = filterCompound.split('~')
rawMd = butler.get('raw_md', **kwargs)
airmass = _airMassFromrRawMd(rawMd)
print(f"Object name: {obj}")
print(f"expTime: {expTime}s")
print(f"imageType: {imageType}")
print(f"Filter: {filt}")
print(f"Grating: {grating}")
print(f"Airmass: {airmass:.3f}")
if imageType not in ['BIAS', 'FLAT', 'DARK']:
simbadObj = Simbad.query_object(obj)
if simbadObj is None:
print(f"Failed to find {obj} in Simbad.")
else:
assert (len(simbadObj.as_array()) == 1)
raStr = simbadObj[0]['RA']
decStr = simbadObj[0]['DEC']
skyLocation = SkyCoord(raStr,
decStr,
unit=(u.hourangle, u.degree),
frame='icrs')
raRad, decRad = skyLocation.ra.rad, skyLocation.dec.rad
print(f"obj RA (str): {raStr}")
print(f"obj DEC (str): {decStr}")
print(f"obj RA (rad): {raRad:5f}")
print(f"obj DEC (rad): {decRad:5f}")
print(f"obj RA (deg): {raRad*180/math.pi:5f}")
print(f"obj DEC (deg): {decRad*180/math.pi:5f}")
if exp is not None: # calc source coords from exp wcs
ra = geom.Angle(raRad)
dec = geom.Angle(decRad)
targetLocation = geom.SpherePoint(ra, dec)
pixCoord = exp.getWcs().skyToPixel(targetLocation)
print(exp.getWcs())
print(f"Source location: {pixCoord} using exp provided")
else: # try to find one, but not too hard
datasetTypes = ['calexp', 'quickLookExp', 'postISRCCD']
for datasetType in datasetTypes:
wcs = None
typeUsed = None
try:
wcs = butler.get(datasetType + '_wcs', **kwargs)
typeUsed = datasetType
break
except butlerExcept.NoResults:
pass
if wcs is not None:
ra = geom.Angle(raRad)
dec = geom.Angle(decRad)
targetLocation = geom.SpherePoint(ra, dec)
pixCoord = wcs.skyToPixel(targetLocation)
print(wcs)
print(f"Source location: {pixCoord} using {typeUsed}")
if extendedSummary:
print('\n--- Extended Summary ---')
ranIsr = False
if exp is None:
print("Running isr to compute image stats...")
# catch all the ISR chat
# logRedirection1 = LogRedirect(1, open(os.devnull, 'w'))
# logRedirection2 = LogRedirect(2, open(os.devnull, 'w'))
# import ipdb as pdb; pdb.set_trace()
from lsst.ip.isr.isrTask import IsrTask
isrConfig = IsrTask.ConfigClass()
isrConfig.doLinearize = False
isrConfig.doBias = False
isrConfig.doFlat = False
isrConfig.doDark = False
isrConfig.doFringe = False
isrConfig.doDefect = False
isrConfig.doWrite = False
isrTask = IsrTask(config=isrConfig)
dataRef = butler.dataRef('raw', **kwargs)
exp = isrTask.runDataRef(dataRef).exposure
wcs = exp.getWcs()
ranIsr = True
# logRedirection1.finish() # end re-direct
# logRedirection2.finish() # end re-direct
print(wcs)
if simbadObj and ranIsr:
ra = geom.Angle(raRad)
dec = geom.Angle(decRad)
targetLocation = geom.SpherePoint(ra, dec)
pixCoord = wcs.skyToPixel(targetLocation)
print(
f"Source location: {pixCoord} using postISR just-reconstructed wcs"
)
print(
f'\nImage stats from {"just-constructed" if ranIsr else "provided"} exp:\n'
)
print(f'Image mean: {np.mean(exp.image.array):.2f}')
print(f'Image median: {np.median(exp.image.array):.2f}')
print(f'Image min: {np.min(exp.image.array):.2f}')
print(f'Image max: {np.max(exp.image.array):.2f}')
# TODO: quartiles/percentiles here
# number of masked pixels, saturated pixels
print()
print(f'BAD pixels: {countPixels(exp.maskedImage, "BAD")}')
print(f'SAT pixels: {countPixels(exp.maskedImage, "SAT")}')
print(f'CR pixels: {countPixels(exp.maskedImage, "CR")}')
print(f'INTRP pixels: {countPixels(exp.maskedImage, "INTRP")}')
print(f'DETECTED pixels: {countPixels(exp.maskedImage, "DETECTED")}')
# detector = exp.getDetector()
visitInfo = exp.getInfo().getVisitInfo()
rotAngle = visitInfo.getBoresightRotAngle()
boresight = visitInfo.getBoresightRaDec()
md = butler.get('raw_md', **kwargs)
print("\n From VisitInfo:")
print(f"boresight: {boresight}")
print(f"rotAngle: {rotAngle}")
print(f" → {rotAngle.asDegrees():.4f} deg")
print("\n From raw_md:")
print(f"ROTPA: {md['ROTPA']} deg")
print(f" → {(md['ROTPA']*math.pi/180):.6f} rad")
def setUp(self):
"""Constructs a CCD with two amplifiers and prepares for ISR"""
np.random.seed(12345)
baseValue = 100.0
gain = 1.0
readNoise = 123456789.0
saturation = 987654321.0
height = 234
imageSize = Extent2I(123, height)
overscanSize = Extent2I(16, height)
self.sigma = 1.234
# Set up the various regions
overscan1 = Box2I(Point2I(0, 0), overscanSize)
image1 = Box2I(Point2I(overscanSize[0], 0), imageSize)
image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize)
overscan2 = Box2I(Point2I(overscanSize[0] + 2 * imageSize[0], 0),
overscanSize)
leftBox = Box2I(
overscan1.getMin(),
Extent2I(overscan1.getWidth() + image1.getWidth(), height))
rightBox = Box2I(
image2.getMin(),
Extent2I(image2.getWidth() + overscan2.getWidth(), height))
target1 = Box2I(Point2I(0, 0), imageSize)
target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize)
# Set the pixels
exposure = ExposureF(
Box2I(Point2I(0, 0),
Extent2I(imageSize[0] * 2 + overscanSize[0] * 2, height)))
yy = np.arange(0, height, 1, dtype=np.float32)
leftImage = ExposureF(exposure, leftBox)
leftImage.image.array[:] = baseValue + yy[:, np.newaxis]
rightImage = ExposureF(exposure, rightBox)
rightImage.image.array[:] = baseValue - yy[:, np.newaxis]
leftOverscan = ExposureF(exposure, overscan1)
leftOverscan.image.array += np.random.normal(
0.0, self.sigma, leftOverscan.image.array.shape)
rightOverscan = ExposureF(exposure, overscan2)
rightOverscan.image.array += np.random.normal(
0.0, self.sigma, leftOverscan.image.array.shape)
exposure.mask.array[:] = 0.0
exposure.variance.array[:] = np.nan
# Construct the detectors
amp1 = makeAmplifier("left", target1, image1, overscan1, gain,
readNoise, saturation)
amp2 = makeAmplifier("right", target2, image2, overscan2, gain,
readNoise, saturation)
ccdBox = Box2I(Point2I(0, 0),
Extent2I(image1.getWidth() + image2.getWidth(), height))
camBuilder = cameraGeom.Camera.Builder("fakeCam")
detBuilder = camBuilder.add("detector", 1)
detBuilder.setSerial("det1")
detBuilder.setBBox(ccdBox)
detBuilder.setPixelSize(Extent2D(1.0, 1.0))
detBuilder.setOrientation(cameraGeom.Orientation())
detBuilder.append(amp1)
detBuilder.append(amp2)
cam = camBuilder.finish()
exposure.setDetector(cam.get('detector'))
header = PropertyList()
header.add("EXPTIME", 0.0)
exposure.getInfo().setVisitInfo(VisitInfo(header))
self.exposure = exposure
self.config = IsrTask.ConfigClass()
# Disable everything we don't care about
self.config.doBias = False
self.config.doDark = False
self.config.doFlat = False
self.config.doFringe = False
self.config.doDefect = False
self.config.doWrite = False
self.config.expectWcs = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doBrighterFatter = False
self.config.doAttachTransmissionCurve = False
self.config.doAssembleCcd = False
self.config.doNanMasking = False
self.config.doInterpolate = False
self.config.maskNegativeVariance = False # This runs on mocks.
# Set the things that match our test setup
self.config.overscan.fitType = "CHEB"
self.config.overscan.order = 1
self.config.doEmpiricalReadNoise = True
self.task = IsrTask(config=self.config)
class EmpiricalVarianceTestCast(lsst.utils.tests.TestCase):
def setUp(self):
"""Constructs a CCD with two amplifiers and prepares for ISR"""
np.random.seed(12345)
baseValue = 100.0
gain = 1.0
readNoise = 123456789.0
saturation = 987654321.0
height = 234
imageSize = Extent2I(123, height)
overscanSize = Extent2I(16, height)
self.sigma = 1.234
# Set up the various regions
overscan1 = Box2I(Point2I(0, 0), overscanSize)
image1 = Box2I(Point2I(overscanSize[0], 0), imageSize)
image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize)
overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize)
leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height))
rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height))
target1 = Box2I(Point2I(0, 0), imageSize)
target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize)
# Set the pixels
exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height)))
yy = np.arange(0, height, 1, dtype=np.float32)
leftImage = ExposureF(exposure, leftBox)
leftImage.image.array[:] = baseValue + yy[:, np.newaxis]
rightImage = ExposureF(exposure, rightBox)
rightImage.image.array[:] = baseValue - yy[:, np.newaxis]
leftOverscan = ExposureF(exposure, overscan1)
leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
rightOverscan = ExposureF(exposure, overscan2)
rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
exposure.mask.array[:] = 0.0
exposure.variance.array[:] = np.nan
# Construct the detectors
amp1 = makeAmplifier("left", target1, image1, overscan1, gain, readNoise, saturation)
amp2 = makeAmplifier("right", target2, image2, overscan2, gain, readNoise, saturation)
ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height))
camBuilder = cameraGeom.Camera.Builder("fakeCam")
detBuilder = camBuilder.add("detector", 1)
detBuilder.setSerial("det1")
detBuilder.setBBox(ccdBox)
detBuilder.setPixelSize(Extent2D(1.0, 1.0))
detBuilder.setOrientation(cameraGeom.Orientation())
detBuilder.append(amp1)
detBuilder.append(amp2)
cam = camBuilder.finish()
exposure.setDetector(cam.get('detector'))
header = PropertyList()
header.add("EXPTIME", 0.0)
exposure.getInfo().setVisitInfo(VisitInfo(header))
self.exposure = exposure
self.config = IsrTask.ConfigClass()
# Disable everything we don't care about
self.config.doBias = False
self.config.doDark = False
self.config.doFlat = False
self.config.doFringe = False
self.config.doDefect = False
self.config.doAddDistortionModel = False
self.config.doWrite = False
self.config.expectWcs = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doBrighterFatter = False
self.config.doAttachTransmissionCurve = False
self.config.doAssembleCcd = False
self.config.doNanMasking = False
self.config.doInterpolate = False
# Set the things that match our test setup
self.config.overscanFitType = "CHEB"
self.config.overscanOrder = 1
self.config.doEmpiricalReadNoise = True
self.task = IsrTask(config=self.config)
def tearDown(self):
del self.exposure
def testEmpiricalVariance(self):
results = self.task.run(self.exposure)
postIsr = results.exposure
self.assertFloatsEqual(postIsr.mask.array, 0)
# Image is not exactly zero because the noise in the overscan (required to be able to set
# the empirical variance) leads to a slight misestimate in the polynomial fit.
self.assertFloatsAlmostEqual(np.median(postIsr.image.array), 0.0, atol=5.0e-2)
self.assertFloatsAlmostEqual(np.nanmedian(postIsr.variance.array), self.sigma**2, rtol=5.0e-2)
class IsrTaskTestCases(lsst.utils.tests.TestCase):
"""Test IsrTask methods with trimmed raw data.
"""
def setUp(self):
self.config = IsrTaskConfig()
self.config.qa = IsrQaConfig()
self.task = IsrTask(config=self.config)
self.dataRef = isrMock.DataRefMock()
self.camera = isrMock.IsrMock().getCamera()
self.inputExp = isrMock.TrimmedRawMock().run()
self.amp = self.inputExp.getDetector()[0]
self.mi = self.inputExp.getMaskedImage()
def validateIsrData(self, results):
"""results should be a struct with components that are
not None if included in the configuration file.
"""
self.assertIsInstance(results, Struct)
if self.config.doBias is True:
self.assertIsNotNone(results.bias)
if self.config.doDark is True:
self.assertIsNotNone(results.dark)
if self.config.doFlat is True:
self.assertIsNotNone(results.flat)
if self.config.doFringe is True:
self.assertIsNotNone(results.fringes)
if self.config.doDefect is True:
self.assertIsNotNone(results.defects)
if self.config.doBrighterFatter is True:
self.assertIsNotNone(results.bfKernel)
if self.config.doAttachTransmissionCurve is True:
self.assertIsNotNone(results.opticsTransmission)
self.assertIsNotNone(results.filterTransmission)
self.assertIsNotNone(results.sensorTransmission)
self.assertIsNotNone(results.atmosphereTransmission)
def test_readIsrData_noTrans(self):
"""Test that all necessary calibration frames are retrieved.
"""
self.config.doAttachTransmissionCurve = False
self.task = IsrTask(config=self.config)
results = self.task.readIsrData(self.dataRef, self.inputExp)
self.validateIsrData(results)
def test_readIsrData_withTrans(self):
"""Test that all necessary calibration frames are retrieved.
"""
self.config.doAttachTransmissionCurve = True
self.task = IsrTask(config=self.config)
results = self.task.readIsrData(self.dataRef, self.inputExp)
self.validateIsrData(results)
def test_ensureExposure(self):
"""Test that an exposure has a usable instance class.
"""
self.assertIsInstance(
self.task.ensureExposure(self.inputExp, self.camera, 0),
afwImage.Exposure)
def test_convertItoF(self):
"""Test conversion from integer to floating point pixels.
"""
result = self.task.convertIntToFloat(self.inputExp)
self.assertEqual(result.getImage().getArray().dtype,
np.dtype("float32"))
self.assertEqual(result, self.inputExp)
def test_updateVariance(self):
"""Expect The variance image should have a larger median value after
this operation.
"""
statBefore = computeImageMedianAndStd(
self.inputExp.variance[self.amp.getBBox()])
self.task.updateVariance(self.inputExp, self.amp)
statAfter = computeImageMedianAndStd(
self.inputExp.variance[self.amp.getBBox()])
self.assertGreater(statAfter[0], statBefore[0])
self.assertFloatsAlmostEqual(statBefore[0], 0.0, atol=1e-2)
self.assertFloatsAlmostEqual(statAfter[0], 8170.0195, atol=1e-2)
def test_darkCorrection(self):
"""Expect the median image value should decrease after this operation.
"""
darkIm = isrMock.DarkMock().run()
statBefore = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.task.darkCorrection(self.inputExp, darkIm)
statAfter = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.assertLess(statAfter[0], statBefore[0])
self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
def test_darkCorrection_noVisitInfo(self):
"""Expect the median image value should decrease after this operation.
"""
darkIm = isrMock.DarkMock().run()
darkIm.getInfo().setVisitInfo(None)
statBefore = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.task.darkCorrection(self.inputExp, darkIm)
statAfter = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.assertLess(statAfter[0], statBefore[0])
self.assertFloatsAlmostEqual(statBefore[0], 8070.0195, atol=1e-2)
self.assertFloatsAlmostEqual(statAfter[0], 8045.7773, atol=1e-2)
def test_flatCorrection(self):
"""Expect the image median should increase (divide by < 1).
"""
flatIm = isrMock.FlatMock().run()
statBefore = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.task.flatCorrection(self.inputExp, flatIm)
statAfter = computeImageMedianAndStd(
self.inputExp.image[self.amp.getBBox()])
self.assertGreater(statAfter[1], statBefore[1])
self.assertFloatsAlmostEqual(statAfter[1], 147407.02, atol=1e-2)
self.assertFloatsAlmostEqual(statBefore[1], 147.55304, atol=1e-2)
def test_saturationDetection(self):
"""Expect the saturation level detection/masking to scale with
threshold.
"""
ampB = self.amp.rebuild()
ampB.setSaturation(9000.0)
self.task.saturationDetection(self.inputExp, ampB.finish())
countBefore = countMaskedPixels(self.mi, "SAT")
ampB.setSaturation(8250.0)
self.task.saturationDetection(self.inputExp, ampB.finish())
countAfter = countMaskedPixels(self.mi, "SAT")
self.assertLessEqual(countBefore, countAfter)
self.assertEqual(countBefore, 43)
self.assertEqual(countAfter, 136)
def test_measureBackground(self):
"""Expect the background measurement runs successfully and to save
metadata values.
"""
self.config.qa.flatness.meshX = 20
self.config.qa.flatness.meshY = 20
self.task.measureBackground(self.inputExp, self.config.qa)
self.assertIsNotNone(self.inputExp.getMetadata().getScalar('SKYLEVEL'))
def test_flatContext(self):
"""Expect the flat context manager runs successfully (applying both
flat and dark within the context), and results in the same
image data after completion.
"""
darkExp = isrMock.DarkMock().run()
flatExp = isrMock.FlatMock().run()
mi = self.inputExp.getMaskedImage().clone()
with self.task.flatContext(self.inputExp, flatExp, darkExp):
contextStat = computeImageMedianAndStd(
self.inputExp.getMaskedImage().getImage())
self.assertFloatsAlmostEqual(contextStat[0], 37165.594, atol=1e-2)
self.assertMaskedImagesAlmostEqual(mi, self.inputExp.getMaskedImage())
class IsrTaskUnTrimmedTestCases(lsst.utils.tests.TestCase):
"""Test IsrTask methods using untrimmed raw data.
"""
def setUp(self):
self.config = IsrTaskConfig()
self.config.qa = IsrQaConfig()
self.task = IsrTask(config=self.config)
self.mockConfig = isrMock.IsrMockConfig()
self.mockConfig.isTrimmed = False
self.doGenerateImage = True
self.dataRef = isrMock.DataRefMock(config=self.mockConfig)
self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera()
self.inputExp = isrMock.RawMock(config=self.mockConfig).run()
self.amp = self.inputExp.getDetector()[0]
self.mi = self.inputExp.getMaskedImage()
def batchSetConfiguration(self, value):
"""Set the configuration state to a consistent value.
Disable options we do not need as well.
Parameters
----------
value : `bool`
Value to switch common ISR configuration options to.
"""
self.config.qa.flatness.meshX = 20
self.config.qa.flatness.meshY = 20
self.config.doWrite = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doConvertIntToFloat = value
self.config.doSaturation = value
self.config.doSuspect = value
self.config.doSetBadRegions = value
self.config.doOverscan = value
self.config.doBias = value
self.config.doVariance = value
self.config.doWidenSaturationTrails = value
self.config.doBrighterFatter = value
self.config.doDefect = value
self.config.doSaturationInterpolation = value
self.config.doDark = value
self.config.doStrayLight = value
self.config.doFlat = value
self.config.doFringe = value
self.config.doMeasureBackground = value
self.config.doVignette = value
self.config.doAttachTransmissionCurve = value
self.config.doUseOpticsTransmission = value
self.config.doUseFilterTransmission = value
self.config.doUseSensorTransmission = value
self.config.doUseAtmosphereTransmission = value
self.config.qa.saveStats = value
self.config.qa.doThumbnailOss = value
self.config.qa.doThumbnailFlattened = value
self.config.doApplyGains = not value
self.config.doCameraSpecificMasking = value
def validateIsrResults(self):
"""results should be a struct with components that are
not None if included in the configuration file.
Returns
-------
results : `pipeBase.Struct`
Results struct generated from the current ISR configuration.
"""
self.task = IsrTask(config=self.config)
results = self.task.run(
self.inputExp,
camera=self.camera,
bias=self.dataRef.get("bias"),
dark=self.dataRef.get("dark"),
flat=self.dataRef.get("flat"),
bfKernel=self.dataRef.get("bfKernel"),
defects=self.dataRef.get("defects"),
fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
opticsTransmission=self.dataRef.get("transmission_"),
filterTransmission=self.dataRef.get("transmission_"),
sensorTransmission=self.dataRef.get("transmission_"),
atmosphereTransmission=self.dataRef.get("transmission_"))
self.assertIsInstance(results, Struct)
self.assertIsInstance(results.exposure, afwImage.Exposure)
return results
def test_overscanCorrection(self):
"""Expect that this should reduce the image variance with a full fit.
The default fitType of MEDIAN will reduce the median value.
This needs to operate on a RawMock() to have overscan data to use.
The output types may be different when fitType != MEDIAN.
"""
statBefore = computeImageMedianAndStd(
self.inputExp.image[self.amp.getRawDataBBox()])
oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
self.assertIsInstance(oscanResults, Struct)
self.assertIsInstance(oscanResults.imageFit, float)
self.assertIsInstance(oscanResults.overscanFit, float)
self.assertIsInstance(oscanResults.overscanImage,
afwImage.MaskedImageF)
statAfter = computeImageMedianAndStd(
self.inputExp.image[self.amp.getRawDataBBox()])
self.assertLess(statAfter[0], statBefore[0])
def test_overscanCorrectionMedianPerRow(self):
"""Expect that this should reduce the image variance with a full fit.
fitType of MEDIAN_PER_ROW will reduce the median value.
This needs to operate on a RawMock() to have overscan data to use.
The output types may be different when fitType != MEDIAN_PER_ROW.
"""
self.config.overscan.fitType = 'MEDIAN_PER_ROW'
statBefore = computeImageMedianAndStd(
self.inputExp.image[self.amp.getRawDataBBox()])
oscanResults = self.task.overscanCorrection(self.inputExp, self.amp)
self.assertIsInstance(oscanResults, Struct)
self.assertIsInstance(oscanResults.imageFit, afwImage.ImageF)
self.assertIsInstance(oscanResults.overscanFit, afwImage.ImageF)
self.assertIsInstance(oscanResults.overscanImage,
afwImage.MaskedImageF)
statAfter = computeImageMedianAndStd(
self.inputExp.image[self.amp.getRawDataBBox()])
self.assertLess(statAfter[0], statBefore[0])
def test_runDataRef(self):
"""Expect a dataRef to be handled correctly.
"""
self.config.doLinearize = False
self.config.doWrite = False
self.task = IsrTask(config=self.config)
results = self.task.runDataRef(self.dataRef)
self.assertIsInstance(results, Struct)
self.assertIsInstance(results.exposure, afwImage.Exposure)
def test_run_allTrue(self):
"""Expect successful run with expected outputs when all non-exclusive
configuration options are on.
Output results should be tested more precisely by the
individual function tests.
"""
self.batchSetConfiguration(True)
self.validateIsrResults()
def test_run_allFalse(self):
"""Expect successful run with expected outputs when all non-exclusive
configuration options are off.
Output results should be tested more precisely by the
individual function tests.
"""
self.batchSetConfiguration(False)
self.validateIsrResults()
def test_failCases(self):
"""Expect failure with crosstalk enabled.
Output results should be tested more precisely by the
individual function tests.
"""
self.batchSetConfiguration(True)
# This breaks it
self.config.doCrosstalk = True
with self.assertRaises(RuntimeError):
self.validateIsrResults()
def test_maskingCase_negativeVariance(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.doSaturation = False
self.config.doWidenSaturationTrails = False
self.config.doSaturationInterpolation = False
self.config.doSuspect = False
self.config.doSetBadRegions = False
self.config.doDefect = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = True
self.config.doInterpolate = False
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 40800)
def test_maskingCase_noMasking(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.doSaturation = False
self.config.doWidenSaturationTrails = False
self.config.doSaturationInterpolation = False
self.config.doSuspect = False
self.config.doSetBadRegions = False
self.config.doDefect = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False
self.config.doInterpolate = False
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
def test_maskingCase_satMasking(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = False
self.config.doSuspect = False
self.config.doSetBadRegions = False
self.config.doDefect = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
def test_maskingCase_satMaskingAndInterp(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = True
self.config.doSuspect = False
self.config.doSetBadRegions = False
self.config.doDefect = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
def test_maskingCase_throughEdge(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = True
self.config.numEdgeSuspect = 5
self.config.doSuspect = True
self.config.doSetBadRegions = False
self.config.doDefect = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 0)
def test_maskingCase_throughDefects(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = True
self.config.numEdgeSuspect = 5
self.config.doSuspect = True
self.config.doDefect = True
self.config.doSetBadRegions = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 3940)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
def test_maskingCase_throughDefectsAmpEdges(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = True
self.config.numEdgeSuspect = 5
self.config.doSuspect = True
self.config.doDefect = True
self.config.edgeMaskLevel = 'AMP'
self.config.doSetBadRegions = False
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 11280)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
def test_maskingCase_throughBad(self):
"""Test masking cases of configuration parameters.
"""
self.batchSetConfiguration(True)
self.config.overscanFitType = "POLY"
self.config.overscanOrder = 1
self.config.saturation = 20000.0
self.config.doSaturation = True
self.config.doWidenSaturationTrails = True
self.config.doSaturationInterpolation = True
self.config.doSuspect = True
self.config.doDefect = True
self.config.doSetBadRegions = True
self.config.doBrighterFatter = False
self.config.maskNegativeVariance = False # These are mock images.
results = self.validateIsrResults()
self.assertEqual(countMaskedPixels(results.exposure, "SAT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "INTRP"), 2000)
self.assertEqual(countMaskedPixels(results.exposure, "SUSPECT"), 0)
self.assertEqual(countMaskedPixels(results.exposure, "BAD"), 2000)
Popen.wait(ingest)
print("finished ingesting spots images")
spots_butler = Butler(SPOTS_REPO_DIR)
visits = []
my_metaData = spots_butler.queryMetadata('raw', ['visit', 'dateObs'])
for item in my_metaData:
visits.append(item[0])
byamp_results = []
byamp_corrected_results = []
for visit in visits:
print("Getting exposure # %d"%visit)
sys.stdout.flush()
exposure = spots_butler.get('raw', dataId={'visit': visit, 'detector': DETECTOR})
# Perform the instrument signature removal (mainly assembling the CCD)
isrTask = IsrTask(config=isrConfig)
exposure_isr = isrTask.run(exposure).exposure
# For now, we're applying the gain manually
ccd = exposure_isr.getDetector()
for do_bf_corr in [False, True]:
exposure_copy=exposure_isr.clone()
for amp in ccd:
if spots_bbox.overlaps(amp.getBBox()):
gain = gain_data[amp.getName()]
img = exposure_copy.image
sim = img.Factory(img, amp.getBBox())
sim *= gain
print(amp.getName(), gain, amp.getBBox())
sys.stdout.flush()
if do_bf_corr:
brighterFatterCorrection(exposure_copy[amp.getBBox()],bf_kernel.kernel[amp.getName()],20,10,False)