def transposeDefectList(self, defectList, checkBbox=None):
retDefectList = measAlg.DefectListT()
for defect in defectList:
bbox = defect.getBBox()
nbbox = afwGeom.Box2I(afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
afwGeom.Extent2I(bbox.getDimensions()[1], bbox.getDimensions()[0]))
if checkBbox:
if checkBbox.overlaps(bbox):
retDefectList.push_back(measAlg.Defect(nbbox))
else:
pass
else:
retDefectList.push_back(measAlg.Defect(nbbox))
return retDefectList
def test1295(self):
"""A test case for #1295 (failure to interpolate over groups of defects."""
im = afwImage.ImageF(lsst.geom.ExtentI(100, 100))
mi = afwImage.makeMaskedImage(im)
mi.set(100)
flat = afwImage.ImageF(im.getDimensions())
flat.set(1)
flat[50:51, :, afwImage.LOCAL] = 0.0
flat[55:56, :, afwImage.LOCAL] = 0.0
flat[58:59, :, afwImage.LOCAL] = 0.0
flat[51:60, 51:, afwImage.LOCAL] = 0.0
mi /= flat
if display:
afwDisplay.Display(frame=0).mtv(mi,
title=self._testMethodName +
": Raw")
defectList = []
bbox = lsst.geom.BoxI(lsst.geom.PointI(50, 0),
lsst.geom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = lsst.geom.BoxI(lsst.geom.PointI(55, 0),
lsst.geom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = lsst.geom.BoxI(lsst.geom.PointI(58, 0),
lsst.geom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = lsst.geom.BoxI(lsst.geom.PointI(51, 51),
lsst.geom.ExtentI(9, 49))
defectList.append(algorithms.Defect(bbox))
psf = algorithms.DoubleGaussianPsf(
15, 15, 1. / (2 * math.sqrt(2 * math.log(2))))
algorithms.interpolateOverDefects(mi, psf, defectList, 50.)
if display:
afwDisplay.Display(frame=1).mtv(mi,
title=self._testMethodName +
": Interpolated")
self.assertTrue(np.isfinite(mi.image[56, 51, afwImage.LOCAL]))
def addDefects(exp, nBadCols=10):
img = exp.getMaskedImage().getImage()
(xsize, ysize) = img.getDimensions()
defectList = measAlg.DefectListT()
# set some bad cols and add them to a defect list
for xi in numpy.random.randint(0, xsize, nBadCols):
yi = numpy.random.randint(0, ysize)
xi, yi = int(xi), int(yi)
bbox = afwGeom.Box2I(afwGeom.PointI(xi, 0), afwGeom.ExtentI(1, yi + 1))
subIm = afwImage.ImageF(img, bbox)
subIm.set(1e7)
defectList.push_back(measAlg.Defect(bbox))
# set a 15 pixel box of defects at the upper left corner to demonstrate fallbackValue
bbox = afwGeom.Box2I(afwGeom.PointI(0, ysize - 15),
afwGeom.ExtentI(15, 15))
subIm = afwImage.ImageF(img, bbox)
subIm.set(1e7)
defectList.push_back(measAlg.Defect(bbox))
return defectList
def writeDefectsFile(bboxList, path):
maskBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1))
for box in bboxList:
maskBBox.include(box)
defectsMaskedImage = afwImage.MaskedImageF(maskBBox)
defectList = measAlg.DefectListT()
for bbox in bboxList:
nd = measAlg.Defect(bbox)
defectList.append(nd)
maskPixelsFromDefectList(defectsMaskedImage, defectList, maskName='BAD')
defectsMaskedImage.getMask().writeFits(MaskFileName)
print("wrote %s with bbox %s" % (MaskFileName, maskBBox,))
def defects(self, exposure):
"""Mask defects
@param exposure Exposure to process
@return Defect list
"""
assert exposure, "No exposure provided"
policy = self.config['defects']
defects = measAlg.DefectListT()
ccd = pipUtil.getCcd(exposure)
statics = ccd.getDefects() # Static defects
for defect in statics:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
ipIsr.maskBadPixelsDef(exposure, defects, fwhm=None, interpolate=False, maskName='BAD')
self.log.log(self.log.INFO, "Masked %d static defects." % len(statics))
grow = policy['grow']
sat = ipIsr.defectListFromMask(exposure, growFootprints=grow, maskName='SAT') # Saturated defects
self.log.log(self.log.INFO, "Added %d saturation defects." % len(sat))
for defect in sat:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
exposure.getMaskedImage().getMask().addMaskPlane("UNMASKEDNAN")
nanMasker = ipIsr.UnmaskedNanCounterF()
nanMasker.apply(exposure.getMaskedImage())
nans = ipIsr.defectListFromMask(exposure, maskName='UNMASKEDNAN')
self.log.log(self.log.INFO, "Added %d unmasked NaNs." % nanMasker.getNpix())
for defect in nans:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
return defects
def test1295(self):
"""A test case for #1295 (failure to interpolate over groups of defects"""
im = afwImage.ImageF(afwGeom.ExtentI(100, 100))
mi = afwImage.makeMaskedImage(im)
mi.set(100)
flat = afwImage.ImageF(im.getDimensions())
flat.set(1)
for i in range(100):
for j in range(100):
if i == 50 or i == 55 or i == 58:
flat.set(i, j, 0)
if i < 60 and i > 50 and j > 50:
flat.set(i, j, 0)
mi /= flat
if display:
ds9.mtv(mi, frame=0, title="Raw")
defectList = algorithms.DefectListT()
bbox = afwGeom.BoxI(afwGeom.PointI(50, 0), afwGeom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = afwGeom.BoxI(afwGeom.PointI(55, 0), afwGeom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = afwGeom.BoxI(afwGeom.PointI(58, 0), afwGeom.ExtentI(1, 100))
defectList.append(algorithms.Defect(bbox))
bbox = afwGeom.BoxI(afwGeom.PointI(51, 51), afwGeom.ExtentI(9, 49))
defectList.append(algorithms.Defect(bbox))
psf = algorithms.DoubleGaussianPsf(
15, 15, 1. / (2 * math.sqrt(2 * math.log(2))))
algorithms.interpolateOverDefects(mi, psf, defectList, 50.)
if display:
ds9.mtv(mi, frame=1, title="Interpolated")
self.assertTrue(numpy.isfinite(mi.getImage().get(56, 51)))
def testDefectBase(self):
"""Test DefectBases"""
defectList = []
ccdImage = afwImage.MaskedImageF(250, 225)
ccdImage.set(self.setVal, 0, self.setVal)
#
# Insert some defects into the Ccd
#
for x0, y0, x1, y1 in [
(34, 0, 35, 80),
(34, 81, 34, 100),
(180, 100, 182, 130),
]:
bbox = afwGeom.Box2I(afwGeom.Point2I(x0, y0),
afwGeom.Point2I(x1, y1))
defectList.append(measAlg.Defect(bbox))
bad = ccdImage.Factory(ccdImage, bbox, afwImage.LOCAL)
bad.set(100)
ipIsr.maskPixelsFromDefectList(ccdImage, defectList, maskName='BAD')
mask = ccdImage.getMask()
bitMask = mask.getPlaneBitMask('BAD')
for d in defectList:
bad = mask.Factory(mask, d.getBBox(), afwImage.LOCAL)
self.assertTrue((bad.getArray() & bitMask == bitMask).all())
if display:
ds9.mtv(ccdImage.getImage(), title="Defects")
for d in defectList:
displayUtils.drawBBox(d.getBBox(),
ctype=ds9.CYAN,
borderWidth=.5)
ds9.incrDefaultFrame()
ipIsr.interpolateDefectList(ccdImage, defectList, 2.)
im = ccdImage.getImage()
for d in defectList:
intrp = im.Factory(im, d.getBBox())
expect = np.empty_like(intrp.getArray())
expect[:] = self.setVal
self.assertImagesEqual(intrp, expect)
if display:
ds9.mtv(ccdImage.getImage(), title="Defects Interpolated")
for d in defectList:
displayUtils.drawBBox(d.getBBox(),
ctype=ds9.CYAN,
borderWidth=.5)
ds9.incrDefaultFrame()
def transposeDefectList(defectList):
"""Make a transposed copy of a defect list
@param[in] defectList defect list
@return meas.algorithms.DefectListT with transposed defects
"""
retDefectList = measAlg.DefectListT()
for defect in defectList:
bbox = defect.getBBox()
nbbox = afwGeom.Box2I(
afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
afwGeom.Extent2I(bbox.getDimensions()[1],
bbox.getDimensions()[0]))
retDefectList.push_back(measAlg.Defect(nbbox))
return retDefectList
def transposeDefectList(defectList):
"""Make a transposed copy of a defect list
@param[in] defectList a list of defects (afw.meas.algorithms.Defect)
@return a defect list with transposed defects
"""
retDefectList = []
for defect in defectList:
bbox = defect.getBBox()
nbbox = afwGeom.Box2I(
afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
afwGeom.Extent2I(bbox.getDimensions()[1],
bbox.getDimensions()[0]))
retDefectList.append(measAlg.Defect(nbbox))
return retDefectList
def XXXtest818(self):
"""A test case for #818; the full test is in /lsst/DC3root/ticketFiles/818"""
badPixels = algorithms.DefectListT()
defects = [((82, 663), 6, 8),
((83, 659), 9, 6),
((85, 660), 10, 11),
((87, 669), 3, 3),
]
for xy0, width, height in defects:
x0, y0 = xy0
bbox = afwImage.BBox(afwImage.PointI(x0, y0), width, height)
badPixels.push_back(algorithms.Defect(bbox))
mi = afwImage.MaskedImageF(517, 800)
algorithms.interpolateOverDefects(mi, self.psf, badPixels)
def test818(self):
"""A test case for #818; the full test is in /lsst/DC3root/ticketFiles/818"""
badPixels = []
defects = [((82, 663), 6, 8),
((83, 659), 9, 6),
((85, 660), 10, 11),
((87, 669), 3, 3),
]
for xy0, width, height in defects:
x0, y0 = xy0
bbox = afwGeom.BoxI(afwGeom.PointI(x0, y0), afwGeom.ExtentI(width, height))
badPixels.append(algorithms.Defect(bbox))
mi = afwImage.MaskedImageF(517, 800)
algorithms.interpolateOverDefects(mi, self.psf, badPixels)
def defectListFromFootprintList(fpList, growFootprints=1):
"""Compute a defect list from a footprint list, optionally growing the footprints
@param[in] fpList footprint list
@param[in] growFootprints amount by which to grow footprints of detected regions
@return meas.algorithms.DefectListT
"""
defectList = measAlg.DefectListT()
for fp in fpList:
if growFootprints > 0:
# if "True", growing requires a convolution
# if "False", its faster
fpGrow = afwDetection.growFootprint(fp, growFootprints, False)
else:
fpGrow = fp
for bbox in afwDetection.footprintToBBoxList(fpGrow):
defect = measAlg.Defect(bbox)
defectList.push_back(defect)
return defectList
def maskAndInterpDefect(self, ccdExposure, defectBaseList):
"""!Mask defects using mask plane "BAD" and interpolate over them, in place
\param[in,out] ccdExposure exposure to process
\param[in] defectBaseList a list of defects to mask and interpolate
\warning: call this after CCD assembly, since defects may cross amplifier boundaries
"""
maskedImage = ccdExposure.getMaskedImage()
defectList = measAlg.DefectListT()
for d in defectBaseList:
bbox = d.getBBox()
nd = measAlg.Defect(bbox)
defectList.append(nd)
isr.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
isr.interpolateDefectList(
maskedImage=maskedImage,
defectList=defectList,
fwhm=self.config.fwhm,
)
def defectListFromFootprintList(fpList, growFootprints=1):
"""Compute a defect list from a footprint list, optionally growing the footprints
@param[in] fpList footprint list
@param[in] growFootprints amount by which to grow footprints of detected regions
@return a list of defects (meas.algorithms.Defect)
"""
defectList = []
for fp in fpList:
if growFootprints > 0:
# if "True", growing requires a convolution
# if "False", its faster
tempSpans = fp.spans.dilated(growFootprints,
afwGeom.Stencil.MANHATTAN)
fpGrow = afwDetection.Footprint(tempSpans, fp.getRegion())
else:
fpGrow = fp
for bbox in afwDetection.footprintToBBoxList(fpGrow):
defect = measAlg.Defect(bbox)
defectList.append(defect)
return defectList
def process(self, clipboard):
"""
"""
self.log.log(Log.INFO, "Doing CCD defect mask.")
#grab exposure from clipboard
exposure = clipboard.get(
self.policy.getString("inputKeys.ccdExposure"))
#get known defects from camera class
defectBaseList = cameraGeom.cast_Ccd(
exposure.getDetector()).getDefects()
id = exposure.getDetector().getId()
defectList = measAlg.DefectListT()
#create master list of defects and add those from the camera class
for d in defectBaseList:
bbox = d.getBBox()
nd = measAlg.Defect(bbox)
defectList.append(nd)
fwhm = self.policy.getDouble("parameters.defaultFwhm")
#get saturated pixels from the mask
sdefects = ipIsr.defectListFromMask(exposure,
growFootprints=1,
maskName='SAT')
#mask bad pixels in the camera class
ipIsr.maskBadPixelsDef(exposure,
defectList,
fwhm,
interpolate=False,
maskName='BAD')
#add saturated pixels to master defect list
for d in sdefects:
bbox = d.getBBox()
nd = measAlg.Defect(bbox)
defectList.append(nd)
#find unmasked bad pixels and mask them
exposure.getMaskedImage().getMask().addMaskPlane("UNMASKEDNAN")
unc = ipIsr.UnmaskedNanCounterF()
unc.apply(exposure.getMaskedImage())
nnans = unc.getNpix()
metadata = exposure.getMetadata()
metadata.set("NUMNANS", nnans)
if nnans == 0:
self.log.log(Log.INFO,
"Zero unmasked nans/infs were found, which is good.")
else:
self.log.log(
Log.INFO, "%i unmasked nans/infs found in ccd exposure: %s" %
(nnans, id.__str__()))
#get footprints of bad pixels not in the camera class
undefects = ipIsr.defectListFromMask(exposure,
growFootprints=0,
maskName='UNMASKEDNAN')
for d in undefects:
bbox = d.getBBox()
nd = measAlg.Defect(bbox)
defectList.append(nd)
#interpolate all bad pixels
ipIsr.interpolateDefectList(exposure, defectList, fwhm)
#output products
clipboard.put(self.policy.get("outputKeys.defectMaskedCcdExposure"),
exposure)
def testEdge(self):
"""Test that we can interpolate to the edge"""
mi = afwImage.MaskedImageF(80, 30)
ima = mi.getImage().getArray()
#
# Loop over number of bad columns at left or right edge of image
#
for nBadCol in range(0, 20):
mi.set((0, 0x0, 0))
np.random.seed(666)
ima[:] = np.random.uniform(-1, 1, ima.shape)
defects = []
if nBadCol > 0:
#
# Bad left edge
#
ima[:, 0:nBadCol] = 10
defects.append(afwGeom.BoxI(afwGeom.PointI(0, 0),
afwGeom.ExtentI(nBadCol, mi.getHeight())))
#
# With another bad set of columns next to bad left edge
#
ima[:, -nBadCol:] = 10
defects.append(afwGeom.BoxI(afwGeom.PointI(mi.getWidth() - nBadCol, 0),
afwGeom.ExtentI(nBadCol, mi.getHeight())))
#
# Bad right edge
#
ima[0:10, nBadCol+1:nBadCol+4] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(nBadCol+1, 0),
afwGeom.ExtentI(3, 10)))
#
# With another bad set of columns next to bad right edge
#
ima[0:10, -nBadCol-4:-nBadCol-1] = 100
defects.append((afwGeom.BoxI(afwGeom.PointI(mi.getWidth() - nBadCol - 4, 0),
afwGeom.ExtentI(3, 10))))
#
# Test cases that left and right bad patches nearly (or do) coalesce
#
ima[-3:, 0:mi.getWidth()//2-1] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(0, mi.getHeight() - 3),
afwGeom.ExtentI(mi.getWidth()//2-1, 1)))
ima[-3:, mi.getWidth()//2+1:] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(mi.getWidth()//2 + 1, mi.getHeight() - 3),
afwGeom.ExtentI(mi.getWidth()//2 - 1, 1)))
ima[-2:, 0:mi.getWidth()//2] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(0, mi.getHeight() - 2),
afwGeom.ExtentI(mi.getWidth()//2, 1)))
ima[-2:, mi.getWidth()//2+1:] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(mi.getWidth()//2 + 1, mi.getHeight() - 2),
afwGeom.ExtentI(mi.getWidth()//2 - 1, 1)))
ima[-1:, :] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(0, mi.getHeight() - 1),
afwGeom.ExtentI(mi.getWidth(), 1)))
# Test fix for HSC-978: long defect stops one pixel shy of the edge (when nBadCol == 0)
ima[13, :-1] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(0, 13), afwGeom.ExtentI(mi.getWidth() - 1, 1)))
ima[14, 1:] = 100
defects.append(afwGeom.BoxI(afwGeom.PointI(1, 14), afwGeom.ExtentI(mi.getWidth() - 1, 1)))
#
# Build list of defects to interpolate over
#
defectList = []
for bbox in defects:
defectList.append(algorithms.Defect(bbox))
#
# Guess a PSF and do the work
#
if display:
ds9.mtv(mi, frame=0)
psf = algorithms.DoubleGaussianPsf(15, 15, 1./(2*math.sqrt(2*math.log(2))))
algorithms.interpolateOverDefects(mi, psf, defectList, 0, True)
if display:
ds9.mtv(mi, frame=1)
self.assertGreater(np.min(ima), -2)
self.assertGreater(2, np.max(ima))
def test_defects(self):
defects = Defects()
defects.append(
algorithms.Defect(
lsst.geom.Box2I(lsst.geom.Point2I(5, 6),
lsst.geom.Point2I(41, 50))))
defects.append(
lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(4, 5)))
defects.append(lsst.geom.Point2I(50, 50))
defects.append(
afwImage.DefectBase(
lsst.geom.Box2I(lsst.geom.Point2I(100, 200),
lsst.geom.Extent2I(5, 5))))
self.assertEqual(len(defects), 4)
for d in defects:
self.assertIsInstance(d, algorithms.Defect)
# Transposition
transposed = defects.transpose()
self.assertEqual(len(transposed), len(defects))
# Check that an individual defect is found properly transposed within
# the outputs.
found = False
for defect in transposed:
if defect.getBBox() == lsst.geom.Box2I(lsst.geom.Point2I(6, 5),
lsst.geom.Extent2I(45, 37)):
found = True
break
self.assertTrue(found)
# Serialization round trip
meta = PropertyList()
meta["TESTHDR"] = "testing"
defects.setMetadata(meta)
table = defects.toFitsRegionTable()
defects2 = Defects.fromTable([table])
self.assertEqual(defects2, defects)
# via FITS
with lsst.utils.tests.getTempFilePath(".fits") as tmpFile:
defects.writeFits(tmpFile)
defects2 = Defects.readFits(tmpFile)
# Equality tests the bounding boxes so metadata is tested separately.
self.assertEqual(defects2, defects)
self.assertMetadata(defects2, defects)
# via text file
with lsst.utils.tests.getTempFilePath(".ecsv") as tmpFile:
defects.writeText(tmpFile)
defects2 = Defects.readText(tmpFile)
# Equality tests the bounding boxes so metadata is tested separately.
self.assertEqual(defects2, defects)
self.assertMetadata(defects2, defects)
# Check bad values
with self.assertRaises(ValueError):
defects.append(
lsst.geom.Box2D(lsst.geom.Point2D(0., 0.),
lsst.geom.Point2D(3.1, 3.1)))
with self.assertRaises(ValueError):
defects.append("defect")
def process(self, clipboard):
"""
"""
self.log.log(Log.INFO, "Doing Saturation correction.")
#grab exposure from clipboard
exposure = clipboard.get(self.policy.getString("inputKeys.exposure"))
exposure = ipIsr.convertImageForIsr(exposure)
fwhm = self.policy.getDouble("parameters.defaultFwhm")
amp = cameraGeom.cast_Amp(exposure.getDetector())
dataBbox = amp.getDataSec(True)
x = dataBbox.getMinX()
y = dataBbox.getMinY()
height = dataBbox.getMaxY() - dataBbox.getMinY()
width = dataBbox.getMaxY() - dataBbox.getMinX()
dl = measAlg.DefectListT()
defectList = cameraGeom.cast_Ccd(amp.getParent()).getDefects()
if amp.getId().getIndex()[1] == 1:
for d in defectList:
d1 = measAlg.Defect(d.getBBox())
d1.shift(-x, -y)
bbox = d1.getBBox()
if bbox.getMinX() - 4 > width or bbox.getMaxY() - 4 < 0 or \
height - bbox.getMinY() - 1 > height or height - bbox.getMaxY() - 1 < 0:
pass
else:
nd = measAlg.Defect(
afwGeom.Box2I(
afwGeom.Point2I(bbox.getMinX() + 4,
height - bbox.getMinY() + 1),
bbox.getDimensions()))
dl.append(nd)
else:
for d in defectList:
d1 = measAlg.Defect(d.getBBox())
d1.shift(-x, -y)
bbox = d1.getBBox()
if bbox.getMinX() - 4 > width or bbox.getMaxY() - 4 < 0 or \
bbox.getMinY() - 1 > height or bbox.getMaxY() - 1 < 0:
pass
else:
nd = measAlg.Defect(
afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(4, 1),
bbox.getDimensions()))
dl.append(nd)
saturation = amp.getElectronicParams().getSaturationLevel()
ipIsr.maskBadPixelsDef(exposure,
dl,
fwhm,
interpolate=True,
maskName='BAD')
ipIsr.saturationCorrection(exposure,
int(saturation),
fwhm,
growSaturated=False,
interpolate=True)
#ds9.mtv(exposure, frame = 0, title = "my Amp")
#exposure.writeFits("Amp.fits")
#output products
clipboard.put(
self.policy.get("outputKeys.saturationCorrectedExposure"),
exposure)
