def testLsstTextfile(self):
"""Read legacy LSST text file format"""
with lsst.utils.tests.getTempFilePath(".txt") as tmpFile:
with open(tmpFile, "w") as fh:
print("""# X0 Y0 width height
996 0 56 24
0 4156 2048 20
0 0 17 4176
1998 4035 50 141
1023 0 2 4176
2027 0 21 4176
0 4047 37 129
# Some rows without fixed column widths
14 20 2000 50
10 10 10 10
""",
file=fh)
defects = Defects.readLsstDefectsFile(tmpFile,
normalize_on_init=True)
# Although there are 9 defects listed above, we record 11 after
# normalization. This is due to non-optimal behaviour in
# Defects.fromMask; see DM-24781.
self.assertEqual(len(defects), 11)
def testAstropyRegion(self):
"""Read a FITS region file created by Astropy regions."""
# The file contains three regions:
#
# - Point2I(340, 344)
# - Point2I(340, 344)
# - Box2I(minimum=Point2I(5, -5), dimensions=Extent2I(10, 20))
#
# The two coincident points are combined on read, so we end up with two defects.
with self.assertLogs():
defects = Defects.readFits(os.path.join(TESTDIR, "data",
"fits_region.fits"),
normalize_on_init=True)
self.assertEqual(len(defects), 2)
def test_defects(self):
defects = Defects()
defects.append(
algorithms.Defect(
lsst.geom.Box2I(lsst.geom.Point2I(5, 6),
lsst.geom.Point2I(41, 50))))
defects.append(
lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(4, 5)))
defects.append(lsst.geom.Point2I(50, 50))
defects.append(
afwImage.DefectBase(
lsst.geom.Box2I(lsst.geom.Point2I(100, 200),
lsst.geom.Extent2I(5, 5))))
self.assertEqual(len(defects), 4)
for d in defects:
self.assertIsInstance(d, algorithms.Defect)
# Transposition
transposed = defects.transpose()
self.assertEqual(len(transposed), len(defects))
# Check that an individual defect is found properly transposed within
# the outputs.
found = False
for defect in transposed:
if defect.getBBox() == lsst.geom.Box2I(lsst.geom.Point2I(6, 5),
lsst.geom.Extent2I(45, 37)):
found = True
break
self.assertTrue(found)
# Serialization round trip
meta = PropertyList()
meta["TESTHDR"] = "testing"
defects.setMetadata(meta)
table = defects.toFitsRegionTable()
defects2 = Defects.fromTable([table])
self.assertEqual(defects2, defects)
# via FITS
with lsst.utils.tests.getTempFilePath(".fits") as tmpFile:
defects.writeFits(tmpFile)
defects2 = Defects.readFits(tmpFile)
# Equality tests the bounding boxes so metadata is tested separately.
self.assertEqual(defects2, defects)
self.assertMetadata(defects2, defects)
# via text file
with lsst.utils.tests.getTempFilePath(".ecsv") as tmpFile:
defects.writeText(tmpFile)
defects2 = Defects.readText(tmpFile)
# Equality tests the bounding boxes so metadata is tested separately.
self.assertEqual(defects2, defects)
self.assertMetadata(defects2, defects)
# Check bad values
with self.assertRaises(ValueError):
defects.append(
lsst.geom.Box2D(lsst.geom.Point2D(0., 0.),
lsst.geom.Point2D(3.1, 3.1)))
with self.assertRaises(ValueError):
defects.append("defect")
def test_normalize_defects(self):
"""A test for the lsst.meas.algorithms.Defect.normalize() method.
"""
defects = Defects()
# First series of 1-pixel contiguous defects
for yPix in range(1, 6):
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(15, yPix),
dimensions=lsst.geom.Extent2I(1, 1)))
# Defects are normalized as they are added; check that the above have
# been merged into a single bounding box.
self.assertEqual(len(defects), 1)
# Second series of 1-pixel contiguous defects in bulk mode
with defects.bulk_update():
for yPix in range(11, 16):
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(20, yPix),
dimensions=lsst.geom.Extent2I(1, 1)))
# In bulk mode, defects are not normalized.
self.assertEqual(len(defects), 6)
# Normalization applied on exiting bulk mode.
self.assertEqual(len(defects), 2)
boxesMeasured = []
for defect in defects:
boxesMeasured.append(defect.getBBox())
# The normalizing function should have created the following two boxes
# out of the individual 1-pixel defects from above
expectedDefects = [
lsst.geom.Box2I(corner=lsst.geom.Point2I(15, 1),
dimensions=lsst.geom.Extent2I(1, 5)),
lsst.geom.Box2I(corner=lsst.geom.Point2I(20, 11),
dimensions=lsst.geom.Extent2I(1, 5))
]
self.assertEqual(len(expectedDefects), len(boxesMeasured))
for expDef, measDef in zip(expectedDefects, boxesMeasured):
self.assertEqual(expDef, measDef)
# Normalize two distinct sets of Defects and ensure they compare to the
# same thing.
defects = Defects()
# Set 1
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 2),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 3),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 4),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 6),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 7),
dimensions=lsst.geom.Extent2I(1, 1)))
defects.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 8),
dimensions=lsst.geom.Extent2I(1, 1)))
# Set 2
defects2 = Defects()
defects2.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1),
dimensions=lsst.geom.Extent2I(1, 5)))
defects2.append(
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5),
dimensions=lsst.geom.Extent2I(1, 4)))
self.assertEqual(defects, defects2)
boxesMeasured, boxesMeasured2 = [], []
for defect, defect2 in zip(defects, defects2):
boxesMeasured.append(defect.getBBox())
boxesMeasured2.append(defect2.getBBox())
expectedDefects = [
lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1),
dimensions=lsst.geom.Extent2I(1, 8))
]
self.assertEqual(len(expectedDefects), len(boxesMeasured))
for expDef, measDef in zip(expectedDefects, boxesMeasured):
self.assertEqual(expDef, measDef)
self.assertEqual(len(expectedDefects), len(boxesMeasured2))
for expDef, measDef in zip(expectedDefects, boxesMeasured2):
self.assertEqual(expDef, measDef)
detectorName = "0"
"""Detector name."""
detectorSerial = "0000011"
"""Detector serial code"""
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description=
f"""Construct a defects file from the mask plane of a test camera bias frame.
To use this command you must setup ip_isr and astropy.
Output is written to the current directory as file {DefectsPath}, which must not already exist.
""")
parser.add_argument("bias", help="path to bias image for the test camera")
args = parser.parse_args()
biasMI = afwImage.MaskedImageF(args.bias)
defectList = Defects.fromMask(biasMI, "BAD")
valid_start = dateutil.parser.parse('19700101T000000')
md = defectList.getMetadata()
md['INSTRUME'] = 'test'
md['DETECTOR'] = detectorName
md['CALIBDATE'] = valid_start.isoformat()
md['FILTER'] = None
defect_file = defectList.writeText(DefectsPath)
print("wrote defects file %r" % (DefectsPath, ))
test2defectList = Defects.readText(defect_file)
assert defectList == test2defectList
print("verified that defects file %r round trips correctly" %
(DefectsPath, ))
def run(self, inputDefects, camera):
detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
if detectorId is None:
raise RuntimeError("Cannot identify detector id.")
detector = camera[detectorId]
imageTypes = set()
for inDefect in inputDefects:
imageType = inDefect.getMetadata().get('cpDefectGenImageType',
'UNKNOWN')
imageTypes.add(imageType)
# Determine common defect pixels separately for each input image type.
splitDefects = list()
for imageType in imageTypes:
sumImage = afwImage.MaskedImageF(detector.getBBox())
count = 0
for inDefect in inputDefects:
if imageType == inDefect.getMetadata().get(
'cpDefectGenImageType', 'UNKNOWN'):
count += 1
for defect in inDefect:
sumImage.image[defect.getBBox()] += 1.0
sumImage /= count
nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
self.log.info(
"Pre-merge %s pixels with non-zero detections for %s" %
(nDetected, imageType))
if self.config.combinationMode == 'AND':
threshold = 1.0
elif self.config.combinationMode == 'OR':
threshold = 0.0
elif self.config.combinationMode == 'FRACTION':
threshold = self.config.combinationFraction
else:
raise RuntimeError(
f"Got unsupported combinationMode {self.config.combinationMode}"
)
indices = np.where(sumImage.getImage().getArray() > threshold)
BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
sumImage.getMask().getArray()[indices] |= BADBIT
self.log.info("Post-merge %s pixels marked as defects for %s" %
(len(indices[0]), imageType))
partialDefect = Defects.fromMask(sumImage, 'BAD')
splitDefects.append(partialDefect)
# Do final combination of separate image types
finalImage = afwImage.MaskedImageF(detector.getBBox())
for inDefect in splitDefects:
for defect in inDefect:
finalImage.image[defect.getBBox()] += 1
finalImage /= len(splitDefects)
nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
self.log.info("Pre-final merge %s pixels with non-zero detections" %
(nDetected, ))
# This combination is the OR of all image types
threshold = 0.0
indices = np.where(finalImage.getImage().getArray() > threshold)
BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
finalImage.getMask().getArray()[indices] |= BADBIT
self.log.info("Post-final merge %s pixels marked as defects" %
(len(indices[0]), ))
if self.config.edgesAsDefects:
self.log.info("Masking edge pixels as defects.")
# Do the same as IsrTask.maskEdges()
box = detector.getBBox()
subImage = finalImage[box]
box.grow(-self.nPixBorder)
SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
merged = Defects.fromMask(finalImage, 'BAD')
merged.updateMetadata(camera=camera,
detector=detector,
filterName=None,
setCalibId=True,
setDate=True)
return pipeBase.Struct(mergedDefects=merged, )
def findHotAndColdPixels(self, exp, nSigma):
"""Find hot and cold pixels in an image.
Using config-defined thresholds on a per-amp basis, mask
pixels that are nSigma above threshold in dark frames (hot
pixels), or nSigma away from the clipped mean in flats (hot &
cold pixels).
Parameters
----------
exp : `lsst.afw.image.exposure.Exposure`
The exposure in which to find defects.
nSigma : `list [ `float` ]
Detection threshold to use. Positive for DETECTED pixels,
negative for DETECTED_NEGATIVE pixels.
Returns
-------
defects : `lsst.ip.isr.Defect`
The defects found in the image.
"""
self._setEdgeBits(exp)
maskedIm = exp.maskedImage
# the detection polarity for afwDetection, True for positive,
# False for negative, and therefore True for darks as they only have
# bright pixels, and both for flats, as they have bright and dark pix
footprintList = []
for amp in exp.getDetector():
ampImg = maskedIm[amp.getBBox()].clone()
# crop ampImage depending on where the amp lies in the image
if self.config.nPixBorderLeftRight:
if ampImg.getX0() == 0:
ampImg = ampImg[self.config.nPixBorderLeftRight:, :,
afwImage.LOCAL]
else:
ampImg = ampImg[:-self.config.nPixBorderLeftRight, :,
afwImage.LOCAL]
if self.config.nPixBorderUpDown:
if ampImg.getY0() == 0:
ampImg = ampImg[:, self.config.nPixBorderUpDown:,
afwImage.LOCAL]
else:
ampImg = ampImg[:, :-self.config.nPixBorderUpDown,
afwImage.LOCAL]
if self._getNumGoodPixels(
ampImg) == 0: # amp contains no usable pixels
continue
# Remove a background estimate
ampImg -= afwMath.makeStatistics(
ampImg,
afwMath.MEANCLIP,
).getValue()
mergedSet = None
for sigma in nSigma:
nSig = np.abs(sigma)
self.debugHistogram('ampFlux', ampImg, nSig, exp)
polarity = {-1: False, 1: True}[np.sign(sigma)]
threshold = afwDetection.createThreshold(nSig,
'stdev',
polarity=polarity)
footprintSet = afwDetection.FootprintSet(ampImg, threshold)
footprintSet.setMask(
maskedIm.mask,
("DETECTED" if polarity else "DETECTED_NEGATIVE"))
if mergedSet is None:
mergedSet = footprintSet
else:
mergedSet.merge(footprintSet)
footprintList += mergedSet.getFootprints()
self.debugView(
'defectMap', ampImg,
Defects.fromFootprintList(mergedSet.getFootprints()),
exp.getDetector())
defects = Defects.fromFootprintList(footprintList)
defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
return defects