def testLsstTextfile(self): """Read legacy LSST text file format""" with lsst.utils.tests.getTempFilePath(".txt") as tmpFile: with open(tmpFile, "w") as fh: print("""# X0 Y0 width height 996 0 56 24 0 4156 2048 20 0 0 17 4176 1998 4035 50 141 1023 0 2 4176 2027 0 21 4176 0 4047 37 129 # Some rows without fixed column widths 14 20 2000 50 10 10 10 10 """, file=fh) defects = Defects.readLsstDefectsFile(tmpFile, normalize_on_init=True) # Although there are 9 defects listed above, we record 11 after # normalization. This is due to non-optimal behaviour in # Defects.fromMask; see DM-24781. self.assertEqual(len(defects), 11)
def testAstropyRegion(self): """Read a FITS region file created by Astropy regions.""" # The file contains three regions: # # - Point2I(340, 344) # - Point2I(340, 344) # - Box2I(minimum=Point2I(5, -5), dimensions=Extent2I(10, 20)) # # The two coincident points are combined on read, so we end up with two defects. with self.assertLogs(): defects = Defects.readFits(os.path.join(TESTDIR, "data", "fits_region.fits"), normalize_on_init=True) self.assertEqual(len(defects), 2)
def test_defects(self): defects = Defects() defects.append( algorithms.Defect( lsst.geom.Box2I(lsst.geom.Point2I(5, 6), lsst.geom.Point2I(41, 50)))) defects.append( lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(4, 5))) defects.append(lsst.geom.Point2I(50, 50)) defects.append( afwImage.DefectBase( lsst.geom.Box2I(lsst.geom.Point2I(100, 200), lsst.geom.Extent2I(5, 5)))) self.assertEqual(len(defects), 4) for d in defects: self.assertIsInstance(d, algorithms.Defect) # Transposition transposed = defects.transpose() self.assertEqual(len(transposed), len(defects)) # Check that an individual defect is found properly transposed within # the outputs. found = False for defect in transposed: if defect.getBBox() == lsst.geom.Box2I(lsst.geom.Point2I(6, 5), lsst.geom.Extent2I(45, 37)): found = True break self.assertTrue(found) # Serialization round trip meta = PropertyList() meta["TESTHDR"] = "testing" defects.setMetadata(meta) table = defects.toFitsRegionTable() defects2 = Defects.fromTable([table]) self.assertEqual(defects2, defects) # via FITS with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: defects.writeFits(tmpFile) defects2 = Defects.readFits(tmpFile) # Equality tests the bounding boxes so metadata is tested separately. self.assertEqual(defects2, defects) self.assertMetadata(defects2, defects) # via text file with lsst.utils.tests.getTempFilePath(".ecsv") as tmpFile: defects.writeText(tmpFile) defects2 = Defects.readText(tmpFile) # Equality tests the bounding boxes so metadata is tested separately. self.assertEqual(defects2, defects) self.assertMetadata(defects2, defects) # Check bad values with self.assertRaises(ValueError): defects.append( lsst.geom.Box2D(lsst.geom.Point2D(0., 0.), lsst.geom.Point2D(3.1, 3.1))) with self.assertRaises(ValueError): defects.append("defect")
def test_normalize_defects(self): """A test for the lsst.meas.algorithms.Defect.normalize() method. """ defects = Defects() # First series of 1-pixel contiguous defects for yPix in range(1, 6): defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(15, yPix), dimensions=lsst.geom.Extent2I(1, 1))) # Defects are normalized as they are added; check that the above have # been merged into a single bounding box. self.assertEqual(len(defects), 1) # Second series of 1-pixel contiguous defects in bulk mode with defects.bulk_update(): for yPix in range(11, 16): defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(20, yPix), dimensions=lsst.geom.Extent2I(1, 1))) # In bulk mode, defects are not normalized. self.assertEqual(len(defects), 6) # Normalization applied on exiting bulk mode. self.assertEqual(len(defects), 2) boxesMeasured = [] for defect in defects: boxesMeasured.append(defect.getBBox()) # The normalizing function should have created the following two boxes # out of the individual 1-pixel defects from above expectedDefects = [ lsst.geom.Box2I(corner=lsst.geom.Point2I(15, 1), dimensions=lsst.geom.Extent2I(1, 5)), lsst.geom.Box2I(corner=lsst.geom.Point2I(20, 11), dimensions=lsst.geom.Extent2I(1, 5)) ] self.assertEqual(len(expectedDefects), len(boxesMeasured)) for expDef, measDef in zip(expectedDefects, boxesMeasured): self.assertEqual(expDef, measDef) # Normalize two distinct sets of Defects and ensure they compare to the # same thing. defects = Defects() # Set 1 defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 2), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 3), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 4), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 6), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 7), dimensions=lsst.geom.Extent2I(1, 1))) defects.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 8), dimensions=lsst.geom.Extent2I(1, 1))) # Set 2 defects2 = Defects() defects2.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), dimensions=lsst.geom.Extent2I(1, 5))) defects2.append( lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5), dimensions=lsst.geom.Extent2I(1, 4))) self.assertEqual(defects, defects2) boxesMeasured, boxesMeasured2 = [], [] for defect, defect2 in zip(defects, defects2): boxesMeasured.append(defect.getBBox()) boxesMeasured2.append(defect2.getBBox()) expectedDefects = [ lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), dimensions=lsst.geom.Extent2I(1, 8)) ] self.assertEqual(len(expectedDefects), len(boxesMeasured)) for expDef, measDef in zip(expectedDefects, boxesMeasured): self.assertEqual(expDef, measDef) self.assertEqual(len(expectedDefects), len(boxesMeasured2)) for expDef, measDef in zip(expectedDefects, boxesMeasured2): self.assertEqual(expDef, measDef)
detectorName = "0" """Detector name.""" detectorSerial = "0000011" """Detector serial code""" if __name__ == "__main__": parser = argparse.ArgumentParser( description= f"""Construct a defects file from the mask plane of a test camera bias frame. To use this command you must setup ip_isr and astropy. Output is written to the current directory as file {DefectsPath}, which must not already exist. """) parser.add_argument("bias", help="path to bias image for the test camera") args = parser.parse_args() biasMI = afwImage.MaskedImageF(args.bias) defectList = Defects.fromMask(biasMI, "BAD") valid_start = dateutil.parser.parse('19700101T000000') md = defectList.getMetadata() md['INSTRUME'] = 'test' md['DETECTOR'] = detectorName md['CALIBDATE'] = valid_start.isoformat() md['FILTER'] = None defect_file = defectList.writeText(DefectsPath) print("wrote defects file %r" % (DefectsPath, )) test2defectList = Defects.readText(defect_file) assert defectList == test2defectList print("verified that defects file %r round trips correctly" % (DefectsPath, ))
def run(self, inputDefects, camera): detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) if detectorId is None: raise RuntimeError("Cannot identify detector id.") detector = camera[detectorId] imageTypes = set() for inDefect in inputDefects: imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') imageTypes.add(imageType) # Determine common defect pixels separately for each input image type. splitDefects = list() for imageType in imageTypes: sumImage = afwImage.MaskedImageF(detector.getBBox()) count = 0 for inDefect in inputDefects: if imageType == inDefect.getMetadata().get( 'cpDefectGenImageType', 'UNKNOWN'): count += 1 for defect in inDefect: sumImage.image[defect.getBBox()] += 1.0 sumImage /= count nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) self.log.info( "Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) if self.config.combinationMode == 'AND': threshold = 1.0 elif self.config.combinationMode == 'OR': threshold = 0.0 elif self.config.combinationMode == 'FRACTION': threshold = self.config.combinationFraction else: raise RuntimeError( f"Got unsupported combinationMode {self.config.combinationMode}" ) indices = np.where(sumImage.getImage().getArray() > threshold) BADBIT = sumImage.getMask().getPlaneBitMask('BAD') sumImage.getMask().getArray()[indices] |= BADBIT self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) partialDefect = Defects.fromMask(sumImage, 'BAD') splitDefects.append(partialDefect) # Do final combination of separate image types finalImage = afwImage.MaskedImageF(detector.getBBox()) for inDefect in splitDefects: for defect in inDefect: finalImage.image[defect.getBBox()] += 1 finalImage /= len(splitDefects) nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) # This combination is the OR of all image types threshold = 0.0 indices = np.where(finalImage.getImage().getArray() > threshold) BADBIT = finalImage.getMask().getPlaneBitMask('BAD') finalImage.getMask().getArray()[indices] |= BADBIT self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) if self.config.edgesAsDefects: self.log.info("Masking edge pixels as defects.") # Do the same as IsrTask.maskEdges() box = detector.getBBox() subImage = finalImage[box] box.grow(-self.nPixBorder) SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) merged = Defects.fromMask(finalImage, 'BAD') merged.updateMetadata(camera=camera, detector=detector, filterName=None, setCalibId=True, setDate=True) return pipeBase.Struct(mergedDefects=merged, )
def findHotAndColdPixels(self, exp, nSigma): """Find hot and cold pixels in an image. Using config-defined thresholds on a per-amp basis, mask pixels that are nSigma above threshold in dark frames (hot pixels), or nSigma away from the clipped mean in flats (hot & cold pixels). Parameters ---------- exp : `lsst.afw.image.exposure.Exposure` The exposure in which to find defects. nSigma : `list [ `float` ] Detection threshold to use. Positive for DETECTED pixels, negative for DETECTED_NEGATIVE pixels. Returns ------- defects : `lsst.ip.isr.Defect` The defects found in the image. """ self._setEdgeBits(exp) maskedIm = exp.maskedImage # the detection polarity for afwDetection, True for positive, # False for negative, and therefore True for darks as they only have # bright pixels, and both for flats, as they have bright and dark pix footprintList = [] for amp in exp.getDetector(): ampImg = maskedIm[amp.getBBox()].clone() # crop ampImage depending on where the amp lies in the image if self.config.nPixBorderLeftRight: if ampImg.getX0() == 0: ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] else: ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] if self.config.nPixBorderUpDown: if ampImg.getY0() == 0: ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] else: ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] if self._getNumGoodPixels( ampImg) == 0: # amp contains no usable pixels continue # Remove a background estimate ampImg -= afwMath.makeStatistics( ampImg, afwMath.MEANCLIP, ).getValue() mergedSet = None for sigma in nSigma: nSig = np.abs(sigma) self.debugHistogram('ampFlux', ampImg, nSig, exp) polarity = {-1: False, 1: True}[np.sign(sigma)] threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) footprintSet = afwDetection.FootprintSet(ampImg, threshold) footprintSet.setMask( maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) if mergedSet is None: mergedSet = footprintSet else: mergedSet.merge(footprintSet) footprintList += mergedSet.getFootprints() self.debugView( 'defectMap', ampImg, Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) defects = Defects.fromFootprintList(footprintList) defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) return defects