def testIgnoreKeywords(self): """Check that certain keywords are ignored in read/write of headers""" # May not appear at all in the FITS file (cfitsio doesn't write these by default) notAtAll = [ # FITS core keywords "GCOUNT", "PCOUNT", "XTENSION", "BSCALE", "BZERO", "TZERO", "TSCAL", # FITS compression keywords "ZBITPIX", "ZIMAGE", "ZCMPTYPE", "ZSIMPLE", "ZEXTEND", "ZBLANK", "ZDATASUM", "ZHECKSUM", "ZNAXIS", "ZTILE", "ZNAME", "ZVAL", "ZQUANTIZ", # Not essential these be excluded, but will prevent fitsverify warnings "DATASUM", "CHECKSUM", ] # Additional keywords to check; these should go straight through # Some of these are longer/shorter versions of strings above, # to test that the checks for just the start of strings is working. others = ["FOOBAR", "SIMPLETN", "DATASUMX", "NAX", "SIM"] header = PropertyList() for ii, key in enumerate(self.single + notAtAll + others): header.add(key, ii) metadata = self.writeAndRead(header) for key in self.single: self.assertEqual(metadata.valueCount(key), 1, key) for key in notAtAll: self.assertEqual(metadata.valueCount(key), 0, key) for key in others: self.assertEqual(metadata.valueCount(key), 1, key)
def testMultiPlaneFitsReaders(self): """Run tests for MaskedImageFitsReader and ExposureFitsReader. """ metadata = PropertyList() metadata.add("FIVE", 5) metadata.add("SIX", 6.0) wcs = makeSkyWcs(Point2D(2.5, 3.75), SpherePoint(40.0 * degrees, 50.0 * degrees), np.array([[1E-5, 0.0], [0.0, -1E-5]])) defineFilter("test_readers_filter", lambdaEff=470.0) calib = PhotoCalib(2.5E4) psf = GaussianPsf(21, 21, 8.0) polygon = Polygon(Box2D(self.bbox)) apCorrMap = ApCorrMap() visitInfo = VisitInfo(exposureTime=5.0) transmissionCurve = TransmissionCurve.makeIdentity() coaddInputs = CoaddInputs(ExposureTable.makeMinimalSchema(), ExposureTable.makeMinimalSchema()) detector = DetectorWrapper().detector record = coaddInputs.ccds.addNew() record.setWcs(wcs) record.setPhotoCalib(calib) record.setPsf(psf) record.setValidPolygon(polygon) record.setApCorrMap(apCorrMap) record.setVisitInfo(visitInfo) record.setTransmissionCurve(transmissionCurve) record.setDetector(detector) for n, dtypeIn in enumerate(self.dtypes): with self.subTest(dtypeIn=dtypeIn): exposureIn = Exposure(self.bbox, dtype=dtypeIn) shape = exposureIn.image.array.shape exposureIn.image.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.mask.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.variance.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.setMetadata(metadata) exposureIn.setWcs(wcs) exposureIn.setFilter(Filter("test_readers_filter")) exposureIn.setFilterLabel( FilterLabel(physical="test_readers_filter")) exposureIn.setPhotoCalib(calib) exposureIn.setPsf(psf) exposureIn.getInfo().setValidPolygon(polygon) exposureIn.getInfo().setApCorrMap(apCorrMap) exposureIn.getInfo().setVisitInfo(visitInfo) exposureIn.getInfo().setTransmissionCurve(transmissionCurve) exposureIn.getInfo().setCoaddInputs(coaddInputs) exposureIn.setDetector(detector) with lsst.utils.tests.getTempFilePath(".fits") as fileName: exposureIn.writeFits(fileName) self.checkMaskedImageFitsReader(exposureIn, fileName, self.dtypes[n:]) self.checkExposureFitsReader(exposureIn, fileName, self.dtypes[n:])
def testArrayValues(self): """Check that only the final value is used from an array """ metadata = PropertyList() # work around DM-13232 by setting ABOOL one value at a time for value in [True, True, True, False]: metadata.add("ABOOL", value) dataList = [ ("AFLOAT", [1.2e25, -5.6]), ("ANINT", [-5, 752, 1052]), ("ASTRING1", ["value for string", "more"]), ] for name, value in dataList: metadata.set(name, value) header = makeLimitedFitsHeader(metadata) expectedLines = [ # without padding to 80 chars "ABOOL = F", "AFLOAT = -5.6", "ANINT = 1052", "ASTRING1= 'more'", ] expectedHeader = "".join("%-80s" % val for val in expectedLines) self.assertHeadersEqual(header, expectedHeader) self.checkExcludeNames(metadata, expectedLines)
def testIgnoreKeywords(self): """Check that certain keywords are ignored in read/write of headers""" # May appear only once in the FITS file (because cfitsio will insist on putting them there) single = ["SIMPLE", "BITPIX", "EXTEND", "NAXIS"] # May not appear at all in the FITS file (cfitsio doesn't write these by default) notAtAll = [ # FITS core keywords "GCOUNT", "PCOUNT", "XTENSION", "BSCALE", "BZERO", "TZERO", "TSCAL", # FITS compression keywords "ZBITPIX", "ZIMAGE", "ZCMPTYPE", "ZSIMPLE", "ZEXTEND", "ZBLANK", "ZDATASUM", "ZHECKSUM", "ZNAXIS", "ZTILE", "ZNAME", "ZVAL", # Not essential these be excluded, but will prevent fitsverify warnings "DATASUM", "CHECKSUM", ] # Additional keywords to check; these should go straight through # Some of these are longer/shorter versions of strings above, # to test that the checks for just the start of strings is working. others = ["FOOBAR", "SIMPLETN", "DATASUMX", "NAX", "SIM"] header = PropertyList() for ii, key in enumerate(single + notAtAll + others): header.add(key, ii) fitsFile = lsst.afw.fits.MemFileManager() with lsst.afw.fits.Fits(fitsFile, "w") as fits: fits.createEmpty() fits.writeMetadata(header) with lsst.afw.fits.Fits(fitsFile, "r") as fits: metadata = fits.readMetadata() for key in single: self.assertEqual(metadata.valueCount(key), 1, key) for key in notAtAll: self.assertEqual(metadata.valueCount(key), 0, key) for key in others: self.assertEqual(metadata.valueCount(key), 1, key)
def testMultiPlaneFitsReaders(self): """Run tests for MaskedImageFitsReader and ExposureFitsReader. """ metadata = PropertyList() metadata.add("FIVE", 5) metadata.add("SIX", 6.0) wcs = makeSkyWcs(Point2D(2.5, 3.75), SpherePoint(40.0*degrees, 50.0*degrees), np.array([[1E-5, 0.0], [0.0, -1E-5]])) defineFilter("test_readers_filter", lambdaEff=470.0) calib = PhotoCalib(2.5E4) psf = GaussianPsf(21, 21, 8.0) polygon = Polygon(Box2D(self.bbox)) apCorrMap = ApCorrMap() visitInfo = VisitInfo(exposureTime=5.0) transmissionCurve = TransmissionCurve.makeIdentity() coaddInputs = CoaddInputs(ExposureTable.makeMinimalSchema(), ExposureTable.makeMinimalSchema()) detector = DetectorWrapper().detector record = coaddInputs.ccds.addNew() record.setWcs(wcs) record.setCalib(calib) record.setPsf(psf) record.setValidPolygon(polygon) record.setApCorrMap(apCorrMap) record.setVisitInfo(visitInfo) record.setTransmissionCurve(transmissionCurve) record.setDetector(detector) for n, dtypeIn in enumerate(self.dtypes): with self.subTest(dtypeIn=dtypeIn): exposureIn = Exposure(self.bbox, dtype=dtypeIn) shape = exposureIn.image.array.shape exposureIn.image.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.mask.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.variance.array[:, :] = np.random.randint(low=1, high=5, size=shape) exposureIn.setMetadata(metadata) exposureIn.setWcs(wcs) exposureIn.setFilter(Filter("test_readers_filter")) exposureIn.setCalib(calib) exposureIn.setPsf(psf) exposureIn.getInfo().setValidPolygon(polygon) exposureIn.getInfo().setApCorrMap(apCorrMap) exposureIn.getInfo().setVisitInfo(visitInfo) exposureIn.getInfo().setTransmissionCurve(transmissionCurve) exposureIn.getInfo().setCoaddInputs(coaddInputs) exposureIn.setDetector(detector) with lsst.utils.tests.getTempFilePath(".fits") as fileName: exposureIn.writeFits(fileName) self.checkMaskedImageFitsReader(exposureIn, fileName, self.dtypes[n:]) self.checkExposureFitsReader(exposureIn, fileName, self.dtypes[n:])
def testGetCdMatrixFromMetadata(self): cdMatrix = getCdMatrixFromMetadata(self.metadata) for i in range(2): for j in range(2): cardName = "CD%d_%d" % (i + 1, j + 1) self.assertEqual(cdMatrix[i, j], self.metadata.getScalar(cardName)) metadata = PropertyList() with self.assertRaises(TypeError): getCdMatrixFromMetadata(metadata) metadata.add("CD2_1", 0.56) # just one term, with an arbitrary value cdMatrix2 = getCdMatrixFromMetadata(metadata) for i in range(2): for j in range(2): # CD matrix terms use 1-based indexing cardName = "CD%d_%d" % (i + 1, j + 1) if i == 1 and j == 0: self.assertEqual(cdMatrix2[i, j], 0.56) else: self.assertEqual(cdMatrix2[i, j], 0.0)
def testGetCdMatrixFromMetadata(self): cdMatrix = getCdMatrixFromMetadata(self.metadata) for i in range(2): for j in range(2): cardName = "CD%d_%d" % (i + 1, j + 1) self.assertEqual(cdMatrix[i, j], self.metadata.get(cardName)) metadata = PropertyList() with self.assertRaises(TypeError): getCdMatrixFromMetadata(metadata) metadata.add("CD2_1", 0.56) # just one term, with an arbitrary value cdMatrix2 = getCdMatrixFromMetadata(metadata) for i in range(2): for j in range(2): # CD matrix terms use 1-based indexing cardName = "CD%d_%d" % (i + 1, j + 1) if i == 1 and j == 0: self.assertEqual(cdMatrix2[i, j], 0.56) else: self.assertEqual(cdMatrix2[i, j], 0.0)
def getMetadataCircle(self, coord, radius, filterName, calib=None): """!Return metadata about the load This metadata is used for reloading the catalog (e.g., for reconstituting a normalised match list. @param[in] coord ICRS centr of circle (lsst.afw.geom.SpherePoint) @param[in] radius radius of circle (lsst.afw.geom.Angle) @param[in] filterName name of camera filter, or None or blank for the default filter @param[in] calib calibration, or None if unknown @return metadata (lsst.daf.base.PropertyList) """ md = PropertyList() md.add('RA', coord.getRa().asDegrees(), 'field center in degrees') md.add('DEC', coord.getDec().asDegrees(), 'field center in degrees') md.add('RADIUS', radius.asDegrees(), 'field radius in degrees, minimum') md.add('SMATCHV', 1, 'SourceMatchVector version number') filterName = "UNKNOWN" if filterName is None else str(filterName) md.add('FILTER', filterName, 'filter name for photometric data') return md
def createMatchMetadata(exposure): """Create metadata required for unpersisting a match list @param[in] exposure exposure for which to create metadata @return metadata about the field (a daf_base PropertyList) """ matchMeta = PropertyList() bboxd = Box2D(exposure.getBBox()) ctrPos = bboxd.getCenter() wcs = getDistortedWcs(exposure.getInfo()) ctrCoord = wcs.pixelToSky(ctrPos).toIcrs() llCoord = wcs.pixelToSky(bboxd.getMin()) approxRadius = ctrCoord.angularSeparation(llCoord) matchMeta.add('RA', ctrCoord.getRa().asDegrees(), 'field center in degrees') matchMeta.add('DEC', ctrCoord.getDec().asDegrees(), 'field center in degrees') matchMeta.add('RADIUS', approxRadius.asDegrees(), 'field radius in degrees, approximate') matchMeta.add('SMATCHV', 1, 'SourceMatchVector version number') filterName = exposure.getFilter().getName() or None if filterName is not None and filterName not in ("_unknmown_", ""): matchMeta.add('FILTER', filterName, 'filter name for tagalong data') return matchMeta
def setUp(self): """Constructs a CCD with two amplifiers and prepares for ISR""" np.random.seed(12345) baseValue = 100.0 gain = 1.0 readNoise = 123456789.0 saturation = 987654321.0 height = 234 imageSize = Extent2I(123, height) overscanSize = Extent2I(16, height) self.sigma = 1.234 # Set up the various regions overscan1 = Box2I(Point2I(0, 0), overscanSize) image1 = Box2I(Point2I(overscanSize[0], 0), imageSize) image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize) overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize) leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height)) rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height)) target1 = Box2I(Point2I(0, 0), imageSize) target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize) # Set the pixels exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height))) yy = np.arange(0, height, 1, dtype=np.float32) leftImage = ExposureF(exposure, leftBox) leftImage.image.array[:] = baseValue + yy[:, np.newaxis] rightImage = ExposureF(exposure, rightBox) rightImage.image.array[:] = baseValue - yy[:, np.newaxis] leftOverscan = ExposureF(exposure, overscan1) leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) rightOverscan = ExposureF(exposure, overscan2) rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape) exposure.mask.array[:] = 0.0 exposure.variance.array[:] = np.nan # Construct the detectors amps = AmpInfoCatalog(AmpInfoTable.makeMinimalSchema()) makeAmplifier(amps, "left", target1, image1, overscan1, gain, readNoise, saturation) makeAmplifier(amps, "right", target2, image2, overscan2, gain, readNoise, saturation) ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height)) ccd = Detector("detector", 1, SCIENCE, "det1", ccdBox, amps, Orientation(), Extent2D(1.0, 1.0), {}) exposure.setDetector(ccd) header = PropertyList() header.add("EXPTIME", 0.0) exposure.getInfo().setVisitInfo(VisitInfo(header)) self.exposure = exposure self.config = IsrTask.ConfigClass() # Disable everything we don't care about self.config.doBias = False self.config.doDark = False self.config.doFlat = False self.config.doFringe = False self.config.doDefect = False self.config.doAddDistortionModel = False self.config.doWrite = False self.config.expectWcs = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doBrighterFatter = False self.config.doAttachTransmissionCurve = False # Set the things that match our test setup self.config.overscanFitType = "CHEB" self.config.overscanOrder = 1 self.config.doEmpiricalReadNoise = True self.task = IsrTask(config=self.config)
def makePropertyListFromDict(md): result = PropertyList() for k, v in md.items(): result.add(k, v) return result
def setUp(self): """Constructs a CCD with two amplifiers and prepares for ISR""" np.random.seed(12345) baseValue = 100.0 gain = 1.0 readNoise = 123456789.0 saturation = 987654321.0 height = 234 imageSize = Extent2I(123, height) overscanSize = Extent2I(16, height) self.sigma = 1.234 # Set up the various regions overscan1 = Box2I(Point2I(0, 0), overscanSize) image1 = Box2I(Point2I(overscanSize[0], 0), imageSize) image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize) overscan2 = Box2I(Point2I(overscanSize[0] + 2 * imageSize[0], 0), overscanSize) leftBox = Box2I( overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height)) rightBox = Box2I( image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height)) target1 = Box2I(Point2I(0, 0), imageSize) target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize) # Set the pixels exposure = ExposureF( Box2I(Point2I(0, 0), Extent2I(imageSize[0] * 2 + overscanSize[0] * 2, height))) yy = np.arange(0, height, 1, dtype=np.float32) leftImage = ExposureF(exposure, leftBox) leftImage.image.array[:] = baseValue + yy[:, np.newaxis] rightImage = ExposureF(exposure, rightBox) rightImage.image.array[:] = baseValue - yy[:, np.newaxis] leftOverscan = ExposureF(exposure, overscan1) leftOverscan.image.array += np.random.normal( 0.0, self.sigma, leftOverscan.image.array.shape) rightOverscan = ExposureF(exposure, overscan2) rightOverscan.image.array += np.random.normal( 0.0, self.sigma, leftOverscan.image.array.shape) exposure.mask.array[:] = 0.0 exposure.variance.array[:] = np.nan # Construct the detectors amp1 = makeAmplifier("left", target1, image1, overscan1, gain, readNoise, saturation) amp2 = makeAmplifier("right", target2, image2, overscan2, gain, readNoise, saturation) ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height)) camBuilder = cameraGeom.Camera.Builder("fakeCam") detBuilder = camBuilder.add("detector", 1) detBuilder.setSerial("det1") detBuilder.setBBox(ccdBox) detBuilder.setPixelSize(Extent2D(1.0, 1.0)) detBuilder.setOrientation(cameraGeom.Orientation()) detBuilder.append(amp1) detBuilder.append(amp2) cam = camBuilder.finish() exposure.setDetector(cam.get('detector')) header = PropertyList() header.add("EXPTIME", 0.0) exposure.getInfo().setVisitInfo(VisitInfo(header)) self.exposure = exposure self.config = IsrTask.ConfigClass() # Disable everything we don't care about self.config.doBias = False self.config.doDark = False self.config.doFlat = False self.config.doFringe = False self.config.doDefect = False self.config.doWrite = False self.config.expectWcs = False self.config.doLinearize = False self.config.doCrosstalk = False self.config.doBrighterFatter = False self.config.doAttachTransmissionCurve = False self.config.doAssembleCcd = False self.config.doNanMasking = False self.config.doInterpolate = False self.config.maskNegativeVariance = False # This runs on mocks. # Set the things that match our test setup self.config.overscan.fitType = "CHEB" self.config.overscan.order = 1 self.config.doEmpiricalReadNoise = True self.task = IsrTask(config=self.config)