def testIgnoreKeywords(self):
"""Check that certain keywords are ignored in read/write of headers"""
# May not appear at all in the FITS file (cfitsio doesn't write these by default)
notAtAll = [
# FITS core keywords
"GCOUNT", "PCOUNT", "XTENSION", "BSCALE", "BZERO", "TZERO", "TSCAL",
# FITS compression keywords
"ZBITPIX", "ZIMAGE", "ZCMPTYPE", "ZSIMPLE", "ZEXTEND", "ZBLANK", "ZDATASUM", "ZHECKSUM",
"ZNAXIS", "ZTILE", "ZNAME", "ZVAL", "ZQUANTIZ",
# Not essential these be excluded, but will prevent fitsverify warnings
"DATASUM", "CHECKSUM",
]
# Additional keywords to check; these should go straight through
# Some of these are longer/shorter versions of strings above,
# to test that the checks for just the start of strings is working.
others = ["FOOBAR", "SIMPLETN", "DATASUMX", "NAX", "SIM"]
header = PropertyList()
for ii, key in enumerate(self.single + notAtAll + others):
header.add(key, ii)
metadata = self.writeAndRead(header)
for key in self.single:
self.assertEqual(metadata.valueCount(key), 1, key)
for key in notAtAll:
self.assertEqual(metadata.valueCount(key), 0, key)
for key in others:
self.assertEqual(metadata.valueCount(key), 1, key)
def testMultiPlaneFitsReaders(self):
"""Run tests for MaskedImageFitsReader and ExposureFitsReader.
"""
metadata = PropertyList()
metadata.add("FIVE", 5)
metadata.add("SIX", 6.0)
wcs = makeSkyWcs(Point2D(2.5, 3.75),
SpherePoint(40.0 * degrees, 50.0 * degrees),
np.array([[1E-5, 0.0], [0.0, -1E-5]]))
defineFilter("test_readers_filter", lambdaEff=470.0)
calib = PhotoCalib(2.5E4)
psf = GaussianPsf(21, 21, 8.0)
polygon = Polygon(Box2D(self.bbox))
apCorrMap = ApCorrMap()
visitInfo = VisitInfo(exposureTime=5.0)
transmissionCurve = TransmissionCurve.makeIdentity()
coaddInputs = CoaddInputs(ExposureTable.makeMinimalSchema(),
ExposureTable.makeMinimalSchema())
detector = DetectorWrapper().detector
record = coaddInputs.ccds.addNew()
record.setWcs(wcs)
record.setPhotoCalib(calib)
record.setPsf(psf)
record.setValidPolygon(polygon)
record.setApCorrMap(apCorrMap)
record.setVisitInfo(visitInfo)
record.setTransmissionCurve(transmissionCurve)
record.setDetector(detector)
for n, dtypeIn in enumerate(self.dtypes):
with self.subTest(dtypeIn=dtypeIn):
exposureIn = Exposure(self.bbox, dtype=dtypeIn)
shape = exposureIn.image.array.shape
exposureIn.image.array[:, :] = np.random.randint(low=1,
high=5,
size=shape)
exposureIn.mask.array[:, :] = np.random.randint(low=1,
high=5,
size=shape)
exposureIn.variance.array[:, :] = np.random.randint(low=1,
high=5,
size=shape)
exposureIn.setMetadata(metadata)
exposureIn.setWcs(wcs)
exposureIn.setFilter(Filter("test_readers_filter"))
exposureIn.setFilterLabel(
FilterLabel(physical="test_readers_filter"))
exposureIn.setPhotoCalib(calib)
exposureIn.setPsf(psf)
exposureIn.getInfo().setValidPolygon(polygon)
exposureIn.getInfo().setApCorrMap(apCorrMap)
exposureIn.getInfo().setVisitInfo(visitInfo)
exposureIn.getInfo().setTransmissionCurve(transmissionCurve)
exposureIn.getInfo().setCoaddInputs(coaddInputs)
exposureIn.setDetector(detector)
with lsst.utils.tests.getTempFilePath(".fits") as fileName:
exposureIn.writeFits(fileName)
self.checkMaskedImageFitsReader(exposureIn, fileName,
self.dtypes[n:])
self.checkExposureFitsReader(exposureIn, fileName,
self.dtypes[n:])
def testArrayValues(self):
"""Check that only the final value is used from an array
"""
metadata = PropertyList()
# work around DM-13232 by setting ABOOL one value at a time
for value in [True, True, True, False]:
metadata.add("ABOOL", value)
dataList = [
("AFLOAT", [1.2e25, -5.6]),
("ANINT", [-5, 752, 1052]),
("ASTRING1", ["value for string", "more"]),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = F",
"AFLOAT = -5.6",
"ANINT = 1052",
"ASTRING1= 'more'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def testArrayValues(self):
"""Check that only the final value is used from an array
"""
metadata = PropertyList()
# work around DM-13232 by setting ABOOL one value at a time
for value in [True, True, True, False]:
metadata.add("ABOOL", value)
dataList = [
("AFLOAT", [1.2e25, -5.6]),
("ANINT", [-5, 752, 1052]),
("ASTRING1", ["value for string", "more"]),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = F",
"AFLOAT = -5.6",
"ANINT = 1052",
"ASTRING1= 'more'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def testIgnoreKeywords(self):
"""Check that certain keywords are ignored in read/write of headers"""
# May appear only once in the FITS file (because cfitsio will insist on putting them there)
single = ["SIMPLE", "BITPIX", "EXTEND", "NAXIS"]
# May not appear at all in the FITS file (cfitsio doesn't write these by default)
notAtAll = [
# FITS core keywords
"GCOUNT",
"PCOUNT",
"XTENSION",
"BSCALE",
"BZERO",
"TZERO",
"TSCAL",
# FITS compression keywords
"ZBITPIX",
"ZIMAGE",
"ZCMPTYPE",
"ZSIMPLE",
"ZEXTEND",
"ZBLANK",
"ZDATASUM",
"ZHECKSUM",
"ZNAXIS",
"ZTILE",
"ZNAME",
"ZVAL",
# Not essential these be excluded, but will prevent fitsverify warnings
"DATASUM",
"CHECKSUM",
]
# Additional keywords to check; these should go straight through
# Some of these are longer/shorter versions of strings above,
# to test that the checks for just the start of strings is working.
others = ["FOOBAR", "SIMPLETN", "DATASUMX", "NAX", "SIM"]
header = PropertyList()
for ii, key in enumerate(single + notAtAll + others):
header.add(key, ii)
fitsFile = lsst.afw.fits.MemFileManager()
with lsst.afw.fits.Fits(fitsFile, "w") as fits:
fits.createEmpty()
fits.writeMetadata(header)
with lsst.afw.fits.Fits(fitsFile, "r") as fits:
metadata = fits.readMetadata()
for key in single:
self.assertEqual(metadata.valueCount(key), 1, key)
for key in notAtAll:
self.assertEqual(metadata.valueCount(key), 0, key)
for key in others:
self.assertEqual(metadata.valueCount(key), 1, key)
def testMultiPlaneFitsReaders(self):
"""Run tests for MaskedImageFitsReader and ExposureFitsReader.
"""
metadata = PropertyList()
metadata.add("FIVE", 5)
metadata.add("SIX", 6.0)
wcs = makeSkyWcs(Point2D(2.5, 3.75), SpherePoint(40.0*degrees, 50.0*degrees),
np.array([[1E-5, 0.0], [0.0, -1E-5]]))
defineFilter("test_readers_filter", lambdaEff=470.0)
calib = PhotoCalib(2.5E4)
psf = GaussianPsf(21, 21, 8.0)
polygon = Polygon(Box2D(self.bbox))
apCorrMap = ApCorrMap()
visitInfo = VisitInfo(exposureTime=5.0)
transmissionCurve = TransmissionCurve.makeIdentity()
coaddInputs = CoaddInputs(ExposureTable.makeMinimalSchema(), ExposureTable.makeMinimalSchema())
detector = DetectorWrapper().detector
record = coaddInputs.ccds.addNew()
record.setWcs(wcs)
record.setCalib(calib)
record.setPsf(psf)
record.setValidPolygon(polygon)
record.setApCorrMap(apCorrMap)
record.setVisitInfo(visitInfo)
record.setTransmissionCurve(transmissionCurve)
record.setDetector(detector)
for n, dtypeIn in enumerate(self.dtypes):
with self.subTest(dtypeIn=dtypeIn):
exposureIn = Exposure(self.bbox, dtype=dtypeIn)
shape = exposureIn.image.array.shape
exposureIn.image.array[:, :] = np.random.randint(low=1, high=5, size=shape)
exposureIn.mask.array[:, :] = np.random.randint(low=1, high=5, size=shape)
exposureIn.variance.array[:, :] = np.random.randint(low=1, high=5, size=shape)
exposureIn.setMetadata(metadata)
exposureIn.setWcs(wcs)
exposureIn.setFilter(Filter("test_readers_filter"))
exposureIn.setCalib(calib)
exposureIn.setPsf(psf)
exposureIn.getInfo().setValidPolygon(polygon)
exposureIn.getInfo().setApCorrMap(apCorrMap)
exposureIn.getInfo().setVisitInfo(visitInfo)
exposureIn.getInfo().setTransmissionCurve(transmissionCurve)
exposureIn.getInfo().setCoaddInputs(coaddInputs)
exposureIn.setDetector(detector)
with lsst.utils.tests.getTempFilePath(".fits") as fileName:
exposureIn.writeFits(fileName)
self.checkMaskedImageFitsReader(exposureIn, fileName, self.dtypes[n:])
self.checkExposureFitsReader(exposureIn, fileName, self.dtypes[n:])
def testIgnoreKeywords(self):
"""Check that certain keywords are ignored in read/write of headers"""
# May not appear at all in the FITS file (cfitsio doesn't write these by default)
notAtAll = [
# FITS core keywords
"GCOUNT",
"PCOUNT",
"XTENSION",
"BSCALE",
"BZERO",
"TZERO",
"TSCAL",
# FITS compression keywords
"ZBITPIX",
"ZIMAGE",
"ZCMPTYPE",
"ZSIMPLE",
"ZEXTEND",
"ZBLANK",
"ZDATASUM",
"ZHECKSUM",
"ZNAXIS",
"ZTILE",
"ZNAME",
"ZVAL",
"ZQUANTIZ",
# Not essential these be excluded, but will prevent fitsverify warnings
"DATASUM",
"CHECKSUM",
]
# Additional keywords to check; these should go straight through
# Some of these are longer/shorter versions of strings above,
# to test that the checks for just the start of strings is working.
others = ["FOOBAR", "SIMPLETN", "DATASUMX", "NAX", "SIM"]
header = PropertyList()
for ii, key in enumerate(self.single + notAtAll + others):
header.add(key, ii)
metadata = self.writeAndRead(header)
for key in self.single:
self.assertEqual(metadata.valueCount(key), 1, key)
for key in notAtAll:
self.assertEqual(metadata.valueCount(key), 0, key)
for key in others:
self.assertEqual(metadata.valueCount(key), 1, key)
def testGetCdMatrixFromMetadata(self):
cdMatrix = getCdMatrixFromMetadata(self.metadata)
for i in range(2):
for j in range(2):
cardName = "CD%d_%d" % (i + 1, j + 1)
self.assertEqual(cdMatrix[i, j], self.metadata.getScalar(cardName))
metadata = PropertyList()
with self.assertRaises(TypeError):
getCdMatrixFromMetadata(metadata)
metadata.add("CD2_1", 0.56) # just one term, with an arbitrary value
cdMatrix2 = getCdMatrixFromMetadata(metadata)
for i in range(2):
for j in range(2):
# CD matrix terms use 1-based indexing
cardName = "CD%d_%d" % (i + 1, j + 1)
if i == 1 and j == 0:
self.assertEqual(cdMatrix2[i, j], 0.56)
else:
self.assertEqual(cdMatrix2[i, j], 0.0)
def testGetCdMatrixFromMetadata(self):
cdMatrix = getCdMatrixFromMetadata(self.metadata)
for i in range(2):
for j in range(2):
cardName = "CD%d_%d" % (i + 1, j + 1)
self.assertEqual(cdMatrix[i, j], self.metadata.get(cardName))
metadata = PropertyList()
with self.assertRaises(TypeError):
getCdMatrixFromMetadata(metadata)
metadata.add("CD2_1", 0.56) # just one term, with an arbitrary value
cdMatrix2 = getCdMatrixFromMetadata(metadata)
for i in range(2):
for j in range(2):
# CD matrix terms use 1-based indexing
cardName = "CD%d_%d" % (i + 1, j + 1)
if i == 1 and j == 0:
self.assertEqual(cdMatrix2[i, j], 0.56)
else:
self.assertEqual(cdMatrix2[i, j], 0.0)
def getMetadataCircle(self, coord, radius, filterName, calib=None):
"""!Return metadata about the load
This metadata is used for reloading the catalog (e.g., for
reconstituting a normalised match list.
@param[in] coord ICRS centr of circle (lsst.afw.geom.SpherePoint)
@param[in] radius radius of circle (lsst.afw.geom.Angle)
@param[in] filterName name of camera filter, or None or blank for the default filter
@param[in] calib calibration, or None if unknown
@return metadata (lsst.daf.base.PropertyList)
"""
md = PropertyList()
md.add('RA', coord.getRa().asDegrees(), 'field center in degrees')
md.add('DEC', coord.getDec().asDegrees(), 'field center in degrees')
md.add('RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
md.add('SMATCHV', 1, 'SourceMatchVector version number')
filterName = "UNKNOWN" if filterName is None else str(filterName)
md.add('FILTER', filterName, 'filter name for photometric data')
return md
def createMatchMetadata(exposure):
"""Create metadata required for unpersisting a match list
@param[in] exposure exposure for which to create metadata
@return metadata about the field (a daf_base PropertyList)
"""
matchMeta = PropertyList()
bboxd = Box2D(exposure.getBBox())
ctrPos = bboxd.getCenter()
wcs = getDistortedWcs(exposure.getInfo())
ctrCoord = wcs.pixelToSky(ctrPos).toIcrs()
llCoord = wcs.pixelToSky(bboxd.getMin())
approxRadius = ctrCoord.angularSeparation(llCoord)
matchMeta.add('RA', ctrCoord.getRa().asDegrees(), 'field center in degrees')
matchMeta.add('DEC', ctrCoord.getDec().asDegrees(), 'field center in degrees')
matchMeta.add('RADIUS', approxRadius.asDegrees(), 'field radius in degrees, approximate')
matchMeta.add('SMATCHV', 1, 'SourceMatchVector version number')
filterName = exposure.getFilter().getName() or None
if filterName is not None and filterName not in ("_unknmown_", ""):
matchMeta.add('FILTER', filterName, 'filter name for tagalong data')
return matchMeta
def setUp(self):
"""Constructs a CCD with two amplifiers and prepares for ISR"""
np.random.seed(12345)
baseValue = 100.0
gain = 1.0
readNoise = 123456789.0
saturation = 987654321.0
height = 234
imageSize = Extent2I(123, height)
overscanSize = Extent2I(16, height)
self.sigma = 1.234
# Set up the various regions
overscan1 = Box2I(Point2I(0, 0), overscanSize)
image1 = Box2I(Point2I(overscanSize[0], 0), imageSize)
image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize)
overscan2 = Box2I(Point2I(overscanSize[0] + 2*imageSize[0], 0), overscanSize)
leftBox = Box2I(overscan1.getMin(), Extent2I(overscan1.getWidth() + image1.getWidth(), height))
rightBox = Box2I(image2.getMin(), Extent2I(image2.getWidth() + overscan2.getWidth(), height))
target1 = Box2I(Point2I(0, 0), imageSize)
target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize)
# Set the pixels
exposure = ExposureF(Box2I(Point2I(0, 0), Extent2I(imageSize[0]*2 + overscanSize[0]*2, height)))
yy = np.arange(0, height, 1, dtype=np.float32)
leftImage = ExposureF(exposure, leftBox)
leftImage.image.array[:] = baseValue + yy[:, np.newaxis]
rightImage = ExposureF(exposure, rightBox)
rightImage.image.array[:] = baseValue - yy[:, np.newaxis]
leftOverscan = ExposureF(exposure, overscan1)
leftOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
rightOverscan = ExposureF(exposure, overscan2)
rightOverscan.image.array += np.random.normal(0.0, self.sigma, leftOverscan.image.array.shape)
exposure.mask.array[:] = 0.0
exposure.variance.array[:] = np.nan
# Construct the detectors
amps = AmpInfoCatalog(AmpInfoTable.makeMinimalSchema())
makeAmplifier(amps, "left", target1, image1, overscan1, gain, readNoise, saturation)
makeAmplifier(amps, "right", target2, image2, overscan2, gain, readNoise, saturation)
ccdBox = Box2I(Point2I(0, 0), Extent2I(image1.getWidth() + image2.getWidth(), height))
ccd = Detector("detector", 1, SCIENCE, "det1", ccdBox, amps, Orientation(), Extent2D(1.0, 1.0), {})
exposure.setDetector(ccd)
header = PropertyList()
header.add("EXPTIME", 0.0)
exposure.getInfo().setVisitInfo(VisitInfo(header))
self.exposure = exposure
self.config = IsrTask.ConfigClass()
# Disable everything we don't care about
self.config.doBias = False
self.config.doDark = False
self.config.doFlat = False
self.config.doFringe = False
self.config.doDefect = False
self.config.doAddDistortionModel = False
self.config.doWrite = False
self.config.expectWcs = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doBrighterFatter = False
self.config.doAttachTransmissionCurve = False
# Set the things that match our test setup
self.config.overscanFitType = "CHEB"
self.config.overscanOrder = 1
self.config.doEmpiricalReadNoise = True
self.task = IsrTask(config=self.config)
def makePropertyListFromDict(md):
result = PropertyList()
for k, v in md.items():
result.add(k, v)
return result
def makePropertyListFromDict(md):
result = PropertyList()
for k, v in md.items():
result.add(k, v)
return result
def setUp(self):
"""Constructs a CCD with two amplifiers and prepares for ISR"""
np.random.seed(12345)
baseValue = 100.0
gain = 1.0
readNoise = 123456789.0
saturation = 987654321.0
height = 234
imageSize = Extent2I(123, height)
overscanSize = Extent2I(16, height)
self.sigma = 1.234
# Set up the various regions
overscan1 = Box2I(Point2I(0, 0), overscanSize)
image1 = Box2I(Point2I(overscanSize[0], 0), imageSize)
image2 = Box2I(Point2I(overscanSize[0] + imageSize[0], 0), imageSize)
overscan2 = Box2I(Point2I(overscanSize[0] + 2 * imageSize[0], 0),
overscanSize)
leftBox = Box2I(
overscan1.getMin(),
Extent2I(overscan1.getWidth() + image1.getWidth(), height))
rightBox = Box2I(
image2.getMin(),
Extent2I(image2.getWidth() + overscan2.getWidth(), height))
target1 = Box2I(Point2I(0, 0), imageSize)
target2 = Box2I(Point2I(image1.getWidth(), 0), imageSize)
# Set the pixels
exposure = ExposureF(
Box2I(Point2I(0, 0),
Extent2I(imageSize[0] * 2 + overscanSize[0] * 2, height)))
yy = np.arange(0, height, 1, dtype=np.float32)
leftImage = ExposureF(exposure, leftBox)
leftImage.image.array[:] = baseValue + yy[:, np.newaxis]
rightImage = ExposureF(exposure, rightBox)
rightImage.image.array[:] = baseValue - yy[:, np.newaxis]
leftOverscan = ExposureF(exposure, overscan1)
leftOverscan.image.array += np.random.normal(
0.0, self.sigma, leftOverscan.image.array.shape)
rightOverscan = ExposureF(exposure, overscan2)
rightOverscan.image.array += np.random.normal(
0.0, self.sigma, leftOverscan.image.array.shape)
exposure.mask.array[:] = 0.0
exposure.variance.array[:] = np.nan
# Construct the detectors
amp1 = makeAmplifier("left", target1, image1, overscan1, gain,
readNoise, saturation)
amp2 = makeAmplifier("right", target2, image2, overscan2, gain,
readNoise, saturation)
ccdBox = Box2I(Point2I(0, 0),
Extent2I(image1.getWidth() + image2.getWidth(), height))
camBuilder = cameraGeom.Camera.Builder("fakeCam")
detBuilder = camBuilder.add("detector", 1)
detBuilder.setSerial("det1")
detBuilder.setBBox(ccdBox)
detBuilder.setPixelSize(Extent2D(1.0, 1.0))
detBuilder.setOrientation(cameraGeom.Orientation())
detBuilder.append(amp1)
detBuilder.append(amp2)
cam = camBuilder.finish()
exposure.setDetector(cam.get('detector'))
header = PropertyList()
header.add("EXPTIME", 0.0)
exposure.getInfo().setVisitInfo(VisitInfo(header))
self.exposure = exposure
self.config = IsrTask.ConfigClass()
# Disable everything we don't care about
self.config.doBias = False
self.config.doDark = False
self.config.doFlat = False
self.config.doFringe = False
self.config.doDefect = False
self.config.doWrite = False
self.config.expectWcs = False
self.config.doLinearize = False
self.config.doCrosstalk = False
self.config.doBrighterFatter = False
self.config.doAttachTransmissionCurve = False
self.config.doAssembleCcd = False
self.config.doNanMasking = False
self.config.doInterpolate = False
self.config.maskNegativeVariance = False # This runs on mocks.
# Set the things that match our test setup
self.config.overscan.fitType = "CHEB"
self.config.overscan.order = 1
self.config.doEmpiricalReadNoise = True
self.task = IsrTask(config=self.config)
