def checkEmptyExposure(self, algorithm):
"""Check that we can persist an empty Exposure
Parameters
----------
algorithm : `lsst.afw.fits.ImageCompressionOptions.CompressionAlgorithm`
Compression algorithm to try.
"""
exp = lsst.afw.image.ExposureF(0, 0)
degrees = lsst.geom.degrees
cdMatrix = np.array([[1.0e-4, 0.0], [0.0, 1.0e-4]], dtype=float)
exp.setWcs(
lsst.afw.geom.makeSkyWcs(crval=lsst.geom.SpherePoint(
0 * degrees, 0 * degrees),
crpix=lsst.geom.Point2D(0.0, 0.0),
cdMatrix=cdMatrix))
imageOptions = lsst.afw.fits.ImageWriteOptions(
ImageCompressionOptions(algorithm))
maskOptions = lsst.afw.fits.ImageWriteOptions(
exp.getMaskedImage().getMask())
varianceOptions = lsst.afw.fits.ImageWriteOptions(
ImageCompressionOptions(algorithm))
with lsst.utils.tests.getTempFilePath(".fits") as filename:
exp.writeFits(filename, imageOptions, maskOptions, varianceOptions)
unpersisted = type(exp)(filename)
self.assertEqual(unpersisted.getMaskedImage().getDimensions(),
lsst.geom.Extent2I(0, 0))
self.assertEqual(unpersisted.getWcs(), exp.getWcs())
def testLossyFloatOurs(self):
"""Test lossy compression of floating-point images ourselves
We do lossy compression by scaling first. We have full control over
the scaling (multiple scaling algorithms), and we have access to our
own masks when we do statistics.
"""
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
bitpixList = (16, 32)
quantizeList = (4.0, 10.0)
for cls, algorithm, bitpix, quantize in itertools.product(
classList, algorithmList, bitpixList, quantizeList):
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm),
quantizeLevel=0.0)
scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH,
bitpix,
quantizeLevel=quantize,
fuzz=True)
image = self.makeImage(cls)
self.checkCompressedImage(cls,
image,
compression,
scaling,
atol=self.noise / quantize)
def reduceToFits(obj):
"""Pickle to FITS
Intended to be used by the ``__reduce__`` method of a class.
Parameters
----------
obj
any object with a ``writeFits`` method taking a
`~lsst.afw.fits.MemFileManager` and possibly an
`~lsst.afw.fits.ImageWriteOptions`.
Returns
-------
reduced : `tuple` [callable, `tuple`]
a tuple in the format returned by `~object.__reduce__`
"""
manager = MemFileManager()
options = ImageWriteOptions(ImageCompressionOptions(ImageCompressionOptions.NONE))
table = getattr(obj, 'table', None)
if isinstance(table, lsst.afw.table.BaseTable):
# table objects don't take `options`
obj.writeFits(manager)
else:
# MaskedImage and Exposure both require options for each plane (image, mask, variance)
if isinstance(obj, (lsst.afw.image.MaskedImage, lsst.afw.image.Exposure)):
obj.writeFits(manager, options, options, options)
else:
obj.writeFits(manager, options)
size = manager.getLength()
data = manager.getData()
return (unreduceFromFits, (obj.__class__, data, size))
def testQuantization(self):
"""Test that our quantization produces the same values as cfitsio
Our quantization is more configurable (e.g., choice of scaling algorithm,
specifying mask planes) and extensible (logarithmic, asinh scalings)
than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering")
when reading the data, so if we don't want to add random values twice,
we need to be sure that we're using the same random values. To check that,
we write one image with our scaling+compression, and one with cfitsio's
compression using exactly the BSCALE and dither seed we used for our own.
That way, the two codes will quantize independently, and we can compare
the results.
"""
bscaleSet = 1.0
bzeroSet = self.background - 10 * self.noise
algorithm = ImageCompressionOptions.GZIP
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1))
for cls, tiles in itertools.product(classList, tilesList):
tiles = np.array(tiles, dtype=np.int64)
compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet)
original = self.makeImage(cls)
with lsst.utils.tests.getTempFilePath(self.extension) as filename:
with lsst.afw.fits.Fits(filename, "w") as fits:
options = lsst.afw.fits.ImageWriteOptions(compression)
original.writeFits(fits, options)
cfitsio = cls(filename)
header = lsst.afw.fits.readMetadata(filename, 1)
seed = header.getScalar("ZDITHER0")
self.assertEqual(header.getScalar("BSCALE"), bscaleSet)
compression = ImageCompressionOptions(algorithm, tiles, 0.0)
scaling = ImageScalingOptions(ImageScalingOptions.MANUAL,
32, [u"BAD"],
bscale=bscaleSet,
bzero=bzeroSet,
fuzz=True,
seed=seed)
unpersisted = self.checkCompressedImage(cls,
original,
compression,
scaling,
atol=bscaleSet)
oursDiff = unpersisted.getArray() - original.getArray()
cfitsioDiff = cfitsio.getArray() - original.getArray()
self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0)
def testLosslessFloat(self):
"""Test lossless compression of floating-point image"""
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
algorithmList = ("GZIP", "GZIP_SHUFFLE"
) # Lossless float compression requires GZIP
for cls, algorithm in itertools.product(classList, algorithmList):
image = self.makeImage(cls)
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm))
self.checkCompressedImage(cls, image, compression, atol=0.0)
def testLosslessInt(self):
"""Test lossless compression of integer image
We deliberately don't test `lsst.afw.image.ImageL` because
compression of LONGLONG images is unsupported by cfitsio.
"""
classList = (lsst.afw.image.ImageU, lsst.afw.image.ImageI)
algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
for cls, algorithm in itertools.product(classList, algorithmList):
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm))
image = self.makeImage(cls)
self.checkCompressedImage(cls, image, compression, atol=0.0)
def testLongLong(self):
"""Test graceful failure when compressing ImageL
We deliberately don't test `lsst.afw.image.ImageL` because
compression of LONGLONG images is unsupported by cfitsio.
"""
algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
for algorithm in algorithmList:
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm))
cls = lsst.afw.image.ImageL
image = self.makeImage(cls)
with self.assertRaises(lsst.afw.fits.FitsError):
self.checkCompressedImage(cls, image, compression)
def doRoundTrip(self, image, compression=None, scaling=None):
if compression is None:
compression = dict(algorithm=ImageCompressionOptions.NONE)
if scaling is None:
scaling = dict(algorithm=ImageScalingOptions.NONE, bitpix=0)
options = ImageWriteOptions(compression=ImageCompressionOptions(**compression),
scaling=ImageScalingOptions(**scaling))
isCompressed = (compression.get("algorithm", ImageCompressionOptions.NONE)
!= ImageCompressionOptions.NONE)
with lsst.utils.tests.getTempFilePath(f"_{type(image).__name__}.fits") as filename:
image.writeFits(filename, options=options)
readImage = type(image)(filename)
with astropy.io.fits.open(filename) as hduList:
hdu = hduList[1 if isCompressed else 0]
if hdu.data.dtype.byteorder != '=':
hdu.data = hdu.data.byteswap().newbyteorder()
return readImage, hdu
def testMask(self):
"""Test compression of mask
We deliberately don't test PLIO compression (which is designed for
masks) because our default mask type (32) has too much dynamic range
for PLIO (limit of 24 bits).
"""
for algorithm in ("GZIP", "GZIP_SHUFFLE", "RICE"):
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm))
mask = self.makeMask()
unpersisted = self.checkCompressedImage(lsst.afw.image.Mask,
mask,
compression,
atol=0.0)
for mp in mask.getMaskPlaneDict():
self.assertIn(mp, unpersisted.getMaskPlaneDict())
unpersisted.getPlaneBitMask(mp)
def testLossyFloatCfitsio(self):
"""Test lossy compresion of floating-point images with cfitsio
cfitsio does the compression, controlled through the 'quantizeLevel'
parameter. Note that cfitsio doesn't have access to our masks when
it does its statistics.
"""
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
quantizeList = (4.0, 10.0)
for cls, algorithm, quantizeLevel in itertools.product(
classList, algorithmList, quantizeList):
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm),
quantizeLevel=quantizeLevel)
image = self.makeImage(cls)
self.checkCompressedImage(cls,
image,
compression,
atol=self.noise / quantizeLevel)