def testLossyFloatOurs(self):
"""Test lossy compression of floating-point images ourselves
We do lossy compression by scaling first. We have full control over
the scaling (multiple scaling algorithms), and we have access to our
own masks when we do statistics.
"""
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
algorithmList = ("GZIP", "GZIP_SHUFFLE", "RICE")
bitpixList = (16, 32)
quantizeList = (4.0, 10.0)
for cls, algorithm, bitpix, quantize in itertools.product(
classList, algorithmList, bitpixList, quantizeList):
compression = ImageCompressionOptions(
lsst.afw.fits.compressionAlgorithmFromString(algorithm),
quantizeLevel=0.0)
scaling = ImageScalingOptions(ImageScalingOptions.STDEV_BOTH,
bitpix,
quantizeLevel=quantize,
fuzz=True)
image = self.makeImage(cls)
self.checkCompressedImage(cls,
image,
compression,
scaling,
atol=self.noise / quantize)
def checkRange(self, ImageClass, bitpix):
"""Check that the RANGE scaling works
Parameters
----------
ImageClass : `type`, an `lsst.afw.image.Image` class
Class of image to create.
bitpix : `int`
Bits per pixel for FITS image.
"""
scaling = ImageScalingOptions(ImageScalingOptions.RANGE,
bitpix, [u"BAD"],
fuzz=False)
original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(
ImageClass, scaling, False)
numValues = 2**bitpix - 1
numValues -= 2 # Padding on either end
if bitpix == 32:
numValues -= 10
bscaleExpect = (self.highValue - self.lowValue) / numValues
self.assertFloatsAlmostEqual(bscale, bscaleExpect,
atol=1.0e-6) # F32 resolution
rtol = 1.0 / 2**(bitpix - 1)
self.checkSpecialPixels(original,
unpersisted,
maxValue,
minValue,
atol=bscale)
self.assertImagesAlmostEqual(original, unpersisted, rtol=rtol)
def testQuantization(self):
"""Test that our quantization produces the same values as cfitsio
Our quantization is more configurable (e.g., choice of scaling algorithm,
specifying mask planes) and extensible (logarithmic, asinh scalings)
than cfitsio's. However, cfitsio uses its own fuzz ("subtractive dithering")
when reading the data, so if we don't want to add random values twice,
we need to be sure that we're using the same random values. To check that,
we write one image with our scaling+compression, and one with cfitsio's
compression using exactly the BSCALE and dither seed we used for our own.
That way, the two codes will quantize independently, and we can compare
the results.
"""
bscaleSet = 1.0
bzeroSet = self.background - 10 * self.noise
algorithm = ImageCompressionOptions.GZIP
classList = (lsst.afw.image.ImageF, lsst.afw.image.ImageD)
tilesList = ((4, 5), (0, 0), (0, 5), (4, 0), (0, 1))
for cls, tiles in itertools.product(classList, tilesList):
tiles = np.array(tiles, dtype=np.int64)
compression = ImageCompressionOptions(algorithm, tiles, -bscaleSet)
original = self.makeImage(cls)
with lsst.utils.tests.getTempFilePath(self.extension) as filename:
with lsst.afw.fits.Fits(filename, "w") as fits:
options = lsst.afw.fits.ImageWriteOptions(compression)
original.writeFits(fits, options)
cfitsio = cls(filename)
header = lsst.afw.fits.readMetadata(filename, 1)
seed = header.getScalar("ZDITHER0")
self.assertEqual(header.getScalar("BSCALE"), bscaleSet)
compression = ImageCompressionOptions(algorithm, tiles, 0.0)
scaling = ImageScalingOptions(ImageScalingOptions.MANUAL,
32, [u"BAD"],
bscale=bscaleSet,
bzero=bzeroSet,
fuzz=True,
seed=seed)
unpersisted = self.checkCompressedImage(cls,
original,
compression,
scaling,
atol=bscaleSet)
oursDiff = unpersisted.getArray() - original.getArray()
cfitsioDiff = cfitsio.getArray() - original.getArray()
self.assertImagesAlmostEqual(oursDiff, cfitsioDiff, atol=0.0)
def doRoundTrip(self, image, compression=None, scaling=None):
if compression is None:
compression = dict(algorithm=ImageCompressionOptions.NONE)
if scaling is None:
scaling = dict(algorithm=ImageScalingOptions.NONE, bitpix=0)
options = ImageWriteOptions(compression=ImageCompressionOptions(**compression),
scaling=ImageScalingOptions(**scaling))
isCompressed = (compression.get("algorithm", ImageCompressionOptions.NONE)
!= ImageCompressionOptions.NONE)
with lsst.utils.tests.getTempFilePath(f"_{type(image).__name__}.fits") as filename:
image.writeFits(filename, options=options)
readImage = type(image)(filename)
with astropy.io.fits.open(filename) as hduList:
hdu = hduList[1 if isCompressed else 0]
if hdu.data.dtype.byteorder != '=':
hdu.data = hdu.data.byteswap().newbyteorder()
return readImage, hdu
def checkNone(self, ImageClass, bitpix):
"""Check that the NONE scaling algorithm works
Parameters
----------
ImageClass : `type`, an `lsst.afw.image.Image` class
Class of image to create.
bitpix : `int`
Bits per pixel for FITS image.
"""
scaling = ImageScalingOptions(ImageScalingOptions.NONE,
bitpix, [u"BAD"],
fuzz=False)
original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(
ImageClass, scaling)
self.assertFloatsAlmostEqual(bscale, 1.0, atol=0.0)
self.assertFloatsAlmostEqual(bzero, 0.0, atol=0.0)
self.assertImagesAlmostEqual(original, unpersisted, atol=0.0)
def checkManual(self, ImageClass, bitpix):
"""Check that the MANUAL scaling algorithm works
Parameters
----------
ImageClass : `type`, an `lsst.afw.image.Image` class
Class of image to create.
bitpix : `int`
Bits per pixel for FITS image.
"""
bscaleSet = 1.2345
bzeroSet = self.base
scaling = ImageScalingOptions(ImageScalingOptions.MANUAL,
bitpix, [u"BAD"],
bscale=bscaleSet,
bzero=bzeroSet,
fuzz=False)
original, unpersisted, bscale, bzero, minValue, maxValue = self.makeImage(
ImageClass, scaling)
self.assertFloatsAlmostEqual(bscale, bscaleSet, atol=0.0)
self.assertFloatsAlmostEqual(bzero, bzeroSet, atol=0.0)
self.assertImagesAlmostEqual(original, unpersisted, atol=bscale)