def testConvertTiffFloatPixels(tmpdir):
imagePath = datastore.fetch('d042-353.crop.small.float32.tif')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath)
info = tifftools.read_tiff(outputPath)
assert (info['ifds'][0]['tags'][tifftools.Tag.SampleFormat.value]['data'][0] ==
tifftools.constants.SampleFormat.uint.value)
def testConvertFromLargeImage(tmpdir):
imagePath = datastore.fetch('sample_image.jp2')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath)
source = large_image_source_tiff.open(outputPath)
metadata = source.getMetadata()
assert metadata['levels'] == 6
def testConvertToAperio(tmpdir):
imagePath = datastore.fetch('huron.image2_jpeg2k.tif')
outputPath = os.path.join(tmpdir, 'out.svs')
large_image_converter.convert(imagePath, outputPath, format='aperio')
source = large_image.open(outputPath)
assert 'openslide' in source.name
assert 'label' in source.getAssociatedImagesList()
def testConvert(tmpdir, convert_args, taglist):
testDir = os.path.dirname(os.path.realpath(__file__))
imagePath = os.path.join(testDir, 'test_files', 'yb10kx5k.png')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath, **convert_args)
info = tifftools.read_tiff(outputPath)
for key, value in taglist.items():
assert info['ifds'][0]['tags'][key]['data'][0] == value
def testConvertOMETif(tmpdir):
imagePath = datastore.fetch('sample.ome.tif')
outputPath = os.path.join(tmpdir, 'out.tiff')
# Note: change this when we convert multi-frame files differently
large_image_converter.convert(imagePath, outputPath)
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 3
assert len(info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value]['ifds']) == 4
def testConvertOverwrite(tmpdir):
testDir = os.path.dirname(os.path.realpath(__file__))
imagePath = os.path.join(testDir, 'test_files', 'yb10kx5k.png')
outputPath = os.path.join(tmpdir, 'out.tiff')
open(outputPath, 'w').write('placeholder')
with pytest.raises(Exception):
large_image_converter.convert(imagePath, outputPath)
large_image_converter.convert(imagePath, outputPath, overwrite=True)
assert os.path.getsize(outputPath) > 100
def testConvertMultiframeToAperio(tmpdir):
imagePath = datastore.fetch('sample.ome.tif')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath,
outputPath,
format='aperio',
compression='jp2k')
source = large_image.open(outputPath)
assert 'label' in source.getAssociatedImagesList()
def testConvertFromMultiframeImageOnlyOneFrame(tmpdir):
imagePath = datastore.fetch('sample.ome.tif')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath, onlyFrame=2)
source = large_image_source_tiff.open(outputPath)
metadata = source.getMetadata()
assert metadata['levels'] == 5
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 5
def testConvertPTIF(tmpdir):
imagePath = datastore.fetch('sample_image.ptif')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath,
outputPath,
compression='jpeg',
quality=50)
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 11
def testConvertFromTestSourceFrames(tmpdir):
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert('large_image://test?maxLevel=3&frames=4', outputPath)
source = large_image_source_tiff.open(outputPath)
metadata = source.getMetadata()
assert metadata['levels'] == 4
assert len(metadata['frames']) == 4
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 4
def testConvertFromMultiframeImageNoSubIFDS(tmpdir):
imagePath = datastore.fetch('sample.ome.tif')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath, subifds=False)
source = large_image_source_tiff.open(outputPath)
metadata = source.getMetadata()
assert metadata['levels'] == 5
assert len(metadata['frames']) == 3
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 15
def convert_to_cog(cog):
"""Populate ConvertedImage with COG file."""
if not isinstance(cog, ConvertedImage):
cog = ConvertedImage.objects.get(id=cog)
else:
cog.refresh_from_db()
with _processed_image_helper(cog) as (image, output):
with input_output_path_helper(image.file,
output.file,
prefix='cog_',
vsi=True) as (
input_path,
output_path,
):
large_image_converter.convert(str(input_path), str(output_path))
def testConvertGeospatial(tmpdir):
testDir = os.path.dirname(os.path.realpath(__file__))
imagePath = os.path.join(testDir, 'test_files', 'rgb_geotiff.tiff')
inputPath = os.path.join(tmpdir, 'in.geo.tiff')
shutil.copy(imagePath, inputPath)
outputPath = large_image_converter.convert(inputPath, level=5)
assert 'geo.tiff' in outputPath
assert outputPath != inputPath
info = tifftools.read_tiff(outputPath)
assert tifftools.Tag.ModelTiepointTag.value in info['ifds'][0]['tags']
def testConvertJp2kCompression(tmpdir):
imagePath = datastore.fetch('sample_Easy1.png')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath, compression='jp2k')
info = tifftools.read_tiff(outputPath)
assert (info['ifds'][0]['tags'][tifftools.Tag.Compression.value]['data'][0]
== tifftools.constants.Compression.JP2000.value)
source = large_image_source_tiff.open(outputPath)
image, _ = source.getRegion(output={
'maxWidth': 200,
'maxHeight': 200
},
format=constants.TILE_FORMAT_NUMPY)
assert (image[12][167] == [215, 135, 172]).all()
outputPath2 = os.path.join(tmpdir, 'out2.tiff')
large_image_converter.convert(imagePath,
outputPath2,
compression='jp2k',
psnr=50)
assert os.path.getsize(outputPath2) < os.path.getsize(outputPath)
outputPath3 = os.path.join(tmpdir, 'out3.tiff')
large_image_converter.convert(imagePath,
outputPath3,
compression='jp2k',
cr=100)
assert os.path.getsize(outputPath3) < os.path.getsize(outputPath)
assert os.path.getsize(outputPath3) != os.path.getsize(outputPath2)
def main(args=sys.argv[1:]):
parser = get_parser()
opts = parser.parse_args(args=args)
logger = logging.getLogger('large-image-converter')
if not len(logger.handlers):
logger.addHandler(logging.StreamHandler(sys.stderr))
logger.setLevel(max(1,
logging.WARNING - (opts.verbose - opts.silent) * 10))
try:
import large_image
li_logger = large_image.config.getConfig('logger')
li_logger.setLevel(
max(1, logging.CRITICAL - (opts.verbose - opts.silent) * 10))
except ImportError:
pass
logger.debug('Command line options: %r' % opts)
if not os.path.isfile(
opts.source) and not opts.source.startswith('large_image://test'):
logger.error('Source is not a file (%s)', opts.source)
return 1
if opts.compression == 'zip':
opts.compression = 'deflate'
converterOpts = {
k: v
for k, v in vars(opts).items()
if k not in {'source', 'dest', 'verbose', 'silent'} and v is not None
}
start_time = time.time()
dest = large_image_converter.convert(opts.source, opts.dest,
**converterOpts)
end_time = time.time()
if not os.path.isfile(dest):
logger.error('Failed to generate file')
return 1
logger.info('Created %s, %d bytes, %3.1f s', dest, os.path.getsize(dest),
end_time - start_time)
if opts._stats:
import json
import tifftools.commands
info = tifftools.read_tiff(dest)
try:
desc = json.loads(info['ifds'][0]['tags'][
tifftools.Tag.ImageDescription.value]['data'])
except Exception:
logger.debug('Cannot generate statistics.')
return
desc['large_image_converter']['conversion_stats'] = {
'time':
end_time - start_time,
'filesize':
os.path.getsize(dest),
'original_filesize':
os.path.getsize(opts.source),
'compression_ratio':
desc['large_image_converter'].get('frames', 1) *
sum(info['ifds'][0]['tags'][tifftools.Tag.BitsPerSample.value]
['data']) / 8 *
info['ifds'][0]['tags'][tifftools.Tag.ImageWidth.value]['data'][0]
*
info['ifds'][0]['tags'][tifftools.Tag.ImageLength.value]['data'][0]
/ os.path.getsize(dest),
}
if opts._stats == 'full' and opts.compression not in {
'deflate', 'zip', 'lzw', 'zstd', 'packbits', 'none'
}:
compute_error_metrics(
opts.source, dest,
desc['large_image_converter']['conversion_stats'],
converterOpts)
tifftools.commands.tiff_set(
dest,
overwrite=True,
setlist=[('ImageDescription',
json.dumps(desc,
separators=(',', ':'),
sort_keys=True,
default=large_image_converter.json_serial))])
def compute_error_metrics(original, altered, results, converterOpts=None):
"""
Compute the amount of error introduced via conversion compared to
conversion using a lossless method. Note that this is not compared to the
original, as we may not be able to read that in an efficient way. This is
a very time-consuming way to compute error metrics, since it first
reprocesses the input file to a lossless format, then steps through each
tile and computes RSME and SSIM errors per tile, producing a weighted-by-
number-of-pixels of each of these. The RSME is used to compute a PSNR
value.
:param original: the original file path.
:param altered: the path of compressed file to compare.
:param results: a dictionary to store results. Modified.
:param converterOpts: an optional dictionary of parameters used for the
original conversion. Only parameters that would affect the selected
pixels are used.
"""
import math
from tempfile import TemporaryDirectory
import large_image_source_tiff
import numpy
import packaging
import skimage.metrics
lastlog = 0
with TemporaryDirectory() as tempDir:
# TODO: check if the original is geospatial; if so appropriate options
tempPath = os.path.join(tempDir, os.path.basename(original) + '.tiff')
orig = large_image_converter.convert(original,
tempPath,
compression='lzw')
tsOrig = large_image_source_tiff.open(orig)
numFrames = len(tsOrig.getMetadata().get('frames', [0]))
tsAlt = large_image_source_tiff.open(altered)
mse = 0
ssim = 0
ssim_count = 0
maxval = 0
maxdiff = 0
sum = 0
count = 0
tileSize = 2048
for frame in range(numFrames):
tiAlt = tsAlt.tileIterator(tile_size=dict(width=tileSize),
frame=frame)
for tileOrig in tsOrig.tileIterator(tile_size=dict(width=tileSize),
frame=frame):
tileAlt = next(tiAlt)
do = tileOrig['tile']
da = tileAlt['tile']
if do.dtype != da.dtype and da.dtype == numpy.uint8:
da = da.astype(int) * 257
do = do.astype(int)
da = da.astype(int)
maxval = max(maxval, do.max(), da.max())
if do.shape[2] > da.shape[2]:
do = do[:, :, :da.shape[2]]
if da.shape[2] > do.shape[2]:
da = da[:, :, :do.shape[2]]
diff = numpy.absolute(do - da)
maxdiff = max(maxdiff, diff.max())
sum += diff.sum()
count += diff.size
last_mse = numpy.mean(diff**2)
mse += last_mse * diff.size
last_ssim = 0
try:
kwargs = {}
if (packaging.version.parse(skimage.__version__) >=
packaging.version.parse('0.19')):
kwargs['channel_axis'] = 2 if len(
do.shape) > 2 else None
else:
kwargs['multichannel'] = len(do.shape) > 2
last_ssim = skimage.metrics.structural_similarity(
do.astype(float),
da.astype(float),
data_range=255
if tileOrig['tile'].dtype == numpy.uint8 else 65535,
gaussian_weights=True,
sigma=1.5,
use_sample_covariance=False,
**kwargs)
ssim += last_ssim * diff.size
ssim_count += diff.size
except ValueError:
pass
if time.time() - lastlog >= 10 and ssim_count:
logger.debug(
'Calculating error (%d/%d): rmse %4.2f ssim %6.4f '
'last rmse %4.2f ssim %6.4f' %
(tileOrig['tile_position']['position'] + 1 +
tileOrig['iterator_range']['position'] * frame,
tileOrig['iterator_range']['position'] * numFrames,
(mse / count)**0.5, ssim / ssim_count, last_mse**
0.5, last_ssim))
lastlog = time.time()
results['maximum_error'] = maxdiff
results['average_error'] = sum / count
results['rmse'] = (mse / count)**0.5
results['psnr'] = 10 * math.log10(maxval**2 /
(mse / count)) if mse else None
if ssim_count:
results['ssim'] = ssim / ssim_count
logger.debug(
'Calculated error: rmse %4.2f psnr %3.1f ssim %6.4f' %
(results['rmse'], results['psnr'] or 0, results['ssim']))
def create_tiff(self,
inputFile,
outputName=None,
outputDir=None,
quality=90,
tileSize=256,
**kwargs):
"""
Take a source input file, readable by vips, and output a pyramidal tiff
file.
:param inputFile: the path to the input file or base file of a set.
:param outputName: the name of the output file. If None, the name is
based on the input name and current date and time. May be a full path.
:param outputDir: the location to store the output. If unspecified, the
inputFile's directory is used. If the outputName is a fully qualified
path, this is ignored.
:param quality: a jpeg quality passed to vips. 0 is small, 100 is high
quality. 90 or above is recommended.
:param tileSize: the horizontal and vertical tile size.
Optional parameters that can be specified in kwargs:
:param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits', or
'zstd'.
:param level: compression level for zstd, 1-22 (default is 10).
:param predictor: one of 'none', 'horizontal', or 'float' used for lzw and
deflate.
:param inputName: if no output name is specified, and this is specified,
this is used as the basis of the output name instead of extracting the
name from the inputFile path.
:returns: output path.
"""
import large_image_converter
logger = logging.getLogger('large-image-converter')
if not len(logger.handlers):
logger.addHandler(logging.StreamHandler(sys.stdout))
if not logger.level:
logger.setLevel(logging.INFO)
if '_concurrency' not in kwargs:
kwargs['_concurrency'] = -2
inputPath = os.path.abspath(os.path.expanduser(inputFile))
geospatial = large_image_converter.is_geospatial(inputPath)
inputName = kwargs.get('inputName', os.path.basename(inputPath))
suffix = large_image_converter.format_hook('adjust_params', geospatial,
kwargs, **kwargs)
suffix = suffix or ('.tiff' if not geospatial else '.geo.tiff')
if not outputName:
outputName = os.path.splitext(inputName)[0] + suffix
if outputName.endswith('.geo' + suffix):
outputName = outputName[:len(outputName) - len(suffix) -
4] + suffix
if outputName == inputName:
outputName = (os.path.splitext(inputName)[0] + '.' +
time.strftime('%Y%m%d-%H%M%S') + suffix)
renameOutput = outputName
if not outputName.endswith(suffix):
outputName += suffix
if not outputDir:
outputDir = os.path.dirname(inputPath)
outputPath = os.path.join(outputDir, outputName)
large_image_converter.convert(inputPath,
outputPath,
quality=quality,
tileSize=tileSize,
**kwargs)
if not os.path.exists(outputPath):
raise Exception('Conversion command failed to produce output')
if renameOutput != outputName:
renamePath = os.path.join(outputDir, renameOutput)
shutil.move(outputPath, renamePath)
outputPath = renamePath
logger.info('Created a file of size %d' % os.path.getsize(outputPath))
return outputPath
def testConverterMissingTiles(tmpdir):
imagePath = datastore.fetch('one_layer_missing_tiles.tiff')
outputPath = os.path.join(tmpdir, 'out.tiff')
large_image_converter.convert(imagePath, outputPath)
info = tifftools.read_tiff(outputPath)
assert len(info['ifds']) == 6
