def testConvertTiffFloatPixels(tmpdir): imagePath = datastore.fetch('d042-353.crop.small.float32.tif') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath) info = tifftools.read_tiff(outputPath) assert (info['ifds'][0]['tags'][tifftools.Tag.SampleFormat.value]['data'][0] == tifftools.constants.SampleFormat.uint.value)
def testConvertFromLargeImage(tmpdir): imagePath = datastore.fetch('sample_image.jp2') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath) source = large_image_source_tiff.open(outputPath) metadata = source.getMetadata() assert metadata['levels'] == 6
def testConvertToAperio(tmpdir): imagePath = datastore.fetch('huron.image2_jpeg2k.tif') outputPath = os.path.join(tmpdir, 'out.svs') large_image_converter.convert(imagePath, outputPath, format='aperio') source = large_image.open(outputPath) assert 'openslide' in source.name assert 'label' in source.getAssociatedImagesList()
def testConvert(tmpdir, convert_args, taglist): testDir = os.path.dirname(os.path.realpath(__file__)) imagePath = os.path.join(testDir, 'test_files', 'yb10kx5k.png') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, **convert_args) info = tifftools.read_tiff(outputPath) for key, value in taglist.items(): assert info['ifds'][0]['tags'][key]['data'][0] == value
def testConvertOMETif(tmpdir): imagePath = datastore.fetch('sample.ome.tif') outputPath = os.path.join(tmpdir, 'out.tiff') # Note: change this when we convert multi-frame files differently large_image_converter.convert(imagePath, outputPath) info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 3 assert len(info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value]['ifds']) == 4
def testConvertOverwrite(tmpdir): testDir = os.path.dirname(os.path.realpath(__file__)) imagePath = os.path.join(testDir, 'test_files', 'yb10kx5k.png') outputPath = os.path.join(tmpdir, 'out.tiff') open(outputPath, 'w').write('placeholder') with pytest.raises(Exception): large_image_converter.convert(imagePath, outputPath) large_image_converter.convert(imagePath, outputPath, overwrite=True) assert os.path.getsize(outputPath) > 100
def testConvertMultiframeToAperio(tmpdir): imagePath = datastore.fetch('sample.ome.tif') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, format='aperio', compression='jp2k') source = large_image.open(outputPath) assert 'label' in source.getAssociatedImagesList()
def testConvertFromMultiframeImageOnlyOneFrame(tmpdir): imagePath = datastore.fetch('sample.ome.tif') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, onlyFrame=2) source = large_image_source_tiff.open(outputPath) metadata = source.getMetadata() assert metadata['levels'] == 5 info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 5
def testConvertPTIF(tmpdir): imagePath = datastore.fetch('sample_image.ptif') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, compression='jpeg', quality=50) info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 11
def testConvertFromTestSourceFrames(tmpdir): outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert('large_image://test?maxLevel=3&frames=4', outputPath) source = large_image_source_tiff.open(outputPath) metadata = source.getMetadata() assert metadata['levels'] == 4 assert len(metadata['frames']) == 4 info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 4
def testConvertFromMultiframeImageNoSubIFDS(tmpdir): imagePath = datastore.fetch('sample.ome.tif') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, subifds=False) source = large_image_source_tiff.open(outputPath) metadata = source.getMetadata() assert metadata['levels'] == 5 assert len(metadata['frames']) == 3 info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 15
def convert_to_cog(cog): """Populate ConvertedImage with COG file.""" if not isinstance(cog, ConvertedImage): cog = ConvertedImage.objects.get(id=cog) else: cog.refresh_from_db() with _processed_image_helper(cog) as (image, output): with input_output_path_helper(image.file, output.file, prefix='cog_', vsi=True) as ( input_path, output_path, ): large_image_converter.convert(str(input_path), str(output_path))
def testConvertGeospatial(tmpdir): testDir = os.path.dirname(os.path.realpath(__file__)) imagePath = os.path.join(testDir, 'test_files', 'rgb_geotiff.tiff') inputPath = os.path.join(tmpdir, 'in.geo.tiff') shutil.copy(imagePath, inputPath) outputPath = large_image_converter.convert(inputPath, level=5) assert 'geo.tiff' in outputPath assert outputPath != inputPath info = tifftools.read_tiff(outputPath) assert tifftools.Tag.ModelTiepointTag.value in info['ifds'][0]['tags']
def testConvertJp2kCompression(tmpdir): imagePath = datastore.fetch('sample_Easy1.png') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath, compression='jp2k') info = tifftools.read_tiff(outputPath) assert (info['ifds'][0]['tags'][tifftools.Tag.Compression.value]['data'][0] == tifftools.constants.Compression.JP2000.value) source = large_image_source_tiff.open(outputPath) image, _ = source.getRegion(output={ 'maxWidth': 200, 'maxHeight': 200 }, format=constants.TILE_FORMAT_NUMPY) assert (image[12][167] == [215, 135, 172]).all() outputPath2 = os.path.join(tmpdir, 'out2.tiff') large_image_converter.convert(imagePath, outputPath2, compression='jp2k', psnr=50) assert os.path.getsize(outputPath2) < os.path.getsize(outputPath) outputPath3 = os.path.join(tmpdir, 'out3.tiff') large_image_converter.convert(imagePath, outputPath3, compression='jp2k', cr=100) assert os.path.getsize(outputPath3) < os.path.getsize(outputPath) assert os.path.getsize(outputPath3) != os.path.getsize(outputPath2)
def main(args=sys.argv[1:]): parser = get_parser() opts = parser.parse_args(args=args) logger = logging.getLogger('large-image-converter') if not len(logger.handlers): logger.addHandler(logging.StreamHandler(sys.stderr)) logger.setLevel(max(1, logging.WARNING - (opts.verbose - opts.silent) * 10)) try: import large_image li_logger = large_image.config.getConfig('logger') li_logger.setLevel( max(1, logging.CRITICAL - (opts.verbose - opts.silent) * 10)) except ImportError: pass logger.debug('Command line options: %r' % opts) if not os.path.isfile( opts.source) and not opts.source.startswith('large_image://test'): logger.error('Source is not a file (%s)', opts.source) return 1 if opts.compression == 'zip': opts.compression = 'deflate' converterOpts = { k: v for k, v in vars(opts).items() if k not in {'source', 'dest', 'verbose', 'silent'} and v is not None } start_time = time.time() dest = large_image_converter.convert(opts.source, opts.dest, **converterOpts) end_time = time.time() if not os.path.isfile(dest): logger.error('Failed to generate file') return 1 logger.info('Created %s, %d bytes, %3.1f s', dest, os.path.getsize(dest), end_time - start_time) if opts._stats: import json import tifftools.commands info = tifftools.read_tiff(dest) try: desc = json.loads(info['ifds'][0]['tags'][ tifftools.Tag.ImageDescription.value]['data']) except Exception: logger.debug('Cannot generate statistics.') return desc['large_image_converter']['conversion_stats'] = { 'time': end_time - start_time, 'filesize': os.path.getsize(dest), 'original_filesize': os.path.getsize(opts.source), 'compression_ratio': desc['large_image_converter'].get('frames', 1) * sum(info['ifds'][0]['tags'][tifftools.Tag.BitsPerSample.value] ['data']) / 8 * info['ifds'][0]['tags'][tifftools.Tag.ImageWidth.value]['data'][0] * info['ifds'][0]['tags'][tifftools.Tag.ImageLength.value]['data'][0] / os.path.getsize(dest), } if opts._stats == 'full' and opts.compression not in { 'deflate', 'zip', 'lzw', 'zstd', 'packbits', 'none' }: compute_error_metrics( opts.source, dest, desc['large_image_converter']['conversion_stats'], converterOpts) tifftools.commands.tiff_set( dest, overwrite=True, setlist=[('ImageDescription', json.dumps(desc, separators=(',', ':'), sort_keys=True, default=large_image_converter.json_serial))])
def compute_error_metrics(original, altered, results, converterOpts=None): """ Compute the amount of error introduced via conversion compared to conversion using a lossless method. Note that this is not compared to the original, as we may not be able to read that in an efficient way. This is a very time-consuming way to compute error metrics, since it first reprocesses the input file to a lossless format, then steps through each tile and computes RSME and SSIM errors per tile, producing a weighted-by- number-of-pixels of each of these. The RSME is used to compute a PSNR value. :param original: the original file path. :param altered: the path of compressed file to compare. :param results: a dictionary to store results. Modified. :param converterOpts: an optional dictionary of parameters used for the original conversion. Only parameters that would affect the selected pixels are used. """ import math from tempfile import TemporaryDirectory import large_image_source_tiff import numpy import packaging import skimage.metrics lastlog = 0 with TemporaryDirectory() as tempDir: # TODO: check if the original is geospatial; if so appropriate options tempPath = os.path.join(tempDir, os.path.basename(original) + '.tiff') orig = large_image_converter.convert(original, tempPath, compression='lzw') tsOrig = large_image_source_tiff.open(orig) numFrames = len(tsOrig.getMetadata().get('frames', [0])) tsAlt = large_image_source_tiff.open(altered) mse = 0 ssim = 0 ssim_count = 0 maxval = 0 maxdiff = 0 sum = 0 count = 0 tileSize = 2048 for frame in range(numFrames): tiAlt = tsAlt.tileIterator(tile_size=dict(width=tileSize), frame=frame) for tileOrig in tsOrig.tileIterator(tile_size=dict(width=tileSize), frame=frame): tileAlt = next(tiAlt) do = tileOrig['tile'] da = tileAlt['tile'] if do.dtype != da.dtype and da.dtype == numpy.uint8: da = da.astype(int) * 257 do = do.astype(int) da = da.astype(int) maxval = max(maxval, do.max(), da.max()) if do.shape[2] > da.shape[2]: do = do[:, :, :da.shape[2]] if da.shape[2] > do.shape[2]: da = da[:, :, :do.shape[2]] diff = numpy.absolute(do - da) maxdiff = max(maxdiff, diff.max()) sum += diff.sum() count += diff.size last_mse = numpy.mean(diff**2) mse += last_mse * diff.size last_ssim = 0 try: kwargs = {} if (packaging.version.parse(skimage.__version__) >= packaging.version.parse('0.19')): kwargs['channel_axis'] = 2 if len( do.shape) > 2 else None else: kwargs['multichannel'] = len(do.shape) > 2 last_ssim = skimage.metrics.structural_similarity( do.astype(float), da.astype(float), data_range=255 if tileOrig['tile'].dtype == numpy.uint8 else 65535, gaussian_weights=True, sigma=1.5, use_sample_covariance=False, **kwargs) ssim += last_ssim * diff.size ssim_count += diff.size except ValueError: pass if time.time() - lastlog >= 10 and ssim_count: logger.debug( 'Calculating error (%d/%d): rmse %4.2f ssim %6.4f ' 'last rmse %4.2f ssim %6.4f' % (tileOrig['tile_position']['position'] + 1 + tileOrig['iterator_range']['position'] * frame, tileOrig['iterator_range']['position'] * numFrames, (mse / count)**0.5, ssim / ssim_count, last_mse** 0.5, last_ssim)) lastlog = time.time() results['maximum_error'] = maxdiff results['average_error'] = sum / count results['rmse'] = (mse / count)**0.5 results['psnr'] = 10 * math.log10(maxval**2 / (mse / count)) if mse else None if ssim_count: results['ssim'] = ssim / ssim_count logger.debug( 'Calculated error: rmse %4.2f psnr %3.1f ssim %6.4f' % (results['rmse'], results['psnr'] or 0, results['ssim']))
def create_tiff(self, inputFile, outputName=None, outputDir=None, quality=90, tileSize=256, **kwargs): """ Take a source input file, readable by vips, and output a pyramidal tiff file. :param inputFile: the path to the input file or base file of a set. :param outputName: the name of the output file. If None, the name is based on the input name and current date and time. May be a full path. :param outputDir: the location to store the output. If unspecified, the inputFile's directory is used. If the outputName is a fully qualified path, this is ignored. :param quality: a jpeg quality passed to vips. 0 is small, 100 is high quality. 90 or above is recommended. :param tileSize: the horizontal and vertical tile size. Optional parameters that can be specified in kwargs: :param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits', or 'zstd'. :param level: compression level for zstd, 1-22 (default is 10). :param predictor: one of 'none', 'horizontal', or 'float' used for lzw and deflate. :param inputName: if no output name is specified, and this is specified, this is used as the basis of the output name instead of extracting the name from the inputFile path. :returns: output path. """ import large_image_converter logger = logging.getLogger('large-image-converter') if not len(logger.handlers): logger.addHandler(logging.StreamHandler(sys.stdout)) if not logger.level: logger.setLevel(logging.INFO) if '_concurrency' not in kwargs: kwargs['_concurrency'] = -2 inputPath = os.path.abspath(os.path.expanduser(inputFile)) geospatial = large_image_converter.is_geospatial(inputPath) inputName = kwargs.get('inputName', os.path.basename(inputPath)) suffix = large_image_converter.format_hook('adjust_params', geospatial, kwargs, **kwargs) suffix = suffix or ('.tiff' if not geospatial else '.geo.tiff') if not outputName: outputName = os.path.splitext(inputName)[0] + suffix if outputName.endswith('.geo' + suffix): outputName = outputName[:len(outputName) - len(suffix) - 4] + suffix if outputName == inputName: outputName = (os.path.splitext(inputName)[0] + '.' + time.strftime('%Y%m%d-%H%M%S') + suffix) renameOutput = outputName if not outputName.endswith(suffix): outputName += suffix if not outputDir: outputDir = os.path.dirname(inputPath) outputPath = os.path.join(outputDir, outputName) large_image_converter.convert(inputPath, outputPath, quality=quality, tileSize=tileSize, **kwargs) if not os.path.exists(outputPath): raise Exception('Conversion command failed to produce output') if renameOutput != outputName: renamePath = os.path.join(outputDir, renameOutput) shutil.move(outputPath, renamePath) outputPath = renamePath logger.info('Created a file of size %d' % os.path.getsize(outputPath)) return outputPath
def testConverterMissingTiles(tmpdir): imagePath = datastore.fetch('one_layer_missing_tiles.tiff') outputPath = os.path.join(tmpdir, 'out.tiff') large_image_converter.convert(imagePath, outputPath) info = tifftools.read_tiff(outputPath) assert len(info['ifds']) == 6