def test_write_wrong_counts():
path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
info = tifftools.read_tiff(path)
info['ifds'][0]['tags'][tifftools.Tag.StripByteCounts.value]['data'].pop()
with pytest.raises(Exception) as exc:
tifftools.write_tiff(info, '-')
assert 'Offsets and byte counts do not correspond' in str(exc.value)
def test_write_allow_existing(tmp_path):
path = datastore.fetch('d043-200.tif')
info = tifftools.read_tiff(path)
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
len = os.path.getsize(destpath)
tifftools.write_tiff(info, destpath, allowExisting=True)
assert len == os.path.getsize(destpath)
def test_write_already_exists(tmp_path):
path = datastore.fetch('d043-200.tif')
info = tifftools.read_tiff(path)
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
with pytest.raises(Exception) as exc:
tifftools.write_tiff(info, destpath)
assert 'File already exists' in str(exc.value)
def test_write_switch_to_bigtiff(tmp_path):
path = datastore.fetch('hamamatsu.ndpi')
info = tifftools.read_tiff(path)
info['ifds'].extend(info['ifds'])
info['ifds'].extend(info['ifds'])
info['ifds'].extend(info['ifds'])
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
destinfo = tifftools.read_tiff(destpath)
assert destinfo['bigtiff'] is True
def test_write_bigtiff_from_datatype(tmp_path):
path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
info = tifftools.read_tiff(path)
info['ifds'][0]['tags'][23456] = {
'datatype': tifftools.Datatype.LONG8,
'data': [8],
}
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
destinfo = tifftools.read_tiff(destpath)
assert destinfo['bigtiff'] is True
def test_write_bad_strip_offset(tmp_path, caplog):
path = os.path.join(os.path.dirname(__file__), 'data',
'bad_strip_offset.tif')
info = tifftools.read_tiff(path)
destpath = tmp_path / 'sample.tiff'
with caplog.at_level(logging.WARNING):
tifftools.write_tiff(info, destpath)
assert 'from desired offset' in caplog.text
destinfo = tifftools.read_tiff(destpath)
assert destinfo['ifds'][0]['tags'][
tifftools.Tag.StripOffsets.value]['data'][0] == 0
def test_write_bytecount_data(tmp_path):
path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
info = tifftools.read_tiff(path)
# Just use data from within the file itself; an actual sample file with
# compression 6 and defined Q, AC, and DC tables would be better.
info['ifds'][0]['tags'][tifftools.Tag.JPEGQTables.value] = {
'datatype': tifftools.Datatype.LONG,
'data': [8],
}
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
assert os.path.getsize(destpath) > os.path.getsize(path) + 64
def test_write_bigtiff_with_offset_data(tmp_path):
path = datastore.fetch('hamamatsu.ndpi')
info = tifftools.read_tiff(path)
info['ifds'][0]['tags'][tifftools.Tag.FreeOffsets.value] = {
'datatype': tifftools.Datatype.LONG,
'data': [8] * 256,
}
info['ifds'][0]['tags'][tifftools.Tag.FreeByteCounts.value] = {
'datatype': tifftools.Datatype.LONG,
'data': [16777216] * 256,
}
destpath = tmp_path / 'sample.tiff'
tifftools.write_tiff(info, destpath)
destinfo = tifftools.read_tiff(destpath)
assert destinfo['bigtiff'] is True
def test_write_single_subifd(tmp_path):
path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
info = tifftools.read_tiff(path)
info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value] = {
'ifds': [copy.deepcopy(info['ifds'][0])]
}
dest1path = tmp_path / 'sample1.tiff'
tifftools.write_tiff(info, dest1path)
dest1info = tifftools.read_tiff(dest1path)
assert len(dest1info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value]['ifds']
[0]) == 1
info = tifftools.read_tiff(path)
info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value] = {
'ifds': [copy.deepcopy(info['ifds'])]
}
dest2path = tmp_path / 'sample2.tiff'
tifftools.write_tiff(info, dest2path)
dest2info = tifftools.read_tiff(dest2path)
assert len(dest2info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value]['ifds']
[0]) == 1
def create_thumbnail_and_label(tempPath, info, ifdCount, needsLabel,
labelPosition, **kwargs):
"""
Create a thumbnail and, optionally, label image for the aperio file.
:param tempPath: a temporary file in a temporary directory.
:param info: the tifftools info that will be written to the tiff tile;
modified.
:param ifdCount: the number of ifds in the first tiled image. This is 1 if
there are subifds.
:param needsLabel: true if a label image needs to be added.
:param labelPosition: the position in the ifd list where a label image
should be inserted.
"""
thumbnailSize = 1024
labelSize = 640
maxLabelAspect = 1.5
tileSize = info['ifds'][0]['tags'][
tifftools.Tag.TileWidth.value]['data'][0]
levels = int(
math.ceil(
math.log(max(thumbnailSize, labelSize) / tileSize) /
math.log(2))) + 1
neededList = ['thumbnail']
if needsLabel:
neededList[0:0] = ['label']
tiledPath = tempPath + '-overview.tiff'
firstFrameIfds = info['ifds'][max(0, ifdCount - levels):ifdCount]
tifftools.write_tiff(firstFrameIfds, tiledPath)
ts = large_image_source_tiff.open(tiledPath)
for subImage in neededList:
if subImage == 'label':
x = max(0,
(ts.sizeX - min(ts.sizeX, ts.sizeY) * maxLabelAspect) // 2)
y = max(0,
(ts.sizeY - min(ts.sizeX, ts.sizeY) * maxLabelAspect) // 2)
regionParams = {
'output':
dict(maxWidth=labelSize, maxHeight=labelSize),
'region':
dict(left=x, right=ts.sizeX - x, top=y, bottom=ts.sizeY - y),
}
else:
regionParams = {
'output': dict(maxWidth=thumbnailSize, maxHeight=thumbnailSize)
}
image, _ = ts.getRegion(format=large_image.constants.TILE_FORMAT_PIL,
**regionParams)
if image.mode not in {'RGB', 'L'}:
image = image.convert('RGB')
if subImage == 'label':
image = image.rotate(90, expand=True)
imagePath = tempPath + '-image_%s.tiff' % subImage
image.save(imagePath,
'TIFF',
compression='tiff_jpeg',
quality=int(kwargs.get('quality', 90)))
imageInfo = tifftools.read_tiff(imagePath)
ifd = imageInfo['ifds'][0]
if subImage == 'label':
ifd['tags'][tifftools.Tag.Orientation.value] = {
'data': [tifftools.constants.Orientation.RightTop.value],
'datatype': tifftools.Datatype.LONG,
}
description = AperioHeader + AssociatedHeader.format(
name='label',
width=ifd['tags'][tifftools.Tag.ImageWidth.value]['data'][0],
height=ifd['tags'][tifftools.Tag.ImageHeight.value]['data'][0],
)
ifd['tags'][tifftools.Tag.ImageDescription.value] = {
'data': description,
'datatype': tifftools.Datatype.ASCII,
}
ifd['tags'][tifftools.Tag.NewSubfileType.value] = {
'data':
[tifftools.constants.NewSubfileType.ReducedImage.value],
'datatype': tifftools.Datatype.LONG,
}
info['ifds'][labelPosition:labelPosition] = imageInfo['ifds']
else:
fullDesc = info['ifds'][0]['tags'][
tifftools.Tag.ImageDescription.value]['data']
description = fullDesc.split('[', 1)[0] + ThumbnailHeader.format(
width=ifd['tags'][tifftools.Tag.ImageWidth.value]['data'][0],
height=ifd['tags'][tifftools.Tag.ImageHeight.value]['data'][0],
) + fullDesc.split('|', 1)[1]
ifd['tags'][tifftools.Tag.ImageDescription.value] = {
'data': description,
'datatype': tifftools.Datatype.ASCII,
}
info['ifds'][1:1] = imageInfo['ifds']
def _output_tiff(inputs,
outputPath,
tempPath,
lidata,
extraImages=None,
**kwargs):
"""
Given a list of input tiffs and data as parsed by _data_from_large_image,
generate an output tiff file with the associated images, correct scale, and
other metadata.
:param inputs: a list of pyramidal input files.
:param outputPath: the final destination.
:param tempPath: a temporary file in a temporary directory.
:param lidata: large_image data including metadata and associated images.
:param extraImages: an optional dictionary of keys and paths to add as
extra associated images.
"""
logger.debug('Reading %s', inputs[0])
info = tifftools.read_tiff(inputs[0])
ifdIndices = [0]
imgDesc = info['ifds'][0]['tags'].get(tifftools.Tag.ImageDescription.value)
description = _make_li_description(
len(info['ifds']), len(inputs), lidata,
(len(extraImages) if extraImages else 0) +
(len(lidata['images']) if lidata else 0),
imgDesc['data'] if imgDesc else None, **kwargs)
info['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value] = {
'data': description,
'datatype': tifftools.Datatype.ASCII,
}
if lidata:
_set_resolution(info['ifds'], lidata['metadata'])
if len(inputs) > 1:
if kwargs.get('subifds') is not False:
info['ifds'][0]['tags'][tifftools.Tag.SubIFD.value] = {
'ifds': info['ifds'][1:]
}
info['ifds'][1:] = []
for idx, inputPath in enumerate(inputs):
if not idx:
continue
logger.debug('Reading %s', inputPath)
nextInfo = tifftools.read_tiff(inputPath)
if lidata:
_set_resolution(nextInfo['ifds'], lidata['metadata'])
if len(lidata['metadata'].get('frames', [])) > idx:
nextInfo['ifds'][0]['tags'][
tifftools.Tag.ImageDescription.value] = {
'data':
json.dumps(
{'frame': lidata['metadata']['frames'][idx]},
separators=(',', ':'),
sort_keys=True,
default=json_serial),
'datatype':
tifftools.Datatype.ASCII,
}
ifdIndices.append(len(info['ifds']))
if kwargs.get('subifds') is not False:
nextInfo['ifds'][0]['tags'][tifftools.Tag.SubIFD.value] = {
'ifds': nextInfo['ifds'][1:]
}
info['ifds'].append(nextInfo['ifds'][0])
else:
info['ifds'].extend(nextInfo['ifds'])
ifdIndices.append(len(info['ifds']))
assocList = []
if lidata:
assocList += list(lidata['images'].items())
if extraImages:
assocList += list(extraImages.items())
for key, assocPath in assocList:
logger.debug('Reading %s', assocPath)
assocInfo = tifftools.read_tiff(assocPath)
assocInfo['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value] = {
'data': key,
'datatype': tifftools.Datatype.ASCII,
}
info['ifds'] += assocInfo['ifds']
if format_hook('modify_tiff_before_write', info, ifdIndices, tempPath,
lidata, **kwargs) is False:
return
logger.debug('Writing %s', outputPath)
tifftools.write_tiff(info,
outputPath,
bigEndian=False,
bigtiff=False,
allowExisting=True)
def _convert_to_jp2k(path, **kwargs):
"""
Given a tiled tiff file without compression, convert it to jp2k compression
using the gylmur library. This expects a tiff as written by vips without
any subifds.
:param path: the path of the tiff file. The file is altered.
:param psnr: if set, the target psnr. 0 for lossless.
:param cr: is set, the target compression ratio. 1 for lossless.
"""
info = tifftools.read_tiff(path)
jp2kargs = {}
if 'psnr' in kwargs:
jp2kargs['psnr'] = [int(kwargs['psnr'])]
elif 'cr' in kwargs:
jp2kargs['cratios'] = [int(kwargs['cr'])]
tilecount = sum(
len(ifd['tags'][tifftools.Tag.TileOffsets.value]['data'])
for ifd in info['ifds'])
processed = 0
lastlog = 0
tasks = []
lock = threading.Lock()
pool = _get_thread_pool(**kwargs)
with open(path, 'r+b') as fptr:
for ifd in info['ifds']:
ifd['tags'][tifftools.Tag.Compression.value]['data'][0] = (
tifftools.constants.Compression.JP2000)
shape = (
ifd['tags'][tifftools.Tag.TileWidth.value]['data'][0],
ifd['tags'][tifftools.Tag.TileLength.value]['data'][0],
len(ifd['tags'][tifftools.Tag.BitsPerSample.value]['data']))
dtype = numpy.uint16 if ifd['tags'][
tifftools.Tag.BitsPerSample.
value]['data'][0] == 16 else numpy.uint8
for idx, offset in enumerate(
ifd['tags'][tifftools.Tag.TileOffsets.value]['data']):
tmppath = path + '%d.jp2k' % processed
tasks.append(
(ifd, idx, processed, tmppath,
pool.submit(
_convert_to_jp2k_tile, lock, fptr, tmppath, offset,
ifd['tags'][tifftools.Tag.TileByteCounts.value]
['data'][idx], shape, dtype, jp2kargs)))
processed += 1
while len(tasks):
try:
tasks[0][-1].result(0.1)
except concurrent.futures.TimeoutError:
continue
ifd, idx, processed, tmppath, task = tasks.pop(0)
data = open(tmppath, 'rb').read()
os.unlink(tmppath)
# Remove first comment marker. It adds needless bytes
compos = data.find(b'\xff\x64')
if compos >= 0 and compos + 4 < len(data):
comlen = struct.unpack('>H', data[compos + 2:compos + 4])[0]
if compos + 2 + comlen + 1 < len(data) and data[
compos + 2 + comlen] == 0xff:
data = data[:compos] + data[compos + 2 + comlen:]
with lock:
fptr.seek(0, os.SEEK_END)
ifd['tags'][tifftools.Tag.TileOffsets.
value]['data'][idx] = fptr.tell()
ifd['tags'][tifftools.Tag.TileByteCounts.
value]['data'][idx] = len(data)
fptr.write(data)
if time.time() - lastlog >= 10 and tilecount > 1:
logger.debug('Converted %d of %d tiles to jp2k', processed + 1,
tilecount)
lastlog = time.time()
pool.shutdown(False)
fptr.seek(0, os.SEEK_END)
for ifd in info['ifds']:
ifd['size'] = fptr.tell()
info['size'] = fptr.tell()
tmppath = path + '.jp2k.tiff'
tifftools.write_tiff(info, tmppath, bigtiff=False, allowExisting=True)
os.unlink(path)
os.rename(tmppath, path)