def deserialize_uint64_blocks(compressed_blocks, shape):
"""
Reconstitute a volume that was serialized with serialize_uint64_blocks(), above.
NOTE: If the volume is not 64-px aligned, then the output will NOT be C-contiguous.
"""
if (np.array(shape) % 64).any():
padding = 64 - ( np.array(shape) % 64 )
aligned_shape = shape + padding
else:
aligned_shape = shape
aligned_volume = np.empty( aligned_shape, dtype=np.uint64 )
block_view = view_as_blocks( aligned_volume, (64,64,64) )
for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])):
compressed_block = compressed_blocks[bi]
# (See note above regarding recompression with LZ4)
encoded_block = lz4.frame.decompress( compressed_block )
block = decode_label_block( encoded_block )
block_view[zi,yi,xi] = block
if shape == tuple(aligned_shape):
volume = aligned_volume
else:
# Trim
volume = np.asarray(aligned_volume[box_to_slicing((0,0,0), shape)], order='C')
return volume
def test_completely_unique_voxels(self):
"""
Tests the worst possible scenario:
What if every voxel in the block has a unique label?
"""
header_size = 16
global_table_size = (64**3) * 8
num_sublocks = 8**3
voxels_per_subblock = 8**3 # 512 == 2**9
bits_per_voxel = 9
stream_bits_per_subblock = bits_per_voxel * voxels_per_subblock
stream_bytes_per_subblock = (
stream_bits_per_subblock +
7) // 8 # techinically, supposed to round up
# (ok, in this case, it's divisible anyway)
max_block_encoded_size = (
header_size + global_table_size +
num_sublocks * 2 # num labels for each subblock
+ num_sublocks * voxels_per_subblock * 4 # subblock tables
+ num_sublocks * stream_bytes_per_subblock) # subblock bitstreams
a = np.arange(64**3, dtype=np.uint32).reshape(
(64, 64, 64)) # every voxel is unique
a = a.astype(np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == max_block_encoded_size
decoded = decode_label_block(encoded_data)
assert (decoded == a).all()
def deserialize_uint64_blocks(compressed_blocks, shape):
"""
Reconstitute a volume that was serialized with serialize_uint64_blocks(), above.
NOTE: If the volume is not 64-px aligned, then the output will NOT be C-contiguous.
"""
if (np.array(shape) % 64).any():
padding = 64 - ( np.array(shape) % 64 )
aligned_shape = shape + padding
else:
aligned_shape = shape
aligned_volume = np.empty( aligned_shape, dtype=np.uint64 )
block_view = view_as_blocks( aligned_volume, (64,64,64) )
for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])):
compressed_block = compressed_blocks[bi]
# (See note above regarding recompression with LZ4)
encoded_block = lz4.frame.decompress( compressed_block )
block = decode_label_block( encoded_block )
block_view[zi,yi,xi] = block
if shape == tuple(aligned_shape):
volume = aligned_volume
else:
# Trim
volume = np.asarray(aligned_volume[box_to_slicing((0,0,0), shape)], order='C')
return volume
def test_tricky_block(self):
test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs'
tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block']
encoded_data = encode_label_block(tricky_block)
decoded = decode_label_block(encoded_data)
assert (decoded == tricky_block).all()
def test_tricky_block(self):
test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs'
tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block']
encoded_data = encode_label_block(tricky_block)
decoded = decode_label_block(encoded_data)
assert (decoded == tricky_block).all()
def test_completely_unique_voxels(self):
"""
Tests the worst possible scenario:
What if every voxel in the block has a unique label?
"""
header_size = 16
global_table_size = (64**3) * 8
num_sublocks = 8**3
voxels_per_subblock = 8**3 # 512 == 2**9
bits_per_voxel = 9
stream_bits_per_subblock = bits_per_voxel*voxels_per_subblock
stream_bytes_per_subblock = (stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up
# (ok, in this case, it's divisible anyway)
max_block_encoded_size = ( header_size
+ global_table_size
+ num_sublocks*2 # num labels for each subblock
+ num_sublocks*voxels_per_subblock*4 # subblock tables
+ num_sublocks*stream_bytes_per_subblock ) # subblock bitstreams
a = np.arange(64**3, dtype=np.uint32).reshape((64,64,64)) # every voxel is unique
a = a.astype(np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == max_block_encoded_size
decoded = decode_label_block(encoded_data)
assert (decoded == a).all()
def test_random_block_round_trip(self):
a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64)
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_block_round_trip(self):
a = np.ones((64, 64, 64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_random_block_round_trip(self):
a = np.random.randint(0,10, (64,64,64)).astype(np.uint64)
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_block_round_trip(self):
a = np.ones((64,64,64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def deserialize_uint64_blocks(compressed_blocks, shape):
"""
Reconstitute a volume that was serialized with serialize_uint64_blocks(), above.
"""
volume = np.ndarray(shape, dtype=np.uint64)
block_view = view_as_blocks(volume, (64, 64, 64))
for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])):
block = decode_label_block(compressed_blocks[bi])
block_view[zi, yi, xi] = block
return volume
def deserialize_uint64_blocks(compressed_blocks, shape):
"""
Reconstitute a volume that was serialized with serialize_uint64_blocks(), above.
"""
volume = np.ndarray(shape, dtype=np.uint64)
block_view = view_as_blocks( volume, (64,64,64) )
for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])):
block = decode_label_block( compressed_blocks[bi] )
block_view[zi,yi,xi] = block
return volume
def test_5_labels_round_trip(self):
a = np.ones((64, 64, 64), np.uint64)
n_labels = 5
for i in range(n_labels):
a.flat[i::n_labels] = i + 13
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_5_labels_round_trip(self):
a = np.ones((64,64,64), np.uint64)
n_labels = 5
for i in range(n_labels):
a.flat[i::n_labels] = i+13
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self):
# Start with a random block
a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64)
# Change a subblock to a solid label.
# (This is a special case in the spec.)
a[:-8, -8, -8] = 14
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self):
# Start with a random block
a = np.random.randint(0,10, (64,64,64)).astype(np.uint64)
# Change a subblock to a solid label.
# (This is a special case in the spec.)
a[:-8, -8, -8] = 14
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
