def test_tricky_block(self):
test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs'
tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block']
encoded_data = encode_label_block(tricky_block)
decoded = decode_label_block(encoded_data)
assert (decoded == tricky_block).all()
def test_completely_unique_voxels(self):
"""
Tests the worst possible scenario:
What if every voxel in the block has a unique label?
"""
header_size = 16
global_table_size = (64**3) * 8
num_sublocks = 8**3
voxels_per_subblock = 8**3 # 512 == 2**9
bits_per_voxel = 9
stream_bits_per_subblock = bits_per_voxel * voxels_per_subblock
stream_bytes_per_subblock = (
stream_bits_per_subblock +
7) // 8 # techinically, supposed to round up
# (ok, in this case, it's divisible anyway)
max_block_encoded_size = (
header_size + global_table_size +
num_sublocks * 2 # num labels for each subblock
+ num_sublocks * voxels_per_subblock * 4 # subblock tables
+ num_sublocks * stream_bytes_per_subblock) # subblock bitstreams
a = np.arange(64**3, dtype=np.uint32).reshape(
(64, 64, 64)) # every voxel is unique
a = a.astype(np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == max_block_encoded_size
decoded = decode_label_block(encoded_data)
assert (decoded == a).all()
def test_completely_unique_voxels(self):
"""
Tests the worst possible scenario:
What if every voxel in the block has a unique label?
"""
header_size = 16
global_table_size = (64**3) * 8
num_sublocks = 8**3
voxels_per_subblock = 8**3 # 512 == 2**9
bits_per_voxel = 9
stream_bits_per_subblock = bits_per_voxel*voxels_per_subblock
stream_bytes_per_subblock = (stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up
# (ok, in this case, it's divisible anyway)
max_block_encoded_size = ( header_size
+ global_table_size
+ num_sublocks*2 # num labels for each subblock
+ num_sublocks*voxels_per_subblock*4 # subblock tables
+ num_sublocks*stream_bytes_per_subblock ) # subblock bitstreams
a = np.arange(64**3, dtype=np.uint32).reshape((64,64,64)) # every voxel is unique
a = a.astype(np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == max_block_encoded_size
decoded = decode_label_block(encoded_data)
assert (decoded == a).all()
def test_tricky_block(self):
test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs'
tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block']
encoded_data = encode_label_block(tricky_block)
decoded = decode_label_block(encoded_data)
assert (decoded == tricky_block).all()
def test_solid_block_round_trip(self):
a = np.ones((64, 64, 64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_random_block_round_trip(self):
a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64)
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_random_block_round_trip(self):
a = np.random.randint(0,10, (64,64,64)).astype(np.uint64)
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_block_round_trip(self):
a = np.ones((64,64,64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_encode_zeros(self):
"""
A trivial test to verify that the header of
the encoded data is correct for a solid block of ZEROS
"""
a = np.zeros((64,64,64), np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == 24
assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all()
assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1
assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 0
def test_encode_zeros(self):
"""
A trivial test to verify that the header of
the encoded data is correct for a solid block of ZEROS
"""
a = np.zeros((64, 64, 64), np.uint64)
encoded_data = encode_label_block(a)
assert len(encoded_data) == 24
assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all()
assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1
assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 0
def test_encode_solid_block(self):
"""
A trivial test to verify that the header of
the encoded data is correct for a solid block.
"""
a = np.ones((64,64,64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
assert len(encoded_data) == 24
assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all()
assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1
assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 13
def test_encode_solid_block(self):
"""
A trivial test to verify that the header of
the encoded data is correct for a solid block.
"""
a = np.ones((64, 64, 64), np.uint64)
a *= 13 # label 13 everywhere.
encoded_data = encode_label_block(a)
assert len(encoded_data) == 24
assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all()
assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1
assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 13
def test_5_labels_round_trip(self):
a = np.ones((64,64,64), np.uint64)
n_labels = 5
for i in range(n_labels):
a.flat[i::n_labels] = i+13
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_5_labels_round_trip(self):
a = np.ones((64, 64, 64), np.uint64)
n_labels = 5
for i in range(n_labels):
a.flat[i::n_labels] = i + 13
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self):
# Start with a random block
a = np.random.randint(0,10, (64,64,64)).astype(np.uint64)
# Change a subblock to a solid label.
# (This is a special case in the spec.)
a[:-8, -8, -8] = 14
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self):
# Start with a random block
a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64)
# Change a subblock to a solid label.
# (This is a special case in the spec.)
a[:-8, -8, -8] = 14
encoded_data = encode_label_block(a)
decoded_block = decode_label_block(encoded_data)
assert decoded_block.dtype == a.dtype
assert decoded_block.shape == a.shape
assert (decoded_block == a).all()
def serialize_uint64_blocks(volume):
"""
Compress and serialize a volume of uint64.
Preconditions:
- volume.dtype == np.uint64
- volume.ndim == 3
NOTE: If volume.shape is NOT divisible by 64, the input will be copied and padded.
Returns compressed_blocks, where the blocks are a flat list, in scan-order
"""
assert volume.dtype == np.uint64
assert volume.ndim == 3
if (np.array(volume.shape) % 64).any():
padding = 64 - ( np.array(volume.shape) % 64 )
aligned_shape = volume.shape + padding
aligned_volume = np.zeros( aligned_shape, dtype=np.uint64 )
aligned_volume[box_to_slicing((0,0,0), volume.shape)] = volume
else:
aligned_volume = volume
assert (np.array(aligned_volume.shape) % 64 == 0).all()
block_view = view_as_blocks( aligned_volume, (64,64,64) )
compressed_blocks = []
for zi, yi, xi in np.ndindex(*block_view.shape[:3]):
block = block_view[zi,yi,xi].copy('C')
encoded_block = encode_label_block(block)
# We compress AGAIN, with LZ4, because this seems to provide
# an additional 2x size reduction for nearly no slowdown.
compressed_block = lz4.frame.compress( encoded_block )
compressed_blocks.append( compressed_block )
del block
return compressed_blocks
def serialize_uint64_blocks(volume):
"""
Compress and serialize a volume of uint64.
Preconditions:
- volume.dtype == np.uint64
- volume.ndim == 3
NOTE: If volume.shape is NOT divisible by 64, the input will be copied and padded.
Returns compressed_blocks, where the blocks are a flat list, in scan-order
"""
assert volume.dtype == np.uint64
assert volume.ndim == 3
if (np.array(volume.shape) % 64).any():
padding = 64 - ( np.array(volume.shape) % 64 )
aligned_shape = volume.shape + padding
aligned_volume = np.zeros( aligned_shape, dtype=np.uint64 )
aligned_volume[box_to_slicing((0,0,0), volume.shape)] = volume
else:
aligned_volume = volume
assert (np.array(aligned_volume.shape) % 64 == 0).all()
block_view = view_as_blocks( aligned_volume, (64,64,64) )
compressed_blocks = []
for zi, yi, xi in np.ndindex(*block_view.shape[:3]):
block = block_view[zi,yi,xi].copy('C')
encoded_block = encode_label_block(block)
# We compress AGAIN, with LZ4, because this seems to provide
# an additional 2x size reduction for nearly no slowdown.
compressed_block = lz4.frame.compress( encoded_block )
compressed_blocks.append( compressed_block )
del block
return compressed_blocks
def serialize_uint64_blocks(volume):
"""
Compress and serialize a volume of uint64.
Preconditions:
- volume.dtype == np.uint64
- volume.ndim == 3
- volume.shape is divisible by 64
Returns compressed_blocks, where the blocks are a flat list, in scan-order
"""
assert volume.dtype == np.uint64
assert volume.ndim == 3
assert (np.array(volume.shape) % 64 == 0).all()
block_view = view_as_blocks(volume, (64, 64, 64))
compressed_blocks = []
for zi, yi, xi in np.ndindex(*block_view.shape[:3]):
block = block_view[zi, yi, xi].copy('C')
compressed_blocks.append(encode_label_block(block))
del block
return compressed_blocks
def serialize_uint64_blocks(volume):
"""
Compress and serialize a volume of uint64.
Preconditions:
- volume.dtype == np.uint64
- volume.ndim == 3
- volume.shape is divisible by 64
Returns compressed_blocks, where the blocks are a flat list, in scan-order
"""
assert volume.dtype == np.uint64
assert volume.ndim == 3
assert (np.array(volume.shape) % 64 == 0).all()
block_view = view_as_blocks( volume, (64,64,64) )
compressed_blocks = []
for zi, yi, xi in np.ndindex(*block_view.shape[:3]):
block = block_view[zi,yi,xi].copy('C')
compressed_blocks.append( encode_label_block(block) )
del block
return compressed_blocks