def deserialize_uint64_blocks(compressed_blocks, shape): """ Reconstitute a volume that was serialized with serialize_uint64_blocks(), above. NOTE: If the volume is not 64-px aligned, then the output will NOT be C-contiguous. """ if (np.array(shape) % 64).any(): padding = 64 - ( np.array(shape) % 64 ) aligned_shape = shape + padding else: aligned_shape = shape aligned_volume = np.empty( aligned_shape, dtype=np.uint64 ) block_view = view_as_blocks( aligned_volume, (64,64,64) ) for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])): compressed_block = compressed_blocks[bi] # (See note above regarding recompression with LZ4) encoded_block = lz4.frame.decompress( compressed_block ) block = decode_label_block( encoded_block ) block_view[zi,yi,xi] = block if shape == tuple(aligned_shape): volume = aligned_volume else: # Trim volume = np.asarray(aligned_volume[box_to_slicing((0,0,0), shape)], order='C') return volume
def test_completely_unique_voxels(self): """ Tests the worst possible scenario: What if every voxel in the block has a unique label? """ header_size = 16 global_table_size = (64**3) * 8 num_sublocks = 8**3 voxels_per_subblock = 8**3 # 512 == 2**9 bits_per_voxel = 9 stream_bits_per_subblock = bits_per_voxel * voxels_per_subblock stream_bytes_per_subblock = ( stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up # (ok, in this case, it's divisible anyway) max_block_encoded_size = ( header_size + global_table_size + num_sublocks * 2 # num labels for each subblock + num_sublocks * voxels_per_subblock * 4 # subblock tables + num_sublocks * stream_bytes_per_subblock) # subblock bitstreams a = np.arange(64**3, dtype=np.uint32).reshape( (64, 64, 64)) # every voxel is unique a = a.astype(np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == max_block_encoded_size decoded = decode_label_block(encoded_data) assert (decoded == a).all()
def deserialize_uint64_blocks(compressed_blocks, shape): """ Reconstitute a volume that was serialized with serialize_uint64_blocks(), above. NOTE: If the volume is not 64-px aligned, then the output will NOT be C-contiguous. """ if (np.array(shape) % 64).any(): padding = 64 - ( np.array(shape) % 64 ) aligned_shape = shape + padding else: aligned_shape = shape aligned_volume = np.empty( aligned_shape, dtype=np.uint64 ) block_view = view_as_blocks( aligned_volume, (64,64,64) ) for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])): compressed_block = compressed_blocks[bi] # (See note above regarding recompression with LZ4) encoded_block = lz4.frame.decompress( compressed_block ) block = decode_label_block( encoded_block ) block_view[zi,yi,xi] = block if shape == tuple(aligned_shape): volume = aligned_volume else: # Trim volume = np.asarray(aligned_volume[box_to_slicing((0,0,0), shape)], order='C') return volume
def test_tricky_block(self): test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs' tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block'] encoded_data = encode_label_block(tricky_block) decoded = decode_label_block(encoded_data) assert (decoded == tricky_block).all()
def test_tricky_block(self): test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs' tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block'] encoded_data = encode_label_block(tricky_block) decoded = decode_label_block(encoded_data) assert (decoded == tricky_block).all()
def test_completely_unique_voxels(self): """ Tests the worst possible scenario: What if every voxel in the block has a unique label? """ header_size = 16 global_table_size = (64**3) * 8 num_sublocks = 8**3 voxels_per_subblock = 8**3 # 512 == 2**9 bits_per_voxel = 9 stream_bits_per_subblock = bits_per_voxel*voxels_per_subblock stream_bytes_per_subblock = (stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up # (ok, in this case, it's divisible anyway) max_block_encoded_size = ( header_size + global_table_size + num_sublocks*2 # num labels for each subblock + num_sublocks*voxels_per_subblock*4 # subblock tables + num_sublocks*stream_bytes_per_subblock ) # subblock bitstreams a = np.arange(64**3, dtype=np.uint32).reshape((64,64,64)) # every voxel is unique a = a.astype(np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == max_block_encoded_size decoded = decode_label_block(encoded_data) assert (decoded == a).all()
def test_random_block_round_trip(self): a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64) encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_block_round_trip(self): a = np.ones((64, 64, 64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_random_block_round_trip(self): a = np.random.randint(0,10, (64,64,64)).astype(np.uint64) encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_block_round_trip(self): a = np.ones((64,64,64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def deserialize_uint64_blocks(compressed_blocks, shape): """ Reconstitute a volume that was serialized with serialize_uint64_blocks(), above. """ volume = np.ndarray(shape, dtype=np.uint64) block_view = view_as_blocks(volume, (64, 64, 64)) for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])): block = decode_label_block(compressed_blocks[bi]) block_view[zi, yi, xi] = block return volume
def deserialize_uint64_blocks(compressed_blocks, shape): """ Reconstitute a volume that was serialized with serialize_uint64_blocks(), above. """ volume = np.ndarray(shape, dtype=np.uint64) block_view = view_as_blocks( volume, (64,64,64) ) for bi, (zi, yi, xi) in enumerate(np.ndindex(*block_view.shape[:3])): block = decode_label_block( compressed_blocks[bi] ) block_view[zi,yi,xi] = block return volume
def test_5_labels_round_trip(self): a = np.ones((64, 64, 64), np.uint64) n_labels = 5 for i in range(n_labels): a.flat[i::n_labels] = i + 13 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_5_labels_round_trip(self): a = np.ones((64,64,64), np.uint64) n_labels = 5 for i in range(n_labels): a.flat[i::n_labels] = i+13 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self): # Start with a random block a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64) # Change a subblock to a solid label. # (This is a special case in the spec.) a[:-8, -8, -8] = 14 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self): # Start with a random block a = np.random.randint(0,10, (64,64,64)).astype(np.uint64) # Change a subblock to a solid label. # (This is a special case in the spec.) a[:-8, -8, -8] = 14 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()