def test_tricky_block(self): test_inputs_dir = os.path.dirname(__file__) + '/../../tests/inputs' tricky_block = np.load(test_inputs_dir + '/tricky-label-block.npz')['block'] encoded_data = encode_label_block(tricky_block) decoded = decode_label_block(encoded_data) assert (decoded == tricky_block).all()
def test_completely_unique_voxels(self): """ Tests the worst possible scenario: What if every voxel in the block has a unique label? """ header_size = 16 global_table_size = (64**3) * 8 num_sublocks = 8**3 voxels_per_subblock = 8**3 # 512 == 2**9 bits_per_voxel = 9 stream_bits_per_subblock = bits_per_voxel * voxels_per_subblock stream_bytes_per_subblock = ( stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up # (ok, in this case, it's divisible anyway) max_block_encoded_size = ( header_size + global_table_size + num_sublocks * 2 # num labels for each subblock + num_sublocks * voxels_per_subblock * 4 # subblock tables + num_sublocks * stream_bytes_per_subblock) # subblock bitstreams a = np.arange(64**3, dtype=np.uint32).reshape( (64, 64, 64)) # every voxel is unique a = a.astype(np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == max_block_encoded_size decoded = decode_label_block(encoded_data) assert (decoded == a).all()
def test_completely_unique_voxels(self): """ Tests the worst possible scenario: What if every voxel in the block has a unique label? """ header_size = 16 global_table_size = (64**3) * 8 num_sublocks = 8**3 voxels_per_subblock = 8**3 # 512 == 2**9 bits_per_voxel = 9 stream_bits_per_subblock = bits_per_voxel*voxels_per_subblock stream_bytes_per_subblock = (stream_bits_per_subblock + 7) // 8 # techinically, supposed to round up # (ok, in this case, it's divisible anyway) max_block_encoded_size = ( header_size + global_table_size + num_sublocks*2 # num labels for each subblock + num_sublocks*voxels_per_subblock*4 # subblock tables + num_sublocks*stream_bytes_per_subblock ) # subblock bitstreams a = np.arange(64**3, dtype=np.uint32).reshape((64,64,64)) # every voxel is unique a = a.astype(np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == max_block_encoded_size decoded = decode_label_block(encoded_data) assert (decoded == a).all()
def test_solid_block_round_trip(self): a = np.ones((64, 64, 64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_random_block_round_trip(self): a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64) encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_random_block_round_trip(self): a = np.random.randint(0,10, (64,64,64)).astype(np.uint64) encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_block_round_trip(self): a = np.ones((64,64,64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_encode_zeros(self): """ A trivial test to verify that the header of the encoded data is correct for a solid block of ZEROS """ a = np.zeros((64,64,64), np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == 24 assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all() assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1 assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 0
def test_encode_zeros(self): """ A trivial test to verify that the header of the encoded data is correct for a solid block of ZEROS """ a = np.zeros((64, 64, 64), np.uint64) encoded_data = encode_label_block(a) assert len(encoded_data) == 24 assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all() assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1 assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 0
def test_encode_solid_block(self): """ A trivial test to verify that the header of the encoded data is correct for a solid block. """ a = np.ones((64,64,64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) assert len(encoded_data) == 24 assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all() assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1 assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 13
def test_encode_solid_block(self): """ A trivial test to verify that the header of the encoded data is correct for a solid block. """ a = np.ones((64, 64, 64), np.uint64) a *= 13 # label 13 everywhere. encoded_data = encode_label_block(a) assert len(encoded_data) == 24 assert (np.frombuffer(encoded_data[:12], np.uint32) == 8).all() assert np.frombuffer(encoded_data[12:16], np.uint32)[0] == 1 assert np.frombuffer(encoded_data[16:24], np.uint64)[0] == 13
def test_5_labels_round_trip(self): a = np.ones((64,64,64), np.uint64) n_labels = 5 for i in range(n_labels): a.flat[i::n_labels] = i+13 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_5_labels_round_trip(self): a = np.ones((64, 64, 64), np.uint64) n_labels = 5 for i in range(n_labels): a.flat[i::n_labels] = i + 13 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self): # Start with a random block a = np.random.randint(0,10, (64,64,64)).astype(np.uint64) # Change a subblock to a solid label. # (This is a special case in the spec.) a[:-8, -8, -8] = 14 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def test_solid_subblock_round_trip(self): # Start with a random block a = np.random.randint(0, 10, (64, 64, 64)).astype(np.uint64) # Change a subblock to a solid label. # (This is a special case in the spec.) a[:-8, -8, -8] = 14 encoded_data = encode_label_block(a) decoded_block = decode_label_block(encoded_data) assert decoded_block.dtype == a.dtype assert decoded_block.shape == a.shape assert (decoded_block == a).all()
def serialize_uint64_blocks(volume): """ Compress and serialize a volume of uint64. Preconditions: - volume.dtype == np.uint64 - volume.ndim == 3 NOTE: If volume.shape is NOT divisible by 64, the input will be copied and padded. Returns compressed_blocks, where the blocks are a flat list, in scan-order """ assert volume.dtype == np.uint64 assert volume.ndim == 3 if (np.array(volume.shape) % 64).any(): padding = 64 - ( np.array(volume.shape) % 64 ) aligned_shape = volume.shape + padding aligned_volume = np.zeros( aligned_shape, dtype=np.uint64 ) aligned_volume[box_to_slicing((0,0,0), volume.shape)] = volume else: aligned_volume = volume assert (np.array(aligned_volume.shape) % 64 == 0).all() block_view = view_as_blocks( aligned_volume, (64,64,64) ) compressed_blocks = [] for zi, yi, xi in np.ndindex(*block_view.shape[:3]): block = block_view[zi,yi,xi].copy('C') encoded_block = encode_label_block(block) # We compress AGAIN, with LZ4, because this seems to provide # an additional 2x size reduction for nearly no slowdown. compressed_block = lz4.frame.compress( encoded_block ) compressed_blocks.append( compressed_block ) del block return compressed_blocks
def serialize_uint64_blocks(volume): """ Compress and serialize a volume of uint64. Preconditions: - volume.dtype == np.uint64 - volume.ndim == 3 - volume.shape is divisible by 64 Returns compressed_blocks, where the blocks are a flat list, in scan-order """ assert volume.dtype == np.uint64 assert volume.ndim == 3 assert (np.array(volume.shape) % 64 == 0).all() block_view = view_as_blocks(volume, (64, 64, 64)) compressed_blocks = [] for zi, yi, xi in np.ndindex(*block_view.shape[:3]): block = block_view[zi, yi, xi].copy('C') compressed_blocks.append(encode_label_block(block)) del block return compressed_blocks
def serialize_uint64_blocks(volume): """ Compress and serialize a volume of uint64. Preconditions: - volume.dtype == np.uint64 - volume.ndim == 3 - volume.shape is divisible by 64 Returns compressed_blocks, where the blocks are a flat list, in scan-order """ assert volume.dtype == np.uint64 assert volume.ndim == 3 assert (np.array(volume.shape) % 64 == 0).all() block_view = view_as_blocks( volume, (64,64,64) ) compressed_blocks = [] for zi, yi, xi in np.ndindex(*block_view.shape[:3]): block = block_view[zi,yi,xi].copy('C') compressed_blocks.append( encode_label_block(block) ) del block return compressed_blocks