def decompress_file(src, volume_size, block_size, dtype, order): with open(src, "rb") as f: binary = f.read() data = compressed_segmentation.decompress(binary, volume_size, dtype, block_size, order) del binary dest = src.replace(".cseg", "") _, ext = os.path.splitext(dest) if ext != ".npy": dest += ".npy" np.save(dest, data) try: stat = os.stat(dest) if stat.st_size > 0: os.remove(src) else: raise ValueError("File is zero length.") except (FileNotFoundError, ValueError) as err: print(f"cseg: Unable to write {dest}. Aborting.") sys.exit()
def test_recover_random(): for dtype in (np.uint32, np.uint64): print("dtype", dtype) for order in ('C', 'F'): print("order", order) labels = np.random.randint(255, size=(50, 49, 47), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (50, 49, 47), dtype=dtype, order=order) assert type(compressed) in (bytes, str) assert np.all(recovered == labels) labels = np.random.randint(255, size=(50, 49, 47, 1), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (50, 49, 47, 1), dtype=dtype, order=order) assert type(compressed) in (bytes, str) assert np.all(recovered == labels)
def test_zero_size(dtype, order): sx = 0 sy = random.randint(1, 512) sz = random.randint(1, 512) labels = np.random.randint(10, size=(sx, sy, sz), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order) assert labels.shape == recovered.shape
def test_table_offset_error_sequence(dtype, order): sx = 256 sy = 256 sz = 32 labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz), order=order) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order) assert np.all(labels == recovered)
def __decode_compressed_segmentation( self, buffer: bytes, data_type: str, chunk_size: Vec3D, block_size: Optional[Vec3D], ) -> np.ndarray: assert block_size is not None return compressed_segmentation.decompress( buffer, chunk_size.as_tuple(), dtype=data_type, block_size=block_size, order="F", )
def test_remap(dtype, preserve_missing_labels): shape = (61, 63, 67) labels = np.random.randint(0, 15, size=shape).astype(dtype) remap = {i: i + 20 for i in range(15)} binary = cseg.compress(labels) recovered = cseg.decompress(binary, shape, dtype) assert np.all(labels == recovered) assert np.all(cseg.labels(binary, shape, dtype) == list(range(15))) binary2 = cseg.remap( binary, shape, dtype, mapping=remap, preserve_missing_labels=preserve_missing_labels, ) assert np.all(cseg.labels(binary2, shape, dtype) == list(range(20, 35)))
def measure(dtype, order, N=20, block_size=(8, 8, 8)): csec = 0 dsec = 0 voxels = 0 input_bytes = 0 output_bytes = 0 for i in range(N): sx = random.randint(0, 256) sy = random.randint(0, 256) sz = random.randint(0, 128) voxels += sx * sy * sz labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype) labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz), order=order) input_bytes += labels.nbytes s = time.time() compressed = cseg.compress(labels, order=order, block_size=block_size) csec += time.time() - s output_bytes = len(compressed) s = time.time() recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order, block_size=block_size) dsec += time.time() - s assert np.all(labels == recovered) assert labels.flags.f_contiguous == recovered.flags.f_contiguous assert labels.flags.c_contiguous == recovered.flags.c_contiguous print("MVx: {:.2f}, {}, order: {}".format(voxels / 1024 / 1024, dtype, order)) print("Compression Ratio: {:.2f}x (in: {:.2f} kiB out: {:.2f} kiB)".format( input_bytes / output_bytes, input_bytes / 1024, output_bytes / 1024)) print("Compression: {:.2f} sec :: {:.2f} MVx/sec".format( csec, voxels / csec / 1e6)) print("Decompression: {:.2f} sec :: {:.2f} MVx/sec".format( dsec, voxels / dsec / 1e6))
def test_recover_random(dtype, order, block_size, variation): for _ in range(3): sx = random.randint(0, 256) sy = random.randint(0, 256) sz = random.randint(0, 128) labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype) labels = np.copy(labels, order=order) compressed = cseg.compress(labels, block_size=block_size, order=order) check_compressed_headers(compressed, [sx, sy, sz], block_size) recovered = cseg.decompress(compressed, (sx, sy, sz), block_size=block_size, dtype=dtype, order=order) N = np.sum(recovered != labels) if N > 0: print("Non-matching: ", N) assert np.all(labels == recovered)
def decode_compressed_segmentation_c_ext(bytestring, shape, dtype, block_size): return cseg.decompress(bytes(bytestring), shape, dtype, block_size, order='F')