def test_random_access(dtype, block_size): sx = 100 sy = 100 sz = 100 labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype) binary = cseg.compress(labels, block_size=block_size) arr = cseg.CompressedSegmentationArray(binary, shape=(sx, sy, sz), dtype=dtype, block_size=block_size) for i in range(10): x, y, z = np.random.randint(0, sx, size=(3, ), dtype=int) assert arr[x, y, z] == labels[x, y, z] labels = np.zeros((sx, sy, sz), dtype=dtype) binary = cseg.compress(labels, block_size=block_size) arr = cseg.CompressedSegmentationArray(binary, shape=(sx, sy, sz), dtype=dtype, block_size=block_size) for i in range(10): x, y, z = np.random.randint(0, sx, size=(3, ), dtype=int) assert arr[x, y, z] == labels[x, y, z]
def test_table_offset_error_random(dtype, order): sx = 300 sy = 300 sz = 300 labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype) try: cseg.compress(labels, order=order) assert False, "An OverflowError should have been triggered." except OverflowError: pass
def test_recover_random(): for dtype in (np.uint32, np.uint64): print("dtype", dtype) for order in ('C', 'F'): print("order", order) labels = np.random.randint(255, size=(50, 49, 47), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (50, 49, 47), dtype=dtype, order=order) assert type(compressed) in (bytes, str) assert np.all(recovered == labels) labels = np.random.randint(255, size=(50, 49, 47, 1), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (50, 49, 47, 1), dtype=dtype, order=order) assert type(compressed) in (bytes, str) assert np.all(recovered == labels)
def test_zero_size(dtype, order): sx = 0 sy = random.randint(1, 512) sz = random.randint(1, 512) labels = np.random.randint(10, size=(sx, sy, sz), dtype=dtype) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order) assert labels.shape == recovered.shape
def test_table_offset_error_sequence(dtype, order): sx = 256 sy = 256 sz = 32 labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz), order=order) compressed = cseg.compress(labels, order=order) recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order) assert np.all(labels == recovered)
def encode_compressed_segmentation_c_ext(subvol, block_size): if np.dtype(subvol.dtype) not in (np.uint32, np.uint64): raise ValueError("compressed_segmentation only supports uint32 and uint64 datatypes. Got: " + str(subvol.dtype)) subvol = np.squeeze(subvol, axis=3) if subvol.flags.c_contiguous: order = 'C' elif subvol.flags.f_contiguous: order = 'F' else: order = 'F' subvol = np.asfortranarray(subvol) return cseg.compress(subvol, block_size=block_size, order=order)
def test_empty_labels(): labels = np.zeros((0, ), dtype=np.uint32).reshape((0, 0, 0)) compressed = cseg.compress(labels, block_size=(8, 8, 8)) try: cseg.labels(b'', block_size=(8, 8, 8), shape=(0, 0, 0), dtype=np.uint32) assert False except cseg.DecodeError: pass clabels = cseg.labels(compressed, block_size=(8, 8, 8), shape=(0, 0, 0), dtype=np.uint32) assert all(clabels == labels)
def test_labels(dtype, order, variation, block_size): for _ in range(3): sx = random.randint(0, 256) sy = random.randint(0, 256) sz = random.randint(0, 128) labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype) labels = np.copy(labels, order=order) compressed = cseg.compress(labels, block_size=block_size, order=order) uniq = np.unique(labels) uniql = cseg.labels( compressed, shape=(sx, sy, sz), dtype=dtype, block_size=block_size, ) assert np.all(uniq == uniql)
def test_remap(dtype, preserve_missing_labels): shape = (61, 63, 67) labels = np.random.randint(0, 15, size=shape).astype(dtype) remap = {i: i + 20 for i in range(15)} binary = cseg.compress(labels) recovered = cseg.decompress(binary, shape, dtype) assert np.all(labels == recovered) assert np.all(cseg.labels(binary, shape, dtype) == list(range(15))) binary2 = cseg.remap( binary, shape, dtype, mapping=remap, preserve_missing_labels=preserve_missing_labels, ) assert np.all(cseg.labels(binary2, shape, dtype) == list(range(20, 35)))
def measure(dtype, order, N=20, block_size=(8, 8, 8)): csec = 0 dsec = 0 voxels = 0 input_bytes = 0 output_bytes = 0 for i in range(N): sx = random.randint(0, 256) sy = random.randint(0, 256) sz = random.randint(0, 128) voxels += sx * sy * sz labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype) labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz), order=order) input_bytes += labels.nbytes s = time.time() compressed = cseg.compress(labels, order=order, block_size=block_size) csec += time.time() - s output_bytes = len(compressed) s = time.time() recovered = cseg.decompress(compressed, (sx, sy, sz), dtype=dtype, order=order, block_size=block_size) dsec += time.time() - s assert np.all(labels == recovered) assert labels.flags.f_contiguous == recovered.flags.f_contiguous assert labels.flags.c_contiguous == recovered.flags.c_contiguous print("MVx: {:.2f}, {}, order: {}".format(voxels / 1024 / 1024, dtype, order)) print("Compression Ratio: {:.2f}x (in: {:.2f} kiB out: {:.2f} kiB)".format( input_bytes / output_bytes, input_bytes / 1024, output_bytes / 1024)) print("Compression: {:.2f} sec :: {:.2f} MVx/sec".format( csec, voxels / csec / 1e6)) print("Decompression: {:.2f} sec :: {:.2f} MVx/sec".format( dsec, voxels / dsec / 1e6))
def test_recover_random(dtype, order, block_size, variation): for _ in range(3): sx = random.randint(0, 256) sy = random.randint(0, 256) sz = random.randint(0, 128) labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype) labels = np.copy(labels, order=order) compressed = cseg.compress(labels, block_size=block_size, order=order) check_compressed_headers(compressed, [sx, sy, sz], block_size) recovered = cseg.decompress(compressed, (sx, sy, sz), block_size=block_size, dtype=dtype, order=order) N = np.sum(recovered != labels) if N > 0: print("Non-matching: ", N) assert np.all(labels == recovered)
def compress_file(src, block_size): try: data = np.load(src) except ValueError: print(f"cseg: {src} is not a numpy file.") sys.exit() binary = compressed_segmentation.compress(data, block_size=block_size) del data dest = f"{src}.cseg" with open(dest, "wb") as f: f.write(binary) del binary try: stat = os.stat(dest) if stat.st_size > 0: os.remove(src) else: raise ValueError("File is zero length.") except (FileNotFoundError, ValueError) as err: print(f"cseg: Unable to write {dest}. Aborting.") sys.exit()
def encode_compressed_segmentation_c_ext(subvol, block_size): subvol = np.squeeze(subvol, axis=3) subvol = np.copy(subvol, order='C') return cseg.compress(subvol, block_size=block_size, order='F')