def decompress_file(src, volume_size, block_size, dtype, order):
with open(src, "rb") as f:
binary = f.read()
data = compressed_segmentation.decompress(binary, volume_size, dtype,
block_size, order)
del binary
dest = src.replace(".cseg", "")
_, ext = os.path.splitext(dest)
if ext != ".npy":
dest += ".npy"
np.save(dest, data)
try:
stat = os.stat(dest)
if stat.st_size > 0:
os.remove(src)
else:
raise ValueError("File is zero length.")
except (FileNotFoundError, ValueError) as err:
print(f"cseg: Unable to write {dest}. Aborting.")
sys.exit()
def test_recover_random():
for dtype in (np.uint32, np.uint64):
print("dtype", dtype)
for order in ('C', 'F'):
print("order", order)
labels = np.random.randint(255, size=(50, 49, 47), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (50, 49, 47), dtype=dtype, order=order)
assert type(compressed) in (bytes, str)
assert np.all(recovered == labels)
labels = np.random.randint(255, size=(50, 49, 47, 1), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (50, 49, 47, 1), dtype=dtype, order=order)
assert type(compressed) in (bytes, str)
assert np.all(recovered == labels)
def test_zero_size(dtype, order):
sx = 0
sy = random.randint(1, 512)
sz = random.randint(1, 512)
labels = np.random.randint(10, size=(sx, sy, sz), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order)
assert labels.shape == recovered.shape
def test_table_offset_error_sequence(dtype, order):
sx = 256
sy = 256
sz = 32
labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz),
order=order)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order)
assert np.all(labels == recovered)
def __decode_compressed_segmentation(
self,
buffer: bytes,
data_type: str,
chunk_size: Vec3D,
block_size: Optional[Vec3D],
) -> np.ndarray:
assert block_size is not None
return compressed_segmentation.decompress(
buffer,
chunk_size.as_tuple(),
dtype=data_type,
block_size=block_size,
order="F",
)
def test_remap(dtype, preserve_missing_labels):
shape = (61, 63, 67)
labels = np.random.randint(0, 15, size=shape).astype(dtype)
remap = {i: i + 20 for i in range(15)}
binary = cseg.compress(labels)
recovered = cseg.decompress(binary, shape, dtype)
assert np.all(labels == recovered)
assert np.all(cseg.labels(binary, shape, dtype) == list(range(15)))
binary2 = cseg.remap(
binary,
shape,
dtype,
mapping=remap,
preserve_missing_labels=preserve_missing_labels,
)
assert np.all(cseg.labels(binary2, shape, dtype) == list(range(20, 35)))
def measure(dtype, order, N=20, block_size=(8, 8, 8)):
csec = 0
dsec = 0
voxels = 0
input_bytes = 0
output_bytes = 0
for i in range(N):
sx = random.randint(0, 256)
sy = random.randint(0, 256)
sz = random.randint(0, 128)
voxels += sx * sy * sz
labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype)
labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz),
order=order)
input_bytes += labels.nbytes
s = time.time()
compressed = cseg.compress(labels, order=order, block_size=block_size)
csec += time.time() - s
output_bytes = len(compressed)
s = time.time()
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order,
block_size=block_size)
dsec += time.time() - s
assert np.all(labels == recovered)
assert labels.flags.f_contiguous == recovered.flags.f_contiguous
assert labels.flags.c_contiguous == recovered.flags.c_contiguous
print("MVx: {:.2f}, {}, order: {}".format(voxels / 1024 / 1024, dtype,
order))
print("Compression Ratio: {:.2f}x (in: {:.2f} kiB out: {:.2f} kiB)".format(
input_bytes / output_bytes, input_bytes / 1024, output_bytes / 1024))
print("Compression: {:.2f} sec :: {:.2f} MVx/sec".format(
csec, voxels / csec / 1e6))
print("Decompression: {:.2f} sec :: {:.2f} MVx/sec".format(
dsec, voxels / dsec / 1e6))
def test_recover_random(dtype, order, block_size, variation):
for _ in range(3):
sx = random.randint(0, 256)
sy = random.randint(0, 256)
sz = random.randint(0, 128)
labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype)
labels = np.copy(labels, order=order)
compressed = cseg.compress(labels, block_size=block_size, order=order)
check_compressed_headers(compressed, [sx, sy, sz], block_size)
recovered = cseg.decompress(compressed, (sx, sy, sz),
block_size=block_size,
dtype=dtype,
order=order)
N = np.sum(recovered != labels)
if N > 0:
print("Non-matching: ", N)
assert np.all(labels == recovered)
def decode_compressed_segmentation_c_ext(bytestring, shape, dtype, block_size):
return cseg.decompress(bytes(bytestring),
shape,
dtype,
block_size,
order='F')