def test_random_access(dtype, block_size):
sx = 100
sy = 100
sz = 100
labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype)
binary = cseg.compress(labels, block_size=block_size)
arr = cseg.CompressedSegmentationArray(binary,
shape=(sx, sy, sz),
dtype=dtype,
block_size=block_size)
for i in range(10):
x, y, z = np.random.randint(0, sx, size=(3, ), dtype=int)
assert arr[x, y, z] == labels[x, y, z]
labels = np.zeros((sx, sy, sz), dtype=dtype)
binary = cseg.compress(labels, block_size=block_size)
arr = cseg.CompressedSegmentationArray(binary,
shape=(sx, sy, sz),
dtype=dtype,
block_size=block_size)
for i in range(10):
x, y, z = np.random.randint(0, sx, size=(3, ), dtype=int)
assert arr[x, y, z] == labels[x, y, z]
def test_table_offset_error_random(dtype, order):
sx = 300
sy = 300
sz = 300
labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype)
try:
cseg.compress(labels, order=order)
assert False, "An OverflowError should have been triggered."
except OverflowError:
pass
def test_recover_random():
for dtype in (np.uint32, np.uint64):
print("dtype", dtype)
for order in ('C', 'F'):
print("order", order)
labels = np.random.randint(255, size=(50, 49, 47), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (50, 49, 47), dtype=dtype, order=order)
assert type(compressed) in (bytes, str)
assert np.all(recovered == labels)
labels = np.random.randint(255, size=(50, 49, 47, 1), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (50, 49, 47, 1), dtype=dtype, order=order)
assert type(compressed) in (bytes, str)
assert np.all(recovered == labels)
def test_zero_size(dtype, order):
sx = 0
sy = random.randint(1, 512)
sz = random.randint(1, 512)
labels = np.random.randint(10, size=(sx, sy, sz), dtype=dtype)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order)
assert labels.shape == recovered.shape
def test_table_offset_error_sequence(dtype, order):
sx = 256
sy = 256
sz = 32
labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz),
order=order)
compressed = cseg.compress(labels, order=order)
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order)
assert np.all(labels == recovered)
def encode_compressed_segmentation_c_ext(subvol, block_size):
if np.dtype(subvol.dtype) not in (np.uint32, np.uint64):
raise ValueError("compressed_segmentation only supports uint32 and uint64 datatypes. Got: " + str(subvol.dtype))
subvol = np.squeeze(subvol, axis=3)
if subvol.flags.c_contiguous:
order = 'C'
elif subvol.flags.f_contiguous:
order = 'F'
else:
order = 'F'
subvol = np.asfortranarray(subvol)
return cseg.compress(subvol, block_size=block_size, order=order)
def test_empty_labels():
labels = np.zeros((0, ), dtype=np.uint32).reshape((0, 0, 0))
compressed = cseg.compress(labels, block_size=(8, 8, 8))
try:
cseg.labels(b'',
block_size=(8, 8, 8),
shape=(0, 0, 0),
dtype=np.uint32)
assert False
except cseg.DecodeError:
pass
clabels = cseg.labels(compressed,
block_size=(8, 8, 8),
shape=(0, 0, 0),
dtype=np.uint32)
assert all(clabels == labels)
def test_labels(dtype, order, variation, block_size):
for _ in range(3):
sx = random.randint(0, 256)
sy = random.randint(0, 256)
sz = random.randint(0, 128)
labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype)
labels = np.copy(labels, order=order)
compressed = cseg.compress(labels, block_size=block_size, order=order)
uniq = np.unique(labels)
uniql = cseg.labels(
compressed,
shape=(sx, sy, sz),
dtype=dtype,
block_size=block_size,
)
assert np.all(uniq == uniql)
def test_remap(dtype, preserve_missing_labels):
shape = (61, 63, 67)
labels = np.random.randint(0, 15, size=shape).astype(dtype)
remap = {i: i + 20 for i in range(15)}
binary = cseg.compress(labels)
recovered = cseg.decompress(binary, shape, dtype)
assert np.all(labels == recovered)
assert np.all(cseg.labels(binary, shape, dtype) == list(range(15)))
binary2 = cseg.remap(
binary,
shape,
dtype,
mapping=remap,
preserve_missing_labels=preserve_missing_labels,
)
assert np.all(cseg.labels(binary2, shape, dtype) == list(range(20, 35)))
def measure(dtype, order, N=20, block_size=(8, 8, 8)):
csec = 0
dsec = 0
voxels = 0
input_bytes = 0
output_bytes = 0
for i in range(N):
sx = random.randint(0, 256)
sy = random.randint(0, 256)
sz = random.randint(0, 128)
voxels += sx * sy * sz
labels = np.random.randint(10000, size=(sx, sy, sz), dtype=dtype)
labels = np.arange(0, sx * sy * sz, dtype=dtype).reshape((sx, sy, sz),
order=order)
input_bytes += labels.nbytes
s = time.time()
compressed = cseg.compress(labels, order=order, block_size=block_size)
csec += time.time() - s
output_bytes = len(compressed)
s = time.time()
recovered = cseg.decompress(compressed, (sx, sy, sz),
dtype=dtype,
order=order,
block_size=block_size)
dsec += time.time() - s
assert np.all(labels == recovered)
assert labels.flags.f_contiguous == recovered.flags.f_contiguous
assert labels.flags.c_contiguous == recovered.flags.c_contiguous
print("MVx: {:.2f}, {}, order: {}".format(voxels / 1024 / 1024, dtype,
order))
print("Compression Ratio: {:.2f}x (in: {:.2f} kiB out: {:.2f} kiB)".format(
input_bytes / output_bytes, input_bytes / 1024, output_bytes / 1024))
print("Compression: {:.2f} sec :: {:.2f} MVx/sec".format(
csec, voxels / csec / 1e6))
print("Decompression: {:.2f} sec :: {:.2f} MVx/sec".format(
dsec, voxels / dsec / 1e6))
def test_recover_random(dtype, order, block_size, variation):
for _ in range(3):
sx = random.randint(0, 256)
sy = random.randint(0, 256)
sz = random.randint(0, 128)
labels = np.random.randint(variation, size=(sx, sy, sz), dtype=dtype)
labels = np.copy(labels, order=order)
compressed = cseg.compress(labels, block_size=block_size, order=order)
check_compressed_headers(compressed, [sx, sy, sz], block_size)
recovered = cseg.decompress(compressed, (sx, sy, sz),
block_size=block_size,
dtype=dtype,
order=order)
N = np.sum(recovered != labels)
if N > 0:
print("Non-matching: ", N)
assert np.all(labels == recovered)
def compress_file(src, block_size):
try:
data = np.load(src)
except ValueError:
print(f"cseg: {src} is not a numpy file.")
sys.exit()
binary = compressed_segmentation.compress(data, block_size=block_size)
del data
dest = f"{src}.cseg"
with open(dest, "wb") as f:
f.write(binary)
del binary
try:
stat = os.stat(dest)
if stat.st_size > 0:
os.remove(src)
else:
raise ValueError("File is zero length.")
except (FileNotFoundError, ValueError) as err:
print(f"cseg: Unable to write {dest}. Aborting.")
sys.exit()
def encode_compressed_segmentation_c_ext(subvol, block_size):
subvol = np.squeeze(subvol, axis=3)
subvol = np.copy(subvol, order='C')
return cseg.compress(subvol, block_size=block_size, order='F')
