def test_downsample_multi_channel():
offset = (0, 0, 0)
num_channels = 3
size = (32, 32, 10)
source_data = (128 * np.random.randn(num_channels, size[0], size[1],
size[2])).astype("uint8")
file_len = 32
source_info = WkwDatasetInfo(
"testoutput/multi-channel-test",
"color",
1,
wkw.Header(np.uint8, num_channels, file_len=file_len),
)
target_info = WkwDatasetInfo(
"testoutput/multi-channel-test",
"color",
2,
wkw.Header(np.uint8, file_len=file_len),
)
try:
shutil.rmtree(source_info.dataset_path)
shutil.rmtree(target_info.dataset_path)
except:
pass
with open_wkw(source_info) as wkw_dataset:
print("writing source_data shape", source_data.shape)
wkw_dataset.write(offset, source_data)
assert np.any(source_data != 0)
downsample_args = (
source_info,
target_info,
(2, 2, 2),
InterpolationModes.MAX,
tuple(a * WKW_CUBE_SIZE for a in offset),
CUBE_EDGE_LEN,
False,
True,
)
downsample_cube_job(downsample_args)
channels = []
for channel_index in range(num_channels):
channels.append(
downsample_cube(source_data[channel_index], (2, 2, 2),
InterpolationModes.MAX))
joined_buffer = np.stack(channels)
target_buffer = read_wkw(
target_info,
tuple(a * WKW_CUBE_SIZE for a in offset),
list(map(lambda x: x // 2, size)),
)
assert np.any(target_buffer != 0)
assert np.all(target_buffer == joined_buffer)
def test_buffered_slice_writer() -> None:
test_img = np.arange(24 * 24).reshape(24, 24).astype(np.uint16) + 1
dtype = test_img.dtype
origin = Vec3Int.zeros()
layer_name = "color"
mag = Mag(1)
dataset_dir = TESTOUTPUT_DIR / "buffered_slice_writer"
dataset_path = str(dataset_dir / layer_name / mag.to_layer_name())
rmtree(dataset_dir)
ds = Dataset(dataset_dir, voxel_size=(1, 1, 1))
mag_view = ds.add_layer("color", COLOR_CATEGORY,
dtype_per_channel=dtype).add_mag(mag)
with mag_view.get_buffered_slice_writer(absolute_offset=origin) as writer:
for i in range(13):
writer.send(test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
try:
read_data = data.read(origin, (24, 24, 13))
if read_data[read_data.nonzero()].size != 0:
raise AssertionError(
"Nothing should be written on the disk. But found data with shape: {}"
.format(read_data.shape))
except wkw.wkw.WKWException:
pass
for i in range(13, 32):
writer.send(test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
read_data = data.read(origin, (24, 24, 32))
assert np.squeeze(read_data).shape == (24, 24, 32), (
"The read data should have the shape: (24, 24, 32) "
"but has a shape of: {}".format(np.squeeze(read_data).shape))
assert read_data.size == read_data[read_data.nonzero()].size, (
"The read data contains zeros while the "
"written image has no zeros")
for i in range(32, 35):
writer.send(test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
read_data = data.read(origin, (24, 24, 35))
read_data = np.squeeze(read_data)
assert read_data.shape == (24, 24, 35), (
"The read data should have the shape: (24, 24, 35) "
"but has a shape of: {}".format(np.squeeze(read_data).shape))
assert read_data.size == read_data[read_data.nonzero()].size, (
"The read data contains zeros while the "
"written image has no zeros")
test_img_3d = np.zeros((test_img.shape[0], test_img.shape[1], 35))
for i in np.arange(35):
test_img_3d[:, :, i] = test_img
# check if the data are correct
assert np.array_equal(
test_img_3d, read_data), ("The data from the disk is not the same "
"as the data that should be written.")
def test_buffered_slice_writer():
test_img = np.arange(24 * 24).reshape(24, 24).astype(np.uint16) + 1
dtype = test_img.dtype
bbox = {"topleft": (0, 0, 0), "size": (24, 24, 35)}
origin = [0, 0, 0]
dataset_dir = "testoutput/buffered_slice_writer"
layer_name = "color"
mag = Mag(1)
dataset_path = os.path.join(dataset_dir, layer_name, mag.to_layer_name())
with BufferedSliceWriter(dataset_dir, layer_name, dtype, origin,
mag=mag) as writer:
for i in range(13):
writer.write_slice(i, test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
try:
read_data = data.read(origin, (24, 24, 13))
if read_data[read_data.nonzero()].size != 0:
raise AssertionError(
"Nothing should be written on the disk. But found data with shape: {}"
.format(read_data.shape))
except wkw.wkw.WKWException:
pass
for i in range(13, 32):
writer.write_slice(i, test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
read_data = data.read(origin, (24, 24, 32))
assert np.squeeze(read_data).shape == (24, 24, 32), (
"The read data should have the shape: (24, 24, 32) "
"but has a shape of: {}".format(np.squeeze(read_data).shape))
assert read_data.size == read_data[read_data.nonzero()].size, (
"The read data contains zeros while the "
"written image has no zeros")
for i in range(32, 35):
writer.write_slice(i, test_img)
with wkw.Dataset.open(dataset_path, wkw.Header(dtype)) as data:
read_data = data.read(origin, (24, 24, 35))
read_data = np.squeeze(read_data)
assert read_data.shape == (24, 24, 35), (
"The read data should have the shape: (24, 24, 35) "
"but has a shape of: {}".format(np.squeeze(read_data).shape))
assert read_data.size == read_data[read_data.nonzero()].size, (
"The read data contains zeros while the "
"written image has no zeros")
test_img_3d = np.zeros((test_img.shape[0], test_img.shape[1], 35))
for i in np.arange(35):
test_img_3d[:, :, i] = test_img
# transpose because the slice writer takes [y, x] data and transposes it to [x, y] before writing
test_img_3d = np.transpose(test_img_3d, (1, 0, 2))
# check if the data are correct
assert np.array_equal(
test_img_3d, read_data), ("The data from the disk is not the same "
"as the data that should be written.")
def convert_knossos(source_path,
target_path,
layer_name,
dtype,
mag=1,
jobs=1,
args=None):
source_knossos_info = KnossosDatasetInfo(source_path, dtype)
target_wkw_info = WkwDatasetInfo(
target_path, layer_name, mag,
wkw.Header(convert_element_class_to_dtype(dtype)))
ensure_wkw(target_wkw_info)
with open_knossos(source_knossos_info) as source_knossos:
with get_executor_for_args(args) as executor:
knossos_cubes = list(source_knossos.list_cubes())
if len(knossos_cubes) == 0:
logging.error("No input KNOSSOS cubes found.")
exit(1)
knossos_cubes.sort()
job_args = []
for cube_xyz in knossos_cubes:
job_args.append(
(cube_xyz, source_knossos_info, target_wkw_info))
wait_and_ensure_success(
executor.map_to_futures(convert_cube_job, job_args))
def upload_wkw(seg, datasource_path, axes='zyx'):
"""Upload segmentation to datasource_path, and refresh datasource-properties.json."""
# write
ds_path = join(datasource_path, 'segmentation', '1')
os.makedirs(ds_path, exist_ok=True)
if axes == 'zyx':
seg = np.transpose(seg, [2, 1, 0])
elif axes == 'xyz':
pass
else:
raise ValueError('axes has to be xyz or zyx')
# try:
with wkw.Dataset.create(ds_path, wkw.Header(np.uint32)) as ds:
ds.write([0, 0, 0], np.uint32(seg))
# except:
# ValueError('Already Written')
mt = read_metadata_for_layer(datasource_path, 'color')
bbox = mt[0]['boundingBox']
refresh_metadata(datasource_path,
compute_max_id=True,
exact_bounding_box=bbox)
def downsample_mag(
path,
layer_name,
source_mag: Mag,
target_mag: Mag,
interpolation_mode="default",
compress=False,
buffer_edge_len=None,
args=None,
):
interpolation_mode = parse_interpolation_mode(interpolation_mode, layer_name)
source_wkw_info = WkwDatasetInfo(path, layer_name, source_mag.to_layer_name(), None)
with open_wkw(source_wkw_info) as source:
target_wkw_info = WkwDatasetInfo(
path,
layer_name,
target_mag.to_layer_name(),
wkw.Header(source.header.voxel_type),
)
downsample(
source_wkw_info,
target_wkw_info,
source_mag,
target_mag,
interpolation_mode,
compress,
buffer_edge_len,
args,
)
def test_row_major_order():
data_shape = (4, 5, 6)
data = generate_test_data(np.uint8, data_shape)
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)) as dataset:
dataset.write((0, 0, 0), data)
read_data = dataset.read((0, 0, 0), data_shape)
assert np.all(data == read_data)
with wkw.Dataset.create("tests/tmp2", wkw.Header(np.uint8)) as dataset:
fortran_data = np.asfortranarray(data)
dataset.write((0, 0, 0), fortran_data)
fortran_read_data = dataset.read((0, 0, 0), data_shape)
assert np.all(fortran_read_data == read_data)
assert np.all(fortran_read_data == fortran_data)
def __init__(
self,
dataset_path: str,
layer_name: str,
dtype,
origin: Union[Tuple[int, int, int], List[int]],
# buffer_size specifies, how many slices should be aggregated until they are flushed.
buffer_size: int = 32,
# file_len specifies, how many buckets written per dimension into a wkw cube. Using 32,
# results in 1 GB/wkw file for 8-bit data
file_len: int = 32,
mag: Mag = Mag("1"),
):
self.dataset_path = dataset_path
self.layer_name = layer_name
self.buffer_size = buffer_size
layer_path = path.join(self.dataset_path, self.layer_name,
mag.to_layer_name())
self.dtype = dtype
self.dataset = wkw.Dataset.open(layer_path,
wkw.Header(dtype, file_len=file_len))
self.origin = origin
self.buffer = []
self.current_z = None
self.buffer_start_z = None
def test_readwrite_live_compression_should_truncate():
SIZE128 = (128, 128, 128)
file_len = 4
header = wkw.Header(np.uint8,
block_type=wkw.Header.BLOCK_TYPE_LZ4,
file_len=file_len)
test_data = generate_test_data(header.voxel_type, SIZE128)
ones_data = np.ones(SIZE128).astype(header.voxel_type)
with wkw.Dataset.create('tests/tmp', header) as dataset:
dataset.write(POSITION, test_data)
random_compressed_size = path.getsize(
path.join('tests/tmp', 'z0', 'y0', 'x0.wkw'))
with wkw.Dataset.open('tests/tmp') as dataset:
dataset.write(POSITION, ones_data)
empty_compressed_size = path.getsize(
path.join('tests/tmp', 'z0', 'y0', 'x0.wkw'))
assert empty_compressed_size < random_compressed_size
with wkw.Dataset.open('tests/tmp') as dataset:
assert np.all(dataset.read(POSITION, SIZE128) == ones_data)
def test_element_class_conversion(tmp_path: Path) -> None:
test_wkw_path = tmp_path / "test_metadata"
prediction_layer_name = "prediction"
prediction_wkw_info = WkwDatasetInfo(
test_wkw_path,
prediction_layer_name,
1,
wkw.Header(np.float32, num_channels=3, file_len=1),
)
ensure_wkw(prediction_wkw_info)
write_custom_layer(test_wkw_path, "prediction", np.float32, num_channels=3)
write_webknossos_metadata(
test_wkw_path,
"test_metadata",
(11.24, 11.24, 28),
compute_max_id=True,
exact_bounding_box={"topLeft": [0, 0, 0], "width": 4, "height": 4, "depth": 4},
)
write_custom_layer(test_wkw_path, "segmentation", np.float64, num_channels=1)
write_custom_layer(test_wkw_path, "color", np.uint8, num_channels=3)
refresh_metadata(test_wkw_path)
check_element_class_of_layer(
test_wkw_path, "prediction", "float", np.dtype(np.float32)
)
check_element_class_of_layer(
test_wkw_path, "segmentation", "double", np.dtype(np.float64)
)
check_element_class_of_layer(test_wkw_path, "color", "uint24", np.dtype(np.uint8))
def compress_mag(source_path, layer_name, target_path, mag: Mag, args=None):
if path.exists(path.join(target_path, layer_name, str(mag))):
logging.error("Target path '{}' already exists".format(target_path))
exit(1)
if args is not None and hasattr(args, "dtype"):
header = wkw.Header(convert_element_class_to_dtype(args.dtype))
else:
header = None
source_wkw_info = WkwDatasetInfo(source_path, layer_name, mag, header)
target_mag_path = path.join(target_path, layer_name, str(mag))
logging.info("Compressing mag {0} in '{1}'".format(str(mag),
target_mag_path))
with open_wkw(source_wkw_info) as source_wkw:
source_wkw.compress(target_mag_path)
with get_executor_for_args(args) as executor:
job_args = []
for file in source_wkw.list_files():
rel_file = path.relpath(file, source_wkw.root)
job_args.append((file, path.join(target_mag_path, rel_file)))
wait_and_ensure_success(
executor.map_to_futures(compress_file_job, job_args))
logging.info("Mag {0} successfully compressed".format(str(mag)))
def cubing(source_path, target_path, layer_name, dtype, batch_size, args) -> dict:
source_files = find_source_filenames(source_path)
# All images are assumed to have equal dimensions
num_x, num_y = image_reader.read_dimensions(source_files[0])
num_channels = image_reader.read_channel_count(source_files[0])
num_z = len(source_files)
target_mag = Mag(args.target_mag)
target_wkw_info = WkwDatasetInfo(
target_path,
layer_name,
target_mag,
wkw.Header(
convert_element_class_to_dtype(dtype),
num_channels,
file_len=args.wkw_file_len,
),
)
interpolation_mode = parse_interpolation_mode(
args.interpolation_mode, target_wkw_info.layer_name
)
if target_mag != Mag(1):
logging.info(
f"Downsampling the cubed image to {target_mag} in memory with interpolation mode {interpolation_mode}."
)
logging.info("Found source files: count={} size={}x{}".format(num_z, num_x, num_y))
ensure_wkw(target_wkw_info)
start_z = args.start_z
with get_executor_for_args(args) as executor:
job_args = []
# We iterate over all z sections
for z in range(start_z, num_z + start_z, BLOCK_LEN):
# Prepare z batches
max_z = min(num_z + start_z, z + BLOCK_LEN)
z_batch = list(range(z, max_z))
# Prepare job
job_args.append(
(
target_wkw_info,
z_batch,
target_mag,
interpolation_mode,
source_files[z - start_z : max_z - start_z],
batch_size,
(num_x, num_y),
args.pad,
)
)
wait_and_ensure_success(executor.map_to_futures(cubing_job, job_args))
# Return Bounding Box
return {"topLeft": [0, 0, 0], "width": num_x, "height": num_y, "depth": num_z}
def test_row_major_order_with_different_voxel_size():
data_shape = (4, 3, 9)
data = generate_test_data(np.uint16, data_shape)
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint16)) as dataset:
dataset.write((3, 1, 0), data)
read_data = dataset.read((3, 1, 0), data_shape)
assert np.all(data == read_data)
def test_readwrite_live_compression_should_enforce_full_file_write():
with pytest.raises(Exception):
with wkw.Dataset.create(
'tests/tmp', wkw.Header(np.uint8,
block_type=BLOCK_TYPE_LZ4)) as dataset:
test_data = generate_test_data(dataset.header.voxel_type)
dataset.write(POSITION, test_data)
def test_row_major_order_with_offset():
data_shape = (17, 1, 4)
data = generate_test_data(np.uint8, data_shape)
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)) as dataset:
dataset.write((15, 2, 0), data)
read_data = dataset.read((15, 2, 0), data_shape)
assert np.all(data == read_data)
def test_row_major_order_with_channels():
data_shape = (2, 4, 3, 9)
data = generate_test_data(np.uint8, data_shape)
with wkw.Dataset.create("tests/tmp",
wkw.Header(np.uint8, num_channels=2)) as dataset:
dataset.write((3, 1, 0), data)
read_data = dataset.read((3, 1, 0), data_shape[1:])
assert np.all(data == read_data)
def test_view_on_np_array():
data_shape = (4, 4, 9)
data = generate_test_data(np.uint16, data_shape)
data = data[:, ::2]
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint16)) as dataset:
dataset.write((3, 1, 0), data)
read_data = dataset.read((3, 1, 0), data.shape)
assert np.all(data == read_data)
def test_column_major_order_with_channels_and_different_voxel_size():
data_shape = (2, 4, 3, 9)
data = generate_test_data(np.uint16, data_shape, order="F")
with wkw.Dataset.create("tests/tmp",
wkw.Header(np.uint16, num_channels=2)) as dataset:
dataset.write((3, 1, 0), data)
read_data = dataset.read((3, 1, 0), data_shape[1:])
assert np.all(data == read_data)
def test_readwrite():
with wkw.Dataset.create('tests/tmp', wkw.Header(np.uint8)) as dataset:
header_size = path.getsize(path.join('tests/tmp', 'header.wkw'))
test_data = generate_test_data(dataset.header.voxel_type)
dataset.write(POSITION, test_data)
assert path.getsize(path.join('tests/tmp', 'z0', 'y0', 'x0.wkw')) == \
np.prod(SIZE) * (dataset.header.file_len ** 3) + header_size
assert np.all(dataset.read(POSITION, SIZE) == test_data)
def convert_nifti(source_nifti_path,
target_path,
layer_name,
dtype,
scale,
mag=1):
target_wkw_info = WkwDatasetInfo(str(target_path.resolve()), layer_name,
mag, wkw.Header(np.dtype(dtype)))
ensure_wkw(target_wkw_info)
ref_time = time.time()
# Assume no translation
offset = (0, 0, 0)
with open_wkw(target_wkw_info) as target_wkw:
source_nifti = nib.load(str(source_nifti_path.resolve()))
cube_data = np.array(source_nifti.get_fdata())
if len(source_nifti.shape) == 3:
size = list(source_nifti.shape)
cube_data = cube_data.reshape((1, ) + source_nifti.shape)
elif len(source_nifti.shape) == 4:
size = list(source_nifti.shape[:-1])
cube_data = np.transpose(cube_data, (3, 0, 1, 2))
else:
logging.warning(
"Converting of {} failed! Too many or too less dimensions".
format(source_nifti_path))
return
if scale is None:
scale = tuple(map(float, source_nifti.header["pixdim"][:3]))
cube_data = to_target_datatype(cube_data, dtype)
target_wkw.write(offset, cube_data)
logging.debug("Converting of {} took {:.8f}s".format(
source_nifti_path,
time.time() - ref_time))
write_webknossos_metadata(
str(target_path),
source_nifti_path.stem,
scale=scale,
exact_bounding_box={
"topLeft": offset,
"width": size[0],
"height": size[1],
"depth": size[2],
},
)
def test_readwrite():
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)) as dataset:
header_size = path.getsize(path.join("tests/tmp", "header.wkw"))
test_data = generate_test_data(dataset.header.voxel_type)
dataset.write(POSITION, test_data)
assert (path.getsize(path.join("tests/tmp", "z0", "y0",
"x0.wkw")) == np.prod(SIZE) *
(dataset.header.file_len**3) + header_size)
assert np.all(dataset.read(POSITION, SIZE) == test_data)
def write_custom_layer(target_path, layer_name, dtype, num_channels):
data = (
np.arange(4 * 4 * 4 * num_channels)
.reshape((num_channels, 4, 4, 4))
.astype(dtype)
)
prediction_wkw_info = WkwDatasetInfo(
target_path, layer_name, 1, wkw.Header(dtype, num_channels)
)
ensure_wkw(prediction_wkw_info)
with open_wkw(prediction_wkw_info) as dataset:
dataset.write(off=(0, 0, 0), data=data)
def test_big_read():
data = np.ones((10, 10, 764), order="C", dtype=np.uint8)
offset = np.array([0, 0, 640])
bottom = (2000, 2000, 2000)
mem_buffer = np.zeros(bottom, dtype=np.uint8, order="F")
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)) as dataset:
dataset.write(offset, data)
mem_buffer[offset[0]:offset[0] + data.shape[0],
offset[1]:offset[1] + data.shape[1],
offset[2]:offset[2] + data.shape[2], ] = data
read_data = dataset.read((0, 0, 0), bottom)
assert np.all(read_data == mem_buffer)
def tile_cubing(target_path,
layer_name,
dtype,
batch_size,
input_path_pattern,
args=None):
decimal_lengths = get_digit_counts_for_dimensions(input_path_pattern)
(
min_dimensions,
max_dimensions,
arbitrary_file,
file_count,
) = detect_interval_for_dimensions(input_path_pattern, decimal_lengths)
if not arbitrary_file:
logging.error(
f"No source files found. Maybe the input_path_pattern was wrong. You provided: {input_path_pattern}"
)
return
# Determine tile size from first matching file
tile_size = image_reader.read_dimensions(arbitrary_file)
num_channels = image_reader.read_channel_count(arbitrary_file)
tile_size = (tile_size[0], tile_size[1], num_channels)
logging.info("Found source files: count={} with tile_size={}x{}".format(
file_count, tile_size[0], tile_size[1]))
target_wkw_info = WkwDatasetInfo(
target_path,
layer_name,
1,
wkw.Header(convert_element_class_to_dtype(dtype), num_channels),
)
ensure_wkw(target_wkw_info)
with get_executor_for_args(args) as executor:
job_args = []
# Iterate over all z batches
for z_batch in get_regular_chunks(min_dimensions["z"],
max_dimensions["z"], BLOCK_LEN):
job_args.append((
target_wkw_info,
list(z_batch),
input_path_pattern,
batch_size,
tile_size,
min_dimensions,
max_dimensions,
decimal_lengths,
))
wait_and_ensure_success(
executor.map_to_futures(tile_cubing_job, job_args))
def write_and_test_in_given_order(wkw_path, order):
data_shape = (35, 35, 35)
data = generate_test_data(np.uint8, data_shape, order=order)
with wkw.Dataset.create(wkw_path, wkw.Header(np.uint8)) as dataset:
dataset.write((0, 0, 0), np.ones((35, 35, 64), dtype=np.uint8))
dataset.write((1, 2, 3), data)
read_data = dataset.read((1, 2, 3), (35, 35, 35))
before = dataset.read((0, 0, 0), (1, 2, 3))
after = dataset.read((0, 0, 38), (35, 35, 26))
assert np.all(data == read_data)
assert np.all(before == 1)
assert np.all(after == 1)
def test_readwrite_live_compression_should_not_allow_inconsistent_writes():
SIZE129 = (129, 128, 128)
file_len = 4
header = wkw.Header(np.uint8,
block_type=wkw.Header.BLOCK_TYPE_LZ4,
file_len=file_len)
test_data = generate_test_data(header.voxel_type, SIZE129)
empty_data = np.zeros(SIZE129).astype(header.voxel_type)
with wkw.Dataset.create('tests/tmp', header) as dataset:
with pytest.raises(Exception):
dataset.write(POSITION, test_data)
with wkw.Dataset.open('tests/tmp') as dataset:
assert np.all(dataset.read(POSITION, SIZE129) == empty_data)
def test_multiple_writes_and_reads():
mem_buffer = np.zeros((200, 200, 200), dtype=np.uint8, order="F")
with wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)) as dataset:
for i in range(10):
offset = np.random.randint(100, size=(3))
size = np.random.randint(1, 100, size=(3))
order = np.random.choice(["F", "C"])
data = generate_test_data(np.uint8, [1] + list(size), order)
dataset.write(offset, data)
mem_buffer[offset[0]:offset[0] + size[0],
offset[1]:offset[1] + size[1],
offset[2]:offset[2] + size[2], ] = data
read_data = dataset.read((0, 0, 0), (200, 200, 200))
assert np.all(mem_buffer == read_data)
def test_compress():
with wkw.Dataset.create('tests/tmp', wkw.Header(np.uint8)) as dataset:
test_data = generate_test_data(dataset.header.voxel_type)
dataset.write(POSITION, test_data)
with dataset.compress('tests/tmp2', compress_files=True) as dataset2:
assert dataset2.header.voxel_type == np.uint8
assert dataset2.header.block_type == wkw.Header.BLOCK_TYPE_LZ4HC
header_size = path.getsize(path.join('tests/tmp2', 'header.wkw'))
assert path.exists(path.join('tests/tmp2', 'header.wkw'))
assert path.getsize(path.join('tests/tmp2', 'z0', 'y0', 'x0.wkw')) < \
np.prod(SIZE) * (dataset2.header.file_len ** 3) + header_size
assert np.all(dataset2.read(POSITION, SIZE) == test_data)
def test_non_negative_offsets():
wkw.Dataset.create("tests/tmp", wkw.Header(np.uint8)).close()
with pytest.raises(AssertionError):
with wkw.Dataset.open("tests/tmp") as dataset:
dataset.read((-1, 0, 0), (0, 0, 0))
with pytest.raises(AssertionError):
with wkw.Dataset.open("tests/tmp") as dataset:
dataset.read((0, -1, 0), (0, 0, 0))
with pytest.raises(AssertionError):
with wkw.Dataset.open("tests/tmp") as dataset:
dataset.read((0, 0, -1), (0, 0, 0))
with pytest.raises(AssertionError):
with wkw.Dataset.open("tests/tmp") as dataset:
dataset.read((0, 0, 0), (-1, -1, -1))
def test_readwrite_live_compression():
SIZE128 = (128, 128, 128)
file_len = 4
header = wkw.Header(np.uint8,
block_type=wkw.Header.BLOCK_TYPE_LZ4,
file_len=file_len)
with wkw.Dataset.create('tests/tmp', header) as dataset:
header_size = path.getsize(path.join('tests/tmp', 'header.wkw'))
test_data = generate_test_data(dataset.header.voxel_type, SIZE128)
dataset.write(POSITION, test_data)
# The size should be less than if it was not compressed
assert path.getsize(path.join('tests/tmp', 'z0', 'y0', 'x0.wkw')) < \
np.prod(SIZE128) * (dataset.header.file_len ** 3) + header_size
with wkw.Dataset.open('tests/tmp') as dataset:
assert np.all(dataset.read(POSITION, SIZE128) == test_data)
