def test_dtype_conversion_float_to_int(float_dtype):
reference_test_data = np.array(
[-np.inf, -100, 0.4, 0.6, 2**8, 2**16, 2**32, 2**64, np.inf],
dtype=float_dtype)
test_data = np.copy(reference_test_data)
t = get_chunk_dtype_transformer(test_data.dtype, "uint8")
assert np.array_equal(
t(test_data),
np.array([0, 0, 0, 1, 255, 255, 255, 255, 255], dtype="uint8"))
# Ensure that the input data was not modified in-place
assert np.array_equal(test_data, reference_test_data)
t = get_chunk_dtype_transformer(test_data.dtype, "uint16")
assert np.array_equal(
t(test_data),
np.array([0, 0, 0, 1, 256, 65535, 65535, 65535, 65535],
dtype="uint16"))
# Ensure that the input data was not modified in-place
assert np.array_equal(test_data, reference_test_data)
t = get_chunk_dtype_transformer(test_data.dtype, "uint32")
assert np.array_equal(
t(test_data),
np.array([0, 0, 0, 1, 256, 65536, 2**32 - 1, 2**32 - 1, 2**32 - 1],
dtype="uint32"))
# Ensure that the input data was not modified in-place
assert np.array_equal(test_data, reference_test_data)
# Use a different test for uint64: tests for 2**64 and +infinity are
# expected to fail due to a bug in NumPy, see below.
uint64_test_data = np.array([-np.inf, -100, 0.4, 0.6, 2**63],
dtype=float_dtype)
t = get_chunk_dtype_transformer(uint64_test_data.dtype, "uint64")
assert np.array_equal(t(uint64_test_data),
np.array([0, 0, 0, 1, 2**63], dtype="uint64"))
def convert_chunks(source_url, dest_url, copy_info=False,
options={}):
"""Convert precomputed chunks between different encodings"""
source_accessor = neuroglancer_scripts.accessor.get_accessor_for_url(
source_url
)
chunk_reader = precomputed_io.get_IO_for_existing_dataset(source_accessor)
source_info = chunk_reader.info
dest_accessor = neuroglancer_scripts.accessor.get_accessor_for_url(
dest_url, options
)
if copy_info:
chunk_writer = precomputed_io.get_IO_for_new_dataset(
source_info, dest_accessor, encoder_options=options
)
else:
chunk_writer = precomputed_io.get_IO_for_existing_dataset(
dest_accessor, encoder_options=options
)
dest_info = chunk_writer.info
chunk_transformer = data_types.get_chunk_dtype_transformer(
source_info["data_type"], dest_info["data_type"]
)
for scale_index in reversed(range(len(dest_info["scales"]))):
convert_chunks_for_scale(chunk_reader,
dest_info, chunk_writer, scale_index,
chunk_transformer)
def downscale(self, chunk, downscaling_factors):
if not self.check_factors(downscaling_factors):
raise NotImplementedError
dtype = chunk.dtype
# Use a floating-point type for arithmetic
work_dtype = np.promote_types(chunk.dtype, np.floating)
chunk = chunk.astype(work_dtype, casting="safe")
half = work_dtype.type(0.5)
if downscaling_factors[2] == 2:
if chunk.shape[1] % 2 != 0:
chunk = np.pad(chunk, ((0, 0), (0, 1), (0, 0), (0, 0)),
self.padding_mode, **self.pad_kwargs)
chunk = half * (chunk[:, ::2, :, :] + chunk[:, 1::2, :, :])
if downscaling_factors[1] == 2:
if chunk.shape[2] % 2 != 0:
chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 1), (0, 0)),
self.padding_mode, **self.pad_kwargs)
chunk = half * (chunk[:, :, ::2, :] + chunk[:, :, 1::2, :])
if downscaling_factors[0] == 2:
if chunk.shape[3] % 2 != 0:
chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 0), (0, 1)),
self.padding_mode, **self.pad_kwargs)
chunk = half * (chunk[:, :, :, ::2] + chunk[:, :, :, 1::2])
dtype_converter = get_chunk_dtype_transformer(work_dtype, dtype)
return dtype_converter(chunk)
def test_dtype_conversion_integer_upcasting(dtype):
iinfo_uint64 = np.iinfo(np.uint64)
iinfo = np.iinfo(dtype)
# The test will need to be rewritten if NG_INTEGER_DATA_TYPES ever includes
# signed data types
assert (iinfo_uint64.max >= iinfo.max and iinfo_uint64.min <= iinfo.min)
test_data = np.array([iinfo.min, iinfo.max], dtype=dtype)
t = get_chunk_dtype_transformer(test_data.dtype, "uint64")
assert np.array_equal(t(test_data), test_data)
def test_dtype_conversion_identity(dtype):
if np.issubdtype(dtype, np.integer):
iinfo = np.iinfo(dtype)
test_data = np.array([iinfo.min, 0, iinfo.max], dtype=dtype)
else:
finfo = np.finfo(dtype)
test_data = np.array([finfo.min, 0, finfo.max, np.inf], dtype=dtype)
t = get_chunk_dtype_transformer(dtype, dtype)
res = t(test_data)
assert np.array_equal(res, test_data)
def test_dtype_conversion_float_to_uint64():
# Conversion to uint64 may result in loss of precision, because of a known
# bug / approximation in NumPy, where dtype promotion between a 64-bit
# (u)int and any float will return float64, even though this type can only
# hold all integers up to 2**53 (see e.g.
# https://github.com/numpy/numpy/issues/8851).
test_data = np.array([2**64, np.inf])
t = get_chunk_dtype_transformer(test_data.dtype, "uint64")
assert np.array_equal(t(test_data),
np.array([2**64 - 1, 2**64 - 1], dtype="uint64"))
def slices_to_raw_chunks(slice_filename_lists,
dest_url,
input_orientation,
options={}):
"""Convert a list of 2D slices to Neuroglancer pre-computed chunks.
:param dict info: the JSON dictionary that describes the dataset for
Neuroglancer. Only the information from the first scale is used.
:param list slice_filename_lists: a list of lists of filenames. Files from
each inner list are read as 2D images and concatenated along the third
axis. Blocks from the outer list are concatenated along a fourth axis,
representing the image channels.
:param tuple input_axis_inversions: a 3-tuple in (column, row, slice)
order. Each value must be 1 (preserve orientation along the axis) or -1
(invert orientation along the axis).
:param tuple input_axis_permutation: a 3-tuple in (column, row, slice)
order. Each value is 0 for X (L-R axis), 1 for Y (A-P axis), 2 for Z (I-S
axis).
"""
accessor = neuroglancer_scripts.accessor.get_accessor_for_url(
dest_url, options)
pyramid_writer = precomputed_io.get_IO_for_existing_dataset(
accessor, encoder_options=options)
info = pyramid_writer.info
assert len(info["scales"][0]["chunk_sizes"]) == 1 # more not implemented
chunk_size = info["scales"][0]["chunk_sizes"][0] # in RAS order (X, Y, Z)
size = info["scales"][0]["size"] # in RAS order (X, Y, Z)
key = info["scales"][0]["key"]
dtype = np.dtype(info["data_type"]).newbyteorder("<")
num_channels = info["num_channels"]
input_axis_permutation = tuple(AXIS_PERMUTATION_FOR_RAS[a]
for a in input_orientation)
input_axis_inversions = tuple(AXIS_INVERSION_FOR_RAS[a]
for a in input_orientation)
# Here x, y, and z refer to the data orientation in output chunks (which
# should correspond to RAS+ anatomical axes). For the data orientation in
# input slices the terms (column (index along image width), row (index
# along image height), slice) are used.
# permutation_to_input is a 3-tuple in RAS (X, Y, Z) order.
# Each value is 0 column, 1 for row, 2 for slice.
permutation_to_input = invert_permutation(input_axis_permutation)
# input_size and input_chunk_size are in (column, row, slice) order.
input_size = permute(size, input_axis_permutation)
input_chunk_size = permute(chunk_size, input_axis_permutation)
for filename_list in slice_filename_lists:
if len(filename_list) != input_size[2]:
raise ValueError("{} slices found where {} were expected".format(
len(filename_list), input_size[2]))
for slice_chunk_idx in trange(
(input_size[2] - 1) // input_chunk_size[2] + 1,
desc="converting slice groups",
leave=True,
unit="slice groups"):
first_slice_in_order = input_chunk_size[2] * slice_chunk_idx
last_slice_in_order = min(input_chunk_size[2] * (slice_chunk_idx + 1),
input_size[2])
if input_axis_inversions[2] == -1:
first_slice = input_size[2] - first_slice_in_order - 1
last_slice = input_size[2] - last_slice_in_order - 1
else:
first_slice = first_slice_in_order
last_slice = last_slice_in_order
slice_slicing = np.s_[first_slice:last_slice:input_axis_inversions[2]]
tqdm.write("Reading slices {0} to {1} ({2}B memory needed)... ".format(
first_slice, last_slice - input_axis_inversions[2],
readable_count(input_size[0] * input_size[1] *
(last_slice_in_order - first_slice_in_order + 1) *
num_channels * dtype.itemsize)))
def load_z_stack(slice_filenames):
# Loads the data in [slice, row, column] C-contiguous order
block = skimage.io.concatenate_images(
skimage.io.imread(str(filename))
for filename in slice_filenames[slice_slicing])
assert block.shape[2] == input_size[0] # check slice width
assert block.shape[1] == input_size[1] # check slice height
if block.ndim == 4:
# Scikit-image loads multi-channel (e.g. RGB) images in [slice,
# row, column, channel] order, while Neuroglancer expects
# channel to come first (in C-contiguous indexing).
block = np.moveaxis(block, (3, 0, 1, 2), (0, 1, 2, 3))
elif block.ndim == 3:
block = block[np.newaxis, :, :, :]
else:
raise ValueError(
"block has unexpected dimensionality (ndim={})".format(
block.ndim))
return block
# Concatenate all channels from different directories
block = np.concatenate([
load_z_stack(filename_list)
for filename_list in slice_filename_lists
],
axis=0)
assert block.shape[0] == num_channels
# Flip and permute axes to go from input (channel, slice, row, column)
# to Neuroglancer (channel, Z, Y, X)
block = block[:, :, ::input_axis_inversions[1], ::
input_axis_inversions[0]]
block = np.moveaxis(block, (3, 2, 1),
(3 - a for a in input_axis_permutation))
# equivalent: np.transpose(block, axes=([0] + [3 - a for a in
# reversed(invert_permutation(input_axis_permutation))]))
chunk_dtype_transformer = get_chunk_dtype_transformer(
block.dtype, dtype)
progress_bar = tqdm(
total=(((input_size[1] - 1) // input_chunk_size[1] + 1) *
((input_size[0] - 1) // input_chunk_size[0] + 1)),
desc="writing chunks",
unit="chunks",
leave=False)
for row_chunk_idx in range((input_size[1] - 1) // input_chunk_size[1] +
1):
row_slicing = np.s_[input_chunk_size[1] * row_chunk_idx:min(
input_chunk_size[1] * (row_chunk_idx + 1), input_size[1])]
for column_chunk_idx in range((input_size[0] - 1) //
input_chunk_size[0] + 1):
column_slicing = np.s_[input_chunk_size[0] *
column_chunk_idx:min(
input_chunk_size[0] *
(column_chunk_idx +
1), input_size[0])]
input_slicing = (column_slicing, row_slicing, np.s_[:])
x_slicing, y_slicing, z_slicing = permute(
input_slicing, permutation_to_input)
chunk = block[:, z_slicing, y_slicing, x_slicing]
# This variable represents the coordinates with real slice
# numbers, instead of within-block slice numbers.
input_coords = ((column_slicing.start, column_slicing.stop),
(row_slicing.start, row_slicing.stop),
(first_slice_in_order, last_slice_in_order))
x_coords, y_coords, z_coords = permute(input_coords,
permutation_to_input)
assert chunk.size == ((x_coords[1] - x_coords[0]) *
(y_coords[1] - y_coords[0]) *
(z_coords[1] - z_coords[0]) *
num_channels)
chunk_coords = (x_coords[0], x_coords[1], y_coords[0],
y_coords[1], z_coords[0], z_coords[1])
pyramid_writer.write_chunk(
chunk_dtype_transformer(chunk, preserve_input=False), key,
chunk_coords)
progress_bar.update()
# free up memory before reading next block (prevent doubled memory
# usage)
del block