def test_dtype_conversion_float_to_int(float_dtype): reference_test_data = np.array( [-np.inf, -100, 0.4, 0.6, 2**8, 2**16, 2**32, 2**64, np.inf], dtype=float_dtype) test_data = np.copy(reference_test_data) t = get_chunk_dtype_transformer(test_data.dtype, "uint8") assert np.array_equal( t(test_data), np.array([0, 0, 0, 1, 255, 255, 255, 255, 255], dtype="uint8")) # Ensure that the input data was not modified in-place assert np.array_equal(test_data, reference_test_data) t = get_chunk_dtype_transformer(test_data.dtype, "uint16") assert np.array_equal( t(test_data), np.array([0, 0, 0, 1, 256, 65535, 65535, 65535, 65535], dtype="uint16")) # Ensure that the input data was not modified in-place assert np.array_equal(test_data, reference_test_data) t = get_chunk_dtype_transformer(test_data.dtype, "uint32") assert np.array_equal( t(test_data), np.array([0, 0, 0, 1, 256, 65536, 2**32 - 1, 2**32 - 1, 2**32 - 1], dtype="uint32")) # Ensure that the input data was not modified in-place assert np.array_equal(test_data, reference_test_data) # Use a different test for uint64: tests for 2**64 and +infinity are # expected to fail due to a bug in NumPy, see below. uint64_test_data = np.array([-np.inf, -100, 0.4, 0.6, 2**63], dtype=float_dtype) t = get_chunk_dtype_transformer(uint64_test_data.dtype, "uint64") assert np.array_equal(t(uint64_test_data), np.array([0, 0, 0, 1, 2**63], dtype="uint64"))
def convert_chunks(source_url, dest_url, copy_info=False, options={}): """Convert precomputed chunks between different encodings""" source_accessor = neuroglancer_scripts.accessor.get_accessor_for_url( source_url ) chunk_reader = precomputed_io.get_IO_for_existing_dataset(source_accessor) source_info = chunk_reader.info dest_accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options ) if copy_info: chunk_writer = precomputed_io.get_IO_for_new_dataset( source_info, dest_accessor, encoder_options=options ) else: chunk_writer = precomputed_io.get_IO_for_existing_dataset( dest_accessor, encoder_options=options ) dest_info = chunk_writer.info chunk_transformer = data_types.get_chunk_dtype_transformer( source_info["data_type"], dest_info["data_type"] ) for scale_index in reversed(range(len(dest_info["scales"]))): convert_chunks_for_scale(chunk_reader, dest_info, chunk_writer, scale_index, chunk_transformer)
def downscale(self, chunk, downscaling_factors): if not self.check_factors(downscaling_factors): raise NotImplementedError dtype = chunk.dtype # Use a floating-point type for arithmetic work_dtype = np.promote_types(chunk.dtype, np.floating) chunk = chunk.astype(work_dtype, casting="safe") half = work_dtype.type(0.5) if downscaling_factors[2] == 2: if chunk.shape[1] % 2 != 0: chunk = np.pad(chunk, ((0, 0), (0, 1), (0, 0), (0, 0)), self.padding_mode, **self.pad_kwargs) chunk = half * (chunk[:, ::2, :, :] + chunk[:, 1::2, :, :]) if downscaling_factors[1] == 2: if chunk.shape[2] % 2 != 0: chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 1), (0, 0)), self.padding_mode, **self.pad_kwargs) chunk = half * (chunk[:, :, ::2, :] + chunk[:, :, 1::2, :]) if downscaling_factors[0] == 2: if chunk.shape[3] % 2 != 0: chunk = np.pad(chunk, ((0, 0), (0, 0), (0, 0), (0, 1)), self.padding_mode, **self.pad_kwargs) chunk = half * (chunk[:, :, :, ::2] + chunk[:, :, :, 1::2]) dtype_converter = get_chunk_dtype_transformer(work_dtype, dtype) return dtype_converter(chunk)
def test_dtype_conversion_integer_upcasting(dtype): iinfo_uint64 = np.iinfo(np.uint64) iinfo = np.iinfo(dtype) # The test will need to be rewritten if NG_INTEGER_DATA_TYPES ever includes # signed data types assert (iinfo_uint64.max >= iinfo.max and iinfo_uint64.min <= iinfo.min) test_data = np.array([iinfo.min, iinfo.max], dtype=dtype) t = get_chunk_dtype_transformer(test_data.dtype, "uint64") assert np.array_equal(t(test_data), test_data)
def test_dtype_conversion_identity(dtype): if np.issubdtype(dtype, np.integer): iinfo = np.iinfo(dtype) test_data = np.array([iinfo.min, 0, iinfo.max], dtype=dtype) else: finfo = np.finfo(dtype) test_data = np.array([finfo.min, 0, finfo.max, np.inf], dtype=dtype) t = get_chunk_dtype_transformer(dtype, dtype) res = t(test_data) assert np.array_equal(res, test_data)
def test_dtype_conversion_float_to_uint64(): # Conversion to uint64 may result in loss of precision, because of a known # bug / approximation in NumPy, where dtype promotion between a 64-bit # (u)int and any float will return float64, even though this type can only # hold all integers up to 2**53 (see e.g. # https://github.com/numpy/numpy/issues/8851). test_data = np.array([2**64, np.inf]) t = get_chunk_dtype_transformer(test_data.dtype, "uint64") assert np.array_equal(t(test_data), np.array([2**64 - 1, 2**64 - 1], dtype="uint64"))
def slices_to_raw_chunks(slice_filename_lists, dest_url, input_orientation, options={}): """Convert a list of 2D slices to Neuroglancer pre-computed chunks. :param dict info: the JSON dictionary that describes the dataset for Neuroglancer. Only the information from the first scale is used. :param list slice_filename_lists: a list of lists of filenames. Files from each inner list are read as 2D images and concatenated along the third axis. Blocks from the outer list are concatenated along a fourth axis, representing the image channels. :param tuple input_axis_inversions: a 3-tuple in (column, row, slice) order. Each value must be 1 (preserve orientation along the axis) or -1 (invert orientation along the axis). :param tuple input_axis_permutation: a 3-tuple in (column, row, slice) order. Each value is 0 for X (L-R axis), 1 for Y (A-P axis), 2 for Z (I-S axis). """ accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options) pyramid_writer = precomputed_io.get_IO_for_existing_dataset( accessor, encoder_options=options) info = pyramid_writer.info assert len(info["scales"][0]["chunk_sizes"]) == 1 # more not implemented chunk_size = info["scales"][0]["chunk_sizes"][0] # in RAS order (X, Y, Z) size = info["scales"][0]["size"] # in RAS order (X, Y, Z) key = info["scales"][0]["key"] dtype = np.dtype(info["data_type"]).newbyteorder("<") num_channels = info["num_channels"] input_axis_permutation = tuple(AXIS_PERMUTATION_FOR_RAS[a] for a in input_orientation) input_axis_inversions = tuple(AXIS_INVERSION_FOR_RAS[a] for a in input_orientation) # Here x, y, and z refer to the data orientation in output chunks (which # should correspond to RAS+ anatomical axes). For the data orientation in # input slices the terms (column (index along image width), row (index # along image height), slice) are used. # permutation_to_input is a 3-tuple in RAS (X, Y, Z) order. # Each value is 0 column, 1 for row, 2 for slice. permutation_to_input = invert_permutation(input_axis_permutation) # input_size and input_chunk_size are in (column, row, slice) order. input_size = permute(size, input_axis_permutation) input_chunk_size = permute(chunk_size, input_axis_permutation) for filename_list in slice_filename_lists: if len(filename_list) != input_size[2]: raise ValueError("{} slices found where {} were expected".format( len(filename_list), input_size[2])) for slice_chunk_idx in trange( (input_size[2] - 1) // input_chunk_size[2] + 1, desc="converting slice groups", leave=True, unit="slice groups"): first_slice_in_order = input_chunk_size[2] * slice_chunk_idx last_slice_in_order = min(input_chunk_size[2] * (slice_chunk_idx + 1), input_size[2]) if input_axis_inversions[2] == -1: first_slice = input_size[2] - first_slice_in_order - 1 last_slice = input_size[2] - last_slice_in_order - 1 else: first_slice = first_slice_in_order last_slice = last_slice_in_order slice_slicing = np.s_[first_slice:last_slice:input_axis_inversions[2]] tqdm.write("Reading slices {0} to {1} ({2}B memory needed)... ".format( first_slice, last_slice - input_axis_inversions[2], readable_count(input_size[0] * input_size[1] * (last_slice_in_order - first_slice_in_order + 1) * num_channels * dtype.itemsize))) def load_z_stack(slice_filenames): # Loads the data in [slice, row, column] C-contiguous order block = skimage.io.concatenate_images( skimage.io.imread(str(filename)) for filename in slice_filenames[slice_slicing]) assert block.shape[2] == input_size[0] # check slice width assert block.shape[1] == input_size[1] # check slice height if block.ndim == 4: # Scikit-image loads multi-channel (e.g. RGB) images in [slice, # row, column, channel] order, while Neuroglancer expects # channel to come first (in C-contiguous indexing). block = np.moveaxis(block, (3, 0, 1, 2), (0, 1, 2, 3)) elif block.ndim == 3: block = block[np.newaxis, :, :, :] else: raise ValueError( "block has unexpected dimensionality (ndim={})".format( block.ndim)) return block # Concatenate all channels from different directories block = np.concatenate([ load_z_stack(filename_list) for filename_list in slice_filename_lists ], axis=0) assert block.shape[0] == num_channels # Flip and permute axes to go from input (channel, slice, row, column) # to Neuroglancer (channel, Z, Y, X) block = block[:, :, ::input_axis_inversions[1], :: input_axis_inversions[0]] block = np.moveaxis(block, (3, 2, 1), (3 - a for a in input_axis_permutation)) # equivalent: np.transpose(block, axes=([0] + [3 - a for a in # reversed(invert_permutation(input_axis_permutation))])) chunk_dtype_transformer = get_chunk_dtype_transformer( block.dtype, dtype) progress_bar = tqdm( total=(((input_size[1] - 1) // input_chunk_size[1] + 1) * ((input_size[0] - 1) // input_chunk_size[0] + 1)), desc="writing chunks", unit="chunks", leave=False) for row_chunk_idx in range((input_size[1] - 1) // input_chunk_size[1] + 1): row_slicing = np.s_[input_chunk_size[1] * row_chunk_idx:min( input_chunk_size[1] * (row_chunk_idx + 1), input_size[1])] for column_chunk_idx in range((input_size[0] - 1) // input_chunk_size[0] + 1): column_slicing = np.s_[input_chunk_size[0] * column_chunk_idx:min( input_chunk_size[0] * (column_chunk_idx + 1), input_size[0])] input_slicing = (column_slicing, row_slicing, np.s_[:]) x_slicing, y_slicing, z_slicing = permute( input_slicing, permutation_to_input) chunk = block[:, z_slicing, y_slicing, x_slicing] # This variable represents the coordinates with real slice # numbers, instead of within-block slice numbers. input_coords = ((column_slicing.start, column_slicing.stop), (row_slicing.start, row_slicing.stop), (first_slice_in_order, last_slice_in_order)) x_coords, y_coords, z_coords = permute(input_coords, permutation_to_input) assert chunk.size == ((x_coords[1] - x_coords[0]) * (y_coords[1] - y_coords[0]) * (z_coords[1] - z_coords[0]) * num_channels) chunk_coords = (x_coords[0], x_coords[1], y_coords[0], y_coords[1], z_coords[0], z_coords[1]) pyramid_writer.write_chunk( chunk_dtype_transformer(chunk, preserve_input=False), key, chunk_coords) progress_bar.update() # free up memory before reading next block (prevent doubled memory # usage) del block