def convert_chunks(source_url, dest_url, copy_info=False, options={}): """Convert precomputed chunks between different encodings""" source_accessor = neuroglancer_scripts.accessor.get_accessor_for_url( source_url ) chunk_reader = precomputed_io.get_IO_for_existing_dataset(source_accessor) source_info = chunk_reader.info dest_accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options ) if copy_info: chunk_writer = precomputed_io.get_IO_for_new_dataset( source_info, dest_accessor, encoder_options=options ) else: chunk_writer = precomputed_io.get_IO_for_existing_dataset( dest_accessor, encoder_options=options ) dest_info = chunk_writer.info chunk_transformer = data_types.get_chunk_dtype_transformer( source_info["data_type"], dest_info["data_type"] ) for scale_index in reversed(range(len(dest_info["scales"]))): convert_chunks_for_scale(chunk_reader, dest_info, chunk_writer, scale_index, chunk_transformer)
def volume_file_to_precomputed(volume_filename, dest_url, dset_name=None, ignore_scaling=False, input_min=None, input_max=None, load_full_volume=True, options={}): if dset_name is None: img = nibabel.load(volume_filename) accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options ) print(img) try: precomputed_writer = precomputed_io.get_IO_for_existing_dataset( accessor ) except neuroglancer_scripts.accessor.DataAccessError as exc: logger.error("No 'info' file was found (%s). You can generate one by " "running this program with the --generate-info option, " "then using generate_scales_info.py on the result", exc) return 1 except ValueError as exc: # TODO use specific exception for invalid JSON logger.error("Invalid 'info' file: %s", exc) return 1 return nibabel_image_to_precomputed(img, precomputed_writer, ignore_scaling, input_min, input_max, load_full_volume, options) else: if ( os.path.splitext(volume_filename)[-1] == ".zarr"): img = zarr.open_group(volume_filename,'r') else: img = h5py.File(volume_filename,'r') accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options ) try: precomputed_writer = precomputed_io.get_IO_for_existing_dataset( accessor ) except neuroglancer_scripts.accessor.DataAccessError as exc: logger.error("No 'info' file was found (%s). You can generate one by " "running this program with the --generate-info option, " "then using generate_scales_info.py on the result", exc) return 1 except ValueError as exc: # TODO use specific exception for invalid JSON logger.error("Invalid 'info' file: %s", exc) return 1 return h5py_file_to_precomputed(img,dset_name, precomputed_writer, load_full_volume)
def make_mesh_fragment_links(input_csv, dest_url, no_colon_suffix=False, options={}): if no_colon_suffix: filename_format = "{0}/{1}" else: filename_format = "{0}/{1}:0" accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options) info = precomputed_io.get_IO_for_existing_dataset(accessor).info if "mesh" not in info: logger.critical('The info file is missing the "mesh" key, please ' 'use mesh-to-precomputed first.') return 1 mesh_dir = info["mesh"] with open(input_csv, newline="") as csv_file: for line in csv.reader(csv_file): numeric_label = int(line[0]) fragment_list = line[1:] # Output a warning for missing fragments for fragment_name in fragment_list: if not accessor.file_exists(mesh_dir + "/" + fragment_name): logger.warning("missing fragment %s", fragment_name) relative_filename = filename_format.format(mesh_dir, numeric_label) json_str = json.dumps({"fragments": fragment_list}, separators=(",", ":")) accessor.store_file(relative_filename, json_str.encode("utf-8"), mime_type="application/json")
def mesh_file_to_precomputed(input_path, dest_url, mesh_name=None, mesh_dir=None, coord_transform=None, options={}): """Convert a mesh read by nibabel to Neuroglancer precomputed format""" input_path = pathlib.Path(input_path) accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options) info = precomputed_io.get_IO_for_existing_dataset(accessor).info if mesh_dir is None: mesh_dir = "mesh" # default value if "mesh" not in info: info["mesh"] = mesh_dir # Write the updated info file precomputed_io.get_IO_for_new_dataset(info, accessor, overwrite_info=True) if mesh_dir != info["mesh"]: logger.critical("The provided --mesh-dir value does not match the " "value stored in the info file") return 1 if info["type"] != "segmentation": logger.warning('The dataset has type "image" instead of ' '"segmentation", Neuroglancer will not use the meshes.') if mesh_name is None: mesh_name = input_path.stem mesh = nibabel.load(str(input_path)) points_list = mesh.get_arrays_from_intent("NIFTI_INTENT_POINTSET") assert len(points_list) == 1 points = points_list[0].data triangles_list = mesh.get_arrays_from_intent("NIFTI_INTENT_TRIANGLE") assert len(triangles_list) == 1 triangles = triangles_list[0].data if coord_transform is not None: points_dtype = points.dtype points, triangles = neuroglancer_scripts.mesh.affine_transform_mesh( points, triangles, coord_transform) # Convert vertices back to their original type to avoid the warning # that save_mesh_as_precomputed prints when downcasting to float32. points = points.astype(np.promote_types(points_dtype, np.float32), casting="same_kind") # Gifti uses millimetres, Neuroglancer expects nanometres. # points can be a read-only array, so we cannot use the *= operator. points = 1e6 * points io_buf = io.BytesIO() neuroglancer_scripts.mesh.save_mesh_as_precomputed( io_buf, points, triangles.astype("uint32")) accessor.store_file(mesh_dir + "/" + mesh_name, io_buf.getvalue(), mime_type="application/octet-stream")
def compute_scales(work_dir=".", downscaling_method="average", options={}): """Generate lower scales following an input info file""" accessor = neuroglancer_scripts.accessor.get_accessor_for_url( work_dir, options) pyramid_io = precomputed_io.get_IO_for_existing_dataset( accessor, encoder_options=options) downscaler = neuroglancer_scripts.downscaling.get_downscaler( downscaling_method, options) neuroglancer_scripts.dyadic_pyramid.compute_dyadic_scales( pyramid_io, downscaler)
def test_precomputed_IO_chunk_roundtrip(tmpdir): accessor = get_accessor_for_url(str(tmpdir)) # Minimal info file io = get_IO_for_new_dataset(DUMMY_INFO, accessor) dummy_chunk = np.arange(8 * 3 * 7, dtype="uint16").reshape(1, 7, 3, 8) chunk_coords = (0, 8, 0, 3, 8, 15) io.write_chunk(dummy_chunk, "key", chunk_coords) assert np.array_equal(io.read_chunk("key", chunk_coords), dummy_chunk) io2 = get_IO_for_existing_dataset(accessor) assert io2.info == DUMMY_INFO assert np.array_equal(io2.read_chunk("key", chunk_coords), dummy_chunk)
def test_precomputed_IO_info_error(tmpdir): with (tmpdir / "info").open("w") as f: f.write("invalid JSON") accessor = get_accessor_for_url(str(tmpdir)) with pytest.raises(InvalidInfoError): get_IO_for_existing_dataset(accessor)
def slices_to_raw_chunks(slice_filename_lists, dest_url, input_orientation, options={}): """Convert a list of 2D slices to Neuroglancer pre-computed chunks. :param dict info: the JSON dictionary that describes the dataset for Neuroglancer. Only the information from the first scale is used. :param list slice_filename_lists: a list of lists of filenames. Files from each inner list are read as 2D images and concatenated along the third axis. Blocks from the outer list are concatenated along a fourth axis, representing the image channels. :param tuple input_axis_inversions: a 3-tuple in (column, row, slice) order. Each value must be 1 (preserve orientation along the axis) or -1 (invert orientation along the axis). :param tuple input_axis_permutation: a 3-tuple in (column, row, slice) order. Each value is 0 for X (L-R axis), 1 for Y (A-P axis), 2 for Z (I-S axis). """ accessor = neuroglancer_scripts.accessor.get_accessor_for_url( dest_url, options) pyramid_writer = precomputed_io.get_IO_for_existing_dataset( accessor, encoder_options=options) info = pyramid_writer.info assert len(info["scales"][0]["chunk_sizes"]) == 1 # more not implemented chunk_size = info["scales"][0]["chunk_sizes"][0] # in RAS order (X, Y, Z) size = info["scales"][0]["size"] # in RAS order (X, Y, Z) key = info["scales"][0]["key"] dtype = np.dtype(info["data_type"]).newbyteorder("<") num_channels = info["num_channels"] input_axis_permutation = tuple(AXIS_PERMUTATION_FOR_RAS[a] for a in input_orientation) input_axis_inversions = tuple(AXIS_INVERSION_FOR_RAS[a] for a in input_orientation) # Here x, y, and z refer to the data orientation in output chunks (which # should correspond to RAS+ anatomical axes). For the data orientation in # input slices the terms (column (index along image width), row (index # along image height), slice) are used. # permutation_to_input is a 3-tuple in RAS (X, Y, Z) order. # Each value is 0 column, 1 for row, 2 for slice. permutation_to_input = invert_permutation(input_axis_permutation) # input_size and input_chunk_size are in (column, row, slice) order. input_size = permute(size, input_axis_permutation) input_chunk_size = permute(chunk_size, input_axis_permutation) for filename_list in slice_filename_lists: if len(filename_list) != input_size[2]: raise ValueError("{} slices found where {} were expected".format( len(filename_list), input_size[2])) for slice_chunk_idx in trange( (input_size[2] - 1) // input_chunk_size[2] + 1, desc="converting slice groups", leave=True, unit="slice groups"): first_slice_in_order = input_chunk_size[2] * slice_chunk_idx last_slice_in_order = min(input_chunk_size[2] * (slice_chunk_idx + 1), input_size[2]) if input_axis_inversions[2] == -1: first_slice = input_size[2] - first_slice_in_order - 1 last_slice = input_size[2] - last_slice_in_order - 1 else: first_slice = first_slice_in_order last_slice = last_slice_in_order slice_slicing = np.s_[first_slice:last_slice:input_axis_inversions[2]] tqdm.write("Reading slices {0} to {1} ({2}B memory needed)... ".format( first_slice, last_slice - input_axis_inversions[2], readable_count(input_size[0] * input_size[1] * (last_slice_in_order - first_slice_in_order + 1) * num_channels * dtype.itemsize))) def load_z_stack(slice_filenames): # Loads the data in [slice, row, column] C-contiguous order block = skimage.io.concatenate_images( skimage.io.imread(str(filename)) for filename in slice_filenames[slice_slicing]) assert block.shape[2] == input_size[0] # check slice width assert block.shape[1] == input_size[1] # check slice height if block.ndim == 4: # Scikit-image loads multi-channel (e.g. RGB) images in [slice, # row, column, channel] order, while Neuroglancer expects # channel to come first (in C-contiguous indexing). block = np.moveaxis(block, (3, 0, 1, 2), (0, 1, 2, 3)) elif block.ndim == 3: block = block[np.newaxis, :, :, :] else: raise ValueError( "block has unexpected dimensionality (ndim={})".format( block.ndim)) return block # Concatenate all channels from different directories block = np.concatenate([ load_z_stack(filename_list) for filename_list in slice_filename_lists ], axis=0) assert block.shape[0] == num_channels # Flip and permute axes to go from input (channel, slice, row, column) # to Neuroglancer (channel, Z, Y, X) block = block[:, :, ::input_axis_inversions[1], :: input_axis_inversions[0]] block = np.moveaxis(block, (3, 2, 1), (3 - a for a in input_axis_permutation)) # equivalent: np.transpose(block, axes=([0] + [3 - a for a in # reversed(invert_permutation(input_axis_permutation))])) chunk_dtype_transformer = get_chunk_dtype_transformer( block.dtype, dtype) progress_bar = tqdm( total=(((input_size[1] - 1) // input_chunk_size[1] + 1) * ((input_size[0] - 1) // input_chunk_size[0] + 1)), desc="writing chunks", unit="chunks", leave=False) for row_chunk_idx in range((input_size[1] - 1) // input_chunk_size[1] + 1): row_slicing = np.s_[input_chunk_size[1] * row_chunk_idx:min( input_chunk_size[1] * (row_chunk_idx + 1), input_size[1])] for column_chunk_idx in range((input_size[0] - 1) // input_chunk_size[0] + 1): column_slicing = np.s_[input_chunk_size[0] * column_chunk_idx:min( input_chunk_size[0] * (column_chunk_idx + 1), input_size[0])] input_slicing = (column_slicing, row_slicing, np.s_[:]) x_slicing, y_slicing, z_slicing = permute( input_slicing, permutation_to_input) chunk = block[:, z_slicing, y_slicing, x_slicing] # This variable represents the coordinates with real slice # numbers, instead of within-block slice numbers. input_coords = ((column_slicing.start, column_slicing.stop), (row_slicing.start, row_slicing.stop), (first_slice_in_order, last_slice_in_order)) x_coords, y_coords, z_coords = permute(input_coords, permutation_to_input) assert chunk.size == ((x_coords[1] - x_coords[0]) * (y_coords[1] - y_coords[0]) * (z_coords[1] - z_coords[0]) * num_channels) chunk_coords = (x_coords[0], x_coords[1], y_coords[0], y_coords[1], z_coords[0], z_coords[1]) pyramid_writer.write_chunk( chunk_dtype_transformer(chunk, preserve_input=False), key, chunk_coords) progress_bar.update() # free up memory before reading next block (prevent doubled memory # usage) del block
def show_scale_file_info(url, options={}): """Show information about a list of scales.""" accessor = neuroglancer_scripts.accessor.get_accessor_for_url(url, options) io = precomputed_io.get_IO_for_existing_dataset(accessor) info = io.info show_scales_info(info)
def _bootstrap(self): accessor = get_accessor_for_url(self.url) self._io = get_IO_for_existing_dataset(accessor) self._scales_cached = sorted( [NeuroglancerScale(self, i) for i in self._io.info["scales"]])