def get_sub_neuron(self, bounding_box, spacing=None, origin=None):
"""Returns sub-neuron with node coordinates bounded by start and end
Arguments
----------
bounding_box : tuple or list or None
Defines a bounding box around a sub-region around the neuron. Length 2
tuple/list. First element is the coordinate of one corner (inclusive)
and second element is the coordinate of the opposite corner (exclusive).
Both coordinates are numpy.array([x,y,z])in voxel units.
spacing : None, :class:`numpy.array` (default = None)
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z]).
Provided if graph should convert to voxel coordinates first. Default is None.
origin : :class:`numpy.array`
Origin of the spatial coordinate, if converting to voxels. Default is None.
Assumed to be np.array([x,y,z])
Returns
-------
G_sub : :class:`networkx.classes.digraph.DiGraph`
Neuron from swc represented as directed graph. Coordinates x,y,z are
node attributes accessed by keys 'x','y','z' respectively.
Example
-------
>>> bounding_box=[[1,2,4],[1,2,3]]
>>> #swc input, no spacing and origin
>>> swc_trace.get_sub_neuron(bounding_box)
>>> <networkx.classes.digraph.DiGraph at 0x7f81a95d1e50>
"""
check_type(bounding_box, (tuple, list))
if len(bounding_box) != 2:
raise ValueError("Bounding box must be length 2")
check_type(spacing, (type(None), np.ndarray))
check_type(spacing, (type(None), np.ndarray))
if type(spacing) == np.ndarray:
check_size(spacing)
check_type(origin, (type(None), np.ndarray))
if type(origin) == np.ndarray:
check_size(origin)
# if origin isn't specified but spacing is, set origin to np.array([0, 0, 0])
if type(spacing) == np.ndarray and origin is None:
origin = np.array([0, 0, 0])
# voxel conversion option
if type(spacing) == np.ndarray:
df_voxel = self._df_in_voxel(self.df, spacing, origin)
G = self._df_to_graph(df_voxel)
# no voxel conversion option
else:
G = self._df_to_graph(self.df)
G_sub = self._get_sub_neuron(G, bounding_box)
return G_sub
def create_skel_segids(
swc_dir: str,
origin: Sequence[Union[int, float]]) -> Tuple[Skeleton, List[int]]:
"""Create skeletons to be uploaded as precomputed format
Arguments:
swc_dir: Path to consensus swc files.
origin: x,y,z coordinate of coordinate frame in space in mircons.
Returns:
skeletons: .swc skeletons to be pushed to bucket.
segids: List of ints for each swc's label.
"""
check_type(swc_dir, str)
check_size(origin)
p = Path(swc_dir)
files = [str(i) for i in p.glob("*.swc")]
if len(files) == 0:
raise FileNotFoundError(f"No .swc files found in {swc_dir}.")
skeletons = []
segids = []
for i in tqdm(files, desc="converting swcs to neuroglancer format..."):
skeletons.append(swc2skeleton(i, origin=origin))
segids.append(skeletons[-1].id)
return skeletons, segids
def get_data_ranges(
bin_path: List[List[str]],
chunk_size: Tuple[int, int,
int]) -> Tuple[List[int], List[int], List[int]]:
"""Get ranges (x,y,z) for chunks to be stitched together in volume
Arguments:
bin_path: Binary paths to files.
chunk_size: The size of chunk to get ranges over.
Returns:
x_range: x-coord int bounds.
y_range: y-coord int bounds.
z_range: z-coord int bounds.
"""
for b in bin_path:
check_binary_path(b)
check_size(chunk_size)
x_curr, y_curr, z_curr = 0, 0, 0
tree_level = len(bin_path)
print(bin_path)
for idx, i in enumerate(bin_path):
print(i)
scale_factor = 2**(tree_level - idx - 1)
x_curr += int(i[2]) * chunk_size[0] * scale_factor
y_curr += int(i[1]) * chunk_size[1] * scale_factor
# flip z axis so chunks go anterior to posterior
z_curr += int(i[0]) * chunk_size[2] * scale_factor
x_range = [x_curr, x_curr + chunk_size[0]]
y_range = [y_curr, y_curr + chunk_size[1]]
z_range = [z_curr, z_curr + chunk_size[2]]
return x_range, y_range, z_range
def create_skel_segids(
swc_dir: str,
origin: Sequence[Union[int, float]],
benchmarking: Optional[bool] = False,
) -> Tuple[Skeleton, List[int]]:
"""Create skeletons to be uploaded as precomputed format
Arguments:
swc_dir: Path to consensus swc files.
origin: x,y,z coordinate of coordinate frame in space in mircons.
benchmarking: Optional, scales swc benchmarking data.
Returns:
skeletons: .swc skeletons to be pushed to bucket.
segids: List of ints for each swc's label.
"""
check_type(swc_dir, str)
check_size(origin)
check_type(benchmarking, bool)
p = Path(swc_dir)
files = [str(i) for i in p.glob("*.swc")]
if len(files) == 0:
raise FileNotFoundError(f"No .swc files found in {swc_dir}.")
skeletons = []
segids = []
for i in tqdm(files, desc="converting swcs to neuroglancer format..."):
swc_trace = NeuronTrace(path=i)
skel = swc_trace.get_skel(benchmarking, origin=np.asarray(origin))
skeletons.append(skel)
segids.append(skeletons[-1].id)
return skeletons, segids
def tubes_from_paths(
size: Tuple[int, int, int],
paths: List[List[int]],
radius: Optional[Union[float, int]] = None,
):
"""Constructs tubes from list of paths.
Returns densely labeled paths within the shape of the image.
Arguments:
size: The size of image to consider.
paths: The list of paths. Each path is a list of points along the path (non-dense).
radius: The radius of the line to draw. Default is None = 1 pixel wide line.
"""
check_size(size)
for path in paths:
[check_iterable_type(vert, (int, np.integer)) for vert in path]
if radius is not None:
check_type(radius, (int, np.integer, float, np.float))
if radius <= 0:
raise ValueError(f"Radius {radius} must be positive.")
def _within_img(line, size):
arrline = np.array(line).astype(int)
arrline = arrline[:, arrline[0, :] < size[0]]
arrline = arrline[:, arrline[0, :] >= 0]
arrline = arrline[:, arrline[1, :] < size[1]]
arrline = arrline[:, arrline[1, :] >= 0]
arrline = arrline[:, arrline[2, :] < size[2]]
arrline = arrline[:, arrline[2, :] >= 0]
return (arrline[0, :], arrline[1, :], arrline[2, :])
coords = [[], [], []]
for path in tqdm(paths):
for i in range(len(path) - 1):
line = draw.line_nd(path[i], path[i + 1])
line = _within_img(line, size)
if len(line) > 0:
coords[0] = np.concatenate((coords[0], line[0]))
coords[1] = np.concatenate((coords[1], line[1]))
coords[2] = np.concatenate((coords[2], line[2]))
try:
coords = (coords[0].astype(int), coords[1].astype(int),
coords[2].astype(int))
except AttributeError: # if a list was passed
coords = (coords[0], coords[1], coords[2])
if radius is not None:
line_array = np.ones(size, dtype=int)
line_array[coords] = 0
seg = distance_transform_edt(line_array)
labels = np.where(seg <= radius, 1, 0)
else:
labels = np.zeros(size, dtype=int)
labels[coords] = 1
return labels
def get_paths(self, spacing=None, origin=None):
"""Converts dataframe in either spatial or voxel coordinates into a list of paths.
Will convert to voxel coordinates if spacing is specified.
Arguments
----------
spacing : None, :class:`numpy.array` (default = None)
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z]).
Provided if graph should convert to voxel coordinates first. Default is None.
origin : None, :class:`numpy.array`
Origin of the spatial coordinate, if converting to voxels. Default is None.
Assumed to be np.array([x,y,z])
Returns
-------
paths : list
List of Nx3 numpy.array. Rows of the array are 3D coordinates in voxel
units. Each array is one path.
Example
-------
>>> swc_trace.get_paths()[0][1:10]
>>> array([[-52, -1, -1],
[-51, -1, 0],
[-51, -1, 0],
[-50, 0, 0],
[-50, 0, 0],
[-49, 0, 0],
[-48, 0, 0],
[-46, 0, 0],
[-46, 0, 0]], dtype=object)
"""
check_type(spacing, (type(None), np.ndarray))
if type(spacing) == np.ndarray:
check_size(spacing)
check_type(origin, (type(None), np.ndarray))
if type(origin) == np.ndarray:
check_size(origin)
# if origin isn't specified but spacing is, set origin to np.array([0, 0, 0])
if type(spacing) == np.ndarray and origin is None:
origin = np.array([0, 0, 0])
# voxel conversion option
if type(spacing) == np.ndarray:
df_voxel = self._df_in_voxel(self.df, spacing, origin)
G = self._df_to_graph(df_voxel)
# no voxel conversion option
else:
G = self._df_to_graph(self.df)
paths = self._graph_to_paths(G)
return paths
def get_graph(self, spacing=None, origin=None):
"""Converts dataframe in either spatial or voxel coordinates into a directed graph.
Will convert to voxel coordinates if spacing is specified.
Arguments
----------
spacing : None, :class:`numpy.array` (default = None)
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z]).
Provided if graph should convert to voxel coordinates first. Default is None.
origin : None, :class:`numpy.array` (default = None)
Origin of the spatial coordinate, if converting to voxels. Default is None.
Assumed to be np.array([x,y,z])
Returns
-------
G : :class:`networkx.classes.digraph.DiGraph`
Neuron from swc represented as directed graph. Coordinates x,y,z are
node attributes accessed by keys 'x','y','z' respectively.
Example
-------
>>> swc_trace.get_graph()
>>> <networkx.classes.digraph.DiGraph at 0x7f81a83937f0>
"""
check_type(spacing, (type(None), np.ndarray))
if type(spacing) == np.ndarray:
check_size(spacing)
check_type(origin, (type(None), np.ndarray))
if type(origin) == np.ndarray:
check_size(origin)
# if origin isn't specified but spacing is, set origin to np.array([0, 0, 0])
if type(spacing) == np.ndarray and origin is None:
origin = np.array([0, 0, 0])
# voxel conversion option
if type(spacing) == np.ndarray:
df_voxel = self._df_in_voxel(self.df, spacing, origin)
G = self._df_to_graph(df_voxel)
# no voxel conversion option
else:
G = self._df_to_graph(self.df)
return G
def get_df_voxel(self, spacing, origin=np.array([0, 0, 0])):
"""Converts coordinates in pd.DataFrame from spatial units to voxel units
Arguments
----------
spacing : :class:`numpy.array`
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z])
origin : :class:`numpy.array`
Origin of the spatial coordinate. Default is (0,0,0). Assumed to be
np.array([x,y,z])
Returns
-------
df_voxel : :class:`pandas.DataFrame`
Indicies, coordinates, and parents of each node in the swc. Coordinates
are in voxel units.
Example
-------
>>> swc_trace.get_df_voxel(spacing=np.asarray([2,2,2]))
>>> sample structure x y z r parent
0 1 0 -26 -1 -1 1.0 -1
1 2 0 -26 -1 -1 1.0 1
2 3 0 -26 -1 0 1.0 2
3 4 0 -26 -1 0 1.0 3
4 5 0 -25 0 0 1.0 4
... ... ... ... ... ... ... ...
148 149 0 23 7 -4 1.0 148
149 150 0 23 7 -4 1.0 149
150 151 0 23 7 -4 1.0 150
151 152 0 24 8 -4 1.0 151
152 153 6 24 8 -4 1.0 152
153 rows × 7 columns
"""
check_type(spacing, np.ndarray)
check_size(spacing)
check_type(origin, np.ndarray)
check_size(origin)
df_voxel = self._df_in_voxel(self.df, spacing, origin)
return df_voxel
def subsample(
arr: np.ndarray, orig_shape: List[int], dest_shape: List[int]
) -> np.ndarray:
"""Subsamples a flattened neighborhood to a smaller flattened neighborhood.
Arguments:
arr: The flattened array
orig_shape: The original shape of the array before flattening
dest_shape: The desired shape of the array before flattening
"""
check_type(arr, np.ndarray)
if len(orig_shape) != len(dest_shape):
raise ValueError("Mismatched in and out dimensions.")
if np.prod(orig_shape) != len(arr):
raise ValueError("Original shape is incorrect.")
if len(orig_shape) == 3:
check_size(orig_shape, dim=3)
elif len(orig_shape) == 2:
check_size(dest_shape, dim=2)
else:
raise NotImplementedError("Only 2 and 3 dimensions supported.")
start = np.subtract(orig_shape, dest_shape) // 2
end = start + dest_shape
if len(orig_shape) == 2:
idx = np.ravel_multi_index(
(np.mgrid[start[0] : end[0], start[1] : end[1]].reshape(2, -1)), orig_shape
)
elif len(orig_shape) == 3:
idx = np.ravel_multi_index(
(
np.mgrid[
start[0] : end[0], start[1] : end[1], start[2] : end[2]
].reshape(3, -1)
),
orig_shape,
)
return arr[idx]
def get_skel(self, benchmarking=False, origin=None):
"""Gets a skeleton version of dataframe, if swc input is provided
Arguments
----------
origin : None, numpy array with shape (3,1) (default = None)
origin of coordinate frame in microns, (default: None assumes (0,0,0) origin)
benchmarking : bool
For swc files, specifies whether swc file is from benchmarking dataset, to obtain skeleton ID
Returns
--------
skel : cloudvolume.Skeleton
Skeleton object of given SWC file
Example
-------
>>> swc_trace.get_skel(benchmarking=True)
>>> Skeleton(segid=, vertices=(shape=153, float32), edges=(shape=152, uint32), radius=(153, float32), vertex_types=(153, uint8), vertex_color=(153, float32), space='physical' transform=[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]])
"""
check_type(origin, (type(None), np.ndarray))
check_type(benchmarking, bool)
if type(origin) == np.ndarray:
check_size(origin)
if self.input_type == "swc":
skel = self._swc2skeleton(self.path, benchmarking, origin)
return skel
elif self.input_type == "skel":
cv = CloudVolume(
self.path,
mip=self.mip,
fill_missing=self.fill_missing,
use_https=self.use_https,
)
skel = cv.skeleton.get(self.seg_id)
return skel
def get_sub_neuron_paths(self, bounding_box, spacing=None, origin=None):
"""Returns sub-neuron with node coordinates bounded by start and end
Arguments
----------
bounding_box : tuple or list or None
Defines a bounding box around a sub-region around the neuron. Length 2
tuple/list. First element is the coordinate of one corner (inclusive)
and second element is the coordinate of the opposite corner (exclusive).
Both coordinates are numpy.array([x,y,z])in voxel units.
spacing : None, :class:`numpy.array` (default = None)
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z]).
Provided if graph should convert to voxel coordinates first. Default is None.
origin : :class:`numpy.array`
Origin of the spatial coordinate, if converting to voxels. Default is None.
Assumed to be np.array([x,y,z])
Returns
-------
paths : list
List of Nx3 numpy.array. Rows of the array are 3D coordinates in voxel
units. Each array is one path.
Example
-------
>>> bounding_box=[[1,2,4],[1,2,3]]
>>> #swc input, no spacing and origin
>>> swc_trace.get_sub_neuron_paths(bounding_box)
>>> array([], dtype=object)
"""
check_type(bounding_box, (tuple, list))
if len(bounding_box) != 2:
raise ValueError("Bounding box must be length 2")
check_type(spacing, (type(None), np.ndarray))
check_type(spacing, (type(None), np.ndarray))
if type(spacing) == np.ndarray:
check_size(spacing)
check_type(origin, (type(None), np.ndarray))
if type(origin) == np.ndarray:
check_size(origin)
# if origin isn't specified but spacing is, set origin to np.array([0, 0, 0])
if type(spacing) == np.ndarray and origin is None:
origin = np.array([0, 0, 0])
# voxel conversion option
if type(spacing) == np.ndarray:
df_voxel = self._df_in_voxel(self.df, spacing, origin)
G = self._df_to_graph(df_voxel)
# no voxel conversion option
else:
G = self._df_to_graph(self.df)
G_sub = self._get_sub_neuron(G, bounding_box)
paths = self._graph_to_paths(G_sub)
return paths
def get_bfs_subgraph(self,
node_id,
depth,
df=None,
spacing=None,
origin=None):
"""
Creates a spanning subgraph from a seed node and parent graph using BFS.
Arguments
----------
node_id : int
The id of the node to use as a seed.
If df is not None this become the node index.
depth : int
The max depth for BFS to traven in each direction.
df : None, DataFrame (default = None)
Dataframe storing indices.
In some cases indexing by row number is preferred.
spacing : None, :class:`numpy.array` (default = None)
Conversion factor (spatial units/voxel). Assumed to be np.array([x,y,z]).
Provided if graph should convert to voxel coordinates first. Default is None.
origin : :class:`numpy.array`
Origin of the spatial coordinate, if converting to voxels. Default is None.
Assumed to be np.array([x,y,z])
Returns
-------
G_sub : :class:`networkx.classes.digraph.DiGraph`
Subgraph
tree : DiGraph
The tree returned by BFS.
paths : list
List of Nx3 numpy.array. Rows of the array are 3D coordinates in voxel
units. Each array is one path.
Example
-------
>>> #swc input, specify node_id and depth
>>> swc_trace.get_bfs_subgraph(node_id=11,depth=2)
>>>(<networkx.classes.digraph.DiGraph at 0x7f7f2ce65670>,
<networkx.classes.digraph.DiGraph at 0x7f7f2ce65370>,
array([array([[4727, 4440, 3849],
[4732, 4442, 3850],
[4739, 4455, 3849]]),
array([[4732, 4442, 3850],
[4749, 4439, 3856]])], dtype=object))
"""
check_type(node_id, (list, int))
check_type(depth, int)
check_type(df, (type(None), pd.core.frame.DataFrame))
check_type(spacing, (type(None), np.ndarray))
if type(spacing) == np.ndarray:
check_size(spacing)
check_type(origin, (type(None), np.ndarray))
if type(origin) == np.ndarray:
check_size(origin)
# if origin isn't specified but spacing is, set origin to np.array([0, 0, 0])
if type(spacing) == np.ndarray and origin is None:
origin = np.array([0, 0, 0])
# voxel conversion option
if type(spacing) == np.ndarray:
df_voxel = self._df_in_voxel(self.df, spacing, origin)
G = self._df_to_graph(df_voxel)
# no voxel conversion option
else:
G = self._df_to_graph(self.df)
G_sub, tree = self._get_bfs_subgraph(G, node_id, depth, df)
paths = self._graph_to_paths(G_sub)
return G_sub, tree, paths
def create_cloud_volume(
precomputed_path: str,
img_size: Sequence[int],
voxel_size: Sequence[Union[int, float]],
num_resolutions: int,
chunk_size: Optional[Sequence[int]] = None,
parallel: Optional[bool] = False,
layer_type: Optional[str] = "image",
dtype: Optional[str] = None,
commit_info: Optional[bool] = True,
) -> CloudVolumePrecomputed:
"""Create CloudVolume object and info file.
Handles both image volumes and segmentation volumes from octree structure.
Arguments:
precomputed_path: cloudvolume path
img_size: x, y, z voxel dimensions of tiff images.
voxel_size: x, y, z dimensions of highest res voxel size (nm).
num_resolutions: The number of resolutions to upload.
chunk_size: The size of chunks to use for upload. If None, uses img_size/2.
parallel: Whether to upload chunks in parallel.
layer_type: The type of cloudvolume object to create.
dtype: The data type of the volume. If None, uses default for layer type.
commit_info: Whether to create an info file at the path, defaults to True.
Returns:
vol: Volume designated for upload.
"""
# defaults
if chunk_size is None:
chunk_size = [int(i / 4) for i in img_size] # /2 took 42 hrs
if dtype is None:
if layer_type == "image":
dtype = "uint16"
elif layer_type == "segmentation" or layer_type == "annotation":
dtype = "uint64"
else:
raise ValueError(
f"layer type is {layer_type}, when it should be image or str")
# check inputs
check_precomputed(precomputed_path)
check_size(img_size, allow_float=False)
check_size(voxel_size)
check_type(num_resolutions, (int, np.integer))
if num_resolutions < 1:
raise ValueError(
f"Number of resolutions should be > 0, not {num_resolutions}")
check_size(chunk_size)
check_type(parallel, bool)
check_type(layer_type, str)
if layer_type not in ["image", "segmentation", "annotation"]:
raise ValueError(
f"{layer_type} should be 'image', 'segmentation', or 'annotation'")
check_type(dtype, str)
if dtype not in ["uint16", "uint64"]:
raise ValueError(f"{dtype} should be 'uint16' or 'uint64'")
check_type(commit_info, bool)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=dtype, # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=voxel_size, # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
chunk_size=chunk_size, # units are voxels
volume_size=[i * 2**(num_resolutions - 1) for i in img_size],
)
vol = CloudVolume(precomputed_path, info=info, parallel=parallel)
[
vol.add_scale((2**i, 2**i, 2**i), chunk_size=chunk_size)
for i in range(num_resolutions)
]
if commit_info:
vol.commit_info()
if layer_type == "image" or layer_type == "annotation":
vols = [
CloudVolume(precomputed_path, mip=i, parallel=parallel)
for i in range(num_resolutions - 1, -1, -1)
]
elif layer_type == "segmentation":
info.update(skeletons="skeletons")
skel_info = {
"@type":
"neuroglancer_skeletons",
"transform": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0],
"vertex_attributes": [
{
"id": "radius",
"data_type": "float32",
"num_components": 1
},
{
"id": "vertex_types",
"data_type": "float32",
"num_components": 1
},
{
"id": "vertex_color",
"data_type": "float32",
"num_components": 4
},
],
}
with storage.SimpleStorage(vol.cloudpath) as stor:
stor.put_json(str(Path("skeletons") / "info"), skel_info)
vols = [vol]
return vols