def get_link_edges(mesh,
seg_id,
datastack_name=None,
close_map_distance=300,
server_address=None,
verbose=False,
client=None):
"""function to get a set of edges that should be added to a mesh
Parameters
----------
mesh : trimesh_io.Mesh
the mesh to add edges to
seg_id : np.uint64 or int
the seg_id to query the PCG endpoint for merges
dataset_name: str
the name of the dataset to query
close_map_distance: int or float
the maximum distance to map (default 300 in units of mesh.vertices)
server_address: str
the url to the root of the framework deployment
verbose: bool
whether to print debug statements
client : annotationframeworkclient.ChunkedGraphClient
Returns
-------
np.array
link_edges, a Kx2 array of mesh.vertices indices representing edges that should be added to the mesh graph
"""
# initialize a chunkedgraph client
if client is None:
client = FrameworkClient(datastack_name,
server_address=server_address).chunkedgraph
# get the merge log
if type(seg_id) is np.int64:
seg_id = int(seg_id)
merge_log = client.get_merge_log(seg_id)
return merge_log_edges(mesh,
merge_log,
client.base_resolution,
close_map_distance=close_map_distance,
verbose=verbose)
def generate_proofreading_state(
datastack,
root_id,
branch_points=True,
end_points=True,
root_loc=None,
root_is_soma=False,
auth_token=None,
return_as='html',
min_mesh_component_size=1000,
):
"""Go through the steps to generate a proofreading state from the root id"""
client = FrameworkClient(datastack, auth_token=auth_token)
mm = trimesh_io.MeshMeta(cache_size=0,
cv_path=client.info.segmentation_source())
mesh = process_mesh(root_id,
client,
mm,
min_component_size=min_mesh_component_size)
skf = process_skeleton_from_mesh(mesh,
root_loc=root_loc,
root_is_soma=root_is_soma,
sk_kwargs=SK_KWARGS)
pf_sb, sb_dfs = process_points_from_skeleton(root_id, skf, client)
state = pf_sb.render_state(sb_dfs,
return_as=return_as,
url_prefix=client.info.viewer_site())
return state
def chunk_index_mesh(
root_id,
client=None,
datastack_name=None,
cv=None,
return_l2dict=False,
):
"""Download a mesh with chunk index vertices
Parameters
----------
root_id : int
Root id to download.
client : FrameworkClient, optional
Preset FrameworkClient, by default None.
datastack_name : str or None, optional
Datastack to use to initialize a FrameworkClient, by default None.
cv : cloudvolume.CloudVolume or None, optional
Cloudvolume instance, by default None.
return_l2dict : bool, optional
If True, returns both a l2id to vertex dict and the reverse, by default False.
Returns
-------
mesh : trimesh_io.Mesh
Chunk graph represented as a mesh, with vertices at chunk index locations and edges in the link_edges attribute.
l2dict_mesh : dict
l2 id to mesh vertex index dictionary. Only returned if return_l2dict is True.
l2dict_r_mesh : dict
Mesh vertex index to l2 id dictionary. Only returned if return_l2dict is True.
"""
if client is None:
client = FrameworkClient(datastack_name)
if cv is None:
cv = cloudvolume.CloudVolume(
client.info.segmentation_source(),
use_https=True,
progress=False,
bounded=False,
fill_missing=True,
secrets={"token": client.auth.token},
)
lvl2_eg = client.chunkedgraph.level2_chunk_graph(root_id)
eg, l2dict_mesh, l2dict_r_mesh, x_ch = build_spatial_graph(lvl2_eg, cv)
mesh_chunk = trimesh_io.Mesh(
vertices=x_ch,
faces=[[0, 0, 0]], # Some functions fail if no faces are set.
link_edges=eg,
)
if return_l2dict:
return mesh_chunk, l2dict_mesh, l2dict_r_mesh
else:
return mesh_chunk
def generate_lvl2_proofreading(datastack,
root_id,
root_point=None,
point_radius=200,
invalidation_d=3,
auto_remesh=True,
verbose=True):
if verbose:
t0 = time.time()
client = FrameworkClient(datastack)
l2_sk, l2dict_reversed = get_lvl2_skeleton(client,
root_id,
root_point=root_point,
refine_branch_points=True,
convert_to_nm=True,
point_radius=point_radius,
invalidation_d=invalidation_d,
verbose=verbose,
auto_remesh=auto_remesh)
sbs = []
dfs = []
base_sb, base_df = base_sb_data(client,
root_id,
focus_loc=root_point,
fixed_ids=[root_id])
sbs.append(base_sb)
dfs.append(base_df)
rt_sb, rt_df = root_sb_data(l2_sk, set_position=True)
sbs.append(rt_sb)
dfs.append(rt_df)
lbls = process_node_groups(l2_sk, cp_max_thresh=200_000)
bp_sb, bp_df = branch_sb_data(client,
root_id,
l2_sk,
labels=lbls,
tags=BP_PROOFREADING_TAGS,
set_position=False,
active=True)
sbs.append(bp_sb)
dfs.append(bp_df)
sb_pf = sb.ChainedStateBuilder(sbs)
if verbose:
print('\nComplete time: ', time.time() - t0)
return sb_pf.render_state(dfs,
return_as='url',
url_prefix=client.info.viewer_site())
def lvl2_statebuilder(datastack, root_id):
client = FrameworkClient(datastack)
img = sb.ImageLayerConfig(client.info.image_source(),
contrast_controls=True,
black=0.35,
white=0.66)
seg = sb.SegmentationLayerConfig(client.info.segmentation_source(),
fixed_ids=[root_id])
anno = sb.AnnotationLayerConfig('branch_points',
mapping_rules=sb.PointMapper(),
array_data=True,
active=True)
bp_statebuilder = sb.StateBuilder([img, seg, anno],
url_prefix=client.info.viewer_site())
return bp_statebuilder
def l2_graph(root_id, progress=True, dataset='production'):
"""Fetch L2 graph(s).
Parameters
----------
root_id : int | list of ints
FlyWire root ID(s) for which to fetch the L2 graphs.
Returns
-------
networkx.Graph
The L2 graph.
Examples
--------
>>> from fafbseg import flywire
>>> n = flywire.l2_graph(720575940614131061)
"""
if navis.utils.is_iterable(root_id):
graphs = []
for id in navis.config.tqdm(root_id, desc='Fetching',
disable=not progress, leave=False):
n = l2_graph(id, dataset=dataset)
graphs.append(n)
return graphs
# Hard-coded datastack names
ds = {"production": "flywire_fafb_production",
"sandbox": "flywire_fafb_sandbox"}
# Note that the default server url is https://global.daf-apis.com/info/
client = FrameworkClient(ds.get(dataset, dataset))
# Load the L2 graph for given root ID
# This is a (N,2) array of edges
l2_eg = np.array(client.chunkedgraph.level2_chunk_graph(root_id))
# Drop duplicate edges
l2_eg = np.unique(np.sort(l2_eg, axis=1), axis=0)
G = nx.Graph()
G.add_edges_from(l2_eg)
return G
def l2skelgenerator(ptgenerator,
rootid,
datastackname,
rootpt,
*genargs,
voxelres=[8, 8, 40],
collapse_soma=True,
n_parallel=8,
pointtypes="bpep",
**genkwargs):
"""L2-skeleton task generation
Extracts points from a PCG L2 skeleton and feeds them to a point
generator function
"""
client = FrameworkClient(datastackname)
refine = "bpep" if pointtypes in ["bpep", "epbp"] else None # crude
print("Extracting PCG L2 skeleton")
start = time.time()
l2skel = pcg_skel.pcg_skeleton(rootid,
client=client,
refine=refine,
root_point=rootpt,
root_point_resolution=voxelres,
collapse_soma=True,
n_parallel=n_parallel)
elapsed = time.time() - start
print(f"Extracting PCG L2 skeleton completed in {elapsed:.3f}s")
points = list()
if "bp" in pointtypes:
bps = l2skel.vertices[l2skel.branch_points] // voxelres
points.extend(map(tuple, bps.astype(int)))
if "ep" in pointtypes:
eps = l2skel.vertices[l2skel.end_points] // voxelres
points.extend(map(tuple, eps.astype(int)))
return ptgenerator(points, *genargs, rootid=rootid, **genkwargs)
def l2_skeleton(root_id, refine=False, drop_missing=True,
threads=10, progress=True, dataset='production', **kwargs):
"""Generate skeleton from L2 graph.
Parameters
----------
root_id : int | list of ints
Root ID(s) of the flywire neuron(s) you want to
skeletonize.
refine : bool
If True, will refine skeleton nodes by moving them in
the center of their corresponding chunk meshes.
Only relevant if ``refine=True``:
drop_missing : bool
If True, will drop nodes that don't have a corresponding
chunk mesh. These are typically chunks that are very
small and dropping them might actually be benefitial.
threads : int
How many parallel threads to use for fetching the
chunk meshes. Reduce the number if you run into
``HTTPErrors``. Only relevant if `use_flycache=False`.
progress : bool
Whether to show a progress bar.
Returns
-------
skeleton : navis.TreeNeuron
The extracted skeleton.
Examples
--------
>>> from fafbseg import flywire
>>> n = flywire.l2_skeleton(720575940614131061)
"""
# TODO:
# - drop duplicate nodes in unrefined skeleton
# - use L2 graph to find soma: highest degree is typically the soma
use_flycache = kwargs.get('use_flycache', False)
if refine and use_flycache and dataset != 'production':
raise ValueError('Unable to use fly cache to fetch L2 centroids for '
'sandbox dataset. Please set `use_flycache=False`.')
if navis.utils.is_iterable(root_id):
nl = []
for id in navis.config.tqdm(root_id, desc='Skeletonizing',
disable=not progress, leave=False):
n = l2_skeleton(id, refine=refine, drop_missing=drop_missing,
threads=threads, progress=progress, dataset=dataset)
nl.append(n)
return navis.NeuronList(nl)
# Get the cloudvolume
vol = parse_volume(dataset)
# Hard-coded datastack names
ds = {"production": "flywire_fafb_production",
"sandbox": "flywire_fafb_sandbox"}
# Note that the default server url is https://global.daf-apis.com/info/
client = FrameworkClient(ds.get(dataset, dataset))
# Load the L2 graph for given root ID
# This is a (N,2) array of edges
l2_eg = np.array(client.chunkedgraph.level2_chunk_graph(root_id))
# Drop duplicate edges
l2_eg = np.unique(np.sort(l2_eg, axis=1), axis=0)
# Unique L2 IDs
l2_ids = np.unique(l2_eg)
# ID to index
l2dict = {l2: ii for ii, l2 in enumerate(l2_ids)}
# Remap edge graph to indices
eg_arr_rm = fastremap.remap(l2_eg, l2dict)
coords = [np.array(vol.mesh.meta.meta.decode_chunk_position(l)) for l in l2_ids]
coords = np.vstack(coords)
# This turns the graph into a hierarchal tree by removing cycles and
# ensuring all edges point towards a root
if sk.__version_vector__[0] < 1:
G = sk.skeletonizers.edges_to_graph(eg_arr_rm)
swc = sk.skeletonizers.make_swc(G, coords=coords)
else:
G = sk.skeletonize.utils.edges_to_graph(eg_arr_rm)
swc = sk.skeletonize.utils.make_swc(G, coords=coords, reindex=False)
# Convert to Euclidian space
# Dimension of a single chunk
ch_dims = chunks_to_nm([1, 1, 1], vol) - chunks_to_nm([0, 0, 0], vol)
ch_dims = np.squeeze(ch_dims)
xyz = swc[['x', 'y', 'z']].values
swc[['x', 'y', 'z']] = chunks_to_nm(xyz, vol) + ch_dims / 2
if refine:
if use_flycache:
token = get_chunkedgraph_secret()
centroids = spine.flycache.get_L2_centroids(l2_ids,
token=token,
progress=progress)
# Drop missing (i.e. [0,0,0]) meshes
centroids = {k: v for k, v in centroids.items() if v != [0, 0, 0]}
else:
# Get the centroids
centroids = get_L2_centroids(l2_ids, vol, threads=threads, progress=progress)
new_co = {l2dict[k]: v for k, v in centroids.items()}
# Map refined coordinates onto the SWC
has_new = swc.node_id.isin(new_co)
swc.loc[has_new, 'x'] = swc.loc[has_new, 'node_id'].map(lambda x: new_co[x][0])
swc.loc[has_new, 'y'] = swc.loc[has_new, 'node_id'].map(lambda x: new_co[x][1])
swc.loc[has_new, 'z'] = swc.loc[has_new, 'node_id'].map(lambda x: new_co[x][2])
# Turn into a proper neuron
tn = navis.TreeNeuron(swc, id=root_id, units='1 nm')
# Drop nodes that are still at their unrefined chunk position
if drop_missing:
tn = navis.remove_nodes(tn, swc.loc[~has_new, 'node_id'].values)
else:
tn = navis.TreeNeuron(swc, id=root_id, units='1 nm')
return tn
def l2_soma(x, dataset='production', progress=True):
"""DOES NOT WORK. Use the L2 graph to guess the soma location.
In a nutshell: we use the connectedness (i.e. the degree) of L2 chunks to
guess a neurons soma. The idea was that L2 chunk with the highest degree is
in the center of the soma. Unfortunately, that only seems to work if the
soma is really really big.
Parameters
----------
x : int | nx.Graph | lists thereof
The neuron(s) for which to find somas. Can be either:
- root ID (s)
- networkx L2 graph(s)
Returns
-------
[x, y, z]
Examples
--------
>>> n = flywire.l2_soma(720575940614131061)
"""
if navis.utils.is_iterable(x):
res = []
for id in navis.config.tqdm(x, desc='Finding somas',
disable=not progress, leave=False):
res.append(l2_soma(id, progress=False))
return res
# Get the cloudvolume
vol = parse_volume(dataset)
# Hard-coded datastack names
ds = {"production": "flywire_fafb_production",
"sandbox": "flywire_fafb_sandbox"}
# Note that the default server url is https://global.daf-apis.com/info/
client = FrameworkClient(ds.get(dataset, dataset))
if not isinstance(x, nx.Graph):
# Load the L2 graph for given root ID
# This is a (N,2) array of edges
l2_eg = np.array(client.chunkedgraph.level2_chunk_graph(x))
# Drop duplicate edges
l2_eg = np.unique(np.sort(l2_eg, axis=1), axis=0)
G = nx.Graph()
G.add_edges_from(l2_eg)
else:
G = x
# Get degrees from the graph
degrees = np.array(list(G.degree))
l2_ids = degrees[:, 0]
l2_degrees = degrees[:, 1]
# Find the node with highest degree
mx_deg = l2_ids[np.argmax(l2_degrees)]
# Get it's centroid
centroid = get_L2_centroids([mx_deg], vol, threads=1, progress=False)
return centroid.get(mx_deg, 'NA')
def pcg_meshwork(
root_id,
datastack_name=None,
client=None,
cv=None,
refine="all",
root_point=None,
root_point_resolution=None,
root_point_search_radius=300,
collapse_soma=False,
collapse_radius=DEFAULT_COLLAPSE_RADIUS,
synapses=None,
synapse_table=None,
remove_self_synapse=True,
live_query=False,
timestamp=None,
invalidation_d=3,
segmentation_fallback=False,
fallback_mip=2,
cache=None,
save_to_cache=False,
n_parallel=None,
):
"""Generate a meshwork file based on the level 2 graph.
Parameters
----------
root_id : int
Root id of an object in the pychunkedgraph.
datastack_name : str or None, optional
Datastack name to use to initialize a client, if none is provided. By default None.
client : annotationframeworkclient.FrameworkClientFull or None, optional
Initialized annotationframeworkclient. If None is given, will use the datastack_name to create one. By default None
cv : cloudvolume.CloudVolume or None, optional
Initialized cloudvolume. If none is given, the client info will be used to create one. By default None
refine : 'all', 'ep', 'bp', 'epbp'/'bpep', or None, optional
Selects how to refine vertex locations by downloading mesh chunks.
Unrefined vertices are placed in the center of their chunk in euclidean space.
* 'all' refines all vertex locations. (Default)
* 'ep' refines end points only
* 'bp' refines branch points only
* 'bpep' or 'epbp' refines both branch and end points.
* 'chunk' keeps vertices in chunk index space.
* None refines no points but maps them to the center of the chunk in euclidean space.
root_point : array-like or None, optional
3 element xyz location for the location to set the root in units set by root_point_resolution,
by default None. If None, a distal tip is selected.
root_point_resolution : array-like, optional
Resolution in euclidean space of the root_point, by default [4, 4, 40]
root_point_search_radius : int, optional
Distance in euclidean space to look for segmentation when finding the root vertex, by default 300
collapse_soma : bool, optional,
If True, collapses vertices within a given radius of the root point into the root vertex, typically to better
represent primary neurite branches. Requires a specified root_point. Default if False.
collapse_radius : float, optional
Max distance in euclidean space for soma collapse. Default is 10,000 nm (10 microns).
synapses : 'pre', 'post', 'all', or None, optional
If not None, queries the synapse_table for presynaptic synapses (if 'pre'), postsynaptic sites (if 'post'), or both (if 'all'). By default None
synapse_table : str, optional
Name of the synapse table to query if synapses are requested, by default None
remove_self_synapse : bool, optional
If True, filters out synapses whose pre- and postsynaptic root ids are the same neuron, by default True
invalidation_d : int, optional
Invalidation radius in hops for the mesh skeletonization along the chunk adjacency graph, by default 3
cache : str or None, optional
Filename to a sqlite database with cached lookups for l2 ids. Optional, default is None.
n_parallel : int, optional
Number of parallel downloads passed to cloudvolume, by default 1
Returns
-------
meshparty.meshwork.Meshwork
Meshwork object with skeleton based on the level 2 graph. See documentation for details.
"""
if client is None:
client = FrameworkClient(datastack_name)
if n_parallel is None:
n_parallel = 1
if cv is None:
cv = cloudvolume.CloudVolume(
client.info.segmentation_source(),
parallel=n_parallel,
use_https=True,
progress=False,
bounded=False,
fill_missing=True,
secrets={"token": client.auth.token},
)
if root_point_resolution is None:
root_point_resolution = cv.mip_resolution(0)
sk_l2, mesh_chunk, (l2dict_mesh, l2dict_mesh_r) = pcg_skeleton(
root_id,
client=client,
cv=cv,
root_point=root_point,
root_point_resolution=root_point_resolution,
root_point_search_radius=root_point_search_radius,
collapse_soma=collapse_soma,
collapse_radius=collapse_radius,
refine=refine,
invalidation_d=invalidation_d,
n_parallel=n_parallel,
return_mesh=True,
return_l2dict_mesh=True,
segmentation_fallback=segmentation_fallback,
fallback_mip=fallback_mip,
cache=cache,
save_to_cache=save_to_cache,
)
nrn = meshwork.Meshwork(mesh_chunk, seg_id=root_id, skeleton=sk_l2)
if synapses is not None and synapse_table is not None:
if synapses == "pre":
pre, post = True, False
elif synapses == "post":
pre, post = False, True
elif synapses == "all":
pre, post = True, True
else:
raise ValueError(
'Synapses must be one of "pre", "post", or "all".')
pre_syn_df, post_syn_df = pcg_anno.get_level2_synapses(
root_id,
l2dict_mesh,
client,
synapse_table,
remove_self=remove_self_synapse,
pre=pre,
post=post,
live_query=live_query,
timestamp=timestamp,
)
if pre_syn_df is not None:
nrn.anno.add_annotations(
"pre_syn",
pre_syn_df,
index_column="pre_pt_mesh_ind",
point_column="ctr_pt_position",
)
if post_syn_df is not None:
nrn.anno.add_annotations(
"post_syn",
post_syn_df,
index_column="post_pt_mesh_ind",
point_column="ctr_pt_position",
)
lvl2_df = pd.DataFrame({
"lvl2_id": list(l2dict_mesh.keys()),
"mesh_ind": list(l2dict_mesh.values())
})
nrn.anno.add_annotations("lvl2_ids", lvl2_df, index_column="mesh_ind")
if refine != "chunk":
_adjust_meshwork(nrn, cv)
return nrn
def chunk_index_skeleton(
root_id,
client=None,
datastack_name=None,
cv=None,
root_point=None,
invalidation_d=3,
return_mesh=False,
return_l2dict=False,
return_mesh_l2dict=False,
root_point_resolution=None,
root_point_search_radius=300,
n_parallel=1,
):
"""Generate a basic skeleton with chunked-graph index vertices.
Parameters
----------
root_id : np.uint64
Neuron root id
client : annotationframeworkclient.FrameworkClient, optional
FrameworkClient for a datastack, by default None. If None, you must specify a datastack name.
datastack_name : str, optional
Datastack name to create a FrameworkClient, by default None. Only used if client is None.
cv : cloudvolume.CloudVolume, optional
CloudVolume associated with the object, by default None. If None, one is created based on the client info.
root_point : array, optional
Point in voxel space to set the root vertex. By default None, which makes a random tip root.
invalidation_d : int, optional
TEASAR invalidation radius in chunk space, by default 3
return_mesh : bool, optional
If True, returns the pre-skeletonization mesh with vertices in chunk index space, by default False
return_l2dict : bool, optional
If True, returns the level 2 id to vertex index dict. By default True
n_parallel : int, optional
Sets number of parallel threads for cloudvolume, by default 1
Returns
-------
sk : meshparty.skeleton.Skeleton
Skeleton object
mesh : meshparty.trimesh_io.Mesh
Mesh object, only if return_mesh is True
level2_dict : dict
Level 2 id to vertex map, only if return_l2dict is True.
"""
if client is None:
client = FrameworkClient(datastack_name)
if n_parallel is None:
n_parallel = 1
if cv is None:
cv = cloudvolume.CloudVolume(
client.info.segmentation_source(),
parallel=n_parallel,
use_https=True,
progress=False,
bounded=False,
fill_missing=True,
secrets={"token": client.auth.token},
)
if root_point_resolution is None:
root_point_resolution = cv.mip_resolution(0)
# lvl2_eg = client.chunkedgraph.level2_chunk_graph(root_id)
# eg, l2dict_mesh, l2dict_r_mesh, x_ch = build_spatial_graph(lvl2_eg, cv)
# mesh_chunk = trimesh_io.Mesh(vertices=x_ch, faces=[], link_edges=eg)
mesh_chunk, l2dict_mesh, l2dict_r_mesh = chunk_index_mesh(
root_id, client=client, cv=cv, return_l2dict=True)
if root_point is not None:
lvl2_root_chid, lvl2_root_loc = chunk_tools.get_closest_lvl2_chunk(
root_point,
root_id,
client=client,
cv=None,
radius=root_point_search_radius,
voxel_resolution=root_point_resolution,
return_point=True,
) # Need to have cv=None because of a cloudvolume inconsistency
root_mesh_index = l2dict_mesh[lvl2_root_chid]
else:
root_mesh_index = None
sk_ch = skeletonize.skeletonize_mesh(
mesh_chunk,
invalidation_d=invalidation_d,
collapse_soma=False,
compute_radius=False,
cc_vertex_thresh=0,
root_index=root_mesh_index,
remove_zero_length_edges=False,
)
l2dict, l2dict_r = sk_utils.filter_l2dict(sk_ch, l2dict_r_mesh)
out_list = [sk_ch]
if return_mesh:
out_list.append(mesh_chunk)
if return_l2dict:
out_list.append((l2dict, l2dict_r))
if return_mesh_l2dict:
out_list.append((l2dict_mesh, l2dict_r_mesh))
if len(out_list) == 1:
return out_list[0]
else:
return tuple(out_list)
Returns
-------
sk_l2 : meshparty.skeleton.Skeleton
Skeleton with vertices in euclidean space
mesh_l2 : meshparty.mesh.Mesh, optional
Mesh with vertices in chunk index space. Only if return_mesh is True.
(l2dict, l2dict_r) : (dict, dict), optional
Mappings between level 2 ids and skeleton indices. Only if return_l2dict is True.
(l2dict_mesh, l2dict_mesh_r) : (dict, dict), optional
Mappings between level 2 ids and mesh indices. Only if return_l2dict_mesh is True.
missing_ids : np.array, optional
List of level 2 ids with missing mesh fragments. Only if return_missing_ids is True.
"""
if client is None:
client = FrameworkClient(datastack_name)
if n_parallel is None:
n_parallel = 1
if cv is None:
cv = cloudvolume.CloudVolume(
client.info.segmentation_source(),
parallel=n_parallel,
fill_missing=True,
use_https=True,
progress=False,
bounded=False,
secrets={"token": client.auth.token},
)
if root_point_resolution is None:
root_point_resolution = cv.mip_resolution(0)