def segments_to_neuron(seg_ids,
autoseg_instance,
name_pattern="Google: {id}",
verbose=True,
raise_none_found=True):
"""Retrieve autoseg neurons of given segmentation ID(s).
If a given segmentation ID has been merged into another fragment, will try
retrieving by annotation.
Parameters
----------
seg_ids : int | list of int
Segmentation ID(s) of autoseg skeletons to retrieve.
autoseg_instance : pymaid.CatmaidInstance
Instance with autoseg skeletons.
name_pattern : str, optional
Segmentation IDs are encoded in the name. Use parameter
this to define that pattern.
raise_none_found : bool, optional
If True and none of the requested segments were found,
will raise ValueError
Returns
-------
CatmaidNeuronList
Neurons representing given segmentation IDs.
"""
assert isinstance(autoseg_instance, pymaid.CatmaidInstance)
seg2skid = segments_to_skids(seg_ids,
autoseg_instance=autoseg_instance,
verbose=verbose)
to_fetch = list(set([v for v in seg2skid.values() if v]))
if not to_fetch:
if raise_none_found:
raise ValueError(
"None of the provided segmentation IDs could be found")
else:
# Return empty list
return pymaid.CatmaidNeuronList([])
nl = pymaid.get_neurons(to_fetch, remote_instance=autoseg_instance)
# Make sure we're dealing with a list of neurons
if isinstance(nl, pymaid.CatmaidNeuron):
nl = pymaid.CatmaidNeuronList(nl)
# Invert seg2skid
skid2seg = {}
for k, v in seg2skid.items():
skid2seg[v] = skid2seg.get(v, []) + [k]
for n in nl:
n.seg_ids = skid2seg[int(n.skeleton_id)]
return nl
def get_affinetransformed_neurons_by_skid(skids, transform_file):
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
transformed_neurons = []
for neuron in neurons:
node_coords = neuron.nodes[['x', 'y', 'z']].copy()
connector_coords = neuron.connectors[['x', 'y', 'z']].copy()
#Append a column of 1s to enable affine transformation
node_coords['c'] = 1
connector_coords['c'] = 1
T = np.loadtxt(transform_file)
#Apply transformation matrix using matrix multiplication
transformed_node_coords = node_coords.dot(T)
transformed_connector_coords = connector_coords.dot(T)
#Restore column names
transformed_node_coords.columns = ['x', 'y', 'z', 'c']
transformed_connector_coords.columns = ['x', 'y', 'z', 'c']
neuron.nodes.loc[:, ['x', 'y', 'z']] = transformed_node_coords[[
'x', 'y', 'z'
]]
neuron.connectors.loc[:,
['x', 'y', 'z']] = transformed_connector_coords[[
'x', 'y', 'z'
]]
neuron.neuron_name += ' - affine transform'
transformed_neurons.append(neuron)
return pymaid.CatmaidNeuronList(transformed_neurons)
def neuron_to_segments(x, **kwargs):
"""Use brainmaps API to return segment IDs overlapping with a given neuron.
This is in essence a higher-level function of brainmappy's
get_seg_at_location.
Parameters
----------
x : CatmaidNeuron/List
Neurons for which to return segment IDs.
**kwargs
Keyword arguments passed to
`brainmappy.get_seg_at_location`.
Returns
-------
overlap_matrix : pandas.DataFrame
DataFrame of segment IDs (rows) and skeleton IDs
(columns) with overlap in nodes as values::
skeleton_id 16 3245
seg_id
10336680915 5 0
10336682132 0 1
"""
if isinstance(x, pymaid.CatmaidNeuron):
x = pymaid.CatmaidNeuronList(x)
assert isinstance(x, pymaid.CatmaidNeuronList)
# We must not perform this on x.nodes as this is a temporary property
nodes = x.nodes
# Get segmentation IDs
nodes['seg_id'] = segmentation.get_seg_ids(nodes[['x', 'y', 'z']].values)
# Count segment IDs
seg_counts = nodes.groupby(['skeleton_id', 'seg_id'
]).treenode_id.count().reset_index(drop=False)
seg_counts.columns = ['skeleton_id', 'seg_id', 'counts']
# Remove seg IDs 0 (glia?)
seg_counts = seg_counts[seg_counts.seg_id != 0]
# Turn into matrix where columns are skeleton IDs, segment IDs are rows
# and values are the overlap counts
matrix = seg_counts.pivot(index='seg_id',
columns='skeleton_id',
values='counts')
return matrix
def test_remove_duplicates(self):
t = pymaid.CatmaidNeuronList([self.nl[0], self.nl[0], self.nl[0]])
t2 = t.remove_duplicates(inplace=False)
self.assertLess(len(t2), len(t))
def test_init(self):
self.assertIsInstance(pymaid.CatmaidNeuron(
config_test.test_skids[0]), pymaid.CatmaidNeuron)
self.assertIsInstance(pymaid.CatmaidNeuronList(
config_test.test_skids), pymaid.CatmaidNeuronList)
def find_fragments(x,
remote_instance,
min_node_overlap=3,
min_nodes=1,
mesh=None):
"""Find manual tracings overlapping with given autoseg neuron.
This function is a generalization of ``find_autoseg_fragments`` and is
designed to not require overlapping neurons to have references (e.g.
in their name) to segmentation IDs:
1. Traverse neurites of ``x`` search within 2.5 microns radius for
potentially overlapping fragments.
2. Either collect segmentation IDs for the input neuron and all
potentially overlapping fragments or (if provided) use the mesh
to check if the candidates are inside that mesh.
3. Return fragments that overlap with at least ``min_overlap`` nodes
with input neuron.
Parameters
----------
x : pymaid.CatmaidNeuron | navis.TreeNeuron
Neuron to collect fragments for.
remote_instance : pymaid.CatmaidInstance
Catmaid instance in which to search for fragments.
min_node_overlap : int, optional
Minimal overlap between `x` and a fragment in nodes. If
the fragment has less total nodes than `min_overlap`,
the threshold will be lowered to:
``min_overlap = min(min_overlap, fragment.n_nodes)``
min_nodes : int, optional
Minimum node count for returned neurons.
mesh : navis.Volume | trimesh.Trimesh | navis.MeshNeuron, optional
Mesh representation of ``x``. If provided will use the
mesh instead of querying the segmentation to determine
if fragments overlap. This is generally the faster.
Return
------
pymaid.CatmaidNeuronList
CatmaidNeurons of the overlapping fragments. Overlap
scores are attached to each neuron as ``.overlap_score``
attribute.
Examples
--------
Setup:
>>> import pymaid
>>> import fafbseg
>>> manual = pymaid.CatmaidInstance('MANUAL_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('AUTO_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> # Set a source for segmentation data
>>> fafbseg.use_google_storage("https://storage.googleapis.com/fafb-ffn1-20190805/segmentation")
Find manually traced fragments overlapping with an autoseg neuron:
>>> x = pymaid.get_neuron(204064470, remote_instance=auto)
>>> frags_of_x = fafbseg.find_fragments(x, remote_instance=manual)
See Also
--------
fafbseg.find_autoseg_fragments
Use this function if you are looking for autoseg
fragments overlapping with a given neuron. Because we
can use the reference to segment IDs (via names &
annotations), this function is much faster than
``find_fragments``.
"""
if not isinstance(x, navis.TreeNeuron):
raise TypeError(
'Expected navis.TreeNeuron or pymaid.CatmaidNeuron, got "{}"'.
format(type(x)))
meshtypes = (type(None), navis.MeshNeuron, navis.Volume, tm.Trimesh)
if not isinstance(mesh, meshtypes):
raise TypeError(f'Unexpected mesh of type "{type(mesh)}"')
# Resample the autoseg neuron to 0.5 microns
x_rs = x.resample(500, inplace=False)
# For each node get skeleton IDs in a 0.25 micron radius
r = 250
# Generate bounding boxes around each node
bboxes = [
np.vstack([co - r, co + r]).T
for co in x_rs.nodes[['x', 'y', 'z']].values
]
# Query each bounding box
urls = [
remote_instance._get_skeletons_in_bbox(minx=min(b[0]),
maxx=max(b[0]),
miny=min(b[1]),
maxy=max(b[1]),
minz=min(b[2]),
maxz=max(b[2]),
min_nodes=min_nodes)
for b in bboxes
]
resp = remote_instance.fetch(urls,
desc='Searching for overlapping neurons')
skids = set([s for l in resp for s in l])
# Return empty NeuronList if no skids found
if not skids:
return pymaid.CatmaidNeuronList([])
# Get nodes for these candidates
tn_table = pymaid.get_node_table(skids,
include_details=False,
convert_ts=False,
remote_instance=remote_instance)
# Keep track of total node counts
node_counts = tn_table.groupby('skeleton_id').node_id.count().to_dict()
# If no mesh, use segmentation
if not mesh:
# Get segment IDs for the input neuron
x.nodes['seg_id'] = locs_to_segments(x.nodes[['x', 'y', 'z']].values,
coordinates='nm',
mip=0)
# Count segment IDs
x_seg_counts = x.nodes.groupby('seg_id').node_id.count().reset_index(
drop=False)
x_seg_counts.columns = ['seg_id', 'counts']
# Remove seg IDs 0
x_seg_counts = x_seg_counts[x_seg_counts.seg_id != 0]
# Generate KDTree for nearest neighbor calculations
tree = navis.neuron2KDTree(x)
# Now remove nodes that aren't even close to our input neuron
dist, ix = tree.query(tn_table[['x', 'y', 'z']].values,
distance_upper_bound=2500)
tn_table = tn_table.loc[dist <= 2500]
# Remove neurons that can't possibly have enough overlap
node_counts2 = tn_table.groupby(
'skeleton_id').node_id.count().to_dict()
to_keep = [
k for k, v in node_counts2.items()
if v >= min(min_node_overlap, node_counts[k])
]
tn_table = tn_table[tn_table.skeleton_id.isin(to_keep)]
# Add segment IDs
tn_table['seg_id'] = locs_to_segments(tn_table[['x', 'y', 'z']].values,
coordinates='nm',
mip=0)
# Now group by neuron and by segment
seg_counts = tn_table.groupby(
['skeleton_id', 'seg_id']).node_id.count().reset_index(drop=False)
# Rename columns
seg_counts.columns = ['skeleton_id', 'seg_id', 'counts']
# Remove seg IDs 0
seg_counts = seg_counts[seg_counts.seg_id != 0]
# Remove segments IDs that are not overlapping with input neuron
seg_counts = seg_counts[np.isin(seg_counts.seg_id.values,
x_seg_counts.seg_id.values)]
# Now go over each candidate and see if there is enough overlap
ol = []
scores = []
for s in seg_counts.skeleton_id.unique():
# Subset to nodes of this neurons
this_counts = seg_counts[seg_counts.skeleton_id == s]
# If the neuron is smaller than min_overlap, lower the threshold
this_min_ol = min(min_node_overlap, node_counts[s])
# Sum up counts for both input neuron and this candidate
c_count = this_counts.counts.sum()
x_count = x_seg_counts[x_seg_counts.seg_id.isin(
this_counts.seg_id.values)].counts.sum()
# If there is enough overlap, keep this neuron
# and add score as `overlap_score`
if (c_count >= this_min_ol) and (x_count >= this_min_ol):
# The score is the minimal overlap
scores.append(min(c_count, x_count))
ol.append(s)
else:
# Check if nodes are inside or outside the mesh
tn_table['in_mesh'] = navis.in_volume(tn_table[['x', 'y', 'z']].values,
mesh).astype(bool)
# Count the number of in-mesh nodes for each neuron
# This also drops skeletons without
ol_counts = tn_table.groupby('skeleton_id',
as_index=False).in_mesh.sum()
# Rename columns
ol_counts.columns = ['skeleton_id', 'counts']
# Now subset to those that are overlapping sufficiently
# First drop all non-overlapping fragments
ol_counts = ol_counts[ol_counts.counts > 0]
# Add column with total node counts
ol_counts['node_count'] = ol_counts.skeleton_id.map(node_counts)
# Generate an threshold array such that the threshold is the minimum
# between the total nodes and min_node_overlap
mno_array = np.repeat([min_node_overlap], ol_counts.shape[0])
mnc_array = ol_counts.skeleton_id.map(node_counts).values
thr_array = np.vstack([mno_array, mnc_array]).min(axis=0)
# Subset to candidates that meet the threshold
ol_counts = ol_counts[ol_counts.counts >= thr_array]
ol = ol_counts.skeleton_id.tolist()
scores = ol_counts.counts.tolist()
if ol:
ol = pymaid.get_neurons(ol, remote_instance=remote_instance)
# Make sure it's a neuronlist
if not isinstance(ol, pymaid.CatmaidNeuronList):
ol = pymaid.CatmaidNeuronList(ol)
for n, s in zip(ol, scores):
n.overlap_score = s
else:
ol = pymaid.CatmaidNeuronList([])
return ol
def find_autoseg_fragments(x,
autoseg_instance,
min_node_overlap=3,
min_nodes=1,
verbose=True,
raise_none_found=True):
"""Find autoseg tracings constituting a given neuron.
This function works by querying the segmentation IDs along the neurites of
your query neuron and then fetching the corresponding skeletons in
``autoseg_instance``.
Parameters
----------
x : navis.Neuron/List
Neuron to collect fragments for.
autoseg_instance : pymaid.CatmaidInstance
Catmaid instance which contains autoseg fragments.
min_node_overlap : int, optional
Minimal overlap between `x` and a segmentation
[in nodes].
min_nodes : int, optional
Minimum node count for returned neurons.
verbose : bool, optional
If True, will be print summary of who has contributed
to the autoseg fragments by merging or tracing nodes.
raise_none_found : bool, optional
If True and none of the requested segments were found,
will raise ValueError
Return
------
pymaid.CatmaidNeuronList
CatmaidNeurons of the overlapping fragments.
Examples
--------
Setup:
>>> import pymaid
>>> import fafbseg
>>> manual = pymaid.CatmaidInstance('MANUAL_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('AUTO_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> # Call one of the fafbseg.use_... to set a source for segmentation IDs
>>> fafbseg.use_google_storage("https://storage.googleapis.com/fafb-ffn1-20190805/segmentation")
Find autoseg fragments overlapping with a manually traced neuron:
>>> x = pymaid.get_neuron(16, remote_instance=manual)
>>> frags_of_x = fafbseg.find_autoseg_fragments(x, remote_instance=auto)
See Also
--------
fafbseg.find_fragments
Generalization of this function that can find neurons
independent of whether they have a reference to the
segmentation ID in e.g. their name or annotations. Use
this to find manual tracings overlapping with a given
neuron e.g. from autoseg.
"""
if not isinstance(x, navis.TreeNeuron):
raise TypeError(
'Expected navis.TreeNeuron or pymaid.CatmaidNeuron, got "{}"'.
format(type(x)))
if not isinstance(autoseg_instance, pymaid.CatmaidInstance):
raise TypeError('Expected pymaid.CatmaidInstance, got "{}"'.format(
type(autoseg_instance)))
# First collect segments constituting this neuron
overlap_matrix = neuron_to_segments(x)
# If none found, return empty neuronlist
if overlap_matrix.empty:
return pymaid.CatmaidNeuronList([])
# Filter
seg_ids = overlap_matrix.loc[
overlap_matrix[x.skeleton_id] >= min_node_overlap].index.tolist()
# Now try finding the corresponding skeletons
nl = segments_to_neuron(seg_ids,
autoseg_instance=autoseg_instance,
verbose=verbose,
raise_none_found=raise_none_found)
nl.sort_values('n_nodes')
# Give contribution summary
if verbose:
# Get annotation details
an_details = pymaid.get_annotation_details(
nl, remote_instance=autoseg_instance)
# Extract merge annotations
merge_an = an_details[an_details.annotation.str.contains('Merged:')]
# Group by user and count
merge_count = merge_an.groupby('user')[['annotation']].count()
# Get manual tracing contributions
contr = pymaid.get_user_contributions(nl,
remote_instance=autoseg_instance)
contr.set_index('user', inplace=True)
# Merge both
summary = pd.merge(merge_count,
contr[['nodes', 'nodes_reviewed']],
left_index=True,
right_index=True).fillna(0)
summary.columns = ['merges', 'nodes_traced', 'nodes_reviewed']
if not summary.empty:
print('Tracer have worked on these neurons:')
print(summary)
else:
print('Nobody has worked on these neurons')
return nl[nl.n_nodes >= min_nodes]
def collapse_nodes2(A, B, limit=2, base_neuron=None):
"""Merge neuron A into neuron(s) B creating a union of both.
This implementation uses edge contraction on the neurons' graph to ensure
maximum connectivity. Only works if the fragments collectively form a
continuous tree (i.e. you must be certain that they partially overlap).
Parameters
----------
A : CatmaidNeuron
Neuron to be collapsed into neurons B.
B : CatmaidNeuronList
Neurons to collapse neuron A into.
limit : int, optional
Max distance [microns] for nearest neighbour search.
base_neuron : skeleton_ID | CatmaidNeuron, optional
Neuron from B to use as template for union. If not
provided, the first neuron in the list is used as
template!
Returns
-------
core.CatmaidNeuron
Union of all input neurons.
new_edges : pandas.DataFrame
Subset of the ``.nodes`` table that represent newly
added edges.
collapsed_nodes : dict
Map of collapsed nodes::
NodeA -collapsed-into-> NodeB
"""
if isinstance(A, pymaid.CatmaidNeuronList):
if len(A) == 1:
A = A[0]
else:
A = pymaid.stitch_neurons(A, method="NONE")
elif not isinstance(A, pymaid.CatmaidNeuron):
raise TypeError('`A` must be a CatmaidNeuron, got "{}"'.format(
type(A)))
if isinstance(B, pymaid.CatmaidNeuron):
B = pymaid.CatmaidNeuronList(B)
elif not isinstance(B, pymaid.CatmaidNeuronList):
raise TypeError('`B` must be a CatmaidNeuronList, got "{}"'.format(
type(B)))
# This is just check on the off-chance that skeleton IDs are not unique
# (e.g. if neurons come from different projects) -> this is relevant because
# we identify the master ("base_neuron") via it's skeleton ID
skids = [n.skeleton_id for n in B + A]
if len(skids) > len(np.unique(skids)):
raise ValueError(
'Duplicate skeleton IDs found. Try manually assigning '
'unique skeleton IDs.')
# Convert distance threshold from microns to nanometres
limit *= 1000
# Before we start messing around, let's make sure we can keep track of
# the origin of each node
for n in B + A:
n.nodes['origin_skeletons'] = n.skeleton_id
# First make a weak union by simply combining the node tables
union_simple = pymaid.stitch_neurons(B + A,
method='NONE',
master=base_neuron)
# Check for duplicate node IDs
if any(union_simple.nodes.treenode_id.duplicated()):
raise ValueError('Duplicate node IDs found.')
# Find nodes in A to be merged into B
tree = pymaid.neuron2KDTree(B, tree_type='c', data='treenodes')
# For each node in A get the nearest neighbor in B
coords = A.nodes[['x', 'y', 'z']].values
nn_dist, nn_ix = tree.query(coords, k=1, distance_upper_bound=limit)
# Find nodes that are close enough to collapse
collapsed = A.nodes.loc[nn_dist <= limit].treenode_id.values
clps_into = B.nodes.iloc[nn_ix[nn_dist <= limit]].treenode_id.values
clps_map = {n1: n2 for n1, n2 in zip(collapsed, clps_into)}
# The fastest way to collapse is to work on the edge list
E = nx.to_pandas_edgelist(union_simple.graph)
# Keep track of which edges were collapsed -> we will use this as weight
# later on to prioritize existing edges over newly generated ones
E['is_new'] = 1
E.loc[(E.source.isin(B.nodes.treenode_id.values)) |
(E.target.isin(B.nodes.treenode_id.values)), 'is_new'] = 0
# Now map collapsed nodes onto the nodes they collapsed into
E['target'] = E.target.map(lambda x: clps_map.get(x, x))
E['source'] = E.source.map(lambda x: clps_map.get(x, x))
# Make sure no self loops after collapsing. This happens if two adjacent
# nodes collapse onto the same target node
E = E[E.source != E.target]
# Remove duplicates. This happens e.g. when two adjaceny nodes merge into
# two other adjaceny nodes: A->B C->D ----> A/B->C/D
# By sorting first, we make sure original edges are kept first
E.sort_values('is_new', ascending=True, inplace=True)
# Because edges may exist in both directions (A->B and A<-B) we have to
# generate a column that's agnostic to directionality using frozensets
E['edge'] = E[['source', 'target']].apply(frozenset, axis=1)
E.drop_duplicates(['edge'], keep='first', inplace=True)
# Regenerate graph from these new edges
G = nx.Graph()
G.add_weighted_edges_from(E[['source', 'target',
'is_new']].values.astype(int))
# At this point there might still be disconnected pieces -> we will create
# separate neurons for each tree
props = union_simple.nodes.loc[union_simple.nodes.treenode_id.isin(
G.nodes)].set_index('treenode_id')
nx.set_node_attributes(G, props.to_dict(orient='index'))
fragments = []
for n in nx.connected_components(G):
c = G.subgraph(n)
tree = nx.minimum_spanning_tree(c)
fragments.append(
pymaid.graph.nx2neuron(tree,
neuron_name=base_neuron.neuron_name,
skeleton_id=base_neuron.skeleton_id))
fragments = pymaid.CatmaidNeuronList(fragments)
if len(fragments) > 1:
# Now heal those fragments using a minimum spanning tree
union = pymaid.stitch_neurons(*fragments, method='ALL')
else:
union = fragments[0]
# Reroot to base neuron's root
union.reroot(base_neuron.root[0], inplace=True)
# Add tags back on
union.tags.update(union_simple.tags)
# Add connectors back on
union.connectors = union_simple.connectors.drop_duplicates(
subset='connector_id').copy()
union.connectors.loc[:, 'treenode_id'] = union.connectors.treenode_id.map(
lambda x: clps_map.get(x, x))
# Find the newly added edges (existing edges should not have been modified
# - except for changing direction due to reroot)
# The basic logic here is that new edges were only added between two
# previously separate skeletons, i.e. where the skeleton ID changes between
# parent and child node
node2skid = union_simple.nodes.set_index(
'treenode_id').skeleton_id.to_dict()
union.nodes['parent_skeleton'] = union.nodes.parent_id.map(node2skid)
new_edges = union.nodes[
union.nodes.origin_skeletons != union.nodes.parent_skeleton]
# Remove root edges
new_edges = new_edges[~new_edges.parent_id.isnull()]
return union, new_edges, clps_map
def collapse_nodes(*x, limit=1, base_neuron=None, priority_nodes=None):
"""Generate the union of a set of neurons.
This implementation uses edge contraction on the neurons' graph to ensure
maximum connectivity. Only works if, taken together, the neurons form a
continuous tree (i.e. you must be certain that they partially overlap).
Parameters
----------
*x : CatmaidNeuron/List
Neurons to be merged.
limit : int, optional
Max distance [microns] for nearest neighbour search.
base_neuron : skeleton_ID | CatmaidNeuron, optional
Neuron to use as template for union. If not provided,
the first neuron in the list is used as template!
priority_nodes : list-like
List of treenode IDs. If provided, these nodes will
have priority when pairwise collapsing nodes. If two
priority nodes are to be collapsed, a new edge between
them is created instead.
Returns
-------
core.CatmaidNeuron
Union of all input neurons.
collapsed_nodes : dict
Map of collapsed nodes::
NodeA -collapsed-into-> NodeB
new_edges : list
List of newly added edges::
[[NodeA, NodeB], ...]
"""
# Unpack neurons in *args
x = pymaid.utils._unpack_neurons(x)
# Make sure we're working on copies and don't change originals
x = pymaid.CatmaidNeuronList([n.copy() for n in x])
if isinstance(priority_nodes, type(None)):
priority_nodes = []
# This is just check on the off-chance that skeleton IDs are not unique
# (e.g. if neurons come from different projects) -> this is relevant because
# we identify the master ("base_neuron") via it's skeleton ID
skids = [n.skeleton_id for n in x]
if len(skids) > len(np.unique(skids)):
raise ValueError(
'Duplicate skeleton IDs found in neurons to be merged. '
'Try manually assigning unique skeleton IDs.')
if any([not isinstance(n, pymaid.CatmaidNeuron) for n in x]):
raise TypeError('Input must only be CatmaidNeurons/List')
if len(x) < 2:
raise ValueError('Need at least 2 neurons to make a union!')
# Convert distance threshold from microns to nanometres
limit *= 1000
# First make a weak union by simply combining the node tables
union_simple = pymaid.stitch_neurons(x, method='NONE', master=base_neuron)
# Check for duplicate node IDs
if any(union_simple.nodes.treenode_id.duplicated()):
raise ValueError('Duplicate node IDs found.')
# Map priority nodes -> this will speed things up later
is_priority = {n: True for n in priority_nodes}
# Go over each pair of fragments and check if they can be collapsed
comb = itertools.combinations(x, 2)
collapse_into = {}
new_edges = []
for c in comb:
tree = pymaid.neuron2KDTree(c[0], tree_type='c', data='treenodes')
# For each node in master get the nearest neighbor in minion
coords = c[1].nodes[['x', 'y', 'z']].values
nn_dist, nn_ix = tree.query(coords, k=1, distance_upper_bound=limit)
clps_left = c[0].nodes.iloc[nn_ix[nn_dist <= limit]].treenode_id.values
clps_right = c[1].nodes.iloc[nn_dist <= limit].treenode_id.values
clps_dist = nn_dist[nn_dist <= limit]
for i, (n1, n2, d) in enumerate(zip(clps_left, clps_right, clps_dist)):
if is_priority.get(n1, False):
# If both nodes are priority nodes, don't collapse
if is_priority.get(n2, False):
new_edges.append([n1, n2, d])
# continue
else:
collapse_into[n2] = n1
else:
collapse_into[n1] = n2
# Get the graph
G = union_simple.graph
# Add the new edges to graph
G.add_weighted_edges_from(new_edges)
# Using an edge list is much more efficient than an adjacency matrix
E = nx.to_pandas_edgelist(G)
# All nodes that collapse into other nodes need to have weight set to
# float("inf") to de-prioritize them when generating the minimum spanning
# tree later
clps_nodes = set(collapse_into.keys())
E.loc[(E.source.isin(clps_nodes)) | (E.target.isin(clps_nodes)),
'weight'] = float('inf')
# Now map collapsed nodes onto the nodes they collapsed into
E['target'] = E.target.map(lambda x: collapse_into.get(x, x))
E['source'] = E.source.map(lambda x: collapse_into.get(x, x))
# Make sure no self loops after collapsing. This happens if two adjacent
# nodes collapse onto the same target node
E = E[E.source != E.target]
# Turn this back into a graph
G_clps = nx.from_pandas_edgelist(E, edge_attr='weight')
# Make sure that we are fully connected
if not nx.is_connected(G_clps):
raise ValueError('Neuron still fragmented after collapsing nodes. '
'Try increasing the `limit` parameter.')
# Under certain conditions, collapsing nodes will introduce cycles:
# Consider for example a graph: A->B->C D->E->F
# Collapsing A and C into D will create a loop between B<->D
# To fix this we have to create a minimum spanning tree.
# In doing so, we need to prioritize existing edges over new edges
# otherwise we would have to cut existing neurons -> this is why we set
# weight of new edges to float("inf") earlier on
# Generate the tree
tree = nx.minimum_spanning_tree(G_clps.to_undirected(as_view=True))
# Add properties to nodes
survivors = np.unique(E[['source', 'target']])
props = union_simple.nodes.set_index('treenode_id').loc[survivors]
nx.set_node_attributes(tree, props.to_dict(orient='index'))
# Recreate neuron
union = pymaid.graph.nx2neuron(tree,
neuron_name=union_simple.neuron_name,
skeleton_id=union_simple.skeleton_id)
# Add tags back on
for n in x:
union.tags.update({
k: union.tags.get(k, []) + [collapse_into.get(a, a) for a in v]
for k, v in n.tags.items()
})
# Add connectors back on
union.connectors = x.connectors.drop_duplicates(subset='connector_id')
union.connectors.treenode_id = union.connectors.treenode_id.map(
lambda x: collapse_into.get(x, x))
# Return the last survivor
return union, collapse_into, new_edges
def find_missed_branches(x,
autoseg_instance,
tag=True,
tag_size_thresh=10,
min_node_overlap=4,
**kwargs):
"""Use autoseg to find (and annotate) potential missed branches.
Parameters
----------
x : pymaid.CatmaidNeuron/List
Neuron(s) to search for missed branches.
autoseg_instance : pymaid.CatmaidInstance
CATMAID instance containing the autoseg skeletons.
tag : bool, optional
If True, will tag nodes of x that might have missed
branches with "missed branch?".
tag_size_thresh : int, optional
Size threshold in microns of cable for tagging
potentially missed branches.
min_node_overlap : int, optional
Minimum number of nodes that input neuron(s) x must
overlap with given segmentation ID for it to be
included.
**kwargs
Keyword arguments passed to
``fafbseg.neuron_from_segments``.
Returns
-------
summary : pandas.DataFrame
DataFrame containing a summary of potentially missed
branches.
If input is a single neuron:
fragments : pymaid.CatmaidNeuronList
Fragments found to be potentially overlapping with the
input neuron.
branches : pymaid.CatmaidNeuronList
Potentially missed branches extracted from ``fragments``.
Examples
--------
Setup
>>> import fafbseg
>>> import brainmaps as bm
>>> import pymaid
>>> # Set up connections to manual and autoseg CATMAID
>>> manual = pymaid.CatmaidInstance('URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> # Initialise brainmaps
>>> session = bm.acquire_credentials()
>>> bm.set_global_volume('772153499790:fafb_v14:fafb-ffn1-20190521')
Find missed branches and tag them
>>> # Fetch a neuron
>>> x = pymaid.get_neuron(16, remote_instance=manual)
>>> # Find and tag missed branches
>>> (summary,
... fragments,
... branches) = fafbseg.find_missed_branches(x, autoseg_instance=auto)
>>> # Show summary of missed branches
>>> summary.head()
n_nodes cable_length node_id
0 110 28.297424 3306395
1 90 23.976504 20676047
2 64 15.851333 23419997
3 29 7.494350 6298769
4 16 3.509739 15307841
>>> # Co-visualize your neuron and potentially overlapping autoseg fragments
>>> x.plot3d(color='w')
>>> fragments.plot3d()
>>> # Visualize the potentially missed branches
>>> pymaid.clear3d()
>>> x.plot3d(color='w')
>>> branches.plot3d(color='r')
"""
if isinstance(x, pymaid.CatmaidNeuronList):
to_concat = []
for n in tqdm(x,
desc='Processing neurons',
disable=not use_pbars,
leave=False):
(summary, frags, branches) = find_missed_branches(
n,
autoseg_instance=autoseg_instance,
tag=tag,
tag_size_thresh=tag_size_thresh,
**kwargs)
summary['skeleton_id'] = n.skeleton_id
to_concat.append(summary)
return pd.concat(to_concat, ignore_index=True)
elif not isinstance(x, pymaid.CatmaidNeuron):
raise TypeError('Input must be CatmaidNeuron/List, got "{}"'.format(
type(x)))
# Find autoseg neurons overlapping with input neuron
nl = find_autoseg_fragments(x,
autoseg_instance=autoseg_instance,
min_node_overlap=min_node_overlap,
verbose=False,
raise_none_found=False)
# Next create a union
if not nl.empty:
for n in nl:
n.nodes['origin'] = 'autoseg'
n.nodes['origin_skid'] = n.skeleton_id
x.nodes['origin'] = 'query'
x.nodes['origin_skid'] = x.skeleton_id
union = pymaid.union_neurons(x,
nl,
base_neuron=x,
limit=2,
non_overlap='stitch')
# Subset to autoseg nodes
autoseg_nodes = union.nodes[union.nodes.origin ==
'autoseg'].treenode_id.values
else:
autoseg_nodes = np.empty((0, 5))
# Process fragments if any autoseg nodes left
data = []
frags = pymaid.CatmaidNeuronList([])
if autoseg_nodes.shape[0]:
autoseg = pymaid.subset_neuron(union, autoseg_nodes)
# Split into fragments
frags = pymaid.break_fragments(autoseg)
# Generate summary
nodes = union.nodes.set_index('treenode_id')
for n in frags:
# Find parent node in union
pn = nodes.loc[n.root[0], 'parent_id']
pn_co = nodes.loc[pn, ['x', 'y', 'z']].values
org_skids = n.nodes.origin_skid.unique().tolist()
data.append([n.n_nodes, n.cable_length, pn, pn_co, org_skids])
df = pd.DataFrame(data,
columns=[
'n_nodes', 'cable_length', 'node_id', 'node_loc',
'autoseg_skids'
])
df.sort_values('cable_length', ascending=False, inplace=True)
if tag and not df.empty:
to_tag = df[df.cable_length >= tag_size_thresh].node_id.values
resp = pymaid.add_tags(to_tag,
tags='missed branch?',
node_type='TREENODE',
remote_instance=x._remote_instance)
if 'error' in resp:
return df, resp
return df, nl, frags
def merge_neuron(x,
target_instance,
tag,
min_node_overlap=4,
min_overlap_size=1,
merge_limit=1,
min_upload_size=0,
min_upload_nodes=1,
update_radii=True,
import_tags=False,
label_joins=True,
sid_from_nodes=True):
"""Merge neuron into target instance.
This function will attempt to:
1. Find fragments in ``target_instance`` that overlap with ``x``
using the brainmaps API.
2. Generate a union of these fragments and ``x``.
3. Make a differential upload of the union leaving existing nodes
untouched.
4. Join uploaded and existing tracings into a single continuous
neuron. This will also upload connectors but no node tags.
Parameters
----------
x : pymaid.CatmaidNeuron/List
Neuron(s)/fragment(s) to commit to ``target_instance``.
target_instance : pymaid.CatmaidInstance
Target Catmaid instance to commit the neuron to.
tag : str
A tag to be added as part of a ``{URL} upload {tag}``
annotation. This should be something identifying your
group - e.g. ``tag='WTCam'`` for the Cambridge Wellcome
Trust group.
min_node_overlap : int, optional
Minimal overlap between `x` and a potentially
overlapping neuron in ``target_instance``. If
the fragment has less total nodes than `min_overlap`,
the threshold will be lowered to:
``min_overlap = min(min_overlap, fragment.n_nodes)``
min_overlap_size : int, optional
Minimum node count for potentially overlapping neurons
in ``target_instance``. Use this to e.g. exclude
single-node synapse orphans.
merge_limit : int, optional
Distance threshold [um] for collapsing nodes of ``x``
into overlapping fragments in target instance. Decreasing
this will help if your neuron has complicated branching
patterns (e.g. uPN dendrites) at the cost of potentially
creating duplicate parallel tracings in the neuron's
backbone.
min_upload_size : float, optional
Minimum size in microns for upload of new branches:
branches found in ``x`` but not in the overlapping
neuron(s) in ``target_instance`` are uploaded in
fragments. Use this parameter to exclude small branches
that might not be worth the additional review time.
min_upload_nodes : int, optional
As ``min_upload_size`` but for number of nodes instead
of cable length.
update_radii : bool, optional
If True, will use radii in ``x`` to update radii of
overlapping fragments if (and only if) the nodes
do not currently have a radius (i.e. radius<=0).
import_tags : bool, optional
If True, will import node tags. Please note that this
will NOT import tags of nodes that have been collapsed
into manual tracings.
label_joins : bool, optional
If True, will label nodes at which old and new
tracings have been joined with tags ("Joined from ..."
and "Joined with ...") and with a lower confidence of
1.
sid_from_nodes : bool, optional
If True and the to-be-merged neuron has a "skeleton_id"
column it will be used to set the ``source_id`` upon
uploading new branches. This is relevant if your neuron
is a virtual chimera of several neurons: in order to
preserve provenance (i.e. correctly associating each
node with a ``source_id`` origin) you should make
sure that your neuron is
Returns
-------
Nothing
If all went well.
dict
If something failed, returns server responses with
error logs.
Examples
--------
Setup
>>> import fafbseg
>>> import brainmaps as bm
>>> import pymaid
>>> # Set up connections to manual and autoseg CATMAID
>>> manual = pymaid.CatmaidInstance('URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> # Initialise brainmaps
>>> session = bm.acquire_credentials()
>>> bm.set_global_volume('772153499790:fafb_v14:fafb-ffn1-20190521')
Merge a neuron from autoseg into v14
>>> # Fetch the autoseg neuron to transfer to v14
>>> x = pymaid.get_neuron(267355161, remote_instance=auto)
>>> # Get the neuron's annotations so that they can be merged too
>>> x.get_annotations(remote_instance=auto)
>>> # Start the commit
>>> # See online documentation for video of merge process
>>> resp = fafbseg.merge_neuron(x, target_instance=manual)
"""
if not isinstance(x, pymaid.CatmaidNeuronList):
if not isinstance(x, pymaid.CatmaidNeuron):
raise TypeError(
'Expected pymaid.CatmaidNeuron/List, got "{}"'.format(type(x)))
x = pymaid.CatmaidNeuronList(x)
if not isinstance(tag, (str, type(None))):
raise TypeError('Tag must be string, got "{}"'.format(type(tag)))
pymaid.set_loggers('WARNING')
# Throttle requests just to play it safe
# On a bad connection one might have to decrease max_threads further
target_instance.max_threads = min(target_instance.max_threads, 50)
# For user convenience, we will do all the stuff that needs user interaction
# first and then run the automatic merge:
# Start by find all overlapping fragments
overlapping = []
for n in tqdm(x,
desc='Pre-processing neuron(s)',
leave=False,
disable=not use_pbars):
ol = find_fragments(n,
min_node_overlap=min_node_overlap,
min_nodes=min_overlap_size,
remote_instance=target_instance)
# Add number of samplers to each neuron
n_samplers = pymaid.get_sampler_counts(ol,
remote_instance=target_instance)
for nn in ol:
nn.sampler_count = n_samplers[str(nn.skeleton_id)]
overlapping.append(ol)
# Now have the user confirm merges before we actually make them
viewer = pymaid.Viewer(title='Confirm merges')
viewer.clear()
overlap_cnf = []
base_neurons = []
for n, ol in zip(x, overlapping):
# This asks user a bunch of questions prior to merge and upload
ol, bn = _confirm_overlap(n, ol, viewer=viewer)
overlap_cnf.append(ol)
base_neurons.append(bn)
viewer.close()
for i, (n, ol, bn) in enumerate(zip(x, overlap_cnf, base_neurons)):
print('Processing neuron "{}" [{}/{}]'.format(n.neuron_name, i,
len(x)),
flush=True)
# If no overlapping neurons proceed with just uploading.
if not ol:
print('No overlapping fragments to merge. Uploading...',
end='',
flush=True)
resp = pymaid.upload_neuron(n,
import_tags=import_tags,
import_annotations=True,
import_connectors=True,
remote_instance=target_instance)
if 'error' in resp:
return resp
# Add annotations
_ = __merge_annotations(n, resp['skeleton_id'], tag,
target_instance)
msg = '\nNeuron "{}" successfully uploaded to target instance as "{}" #{}'
print(msg.format(n.neuron_name, n.neuron_name,
resp['skeleton_id']),
flush=True)
continue
# Check if there are any duplicate node IDs between neuron ``x`` and the
# overlapping fragments and create new IDs for ``x`` if necessary
duplicated = n.nodes[n.nodes.treenode_id.isin(
ol.nodes.treenode_id.values)]
if not duplicated.empty:
print('Duplicate node IDs found. Regenerating node tables... ',
end='',
flush=True)
max_ix = max(ol.nodes.treenode_id.max(),
n.nodes.treenode_id.max()) + 1
new_ids = range(max_ix, max_ix + duplicated.shape[0])
id_map = {
old: new
for old, new in zip(duplicated.treenode_id, new_ids)
}
n.nodes['treenode_id'] = n.nodes.treenode_id.map(
lambda n: id_map.get(n, n))
n.nodes['parent_id'] = n.nodes.parent_id.map(
lambda n: id_map.get(n, n))
n.connectors['treenode_id'] = n.connectors.treenode_id.map(
lambda n: id_map.get(n, n))
print('Done.', flush=True)
# Combining the fragments into a single neuron is actually non-trivial:
# 1. Collapse nodes of our input neuron `x` into within-distance nodes
# in the overlapping fragments (never the other way around!)
# 2. At the same time keep connectivity (i.e. edges) of the input-neuron
# 3. Keep track of the nodes' provenance (i.e. the contractions)
#
# In addition there are a lot of edge-cases to consider. For example:
# - multiple nodes collapsing onto the same node
# - nodes of overlapping fragments that are close enough to be collapsed
# (e.g. orphan synapse nodes)
# Keep track of original skeleton IDs
for a in ol + n:
# Original skeleton of each node
a.nodes['origin_skeletons'] = a.skeleton_id
# Original skeleton of each connector
a.connectors['origin_skeletons'] = a.skeleton_id
print('Generating union of all fragments... ', end='', flush=True)
union, new_edges, collapsed_into = collapse_nodes2(n,
ol,
limit=merge_limit,
base_neuron=bn)
print('Done.', flush=True)
print('Extracting new nodes to upload... ', end='', flush=True)
# Now we have to break the neuron into "new" fragments that we can upload
# First get the new and old nodes
new_nodes = union.nodes[union.nodes.origin_skeletons ==
n.skeleton_id].treenode_id.values
old_nodes = union.nodes[
union.nodes.origin_skeletons != n.skeleton_id].treenode_id.values
# Now remove the already existing nodes from the union
only_new = pymaid.subset_neuron(union, new_nodes)
# And then break into continuous fragments for upload
frags = pymaid.break_fragments(only_new)
print('Done.', flush=True)
# Also get the new edges we need to generate
to_stitch = new_edges[~new_edges.parent_id.isnull()]
# We need this later -> no need to compute this for every uploaded fragment
cond1b = to_stitch.treenode_id.isin(old_nodes)
cond2b = to_stitch.parent_id.isin(old_nodes)
# Now upload each fragment and keep track of new node IDs
tn_map = {}
for f in tqdm(frags,
desc='Uploading new tracings',
leave=False,
disable=not use_pbars):
# In cases of complete merging into existing neurons, the fragment
# will have no nodes
if f.n_nodes < 1:
continue
# Check if fragment is a "linker" and as such can not be skipped
lcond1 = np.isin(f.nodes.treenode_id.values,
new_edges.treenode_id.values)
lcond2 = np.isin(f.nodes.treenode_id.values,
new_edges.parent_id.values)
# If not linker, check skip conditions
if sum(lcond1) + sum(lcond2) <= 1:
if f.cable_length < min_upload_size:
continue
if f.n_nodes < min_upload_nodes:
continue
# Collect origin info for this neuron
source_info = {'source_type': 'segmentation'}
if not sid_from_nodes or 'skeleton_id' not in f.nodes.columns:
source_info['source_id'] = int(n.skeleton_id)
else:
if f.nodes.skeleton_id.unique().shape[0] == 1:
skid = f.nodes.skeleton_id.unique()[0]
else:
print('Warning: uploading chimera fragment with multiple '
'skeleton IDs! Using largest contributor ID.')
# Use the skeleton ID that has the most nodes
by_skid = f.nodes.groupby('skeleton_id').x.count()
skid = by_skid.sort_values(ascending=False).index.values[0]
source_info['source_id'] = int(skid)
if not isinstance(getattr(n, '_remote_instance'), type(None)):
source_info[
'source_project_id'] = n._remote_instance.project_id
source_info['source_url'] = n._remote_instance.server
resp = pymaid.upload_neuron(f,
import_tags=import_tags,
import_annotations=False,
import_connectors=True,
remote_instance=target_instance,
**source_info)
# Stop if there was any error while uploading
if 'error' in resp:
return resp
# Collect old -> new node IDs
tn_map.update(resp['node_id_map'])
# Now check if we can create any of the new edges by joining nodes
# Both treenode and parent ID have to be either existing nodes or
# newly uploaded
cond1a = to_stitch.treenode_id.isin(tn_map)
cond2a = to_stitch.parent_id.isin(tn_map)
to_gen = to_stitch.loc[(cond1a | cond1b) & (cond2a | cond2b)]
# Join nodes
for node in to_gen.itertuples():
# Make sure our base_neuron always come out as winner on top
if node.treenode_id in bn.nodes.treenode_id.values:
winner, looser = node.treenode_id, node.parent_id
else:
winner, looser = node.parent_id, node.treenode_id
# We need to map winner and looser to the new node IDs
winner = tn_map.get(winner, winner)
looser = tn_map.get(looser, looser)
# And now do the join
resp = pymaid.join_nodes(winner,
looser,
no_prompt=True,
tag_nodes=label_joins,
remote_instance=target_instance)
# See if there was any error while uploading
if 'error' in resp:
print('Skipping joining nodes '
'{} and {}: {} - '.format(node.treenode_id,
node.parent_id,
resp['error']))
# Skip changing confidences
continue
# Pop this edge from new_edges and from condition
new_edges.drop(node.Index, inplace=True)
cond1b.drop(node.Index, inplace=True)
cond2b.drop(node.Index, inplace=True)
# Change node confidences at new join
if label_joins:
new_conf = {looser: 1}
resp = pymaid.update_node_confidence(
new_conf, remote_instance=target_instance)
# Add annotations
_ = __merge_annotations(n, bn, tag, target_instance)
# Update node radii
if update_radii:
print('Updating radii of existing nodes... ', end='', flush=True)
resp = update_node_radii(source=n,
target=ol,
remote_instance=target_instance,
limit=merge_limit,
skip_existing=True)
print('Done.', flush=True)
print(
'Neuron "{}" successfully merged into target instance as "{}" #{}'.
format(n.neuron_name, bn.neuron_name, bn.skeleton_id),
flush=True)
return
np.vectorize(CLASS_COLOR_DICT.get)(meta["merge_class"])))
connection_types = ["axon", "dendrite", "unsplittable"]
pal = sns.color_palette("deep", 5)
colors = [pal[1], pal[2], pal[4]]
connection_colors = dict(zip(connection_types, colors))
splits = pymaid.find_treenodes(tags="mw axon split")
splits = splits.set_index("skeleton_id")["treenode_id"].squeeze()
# plot paired neurons
pair_meta = meta[meta["pair_id"] != -1]
pairs = pair_meta["pair_id"].unique()
for p in pairs:
temp_meta = pair_meta[pair_meta["pair_id"] == p]
skids = temp_meta.index.values.astype(int)
neuron_class = temp_meta["merge_class"].iloc[0]
nl = pymaid.get_neurons(skids)
plot_fragments(nl, splits, neuron_class=neuron_class)
stashfig(get_savename(nl, neuron_class=neuron_class))
# plot unpaired neurons
unpair_meta = meta[meta["pair_id"] == -1]
for skid, row in unpair_meta.iterrows():
neuron_class = row["merge_class"]
nl = pymaid.get_neurons([skid])
nl = pymaid.CatmaidNeuronList(nl)
plot_fragments(nl, splits, neuron_class=neuron_class)
stashfig(get_savename(nl, neuron_class=neuron_class))
def find_fragments(x, remote_instance, min_node_overlap=3, min_nodes=1):
"""Find manual tracings overlapping with given autoseg neuron.
This function is a generalization of ``find_autoseg_fragments`` and is
designed to not require overlapping neurons to have references (e.g.
in their name) to segmentation IDs:
1. Traverse neurites of ``x`` search within 2.5 microns radius for
potentially overlapping fragments.
2. Collect segmentation IDs for the input neuron and all potentially
overlapping fragments using the brainmaps API.
3. Return fragments that overlap with at least ``min_overlap`` nodes
with input neuron.
Parameters
----------
x : pymaid.CatmaidNeuron
Neuron to collect fragments for.
remote_instance : pymaid.CatmaidInstance
Catmaid instance in which to search for fragments.
min_node_overlap : int, optional
Minimal overlap between `x` and a fragment in nodes. If
the fragment has less total nodes than `min_overlap`,
the threshold will be lowered to:
``min_overlap = min(min_overlap, fragment.n_nodes)``
min_nodes : int, optional
Minimum node count for returned neurons.
Return
------
pymaid.CatmaidNeuronList
CatmaidNeurons of the overlapping fragments. Overlap
scores are attached to each neuron as ``.overlap_score``
attribute.
Examples
--------
Setup:
>>> import pymaid
>>> import fafbseg
>>> import brainmappy as bm
>>> manual = pymaid.CatmaidInstance('MANUAL_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('AUTO_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> flow = bm.acquire_credentials()
>>> # Note that volume ID must match with the autoseg CatmaidInstance!
>>> bm.set_global_volume('some_volume_id')
Find manually traced fragments overlapping with an autoseg neuron:
>>> x = pymaid.get_neuron(204064470, remote_instance=auto)
>>> frags_of_x = fafbseg.find_fragments(x, remote_instance=manual)
See Also
--------
fafbseg.find_autoseg_fragments
Use this function if you are looking for autoseg
fragments overlapping with a given neuron. Because we
can use the reference to segment IDs (via names &
annotations), this function is much faster than
``find_fragments``.
"""
if not isinstance(x, pymaid.CatmaidNeuron):
raise TypeError('Expected pymaid.CatmaidNeuron, got "{}"'.format(
type(x)))
# Resample the autoseg neuron to 0.5 microns
x_rs = x.resample(500, inplace=False)
# For each node get skeleton IDs in a 0.25 micron radius
r = 250
# Generate bounding boxes around each node
bboxes = [
np.vstack([co - r, co + r]).T
for co in x_rs.nodes[['x', 'y', 'z']].values
]
# Query each bounding box
urls = [
remote_instance._get_skeletons_in_bbox(minx=min(b[0]),
maxx=max(b[0]),
miny=min(b[1]),
maxy=max(b[1]),
minz=min(b[2]),
maxz=max(b[2]),
min_nodes=min_nodes)
for b in bboxes
]
resp = remote_instance.fetch(urls,
desc='Searching for overlapping neurons')
skids = set([s for l in resp for s in l])
# Return empty NeuronList if no skids found
if not skids:
return pymaid.CatmaidNeuronList([])
# Get nodes for these candidates
tn_table = pymaid.get_treenode_table(skids,
include_details=False,
convert_ts=False,
remote_instance=remote_instance)
# Keep track of total node counts
node_counts = tn_table.groupby('skeleton_id').treenode_id.count().to_dict()
# Get segment IDs for the input neuron
x.nodes['seg_id'] = segmentation.get_seg_ids(x.nodes[['x', 'y',
'z']].values)
# Count segment IDs
x_seg_counts = x.nodes.groupby('seg_id').treenode_id.count().reset_index(
drop=False)
x_seg_counts.columns = ['seg_id', 'counts']
# Remove seg IDs 0
x_seg_counts = x_seg_counts[x_seg_counts.seg_id != 0]
# Generate KDTree for nearest neighbor calculations
tree = pymaid.neuron2KDTree(x)
# Now remove nodes that aren't even close to our input neuron
dist, ix = tree.query(tn_table[['x', 'y', 'z']].values,
distance_upper_bound=2500)
tn_table = tn_table.loc[dist <= 2500]
# Remove neurons that can't possibly have enough overlap
node_counts2 = tn_table.groupby(
'skeleton_id').treenode_id.count().to_dict()
to_keep = [
k for k, v in node_counts2.items()
if v >= min(min_node_overlap, node_counts[k])
]
tn_table = tn_table[tn_table.skeleton_id.isin(to_keep)]
# Add segment IDs
tn_table['seg_id'] = segmentation.get_seg_ids(tn_table[['x', 'y',
'z']].values)
# Now group by neuron and by segment
seg_counts = tn_table.groupby(
['skeleton_id', 'seg_id']).treenode_id.count().reset_index(drop=False)
# Rename columns
seg_counts.columns = ['skeleton_id', 'seg_id', 'counts']
# Remove seg IDs 0
seg_counts = seg_counts[seg_counts.seg_id != 0]
# Remove segments IDs that are not overlapping with input neuron
seg_counts = seg_counts[np.isin(seg_counts.seg_id.values,
x_seg_counts.seg_id.values)]
# Now go over each candidate and see if there is enough overlap
ol = []
scores = []
for s in seg_counts.skeleton_id.unique():
# Subset to nodes of this neurons
this_counts = seg_counts[seg_counts.skeleton_id == s]
# If the neuron is smaller than min_overlap, lower the threshold
this_min_ol = min(min_node_overlap, node_counts[s])
# Sum up counts for both input neuron and this candidate
c_count = this_counts.counts.sum()
x_count = x_seg_counts[x_seg_counts.seg_id.isin(
this_counts.seg_id.values)].counts.sum()
# If there is enough overlap, keep this neuron
# and add score as `overlap_score`
if (c_count >= this_min_ol) and (x_count >= this_min_ol):
# The score is the minimal overlap
scores.append(min(c_count, x_count))
ol.append(s)
if ol:
ol = pymaid.get_neurons(ol, remote_instance=remote_instance)
# Make sure it's a neuronlist
if not isinstance(ol, pymaid.CatmaidNeuronList):
ol = pymaid.CatmaidNeuronList(ol)
for n, s in zip(ol, scores):
n.overlap_score = s
else:
ol = pymaid.CatmaidNeuronList([])
return ol
def find_autoseg_fragments(x,
autoseg_instance,
min_node_overlap=3,
min_nodes=1,
verbose=True,
raise_none_found=True):
"""Find autoseg tracings constituting a given neuron.
This function works by querying the segmentation IDs along the neurites and
then fetching the corresponding skeletons in ``autoseg_instance``.
Parameters
----------
x : pymaid.CatmaidNeuron
Neuron to collect fragments for.
autoseg_instance : pymaid.CatmaidInstance
Catmaid instance which contains autoseg fragments.
min_node_overlap : int, optional
Minimal overlap between `x` and a segmentation
[in nodes].
min_nodes : int, optional
Minimum node count for returned neurons.
verbose : bool, optional
raise_none_found : bool, optional
If True and none of the requested segments were found,
will raise ValueError
Return
------
pymaid.CatmaidNeuronList
CatmaidNeurons of the overlapping fragments.
Examples
--------
Setup:
>>> import pymaid
>>> import fafbseg
>>> import brainmappy as bm
>>> manual = pymaid.CatmaidInstance('MANUAL_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('AUTO_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> flow = bm.acquire_credentials()
>>> # Note that volume ID must match with the autoseg CatmaidInstance!
>>> bm.set_global_volume('some_volume_id')
Find autoseg fragments overlapping with a manually traced neuron:
>>> x = pymaid.get_neuron(16, remote_instance=manual)
>>> frags_of_x = fafbseg.find_autoseg_fragments(x, remote_instance=auto)
See Also
--------
fafbseg.find_fragments
Generalization of this function that can find neurons
independent of whether they have a reference to the
segmentation ID in e.g. their name or annotations. Use
this to find manual tracings overlapping with a given
neuron e.g. from autoseg.
"""
if not isinstance(x, pymaid.CatmaidNeuron):
raise TypeError('Expected pymaid.CatmaidNeuron, got "{}"'.format(
type(x)))
if not isinstance(autoseg_instance, pymaid.CatmaidInstance):
raise TypeError('Expected pymaid.CatmaidInstance, got "{}"'.format(
type(autoseg_instance)))
# First collect segments constituting this neuron
overlap_matrix = neuron_to_segments(x)
# If none found, return empty neuronlist
if overlap_matrix.empty:
return pymaid.CatmaidNeuronList([])
# Filter
seg_ids = overlap_matrix.loc[
overlap_matrix[x.skeleton_id] >= min_node_overlap].index.tolist()
# Now try finding the corresponding skeletons
nl = segments_to_neuron(seg_ids,
autoseg_instance=autoseg_instance,
verbose=verbose,
raise_none_found=raise_none_found)
nl.sort_values('n_nodes')
return nl[nl.n_nodes >= min_nodes]
