def update_node_radii(source,
target,
remote_instance,
limit=2,
skip_existing=True):
"""Update node radii in target neuron from their nearest neighbor in source neuron.
Parameters
----------
source : CatmaidNeuron
Neuron which node radii to use to update target neuron.
target : CatmaidNeuron
Neuron which node radii to update.
remote_instance : CatmaidInstance
Catmaid instance in which ``target`` lives.
limit : int, optional
Max distance [um] between source and target neurons for
nearest neighbor search.
skip_existing : bool, optional
If True, will skip nodes in ``source`` that already have
a radius >0.
Returns
-------
dict
Server response.
"""
if not isinstance(source,
(pymaid.CatmaidNeuron, pymaid.CatmaidNeuronList)):
raise TypeError('Expected CatmaidNeuron/List, got "{}"'.format(
type(source)))
if not isinstance(target,
(pymaid.CatmaidNeuron, pymaid.CatmaidNeuronList)):
raise TypeError('Expected CatmaidNeuron/List, got "{}"'.format(
type(target)))
# Turn limit from microns to nanometres
limit *= 1000
# First find the closest neighbor within distance limit for each node in target
# Find nodes in A to be merged into B
tree = pymaid.neuron2KDTree(source, tree_type='c', data='treenodes')
nodes = target.nodes
if skip_existing:
# Extract nodes without a radius
nodes = nodes[nodes.radius <= 0]
# For each node in A get the nearest neighbor in B
coords = nodes[['x', 'y', 'z']].values
nn_dist, nn_ix = tree.query(coords, k=1, distance_upper_bound=limit)
# Find nodes that are close enough to collapse
tn_ids = nodes.loc[nn_dist <= limit].treenode_id.values
new_radii = source.nodes.iloc[nn_ix[nn_dist <= limit]].radius.values
return pymaid.update_radii(dict(zip(tn_ids, new_radii)),
remote_instance=remote_instance)
def collapse_nodes(*x, limit=1, base_neuron=None, priority_nodes=None):
"""Generate the union of a set of neurons.
This implementation uses edge contraction on the neurons' graph to ensure
maximum connectivity. Only works if, taken together, the neurons form a
continuous tree (i.e. you must be certain that they partially overlap).
Parameters
----------
*x : CatmaidNeuron/List
Neurons to be merged.
limit : int, optional
Max distance [microns] for nearest neighbour search.
base_neuron : skeleton_ID | CatmaidNeuron, optional
Neuron to use as template for union. If not provided,
the first neuron in the list is used as template!
priority_nodes : list-like
List of treenode IDs. If provided, these nodes will
have priority when pairwise collapsing nodes. If two
priority nodes are to be collapsed, a new edge between
them is created instead.
Returns
-------
core.CatmaidNeuron
Union of all input neurons.
collapsed_nodes : dict
Map of collapsed nodes::
NodeA -collapsed-into-> NodeB
new_edges : list
List of newly added edges::
[[NodeA, NodeB], ...]
"""
# Unpack neurons in *args
x = pymaid.utils._unpack_neurons(x)
# Make sure we're working on copies and don't change originals
x = pymaid.CatmaidNeuronList([n.copy() for n in x])
if isinstance(priority_nodes, type(None)):
priority_nodes = []
# This is just check on the off-chance that skeleton IDs are not unique
# (e.g. if neurons come from different projects) -> this is relevant because
# we identify the master ("base_neuron") via it's skeleton ID
skids = [n.skeleton_id for n in x]
if len(skids) > len(np.unique(skids)):
raise ValueError(
'Duplicate skeleton IDs found in neurons to be merged. '
'Try manually assigning unique skeleton IDs.')
if any([not isinstance(n, pymaid.CatmaidNeuron) for n in x]):
raise TypeError('Input must only be CatmaidNeurons/List')
if len(x) < 2:
raise ValueError('Need at least 2 neurons to make a union!')
# Convert distance threshold from microns to nanometres
limit *= 1000
# First make a weak union by simply combining the node tables
union_simple = pymaid.stitch_neurons(x, method='NONE', master=base_neuron)
# Check for duplicate node IDs
if any(union_simple.nodes.treenode_id.duplicated()):
raise ValueError('Duplicate node IDs found.')
# Map priority nodes -> this will speed things up later
is_priority = {n: True for n in priority_nodes}
# Go over each pair of fragments and check if they can be collapsed
comb = itertools.combinations(x, 2)
collapse_into = {}
new_edges = []
for c in comb:
tree = pymaid.neuron2KDTree(c[0], tree_type='c', data='treenodes')
# For each node in master get the nearest neighbor in minion
coords = c[1].nodes[['x', 'y', 'z']].values
nn_dist, nn_ix = tree.query(coords, k=1, distance_upper_bound=limit)
clps_left = c[0].nodes.iloc[nn_ix[nn_dist <= limit]].treenode_id.values
clps_right = c[1].nodes.iloc[nn_dist <= limit].treenode_id.values
clps_dist = nn_dist[nn_dist <= limit]
for i, (n1, n2, d) in enumerate(zip(clps_left, clps_right, clps_dist)):
if is_priority.get(n1, False):
# If both nodes are priority nodes, don't collapse
if is_priority.get(n2, False):
new_edges.append([n1, n2, d])
# continue
else:
collapse_into[n2] = n1
else:
collapse_into[n1] = n2
# Get the graph
G = union_simple.graph
# Add the new edges to graph
G.add_weighted_edges_from(new_edges)
# Using an edge list is much more efficient than an adjacency matrix
E = nx.to_pandas_edgelist(G)
# All nodes that collapse into other nodes need to have weight set to
# float("inf") to de-prioritize them when generating the minimum spanning
# tree later
clps_nodes = set(collapse_into.keys())
E.loc[(E.source.isin(clps_nodes)) | (E.target.isin(clps_nodes)),
'weight'] = float('inf')
# Now map collapsed nodes onto the nodes they collapsed into
E['target'] = E.target.map(lambda x: collapse_into.get(x, x))
E['source'] = E.source.map(lambda x: collapse_into.get(x, x))
# Make sure no self loops after collapsing. This happens if two adjacent
# nodes collapse onto the same target node
E = E[E.source != E.target]
# Turn this back into a graph
G_clps = nx.from_pandas_edgelist(E, edge_attr='weight')
# Make sure that we are fully connected
if not nx.is_connected(G_clps):
raise ValueError('Neuron still fragmented after collapsing nodes. '
'Try increasing the `limit` parameter.')
# Under certain conditions, collapsing nodes will introduce cycles:
# Consider for example a graph: A->B->C D->E->F
# Collapsing A and C into D will create a loop between B<->D
# To fix this we have to create a minimum spanning tree.
# In doing so, we need to prioritize existing edges over new edges
# otherwise we would have to cut existing neurons -> this is why we set
# weight of new edges to float("inf") earlier on
# Generate the tree
tree = nx.minimum_spanning_tree(G_clps.to_undirected(as_view=True))
# Add properties to nodes
survivors = np.unique(E[['source', 'target']])
props = union_simple.nodes.set_index('treenode_id').loc[survivors]
nx.set_node_attributes(tree, props.to_dict(orient='index'))
# Recreate neuron
union = pymaid.graph.nx2neuron(tree,
neuron_name=union_simple.neuron_name,
skeleton_id=union_simple.skeleton_id)
# Add tags back on
for n in x:
union.tags.update({
k: union.tags.get(k, []) + [collapse_into.get(a, a) for a in v]
for k, v in n.tags.items()
})
# Add connectors back on
union.connectors = x.connectors.drop_duplicates(subset='connector_id')
union.connectors.treenode_id = union.connectors.treenode_id.map(
lambda x: collapse_into.get(x, x))
# Return the last survivor
return union, collapse_into, new_edges
def collapse_nodes2(A, B, limit=2, base_neuron=None):
"""Merge neuron A into neuron(s) B creating a union of both.
This implementation uses edge contraction on the neurons' graph to ensure
maximum connectivity. Only works if the fragments collectively form a
continuous tree (i.e. you must be certain that they partially overlap).
Parameters
----------
A : CatmaidNeuron
Neuron to be collapsed into neurons B.
B : CatmaidNeuronList
Neurons to collapse neuron A into.
limit : int, optional
Max distance [microns] for nearest neighbour search.
base_neuron : skeleton_ID | CatmaidNeuron, optional
Neuron from B to use as template for union. If not
provided, the first neuron in the list is used as
template!
Returns
-------
core.CatmaidNeuron
Union of all input neurons.
new_edges : pandas.DataFrame
Subset of the ``.nodes`` table that represent newly
added edges.
collapsed_nodes : dict
Map of collapsed nodes::
NodeA -collapsed-into-> NodeB
"""
if isinstance(A, pymaid.CatmaidNeuronList):
if len(A) == 1:
A = A[0]
else:
A = pymaid.stitch_neurons(A, method="NONE")
elif not isinstance(A, pymaid.CatmaidNeuron):
raise TypeError('`A` must be a CatmaidNeuron, got "{}"'.format(
type(A)))
if isinstance(B, pymaid.CatmaidNeuron):
B = pymaid.CatmaidNeuronList(B)
elif not isinstance(B, pymaid.CatmaidNeuronList):
raise TypeError('`B` must be a CatmaidNeuronList, got "{}"'.format(
type(B)))
# This is just check on the off-chance that skeleton IDs are not unique
# (e.g. if neurons come from different projects) -> this is relevant because
# we identify the master ("base_neuron") via it's skeleton ID
skids = [n.skeleton_id for n in B + A]
if len(skids) > len(np.unique(skids)):
raise ValueError(
'Duplicate skeleton IDs found. Try manually assigning '
'unique skeleton IDs.')
# Convert distance threshold from microns to nanometres
limit *= 1000
# Before we start messing around, let's make sure we can keep track of
# the origin of each node
for n in B + A:
n.nodes['origin_skeletons'] = n.skeleton_id
# First make a weak union by simply combining the node tables
union_simple = pymaid.stitch_neurons(B + A,
method='NONE',
master=base_neuron)
# Check for duplicate node IDs
if any(union_simple.nodes.treenode_id.duplicated()):
raise ValueError('Duplicate node IDs found.')
# Find nodes in A to be merged into B
tree = pymaid.neuron2KDTree(B, tree_type='c', data='treenodes')
# For each node in A get the nearest neighbor in B
coords = A.nodes[['x', 'y', 'z']].values
nn_dist, nn_ix = tree.query(coords, k=1, distance_upper_bound=limit)
# Find nodes that are close enough to collapse
collapsed = A.nodes.loc[nn_dist <= limit].treenode_id.values
clps_into = B.nodes.iloc[nn_ix[nn_dist <= limit]].treenode_id.values
clps_map = {n1: n2 for n1, n2 in zip(collapsed, clps_into)}
# The fastest way to collapse is to work on the edge list
E = nx.to_pandas_edgelist(union_simple.graph)
# Keep track of which edges were collapsed -> we will use this as weight
# later on to prioritize existing edges over newly generated ones
E['is_new'] = 1
E.loc[(E.source.isin(B.nodes.treenode_id.values)) |
(E.target.isin(B.nodes.treenode_id.values)), 'is_new'] = 0
# Now map collapsed nodes onto the nodes they collapsed into
E['target'] = E.target.map(lambda x: clps_map.get(x, x))
E['source'] = E.source.map(lambda x: clps_map.get(x, x))
# Make sure no self loops after collapsing. This happens if two adjacent
# nodes collapse onto the same target node
E = E[E.source != E.target]
# Remove duplicates. This happens e.g. when two adjaceny nodes merge into
# two other adjaceny nodes: A->B C->D ----> A/B->C/D
# By sorting first, we make sure original edges are kept first
E.sort_values('is_new', ascending=True, inplace=True)
# Because edges may exist in both directions (A->B and A<-B) we have to
# generate a column that's agnostic to directionality using frozensets
E['edge'] = E[['source', 'target']].apply(frozenset, axis=1)
E.drop_duplicates(['edge'], keep='first', inplace=True)
# Regenerate graph from these new edges
G = nx.Graph()
G.add_weighted_edges_from(E[['source', 'target',
'is_new']].values.astype(int))
# At this point there might still be disconnected pieces -> we will create
# separate neurons for each tree
props = union_simple.nodes.loc[union_simple.nodes.treenode_id.isin(
G.nodes)].set_index('treenode_id')
nx.set_node_attributes(G, props.to_dict(orient='index'))
fragments = []
for n in nx.connected_components(G):
c = G.subgraph(n)
tree = nx.minimum_spanning_tree(c)
fragments.append(
pymaid.graph.nx2neuron(tree,
neuron_name=base_neuron.neuron_name,
skeleton_id=base_neuron.skeleton_id))
fragments = pymaid.CatmaidNeuronList(fragments)
if len(fragments) > 1:
# Now heal those fragments using a minimum spanning tree
union = pymaid.stitch_neurons(*fragments, method='ALL')
else:
union = fragments[0]
# Reroot to base neuron's root
union.reroot(base_neuron.root[0], inplace=True)
# Add tags back on
union.tags.update(union_simple.tags)
# Add connectors back on
union.connectors = union_simple.connectors.drop_duplicates(
subset='connector_id').copy()
union.connectors.loc[:, 'treenode_id'] = union.connectors.treenode_id.map(
lambda x: clps_map.get(x, x))
# Find the newly added edges (existing edges should not have been modified
# - except for changing direction due to reroot)
# The basic logic here is that new edges were only added between two
# previously separate skeletons, i.e. where the skeleton ID changes between
# parent and child node
node2skid = union_simple.nodes.set_index(
'treenode_id').skeleton_id.to_dict()
union.nodes['parent_skeleton'] = union.nodes.parent_id.map(node2skid)
new_edges = union.nodes[
union.nodes.origin_skeletons != union.nodes.parent_skeleton]
# Remove root edges
new_edges = new_edges[~new_edges.parent_id.isnull()]
return union, new_edges, clps_map
def find_fragments(x, remote_instance, min_node_overlap=3, min_nodes=1):
"""Find manual tracings overlapping with given autoseg neuron.
This function is a generalization of ``find_autoseg_fragments`` and is
designed to not require overlapping neurons to have references (e.g.
in their name) to segmentation IDs:
1. Traverse neurites of ``x`` search within 2.5 microns radius for
potentially overlapping fragments.
2. Collect segmentation IDs for the input neuron and all potentially
overlapping fragments using the brainmaps API.
3. Return fragments that overlap with at least ``min_overlap`` nodes
with input neuron.
Parameters
----------
x : pymaid.CatmaidNeuron
Neuron to collect fragments for.
remote_instance : pymaid.CatmaidInstance
Catmaid instance in which to search for fragments.
min_node_overlap : int, optional
Minimal overlap between `x` and a fragment in nodes. If
the fragment has less total nodes than `min_overlap`,
the threshold will be lowered to:
``min_overlap = min(min_overlap, fragment.n_nodes)``
min_nodes : int, optional
Minimum node count for returned neurons.
Return
------
pymaid.CatmaidNeuronList
CatmaidNeurons of the overlapping fragments. Overlap
scores are attached to each neuron as ``.overlap_score``
attribute.
Examples
--------
Setup:
>>> import pymaid
>>> import fafbseg
>>> import brainmappy as bm
>>> manual = pymaid.CatmaidInstance('MANUAL_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> auto = pymaid.CatmaidInstance('AUTO_SERVER_URL', 'HTTP_USER', 'HTTP_PW', 'API_TOKEN')
>>> flow = bm.acquire_credentials()
>>> # Note that volume ID must match with the autoseg CatmaidInstance!
>>> bm.set_global_volume('some_volume_id')
Find manually traced fragments overlapping with an autoseg neuron:
>>> x = pymaid.get_neuron(204064470, remote_instance=auto)
>>> frags_of_x = fafbseg.find_fragments(x, remote_instance=manual)
See Also
--------
fafbseg.find_autoseg_fragments
Use this function if you are looking for autoseg
fragments overlapping with a given neuron. Because we
can use the reference to segment IDs (via names &
annotations), this function is much faster than
``find_fragments``.
"""
if not isinstance(x, pymaid.CatmaidNeuron):
raise TypeError('Expected pymaid.CatmaidNeuron, got "{}"'.format(
type(x)))
# Resample the autoseg neuron to 0.5 microns
x_rs = x.resample(500, inplace=False)
# For each node get skeleton IDs in a 0.25 micron radius
r = 250
# Generate bounding boxes around each node
bboxes = [
np.vstack([co - r, co + r]).T
for co in x_rs.nodes[['x', 'y', 'z']].values
]
# Query each bounding box
urls = [
remote_instance._get_skeletons_in_bbox(minx=min(b[0]),
maxx=max(b[0]),
miny=min(b[1]),
maxy=max(b[1]),
minz=min(b[2]),
maxz=max(b[2]),
min_nodes=min_nodes)
for b in bboxes
]
resp = remote_instance.fetch(urls,
desc='Searching for overlapping neurons')
skids = set([s for l in resp for s in l])
# Return empty NeuronList if no skids found
if not skids:
return pymaid.CatmaidNeuronList([])
# Get nodes for these candidates
tn_table = pymaid.get_treenode_table(skids,
include_details=False,
convert_ts=False,
remote_instance=remote_instance)
# Keep track of total node counts
node_counts = tn_table.groupby('skeleton_id').treenode_id.count().to_dict()
# Get segment IDs for the input neuron
x.nodes['seg_id'] = segmentation.get_seg_ids(x.nodes[['x', 'y',
'z']].values)
# Count segment IDs
x_seg_counts = x.nodes.groupby('seg_id').treenode_id.count().reset_index(
drop=False)
x_seg_counts.columns = ['seg_id', 'counts']
# Remove seg IDs 0
x_seg_counts = x_seg_counts[x_seg_counts.seg_id != 0]
# Generate KDTree for nearest neighbor calculations
tree = pymaid.neuron2KDTree(x)
# Now remove nodes that aren't even close to our input neuron
dist, ix = tree.query(tn_table[['x', 'y', 'z']].values,
distance_upper_bound=2500)
tn_table = tn_table.loc[dist <= 2500]
# Remove neurons that can't possibly have enough overlap
node_counts2 = tn_table.groupby(
'skeleton_id').treenode_id.count().to_dict()
to_keep = [
k for k, v in node_counts2.items()
if v >= min(min_node_overlap, node_counts[k])
]
tn_table = tn_table[tn_table.skeleton_id.isin(to_keep)]
# Add segment IDs
tn_table['seg_id'] = segmentation.get_seg_ids(tn_table[['x', 'y',
'z']].values)
# Now group by neuron and by segment
seg_counts = tn_table.groupby(
['skeleton_id', 'seg_id']).treenode_id.count().reset_index(drop=False)
# Rename columns
seg_counts.columns = ['skeleton_id', 'seg_id', 'counts']
# Remove seg IDs 0
seg_counts = seg_counts[seg_counts.seg_id != 0]
# Remove segments IDs that are not overlapping with input neuron
seg_counts = seg_counts[np.isin(seg_counts.seg_id.values,
x_seg_counts.seg_id.values)]
# Now go over each candidate and see if there is enough overlap
ol = []
scores = []
for s in seg_counts.skeleton_id.unique():
# Subset to nodes of this neurons
this_counts = seg_counts[seg_counts.skeleton_id == s]
# If the neuron is smaller than min_overlap, lower the threshold
this_min_ol = min(min_node_overlap, node_counts[s])
# Sum up counts for both input neuron and this candidate
c_count = this_counts.counts.sum()
x_count = x_seg_counts[x_seg_counts.seg_id.isin(
this_counts.seg_id.values)].counts.sum()
# If there is enough overlap, keep this neuron
# and add score as `overlap_score`
if (c_count >= this_min_ol) and (x_count >= this_min_ol):
# The score is the minimal overlap
scores.append(min(c_count, x_count))
ol.append(s)
if ol:
ol = pymaid.get_neurons(ol, remote_instance=remote_instance)
# Make sure it's a neuronlist
if not isinstance(ol, pymaid.CatmaidNeuronList):
ol = pymaid.CatmaidNeuronList(ol)
for n, s in zip(ol, scores):
n.overlap_score = s
else:
ol = pymaid.CatmaidNeuronList([])
return ol