def find_desyncs(annots=[]):
target_skids = pymaid.get_skids_by_annotation(paper_base_annots + annots, intersect=True, remote_instance=target_project)
target_neurons = pymaid.get_neuron(target_skids, remote_instance=target_project)
for target_neuron in tqdm(target_neurons):
linked_skid = [a.split('skeleton id ')[1].split(' ')[0] for a in target_neuron.annotations if a.startswith('LINKED NEURON')]
assert len(linked_skid) is 1
linked_skid = linked_skid[0]
source_neuron = pymaid.get_neuron(linked_skid, remote_instance=source_project)
#TODO change this from counting things to looking at timestamps, which
#is actually guaranteed to find desyncs whereas counts aren't
if source_neuron.n_nodes - target_neuron.n_nodes is not 0:
print('Node number mismatch for:', source_neuron.neuron_name)
if source_neuron.n_connectors - target_neuron.n_connectors is not 0:
print('Connector number mismatch for:', source_neuron.neuron_name)
def get_radius_pruned_neurons_by_skid(skids,
radius_to_keep=PRIMARY_NEURITE_RADIUS,
keep_larger_radii=True):
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
for neuron in neurons:
if 'radius' in neuron.neuron_name:
raise Exception('Radius pruning was requested for'
f' "{neuron.neuron_name}". You probably didn\'t'
' mean to do this since it was already pruned.'
' Abort!')
if keep_larger_radii:
navis.subset_neuron(neuron,
neuron.nodes.node_id.values[
neuron.nodes.radius >= radius_to_keep],
inplace=True)
else:
navis.subset_neuron(neuron,
neuron.nodes.node_id.values[neuron.nodes.radius
== radius_to_keep],
inplace=True)
if neuron.n_skeletons > 1:
navis.plot2d(neuron)
raise Exception(f'You cut {neuron.neuron_name} into two fragments.'
" That's not supposed to happen.")
neuron.annotations.append('pruned to nodes with radius 500')
neuron.neuron_name = neuron.neuron_name + f' - radius {radius_to_keep}'
return neurons
def copy_neurons_by_skid(skids, **kwargs):
"""
See upload_or_update_neurons for all keyword argument options.
"""
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
kwargs['linking_relation'] = 'copy of'
return upload_or_update_neurons(neurons, **kwargs)
def add_dummy_nodes_by_skid(skids, fake=True, remote_instance=None):
assert remote_instance is not None, 'Must pass a remote_instance. Exiting.'
remote_instance.clear_cache()
neurons = pymaid.get_neuron(skids, remote_instance=remote_instance)
server_responses = []
#TODO can I do this in one API call instead of one call per neuron?
for neuron in neurons:
if len(neuron.nodes) is not 1:
print(
f'Dummy node requested for a neuron with >1 node. Skipping "{neuron.neuron_name}".'
)
continue
if not fake:
server_responses.append(
pymaid.add_node((-1, -1, neuron.nodes.iloc[0].z),
neuron.nodes.iloc[0].node_id,
confidence=1,
remote_instance=remote_instance))
else:
print(f'pymaid.add_node((-1, -1, {neuron.nodes.iloc[0].z}),'
f' {neuron.nodes.iloc[0].node_id}, confidence=1,'
' remote_instance=remote_instance)')
return server_responses
def get_affinetransformed_neurons_by_skid(skids, transform_file):
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
transformed_neurons = []
for neuron in neurons:
node_coords = neuron.nodes[['x', 'y', 'z']].copy()
connector_coords = neuron.connectors[['x', 'y', 'z']].copy()
#Append a column of 1s to enable affine transformation
node_coords['c'] = 1
connector_coords['c'] = 1
T = np.loadtxt(transform_file)
#Apply transformation matrix using matrix multiplication
transformed_node_coords = node_coords.dot(T)
transformed_connector_coords = connector_coords.dot(T)
#Restore column names
transformed_node_coords.columns = ['x', 'y', 'z', 'c']
transformed_connector_coords.columns = ['x', 'y', 'z', 'c']
neuron.nodes.loc[:, ['x', 'y', 'z']] = transformed_node_coords[[
'x', 'y', 'z'
]]
neuron.connectors.loc[:,
['x', 'y', 'z']] = transformed_connector_coords[[
'x', 'y', 'z'
]]
neuron.neuron_name += ' - affine transform'
transformed_neurons.append(neuron)
return pymaid.CatmaidNeuronList(transformed_neurons)
def setUp(self):
self.rm = pymaid.CatmaidInstance(config_test.server_url,
config_test.http_user,
config_test.http_pw,
config_test.token)
self.n = pymaid.get_neuron(config_test.test_skids[0],
remote_instance=self.rm)
def setUp(self):
self.rm = pymaid.CatmaidInstance(
config_test.server_url,
config_test.http_user,
config_test.http_pw,
config_test.token)
self.nl = pymaid.get_neuron('annotation:%s' % config_test.test_annotations[
0], remote_instance=self.rm)
def setUp(self):
self.rm = pymaid.CatmaidInstance(
config_test.server_url,
config_test.http_user,
config_test.http_pw,
config_test.token)
self.n = pymaid.get_neuron(config_test.test_skids[0],
remote_instance=self.rm)
self.cn_table = pymaid.get_partners(config_test.test_skids[0],
remote_instance=self.rm)
self.nB = pymaid.get_neuron(self.cn_table.iloc[0].skeleton_id,
remote_instance=self.rm)
self.adj = pymaid.adjacency_matrix(
self.cn_table[self.cn_table.relation == 'upstream'].iloc[:10].skeleton_id.values)
def add_skeleton_layer(x, scene):
"""Add skeleton as new layer to scene.
Parameters
----------
x : navis.TreeNeuron | pymaid.CatmaidNeuron | int
Neuron to generate a URL for. Integers are interpreted as
CATMAID skeleton IDs. CatmaidNeurons will automatically be
transformed to flywire coordinates. Neurons are expected to
be in nanometers and will be converted to pixels.
scene : dict
Scene to add annotation layer to.
Returns
-------
modified scene : dict
"""
if not isinstance(scene, dict):
raise TypeError(f'`scene` must be dict, got "{type(scene)}"')
scene = scene.copy()
if not isinstance(x, (navis.TreeNeuron, navis.NeuronList, pd.DataFrame)):
x = pymaid.get_neuron(x)
if isinstance(x, navis.NeuronList):
if len(x) > 1:
raise ValueError(f'Expected a single neuron, got {len(x)}')
if isinstance(x, pymaid.CatmaidNeuron):
x = xform.fafb14_to_flywire(x, coordinates='nm')
if not isinstance(x, (navis.TreeNeuron, pd.DataFrame)):
raise TypeError(f'Expected skeleton, got {type(x)}')
if isinstance(x, navis.TreeNeuron):
nodes = x.nodes
else:
nodes = x
# Generate list of segments
not_root = nodes[nodes.parent_id >= 0]
loc1 = not_root[['x', 'y', 'z']].values
loc2 = nodes.set_index('node_id').loc[not_root.parent_id.values,
['x', 'y', 'z']].values
stack = np.dstack((loc1, loc2))
stack = np.transpose(stack, (0, 2, 1))
stack = stack / [4, 4, 40]
return add_annotation_layer(stack, scene)
def get_pruned_by_hardcoded_dict(prune_params=default_prune_params, **kwargs):
neurons = []
for skid in prune_params:
neuron = pymaid.get_neuron(skid)
if 'new root' in prune_params[skid]:
neuron.reroot(prune_params[skid]['new root'], inplace=True)
for prune_point in prune_params[skid]['prune points']:
neuron.prune_distal_to(prune_point, inplace=True)
if 'final root' in prune_params[skid]:
neuron.reroot(prune_params[skid]['final root'], inplace=True)
neuron.plot3d(color=None)
neurons.append(neuron)
upload_or_update_neurons(neurons, **kwargs)
return neurons
def replace_skeleton_from_swc(skid, swc_file, remote_instance=None, fake=True):
assert isinstance(skid, int)
if remote_instance is None:
try:
remote_instance = target_project
print('Performing skeleton replacement in TARGET project.')
except:
remote_instance = source_project
print('Performing skeleton replacement in SOURCE project.')
new_neuron = pymaid.from_swc(swc_file)
old_neuron = pymaid.get_neuron(skid, remote_instance=remote_instance)
dist = lambda old, new: sum((new.nodes[['x', 'y', 'z']].mean() - old.nodes[
['x', 'y', 'z']].mean())**2)**0.5
print(f'Neuron to be replaced: {old_neuron.neuron_name}')
print('Distance between mean coordinate of old neuron and mean'
f' coordinate of new neuron: {dist(old_neuron, new_neuron):.0f}nm')
nid = pymaid.get_neuron_id(skid,
remote_instance=remote_instance)[str(skid)]
if len(old_neuron.connectors) != 0:
print('WARNING: connectors on old neuron will become unlinked'
' (i.e. they will not be linked to the new neuron).')
if len(old_neuron.tags) != 0:
print('WARNING: tags on old neuron will be deleted.')
if fake: return False
old_root_radius = old_neuron.nodes.radius[
old_neuron.nodes.parent_id.isnull()].iloc[0]
new_neuron.nodes.loc[new_neuron.nodes.parent_id.isnull(),
'radius'] = old_root_radius
#new_neuron.annotations = old_neuron.annotations
new_neuron.neuron_name = old_neuron.neuron_name
pymaid.upload_neuron(
new_neuron,
skeleton_id=skid,
neuron_id=nid,
force_id=True,
#import_tags=True,
#import_annotations=True,
#import_connectors=import_connectors,
#reuse_existing_connectors=reuse_existing_connectors,
remote_instance=remote_instance)
def setUp(self):
self.rm = pymaid.CatmaidInstance(config_test.server_url,
config_test.http_user,
config_test.http_pw,
config_test.token)
self.nl = pymaid.get_neuron(config_test.test_skids[0:2],
remote_instance=self.rm)
self.n = self.nl[0]
self.n.reroot(self.n.soma)
# Get some random leaf node
self.leaf_id = self.n.nodes[self.n.nodes.type == 'end'].sample(
1).iloc[0].treenode_id
self.slab_id = self.n.nodes[self.n.nodes.type == 'slab'].sample(
1).iloc[0].treenode_id
def delete_dummy_nodes_by_skid(skids,
dummy_coords=(-1, -1),
fake=True,
remote_instance=None):
assert remote_instance is not None, 'Must pass a remote_instance. Exiting.'
remote_instance.clear_cache()
neurons = pymaid.get_neuron(skids, remote_instance=remote_instance)
neurons_with_dummy_nodes = []
for neuron in neurons:
if len(neuron.nodes) is not 2:
print(
f'"{neuron.neuron_name}" doesn\'t have exactly 2 nodes. Skipping.'
)
continue
# TODO checking for equality between floats is bad.
# Change it to difference < 0.1 or something.
if not (neuron.nodes[['x', 'y']] == dummy_coords).all(axis=1).any():
print(
f'"{neuron.neuron_name}" has no nodes at (x, y) = {dummy_coords}. Skipping.'
)
continue
neurons_with_dummy_nodes.append(neuron)
neurons = pymaid.core.CatmaidNeuronList(neurons_with_dummy_nodes)
if len(neurons) is 0:
raise ValueError('No neurons appear to have dummy nodes.')
server_responses = []
# TODO checking for equality between floats is bad.
# Change it to difference < 0.1 or something
is_at_dummy_coords = (neurons.nodes[['x',
'y']] == dummy_coords).all(axis=1)
nodes_to_delete = neurons.nodes.node_id[is_at_dummy_coords].to_list()
if not fake:
server_responses.append(
pymaid.delete_nodes(nodes_to_delete,
'TREENODE',
remote_instance=remote_instance))
else:
print(f"pymaid.delete_nodes({nodes_to_delete},"
" 'TREENODE', remote_instance=remote_instance)")
return server_responses
def get_translated_neurons_by_skid(skids,
translation,
unit='nm',
pixel_size=(4, 4, 40)):
if len(translation) != 3:
raise ValueError('Expected translation to look like [x, y, z]'
f' and have length 3 but got {translation}')
if unit not in ('nm', 'pixel'):
raise ValueError(f"Expected unit to be 'nm' or 'pixel' but got {unit}")
if unit is 'pixel':
print(f'Translation of ({translation[0]}, {translation[1]},'
f' {translation[2]}) pixels requested. Using pixel size of'
f' {pixel_size} nm to convert to nm. Resulting translation is'
f' ({translation[0]*pixel_size[0]},'
f' {translation[1]*pixel_size[1]},'
f' {translation[2]*pixel_size[2]}) nm.')
translation = (translation[0] * pixel_size[0],
translation[1] * pixel_size[1],
translation[2] * pixel_size[2])
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
for neuron in neurons:
neuron.nodes.x += translation[0]
neuron.nodes.y += translation[1]
neuron.nodes.z += translation[2]
neuron.connectors.x += translation[0]
neuron.connectors.y += translation[1]
neuron.connectors.z += translation[2]
neuron.neuron_name += ' - translated'
return neurons
def test_treenode_tags(self):
n = pymaid.get_neuron(config_test.test_skids[0])
self.assertIsInstance(pymaid.get_node_tags(n.nodes.node_id.values[0:50],
node_type='TREENODE'),
dict)
def test_treenode_info(self):
n = pymaid.get_neuron(config_test.test_skids[0])
self.assertIsInstance(pymaid.get_treenode_info(n.nodes.node_id.values[0:50]),
pd.DataFrame)
def test_skid_from_treenode(self):
n = pymaid.get_neuron(config_test.test_skids[0])
self.assertIsInstance(pymaid.get_skid_from_treenode(n.nodes.iloc[0].node_id),
dict)
def test_node_details(self):
n = pymaid.get_neuron(config_test.test_skids[0])
self.assertIsInstance(pymaid.get_node_details(n.nodes.sample(100).node_id.values),
pd.DataFrame)
def test_get_neuron(self):
self.assertIsInstance(pymaid.get_neuron(config_test.test_skids,
remote_instance=self.rm),
pymaid.CatmaidNeuronList)
def skid_to_id(x,
dataset='production',
progress=True, **kwargs):
"""Find the flywire ID(s) corresponding to given CATMAID skeleton ID(s).
This function works by:
1. Fetch supervoxels for all nodes in the CATMAID skeletons
2. Pick a random sample of ``sample`` of these supervoxels
3. Fetch the most recent root IDs for the sample supervoxels
4. Return the root ID that collectively cover 90% of the supervoxels
Parameters
----------
x : int | list-like | str | TreeNeuron/List
Anything that's not a TreeNeuron/List will be passed
directly to ``pymaid.get_neuron``.
dataset : str | CloudVolume
Against which flywire dataset to query::
- "production" (current production dataset, fly_v31)
- "sandbox" (i.e. fly_v26)
progress : bool
If True, shows progress bar.
Returns
-------
pandas.DataFrame
Mapping of flywire IDs to skeleton IDs with confidence::
flywire_id skeleton_id confidence
0
1
"""
vol = parse_volume(dataset, **kwargs)
if not isinstance(x, (navis.TreeNeuron, navis.NeuronList)):
x = pymaid.get_neuron(x)
if isinstance(x, navis.TreeNeuron):
nodes = x.nodes[['x', 'y', 'z']].copy()
nodes['skeleton_id'] = x.id
elif isinstance(x, navis.NeuronList):
nodes = x.nodes[['x', 'y', 'z']].copy()
else:
raise TypeError(f'Unable to data of type "{type(x)}"')
# XForm coordinates from FAFB14 to FAFB14.1
xformed = xform.fafb14_to_flywire(nodes[['x', 'y', 'z']].values,
coordinates='nm')
# Get the root IDs for each of these locations
roots = locs_to_segments(xformed, coordinates='nm')
# Drop zeros
roots = roots[roots != 0]
# Find unique Ids and count them
unique, counts = np.unique(roots, return_counts=True)
# Get sorted indices
sort_ix = np.argsort(counts)
# New Id is the most frequent ID
new_id = unique[sort_ix[-1]]
# Confidence is the difference between the top and the 2nd most frequent ID
if len(unique) > 1:
diff_1st_2nd = counts[sort_ix[-1]] - counts[sort_ix[-2]]
conf = round(diff_1st_2nd / roots.shape[0], 2)
else:
conf = 1
return pd.DataFrame([[x.id, new_id, conf, x.id != new_id]],
columns=['old_id', 'new_id', 'confidence', 'changed'])
import pymaid
import mushroom_2to3.neurogenesis as neurogenesis
# %run startup_py3.py
# %run load_pn_metadata_v1.py
# pn_skids = cc.get_skids_from_annos(fafb_c, [['right_calyx_PN'], ['has_bouton']], ["multiglomerular PN"])
pns_ms = neurogenesis.init_from_skid_list(fafb_c, pn_skids)
import pickle
path = local_path + "data/pn_bouton_clusters/"
with open(path + "pns_ms.pkl", 'wb') as f:
pickle.dump(pns_ms, f, -1)
nl = [pymaid.get_neuron([str(i) for i in j]) for j in [pn_skids[:40], pn_skids[40:80], pn_skids[80:]]]
pns_pm = nl[0] + nl[1] + nl [2]
with open(path + "pns_pm.pkl", 'wb') as f:
pickle.dump(pns_pm, f, -1)
ca = pymaid.get_volume('MB_CA_R')
with open(path + "ca.pkl", 'wb') as f:
pickle.dump(ca, f, -1)
df = pd.read_excel(local_path + 'data/180613-pn_subtypes.xlsx')
# for loading the pickle pymaid neuronlist data
path = local_path + "data/pn_bouton_clusters/"
def make_rainbow_json_by_position(annotations,
filename,
extract_position=None,
convert_values_to_rank=False,
opacity=1,
**kwargs):
"""
extract_position can be either a lambda function that defines how to
extract a position value from a CatmaidNeuron object, or can be one of the
following strings:
'root_x', 'root_y' (default), 'root_z', 'mean_x', 'mean_y', 'mean_z'
colormap must be a Nx3 array specifying triplets of RGB values. The
smallest extracted position will get mapped to the first element of the
colormap, the largest extracted position will get mapped to the last
element of the colormap, and intermediate positions will take on
intermediate values.
"""
colormap = kwargs.get('colormap', turbo_colormap_data)
if extract_position == 'root_x':
extract_position = lambda n: n.nodes.x[n.nodes.node_id == n.root[0]
].values[0]
elif extract_position in (None, 'root_y'):
extract_position = lambda n: n.nodes.y[n.nodes.node_id == n.root[0]
].values[0]
elif extract_position == 'root_z':
extract_position = lambda n: n.nodes.z[n.nodes.node_id == n.root[0]
].values[0]
elif extract_position == 'mean_x':
extract_position = lambda n: n.nodes.x.mean()
elif extract_position == 'mean_y':
extract_position = lambda n: n.nodes.y.mean()
elif extract_position == 'mean_z':
extract_position = lambda n: n.nodes.z.mean()
elif extract_position in ['rand', 'random']:
import random
extract_position = lambda: random.random()
if 'neurons' in kwargs:
neurons = kwargs['neurons']
skids = neurons.skeleton_id
else:
try:
skids = pymaid.get_skids_by_annotation(
annotations, intersect=True, remote_instance=source_project)
except:
skids = annotations # Allows users to pass skids directly
if extract_position.__code__.co_argcount > 0: # Neuron data needed
# Can I avoid pulling all this neuron data if I only need the root
# position? Is there a way to pull less data even if I need the nodes?
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if extract_position.__code__.co_argcount > 0:
extracted_vals = pd.Series(
{n.skeleton_id: extract_position(n)
for n in neurons})
else:
extracted_vals = pd.Series(
{skid: extract_position()
for skid in skids})
extracted_vals.sort_values(ascending=False, inplace=True)
if convert_values_to_rank:
extracted_vals.iloc[:] = np.arange(len(extracted_vals), 0, -1)
#print(extracted_vals)
max_pos = extracted_vals.iloc[0]
min_pos = extracted_vals.iloc[-1]
scaled_extracted_vals = (extracted_vals - min_pos) / (max_pos - min_pos)
colors = [
RGB_to_catmaidhex(colormap[int(p * (len(colormap) - 1))])
for p in scaled_extracted_vals
]
#print(colors)
skids_to_colors = dict(zip(extracted_vals.index, colors))
filename = expand_filename(filename,
datestamp=kwargs.get('datestamp', False))
write_catmaid_json(skids_to_colors, filename, default_opacity=opacity)
def get_elastictransformed_neurons_by_skid(skids,
transform=None,
include_connectors=True,
y_coordinate_cutoff=None,
**kwargs):
"""
Apply an arbitrary transformation to a neuron.
skids: A skeleton ID or list of skeleton IDs to transform.
transform: a function that takes in an Nx3 numpy array representing a list
of treenodes' coordinates and returns an Nx3 numpy array representing
the transformed coordinates. Defaults to using warp_points_FANC_to_template
from coordinate_transforms.warp_points_between_FANC_and_template.
include_connectors (bool): If True, connectors will be transformed,
otherwise will not be transformed (which saves execution time).
y_coordinate_cutoff (numerical): If not None, filter out treenodes and
connectors with y coordinate < y_coordinate_cutoff before transforming.
kwargs:
left_right_flip (bool): Flips the neuron across the VNC template's
midplane. Only relevant for the default transform function.
"""
left_right_flip = kwargs.get('left_right_flip', False)
if transform is None:
try:
from .coordinate_transforms.warp_points_between_FANC_and_template \
import warp_points_FANC_to_template as warp
except:
from coordinate_transforms.warp_points_between_FANC_and_template \
import warp_points_FANC_to_template as warp
transform = lambda x: warp(x, reflect=left_right_flip)
y_coordinate_cutoff = 322500 # 300000 * 4.3/4. In nm
print('Pulling source neuron data from catmaid')
source_project.clear_cache()
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
if y_coordinate_cutoff is not None:
for neuron in neurons:
kept_rows = neuron.nodes.y >= y_coordinate_cutoff
if neuron.n_nodes == kept_rows.sum(): # No nodes to cut off
continue
print(f'Applying y coordinate cutoff of {y_coordinate_cutoff}'
f' to {neuron.neuron_name}')
kept_node_ids = neuron.nodes[kept_rows].node_id.values
# TODO double check whether this removes synapses as well. I think it does
navis.subset_neuron(neuron, kept_node_ids, inplace=True)
if neuron.n_skeletons > 1:
print(
f'{neuron.neuron_name} is fragmented. Healing before continuing.'
)
navis.heal_fragmented_neuron(neuron, inplace=True)
for neuron in neurons:
print(f'Transforming {neuron.neuron_name}')
neuron.neuron_name += ' - elastic transform'
if left_right_flip:
neuron.neuron_name += ' - flipped'
neuron.annotations.append('left-right flipped')
neuron.nodes[['x', 'y', 'z']] = transform(neuron.nodes[['x', 'y',
'z']])
if include_connectors and neuron.n_connectors > 0:
neuron.connectors[['x', 'y', 'z']] = transform(
neuron.connectors[['x', 'y', 'z']])
return neurons
def get_connection_list(all_pids,noi,confidence = 5):
""" Input: list of all project ideas to get list for
list of neurons of interest
Output: list of all connections
"""
fullConnsList = pd.DataFrame(columns = ['connector_id',
'length',
'dist_from_root',
'neuron',
'project',
'type',])
for project in all_pids:
# open an instance of CATMAID containing data https://zhencatmaid.com
catmaid = pymaid.CatmaidInstance(server = 'https://zhencatmaid.com/',
api_token='c48243e19b85edf37345ced8049ce5d6c5802412',
project_id = project)
for neurName in noi:
print('Working on ' + neurName + ' in project ' + str(project))
try:
catNeur = pymaid.get_neuron(neurName)
except:
print(neurName + " not found in project " + str(project))
continue
if isinstance(catNeur, pymaid.CatmaidNeuron):
catNeur = [catNeur]
for neur in catNeur:
if neur.n_nodes < 10:
continue
skid = neur.id
if pymaid.find_nodes(tags=['nerve_ring_starts'],skeleton_ids=skid).empty:
continue
catNeurnumpy = neur.nodes[["node_id","parent_id","x","y","z"]].to_numpy()
skTree = bf.build_tree(neur)
nr_subtree = bf.crop_tree_nr(skTree,skid)
# define and filter connections
connectors = neur.connectors
filt_conns = connectors[connectors.type.isin([0,1])].reset_index(drop = True)
for i in range(len(nr_subtree)):
if len(nr_subtree) > 1:
strneurName = bf.strip_neurName(list(pymaid.get_names(skid).values())[0]) + "(" + str(i) + ")"
else:
strneurName = bf.strip_neurName(list(pymaid.get_names(skid).values())[0])
bl_output = bf.get_branchList(nr_subtree[i],neur)
trunk = bl_output[2]
connsList = pd.DataFrame(columns = ['connector_id',
'length',
'dist_from_root',
'neuron',
'project',
'type',
'inputs',
'outputs'])
filt_conns.connector_id = filt_conns.connector_id.astype(str)
filt_conns2 = pd.DataFrame(columns = filt_conns.columns)
inputsClean = []
outputsClean = []
for connector in filt_conns.iterrows():
connector = connector[1]
if connector.node_id in nr_subtree[i]:
if connector.type == 1:
if check_confidence(catmaid,project,connector.connector_id,skid,'input'):
inputsClean.append(clean_inputs(catmaid,project,connector))
outputsClean.append(clean_outputs(catmaid,project,connector))
filt_conns2 = filt_conns2.append(connector)
if connector.type == 0:
if check_confidence(catmaid,project,connector.connector_id,skid,'output'):
inputsClean.append(clean_inputs(catmaid,project,connector))
outputsClean.append(clean_outputs(catmaid,project,connector))
filt_conns2 = filt_conns2.append(connector)
filt_conns2 = filt_conns2.reset_index(drop = True)
lengthTemp = []
distTemp = []
for node in filt_conns2.node_id:
lengthTemp.append(cf.get_norm_length(node,catNeurnumpy,trunk))
distTemp.append(cf.get_norm_dist(node,catNeurnumpy,trunk))
connsList.connector_id = filt_conns2.connector_id
connsList.project = project
connsList.neuron = strneurName
connsList.length = lengthTemp
connsList.dist_from_root = distTemp
connsList.type = filt_conns2.type
connsList.inputs = inputsClean
connsList.outputs = outputsClean
fullConnsList = fullConnsList.append(connsList).reset_index(drop = True)
return fullConnsList
def get_volume_pruned_neurons_by_skid(skids,
volume_id,
mode='fele',
resample=0,
only_keep_largest_fragment=False,
verbose=False,
remote_instance=None):
"""
mode : 'fele' - Keep all parts of the neuron between its primary
neurite's First Entry to and Last Exit from the volume.
So if a segment of the primary neurite leaves and then
re-enters the volume, that segment is not removed.
'strict' - All nodes outside the volume are pruned.
resample : If set to a positive value, the neuron will be resampled before
pruning to have treenodes placed every `resample` nanometers. If
left at 0, resampling is not performed.
In both cases, if a branch point is encountered before the first entry or
last exit, that branch point is used as the prune point.
"""
if remote_instance is None:
remote_instance = source_project
#if exit_volume_id is None:
# exit_volume_id = entry_volume_id
neurons = pymaid.get_neuron(skids, remote_instance=source_project)
if volume_id not in volumes:
try:
print(f'Pulling volume {volume_id} from project'
f' {remote_instance.project_id}.')
volumes[volume_id] = pymaid.get_volume(
volume_id, remote_instance=remote_instance)
except:
print(f"Couldn't find volume {volume_id} in project_id"
f" {remote_instance.project_id}! Exiting.")
raise
else:
print(f'Loading volume {volume_id} from cache.')
volume = volumes[volume_id]
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
if resample > 0:
#TODO find the last node on the primary neurite and store its position
neurons.resample(
resample) # This throws out radius info except for root
#TODO find the new node closest to the stored node and set all nodes
#between that node and root to have radius 500
for neuron in neurons:
if 'pruned by vol' in neuron.neuron_name:
raise Exception(
'Volume pruning was requested for '
f' "{neuron.neuron_name}". You probably didn\'t mean to do'
' this since it was already pruned. Exiting.')
continue
print(f'Pruning neuron {neuron.neuron_name}')
if mode == 'fele':
"""
First, find the most distal primary neurite node. Then, walk
backward until either finding a node within the volume or a branch
point. Prune distal to one distal to that point (so it only gets
the primary neurite and not the offshoot).
Then, start from the primary neurite node that's a child of the
soma node, and walk forward (how?) until finding a node within the
volume or a branch point. Prune proximal to that.
"""
nodes = neuron.nodes.set_index('node_id')
# Find end of the primary neurite
nodes['has_fat_child'] = False
for tid in nodes.index:
if nodes.at[tid, 'radius'] == PRIMARY_NEURITE_RADIUS:
parent = nodes.at[tid, 'parent_id']
nodes.at[parent, 'has_fat_child'] = True
is_prim_neurite_end = (~nodes['has_fat_child']) & (
nodes['radius'] == PRIMARY_NEURITE_RADIUS)
prim_neurite_end = nodes.index[is_prim_neurite_end]
if len(prim_neurite_end) is 0:
raise ValueError(f"{neuron.neuron_name} doesn't look like a"
" motor neuron. Exiting.")
elif len(prim_neurite_end) is not 1:
raise ValueError('Multiple primary neurite ends for'
f' {neuron.neuron_name}: {prim_neurite_end}.'
'\nExiting.')
nodes['is_in_vol'] = navis.in_volume(nodes, volume)
# Walk backwards until at a point inside the volume, or at a branch
# point
current_node = prim_neurite_end[0]
parent_node = nodes.at[current_node, 'parent_id']
while (nodes.at[parent_node, 'type'] != 'branch'
and not nodes.at[parent_node, 'is_in_vol']):
current_node = parent_node
if verbose: print(f'Walk back to {current_node}')
parent_node = nodes.at[parent_node, 'parent_id']
if verbose: print(f'Pruning distal to {current_node}')
neuron.prune_distal_to(current_node, inplace=True)
# Start at the first primary neurite node downstream of root
current_node = nodes.index[
(nodes.parent_id == neuron.root[0])
#& (nodes.radius == PRIMARY_NEURITE_RADIUS)][0]
& (nodes.radius > 0)][0]
#Walking downstream is a bit slow, but probably acceptable
while (not nodes.at[current_node, 'is_in_vol']
and nodes.at[current_node, 'type'] == 'slab'):
current_node = nodes.index[nodes.parent_id == current_node][0]
if verbose: print(f'Walk forward to {current_node}')
if not nodes.at[current_node, 'is_in_vol']:
input('WARNING: Hit a branch before hitting the volume for'
f' neuron {neuron.neuron_name}. This is unusual.'
' Press enter to acknowledge.')
if verbose: print(f'Pruning proximal to {current_node}')
neuron.prune_proximal_to(current_node, inplace=True)
elif mode == 'strict':
neuron.prune_by_volume(volume) #This does in-place pruning
if neuron.n_skeletons > 1:
if only_keep_largest_fragment:
print('Neuron has multiple disconnected fragments after'
' pruning. Will only keep the largest fragment.')
frags = morpho.break_fragments(neuron)
neuron.nodes = frags[frags.n_nodes == max(
frags.n_nodes)][0].nodes
#print(neuron)
#else, the neuron will get healed and print a message about being
#healed during upload_neuron, so nothing needs to be done here.
if mode == 'fele':
neuron.annotations.append(
f'pruned (first entry, last exit) by vol {volume_id}')
elif mode == 'strict':
neuron.annotations.append(f'pruned (strict) by vol {volume_id}')
neuron.neuron_name = neuron.neuron_name + f' - pruned by vol {volume_id}'
if verbose: print('\n')
return neurons
#A requirement of this code is access to CATMAID. #This is for API access to the CATMAID servers and to download skeleton information server = http_user = http_pw = token = pymaid.CatmaidInstance( server, http_user, http_pw, token) #This is an integer number that is unique to your neuron of interest Neuron_1_skeleton_id_number = Neuron_1 = pymaid.get_neuron(Neuron_skeleton_id_number) Neuron_2_skeleton_id_number = Neuron_2 = pymaid.get_neuron(Neuron_2_skeleton_id_number) #This function downsamples the neuron - it removes large stretches of skeleton that do not have any branch points. #When the argument preserve_cn_treenodes = True, this preserves the treenodes where connectors (pre/postsynapses) #have been placed. Downsampling is used to reduce the computational time, as some 3D reconstructed neurons can become #very large. Neuron.downsample(1000000, preserve_cn_treenodes = True) Neuron_2.downsample(1000000, preserve_cn_treenodes = True) #Get the connectors between the two neurons of interest #When True, the directional argument will return the connectors (pre and post synapses) #from neuron A to neuron B (A-->B; in this case Neuron_1 to Neuron_2). When False, it will return all
def adj_split_axons_dendrites(all_neurons, split_tag, special_split_tags,
not_split_skids):
# user must login to CATMAID instance before starting
t0 = time.time()
# find today's date and make an output folder with that name
today = date.today()
today = date.strftime(today, '%Y-%m-%d')
output_path = Path(f"data/processed/{today}")
if not os.path.isdir(output_path):
os.mkdir(output_path)
print("Pulling neurons...\n")
ids = all_neurons
batch_size = 20
max_tries = 10
n_batches = int(np.floor(len(ids) / batch_size))
if len(ids) % n_batches > 0:
n_batches += 1
print(f"Batch size: {batch_size}")
print(f"Number of batches: {n_batches}")
print(f"Number of neurons: {len(ids)}")
print(f"Batch product: {n_batches * batch_size}\n")
i = 0
currtime = time.time()
nl = pymaid.get_neuron(ids[i * batch_size:(i + 1) * batch_size],
with_connectors=False)
print(f"{time.time() - currtime:.3f} seconds elapsed for batch {i}.")
for i in range(1, n_batches):
currtime = time.time()
n_tries = 0
success = False
while not success and n_tries < max_tries:
try:
nl += pymaid.get_neuron(ids[i * batch_size:(i + 1) *
batch_size],
with_connectors=False)
success = True
except ChunkedEncodingError:
print(f"Failed pull on batch {i}, trying again...")
n_tries += 1
print(f"{time.time() - currtime:.3f} seconds elapsed for batch {i}.")
print("\nPulled all neurons.\b")
print("\nPickling neurons...")
currtime = time.time()
with open(output_path / "neurons.pickle", "wb") as f:
dump(nl, f)
print(f"{time.time() - currtime:.3f} seconds elapsed to pickle.")
print("Pulling split points and special split neuron ids...")
currtime = time.time()
##########
# double-check for issues with split tags
# find neurons and nodes with split tag
splits = pymaid.find_nodes(tags=split_tag)
splits = splits.set_index("skeleton_id")["node_id"].squeeze()
# find neurons and nodes with special split start tag
special_splits_start = pymaid.find_nodes(tags=special_split_tags[0])
special_splits_start = special_splits_start.set_index(
"skeleton_id")["node_id"].squeeze()
# find neurons and nodes with special split end tag
special_splits_end = pymaid.find_nodes(tags=special_split_tags[1])
special_splits_end = special_splits_end.set_index(
"skeleton_id")["node_id"].squeeze()
# every skeleton with special split start should also have a special split end
special_start_ids = np.unique(special_splits_start.index)
special_end_ids = np.unique(special_splits_end.index)
intersect = np.intersect1d(special_start_ids, special_end_ids)
union = np.union1d(special_start_ids, special_end_ids)
if (len(intersect) != len(union)):
print('Not all neurons with complex splits have the proper tags!')
sys.exit(
f'Check tags in the following skids: {list(np.setdiff1d(union, intersect))}'
)
# split tag skeletons and special split tag skeletons should be mutually exclusive
split_ids = list(splits.index)
special_ids = list(union)
if (len(np.intersect1d(split_ids, special_ids)) != 0):
sys.exit(
f'Splitting error! Check {problem_skids} which have a combination of {split_tag}, {special_split_tags[0]}, and {special_split_tags[1]} tags'
)
print(f"{time.time() - currtime:.3f} elapsed.\n")
# all splittable neurons should have split tags
should_not_split = not_split_skids # unsplittable neurons defined by user
should_split = list(np.setdiff1d(all_neurons, should_not_split))
not_split = list(np.setdiff1d(should_split, split_ids + special_ids))
if len(not_split) > 0:
print(
f"WARNING: {len(not_split)} neurons should have had split tag(s) and didn't:"
)
print(not_split)
sys.exit('Check the above skeleton IDs')
########
# processing data
print("Getting treenode compartment types...")
currtime = time.time()
treenode_types = get_treenode_types(nl, splits, special_ids, output_path)
print(f"{time.time() - currtime:.3f} elapsed.\n")
print("Pulling connectors...\n")
currtime = time.time()
connectors = get_connectors(nl)
print(f"{time.time() - currtime:.3f} elapsed.\n")
explode_cols = ["postsynaptic_to", "postsynaptic_to_node"]
index_cols = np.setdiff1d(connectors.columns, explode_cols)
print("Exploding connector DataFrame...")
# explode the lists within the connectors dataframe
connectors = (connectors.set_index(list(index_cols)).apply(
pd.Series.explode).reset_index())
# TODO figure out these nans
bad_connectors = connectors[connectors.isnull().any(axis=1)]
bad_connectors.to_csv(output_path / "bad_connectors.csv")
# connectors = connectors[~connectors.isnull().any(axis=1)]
print(f"Connectors with errors: {len(bad_connectors)}")
connectors = connectors.astype({
"presynaptic_to": "Int64",
"presynaptic_to_node": "Int64",
"postsynaptic_to": "Int64",
"postsynaptic_to_node": "Int64",
})
print("Applying treenode types to connectors...")
currtime = time.time()
connectors["presynaptic_type"] = connectors["presynaptic_to_node"].map(
treenode_types)
connectors["postsynaptic_type"] = connectors["postsynaptic_to_node"].map(
treenode_types)
connectors["in_subgraph"] = connectors["presynaptic_to"].isin(
ids) & connectors["postsynaptic_to"].isin(ids)
print(f"{time.time() - currtime:.3f} elapsed.\n")
meta = pd.DataFrame(index=all_neurons)
print("Calculating neuron total inputs and outputs...")
axon_output_map = (connectors[connectors["presynaptic_type"] ==
"axon"].groupby("presynaptic_to").size())
axon_input_map = (connectors[connectors["postsynaptic_type"] ==
"axon"].groupby("postsynaptic_to").size())
dendrite_output_map = (connectors[connectors["presynaptic_type"].isin(
["dendrite", "unsplit"])].groupby("presynaptic_to").size())
dendrite_input_map = (connectors[connectors["postsynaptic_type"].isin(
["dendrite", "unsplit"])].groupby("postsynaptic_to").size())
meta["axon_output"] = meta.index.map(axon_output_map).fillna(0.0)
meta["axon_input"] = meta.index.map(axon_input_map).fillna(0.0)
meta["dendrite_output"] = meta.index.map(dendrite_output_map).fillna(0.0)
meta["dendrite_input"] = meta.index.map(dendrite_input_map).fillna(0.0)
print()
# remap the true compartment type mappings to the 4 that we usually use
connectors["compartment_type"] = flatten(
connectors["presynaptic_type"]) + flatten(
connectors["postsynaptic_type"])
subgraph_connectors = connectors[connectors["in_subgraph"]]
meta_data_dict = meta.to_dict(orient="index")
full_g = connectors_to_nx_multi(subgraph_connectors, meta_data_dict)
graph_types = ["aa", "ad", "da", "dd"]
color_multigraphs = {}
color_flat_graphs = {}
for graph_type in graph_types:
color_subgraph_connectors = subgraph_connectors[
subgraph_connectors["compartment_type"] == graph_type]
color_g = connectors_to_nx_multi(color_subgraph_connectors,
meta_data_dict)
color_multigraphs[graph_type] = color_g
flat_color_g = flatten_muligraph(color_g, meta_data_dict)
color_flat_graphs[graph_type] = flat_color_g
flat_g = flatten_muligraph(full_g, meta_data_dict)
########
# saving data
print("Saving metadata as csv...")
meta.to_csv(output_path / "meta_data.csv")
meta.to_csv(output_path / "meta_data_unmodified.csv")
print("Saving connectors as csv...")
connectors.to_csv(output_path / "connectors.csv")
print("Saving each flattened color graph as graphml...")
for graph_type in graph_types:
nx.write_graphml(color_flat_graphs[graph_type],
output_path / f"G{graph_type}.graphml")
nx.write_graphml(flat_g, output_path / "G.graphml")
print("Saving each flattened color graph as txt edgelist...")
for graph_type in graph_types:
nx.write_weighted_edgelist(color_flat_graphs[graph_type],
output_path / f"G{graph_type}_edgelist.txt")
nx.write_weighted_edgelist(flat_g, output_path / "G_edgelist.txt")
##########
# saving data as pandas DataFrame adjacencies
# import graphs of axon-dendrite split data; generated by Ben Pedigo's scripts
G = nx.readwrite.graphml.read_graphml(f'data/processed/{today}/G.graphml',
node_type=int)
Gad = nx.readwrite.graphml.read_graphml(
f'data/processed/{today}/Gad.graphml', node_type=int)
Gaa = nx.readwrite.graphml.read_graphml(
f'data/processed/{today}/Gaa.graphml', node_type=int)
Gdd = nx.readwrite.graphml.read_graphml(
f'data/processed/{today}/Gdd.graphml', node_type=int)
Gda = nx.readwrite.graphml.read_graphml(
f'data/processed/{today}/Gda.graphml', node_type=int)
print("Generating split adjacency matrices as csv...")
# generate adjacency matrices
adj_all = pd.DataFrame(nx.adjacency_matrix(G=G, weight='weight').todense(),
columns=G.nodes,
index=G.nodes)
adj_ad = pd.DataFrame(nx.adjacency_matrix(G=Gad,
weight='weight').todense(),
columns=Gad.nodes,
index=Gad.nodes)
adj_aa = pd.DataFrame(nx.adjacency_matrix(G=Gaa,
weight='weight').todense(),
columns=Gaa.nodes,
index=Gaa.nodes)
adj_dd = pd.DataFrame(nx.adjacency_matrix(G=Gdd,
weight='weight').todense(),
columns=Gdd.nodes,
index=Gdd.nodes)
adj_da = pd.DataFrame(nx.adjacency_matrix(G=Gda,
weight='weight').todense(),
columns=Gda.nodes,
index=Gda.nodes)
# add back in skids with no edges (with rows/columns of 0); simplifies some later analysis
def refill_adjs(adj, adj_all):
skids_diff = np.setdiff1d(adj_all.index, adj.index)
adj = adj.append(
pd.DataFrame([[0] * adj.shape[1]] * len(skids_diff),
index=skids_diff,
columns=adj.columns)) # add in rows with 0s
for skid in skids_diff:
adj[skid] = [0] * len(adj.index)
return (adj)
adj_ad = refill_adjs(adj_ad, adj_all)
adj_aa = refill_adjs(adj_aa, adj_all)
adj_dd = refill_adjs(adj_dd, adj_all)
adj_da = refill_adjs(adj_da, adj_all)
#convert column names to int for easier indexing
adj_all.columns = adj_all.columns.astype(int)
adj_ad.columns = adj_ad.columns.astype(int)
adj_aa.columns = adj_aa.columns.astype(int)
adj_da.columns = adj_da.columns.astype(int)
adj_dd.columns = adj_dd.columns.astype(int)
# export adjacency matrices
adj_all.to_csv(f'data/adj/all-all_{today}.csv')
adj_ad.to_csv(f'data/adj/ad_{today}.csv')
adj_aa.to_csv(f'data/adj/aa_{today}.csv')
adj_dd.to_csv(f'data/adj/dd_{today}.csv')
adj_da.to_csv(f'data/adj/da_{today}.csv')
# import input data and export as simplified csv
meta_data = pd.read_csv(f'data/processed/{today}/meta_data.csv',
index_col=0)
inputs = meta_data.loc[:, ['axon_input', 'dendrite_input']]
outputs = meta_data.loc[:, ['axon_output', 'dendrite_output']]
# exporting input data
inputs.to_csv(f'data/adj/inputs_{today}.csv')
outputs.to_csv(f'data/adj/outputs_{today}.csv')
print()
print()
print("Done!")
elapsed = time.time() - t0
delta = timedelta(seconds=elapsed)
print("----")
print(f"{delta} elapsed for whole script.")
print("----")
sys.stdout.close()
def get_branches(all_pids,
noi,
conn_data_path=None,
ignore_tags=False,
branch_threshold=0.05):
""" Input: list of all project ids
list of neurons of interest
path to conn_data_per_neuron folder
ignore 'not a branch' tags
% of main branch for threshold to consider branch
Output:
"""
project_data = {}
on_branch_per_project = {}
if conn_data_path != None:
for project in all_pids:
project_data[project] = pd.read_csv(conn_data_path + str(project) +
'/' + str(project) + '.csv')
project_data[project]['neuron'] = project_data[project][
'neuron'].str.split('(').str[0]
on_branch_per_project[project] = []
fixed_outputs = []
for sublist in project_data[project].outputs:
fixed_outputs.append(literal_eval(sublist))
project_data[project]['outputs'] = fixed_outputs
fullBranchList = pd.DataFrame(columns=[
'leafnode', 'length', 'dist_from_root', 'neurName', 'project',
'n_conns'
])
for project in all_pids:
# open an instance of CATMAID containing data https://zhencatmaid.com
catmaid = pymaid.CatmaidInstance(
server='https://zhencatmaid.com/',
api_token='c48243e19b85edf37345ced8049ce5d6c5802412',
project_id=project)
if conn_data_path != None:
curr_project = project_data[project]
for neurName in noi:
print('Working on ' + neurName + ' in project ' + str(project))
try:
catNeur = pymaid.get_neuron(neurName)
except:
print(neurName + " not found in project " + str(project))
continue
if isinstance(catNeur, pymaid.CatmaidNeuron):
catNeur = [catNeur]
for neur in catNeur:
if neur.n_nodes < 10:
continue
skid = neur.id
if pymaid.find_nodes(tags=['nerve_ring_starts'],
skeleton_ids=skid).empty:
continue
catNeurnumpy = neur.nodes[[
"node_id", "parent_id", "x", "y", "z"
]].to_numpy()
skTree = bf.build_tree(neur)
nr_subtree = bf.crop_tree_nr(skTree, skid)
for i in range(0, len(nr_subtree)):
strneurName = bf.strip_neurName(
list(pymaid.get_names(skid).values())[0])
if conn_data_path != None:
connsList = curr_project.loc[
curr_project['neuron'].isin([strneurName])]
else:
connsList = None
bl_output = bf.get_branchList(nr_subtree[i], neur,
branch_threshold)
branchList = bl_output[0]
pathList = bl_output[1]
trunk = bl_output[2]
trunklen = bf.cable_length(trunk[-1], catNeurnumpy,
trunk[0])
lengthTemp = []
connTemp = []
for path in pathList:
lengthTemp.append(
bf.cable_length(path[0], catNeurnumpy, trunk[0]))
connTemp.append(
sum_Conns_on_Branch(path, neur, connsList))
branchList['length'] = branchList['length'] / trunklen
branchList['dist_from_root'] = [
i / trunklen for i in lengthTemp
]
branchList['neurName'] = strneurName
branchList['project'] = project
branchList['n_conns'] = connTemp
if not ignore_tags:
try:
branchList = branchList[
~branchList['leafnode'].astype(float).
astype(int).isin(neur.tags['not a branch'])]
except:
pass
fullBranchList = fullBranchList.append(branchList)
return fullBranchList
def plot_neurons(meta, key=None, label=None, barplot=False):
if label is not None:
ids = list(meta[meta[key] == label].index.values)
else:
ids = list(meta.index.values)
ids = [int(i) for i in ids]
new_ids = []
for i in ids:
try:
pymaid.get_neuron(
i, raise_missing=True, with_connectors=False, with_tags=False
)
new_ids.append(i)
except:
print(f"Missing neuron {i}, not plotting it.")
ids = new_ids
meta = meta.loc[ids]
fig = plt.figure(figsize=(30, 10))
gs = plt.GridSpec(2, 3, figure=fig, wspace=0, hspace=0, height_ratios=[0.8, 0.2])
skeleton_color_dict = dict(
zip(meta.index, np.vectorize(CLASS_COLOR_DICT.get)(meta["merge_class"]))
)
ax = fig.add_subplot(gs[0, 0], projection="3d")
pymaid.plot2d(
ids,
color=skeleton_color_dict,
ax=ax,
connectors=False,
method="3d",
autoscale=True,
)
ax.azim = -90
ax.elev = 0
ax.dist = 5
set_axes_equal(ax)
ax = fig.add_subplot(gs[0, 1], projection="3d")
pymaid.plot2d(
ids,
color=skeleton_color_dict,
ax=ax,
connectors=False,
method="3d",
autoscale=True,
)
ax.azim = 0
ax.elev = 0
ax.dist = 5
set_axes_equal(ax)
ax = fig.add_subplot(gs[0, 2], projection="3d")
pymaid.plot2d(
ids,
color=skeleton_color_dict,
ax=ax,
connectors=False,
method="3d",
autoscale=True,
)
ax.azim = -90
ax.elev = 90
ax.dist = 5
set_axes_equal(ax)
if barplot:
ax = fig.add_subplot(gs[1, :])
temp_meta = meta[meta[key] == label]
cat = temp_meta[key + "_side"].values
subcat = temp_meta["merge_class"].values
stacked_barplot(
cat,
subcat,
ax=ax,
color_dict=CLASS_COLOR_DICT,
category_order=np.unique(cat),
)
ax.get_legend().remove()
# fig.suptitle(label)
return fig, ax
def upload_or_update_neurons(neurons,
linking_relation='',
annotate_source_neuron=False,
import_connectors=False,
reuse_existing_connectors=True,
refuse_to_update=True,
verbose=False,
fake=True):
server_responses = []
start_day = time.strftime('%Y-%m-%d')
start_time = time.strftime('%Y-%m-%d %I:%M %p')
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
# There are some pesky corner cases where updates will unintentionally create unlinked
# connectors. When that occurs, the user is warned and asked to investigate manually.
unlinked_connectors_start = find_unlinked_connectors(
remote_instance=target_project)
for source_neuron in neurons:
source_project.clear_cache()
target_project.clear_cache()
# Check if a neuron/skeleton with this neuron's name already exists in the target project
# If so, replace that neuron/skeleton's data with this neuron's data.
skid_to_update = None
nid_to_update = None
force_id = False
if linking_relation is '':
linking_annotation_template = 'LINKED NEURON - skeleton id {skid} in project id {pid} on server {server}'
else:
linking_annotation_template = 'LINKED NEURON - {relation} skeleton id {skid} in project id {pid} on server {server}'
linking_annotation_target = linking_annotation_template.format(
relation=linking_relation,
skid=source_neuron.skeleton_id,
name=source_neuron.neuron_name, #Not used currently
pid=source_project.project_id,
server=source_project.server)
if verbose:
print("Linking annotation is: '{linking_annotation_target}'")
try:
linked_neuron_skid = pymaid.get_skids_by_annotation(
add_escapes(linking_annotation_target),
raise_not_found=False,
remote_instance=target_project)
except Exception as e:
# There appears to be a bug in get_skids_by_annotation where it still
# raises exceptions sometimes even with raise_not_found=False, so
# use this block to continue through any of those cases without raising.
#print(e)
linked_neuron_skid = []
source_neuron.annotations = [
annot for annot in source_neuron.annotations
if 'LINKED NEURON' not in annot
]
if len(linked_neuron_skid) is 0: # Prepare to upload neuron as new
print(f'Uploading "{source_neuron.neuron_name}" to project'
f' {target_project.project_id} as a new skeleton.')
source_neuron.annotations.append(linking_annotation_target)
source_neuron.annotations.append(
f'UPDATED FROM LINKED NEURON - {start_time}')
elif len(linked_neuron_skid) is not 1:
print('Found multiple neurons annotated with'
f' "{linking_annotation_target}" in target project.'
' Go fix that! Skipping upload for this neuron.')
else: # Prepare to update the linked neuron
linked_neuron = pymaid.get_neuron(linked_neuron_skid[0],
remote_instance=target_project)
m = ', connectors,' if import_connectors else ''
print(f'{source_neuron.neuron_name}: Found linked neuron with '
f'skeleton ID {linked_neuron.skeleton_id} in target project.'
f' Updating its treenodes{m} and annotations to match the'
' source neuron.')
# Check whether names match
if not source_neuron.neuron_name == linked_neuron.neuron_name:
user_input = input(
'WARNING: The linked neuron\'s name is'
f' "{linked_neuron.neuron_name}" but was expected to be'
f' "{source_neuron.neuron_name}". Continuing will rename'
' the linked neuron to the expected name. Proceed? [Y/n] ')
if user_input not in ('y', 'Y'):
continue
# TODO
# Check whether there are any nodes or connectors in the source
# neuron with edition dates after the previous upload date. If not,
# skip the upload and tell the user.
# Check whether any edited nodes will be overwritten
linked_node_details = pymaid.get_node_details(
linked_neuron.nodes.node_id.values,
remote_instance=target_project)
is_edited = linked_node_details.edition_time != min(
linked_node_details.edition_time)
if is_edited.any():
edited_nodes = linked_node_details.loc[
is_edited, ['node_id', 'edition_time', 'editor']]
users = pymaid.get_user_list(
remote_instance=target_project).set_index('id')
edited_nodes.loc[:, 'editor'] = [
users.loc[user_id, 'login']
for user_id in edited_nodes.editor
]
print('WARNING: The linked neuron has been manually edited,'
f' with {len(edited_nodes)} nodes modified. Those'
' changes will get thrown away if this update is allowed'
' to continue.')
print(edited_nodes)
user_input = input(
'OK to proceed and throw away the above changes? [Y/n] ')
if user_input not in ('y', 'Y'):
print(f'Skipping update for "{source_neuron.neuron_name}"')
continue
if refuse_to_update:
print('refuse_to_update set to true. Skipping.\n')
continue
# This does NOT annotate the source neuron on the server,
# it only appends to the object in memory
source_neuron.annotations.append(
f'UPDATED FROM LINKED NEURON - {start_time}')
# Make sure to preserve all annotations on the target neuron. This will not
# be necessary once # https://github.com/catmaid/CATMAID/issues/2042 is resolved
source_neuron.annotations.extend([
a for a in linked_neuron.annotations
if a not in source_neuron.annotations
])
skid_to_update = linked_neuron.skeleton_id
nid_to_update = pymaid.get_neuron_id(
linked_neuron.skeleton_id,
remote_instance=target_project)[str(linked_neuron.skeleton_id)]
force_id = True
if not fake:
# Actually do the upload/update:
server_responses.append(
pymaid.upload_neuron(
source_neuron,
skeleton_id=skid_to_update,
neuron_id=nid_to_update,
force_id=force_id,
import_tags=True,
import_annotations=True,
import_connectors=import_connectors,
reuse_existing_connectors=reuse_existing_connectors,
remote_instance=target_project))
if annotate_source_neuron:
try:
upload_skid = server_responses[-1]['skeleton_id']
source_annotation = linking_annotation_template.format(
relation=linking_relation,
skid=server_responses[-1]['skeleton_id'],
name=source_neuron.neuron_name, #Not used currently
pid=target_project.project_id,
server=target_project.server)
try:
server_responses[-1][
'source_annotation'] = pymaid.add_annotations(
source_neuron.skeleton_id,
source_annotation,
remote_instance=source_project)
except:
m = ('WARNING: annotate_source_neuron was requested,'
' but failed. You may not have permissions to'
' annotate the source project through the API')
print(m)
input('(Press enter to acknowledge and continue.)')
server_responses[-1]['source_annotation'] = m
except:
print('WARNING: upload was not successful,'
' so could not annotate source neuron.')
input('(Press enter to acknowledge and continue.)')
print(f'{source_neuron.neuron_name}: Done with upload or update.')
print(' ')
if fake:
print('fake was set to True. Set fake=False to actually run'
' upload_or_update_neurons with settings:\n'
f'annotate_source_neuron={annotate_source_neuron}\n'
f'import_connectors={import_connectors},\n'
f'reuse_existing_connectors={reuse_existing_connectors},\n'
f'refuse_to_update={refuse_to_update}')
else:
# There are some pesky corner cases where updates will unintentionally
# create unlinked connectors. When that occurs, the user is warned and
# asked to investigate manually. Note that if a human tracer is
# annotating in catmaid and happens to make an unlinked connector that
# exists when the following lines are run, this will throw an warning
# despite there being nothing to worry about. Not much I can do there.
target_project.clear_cache()
unlinked_connectors_end = find_unlinked_connectors(
remote_instance=target_project)
newly_unlinked_connectors = set(unlinked_connectors_end).difference(
set(unlinked_connectors_start))
if len(newly_unlinked_connectors) != 0:
print("WARNING: This upload caused some connectors in the "
"target project to become unlinked from any skeleton. "
"(This can harmlessly result from deleting connectors from "
"the source project, or it may indicate a bug in the code.) "
"You may want to go clean up the new unlinked connectors:")
print(newly_unlinked_connectors)
input('(Press enter to acknowledge warning and continue.)')
return server_responses
