def find_desyncs(annots=[]): target_skids = pymaid.get_skids_by_annotation(paper_base_annots + annots, intersect=True, remote_instance=target_project) target_neurons = pymaid.get_neuron(target_skids, remote_instance=target_project) for target_neuron in tqdm(target_neurons): linked_skid = [a.split('skeleton id ')[1].split(' ')[0] for a in target_neuron.annotations if a.startswith('LINKED NEURON')] assert len(linked_skid) is 1 linked_skid = linked_skid[0] source_neuron = pymaid.get_neuron(linked_skid, remote_instance=source_project) #TODO change this from counting things to looking at timestamps, which #is actually guaranteed to find desyncs whereas counts aren't if source_neuron.n_nodes - target_neuron.n_nodes is not 0: print('Node number mismatch for:', source_neuron.neuron_name) if source_neuron.n_connectors - target_neuron.n_connectors is not 0: print('Connector number mismatch for:', source_neuron.neuron_name)
def get_radius_pruned_neurons_by_skid(skids, radius_to_keep=PRIMARY_NEURITE_RADIUS, keep_larger_radii=True): neurons = pymaid.get_neuron(skids, remote_instance=source_project) if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) for neuron in neurons: if 'radius' in neuron.neuron_name: raise Exception('Radius pruning was requested for' f' "{neuron.neuron_name}". You probably didn\'t' ' mean to do this since it was already pruned.' ' Abort!') if keep_larger_radii: navis.subset_neuron(neuron, neuron.nodes.node_id.values[ neuron.nodes.radius >= radius_to_keep], inplace=True) else: navis.subset_neuron(neuron, neuron.nodes.node_id.values[neuron.nodes.radius == radius_to_keep], inplace=True) if neuron.n_skeletons > 1: navis.plot2d(neuron) raise Exception(f'You cut {neuron.neuron_name} into two fragments.' " That's not supposed to happen.") neuron.annotations.append('pruned to nodes with radius 500') neuron.neuron_name = neuron.neuron_name + f' - radius {radius_to_keep}' return neurons
def copy_neurons_by_skid(skids, **kwargs): """ See upload_or_update_neurons for all keyword argument options. """ neurons = pymaid.get_neuron(skids, remote_instance=source_project) kwargs['linking_relation'] = 'copy of' return upload_or_update_neurons(neurons, **kwargs)
def add_dummy_nodes_by_skid(skids, fake=True, remote_instance=None): assert remote_instance is not None, 'Must pass a remote_instance. Exiting.' remote_instance.clear_cache() neurons = pymaid.get_neuron(skids, remote_instance=remote_instance) server_responses = [] #TODO can I do this in one API call instead of one call per neuron? for neuron in neurons: if len(neuron.nodes) is not 1: print( f'Dummy node requested for a neuron with >1 node. Skipping "{neuron.neuron_name}".' ) continue if not fake: server_responses.append( pymaid.add_node((-1, -1, neuron.nodes.iloc[0].z), neuron.nodes.iloc[0].node_id, confidence=1, remote_instance=remote_instance)) else: print(f'pymaid.add_node((-1, -1, {neuron.nodes.iloc[0].z}),' f' {neuron.nodes.iloc[0].node_id}, confidence=1,' ' remote_instance=remote_instance)') return server_responses
def get_affinetransformed_neurons_by_skid(skids, transform_file): neurons = pymaid.get_neuron(skids, remote_instance=source_project) if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) transformed_neurons = [] for neuron in neurons: node_coords = neuron.nodes[['x', 'y', 'z']].copy() connector_coords = neuron.connectors[['x', 'y', 'z']].copy() #Append a column of 1s to enable affine transformation node_coords['c'] = 1 connector_coords['c'] = 1 T = np.loadtxt(transform_file) #Apply transformation matrix using matrix multiplication transformed_node_coords = node_coords.dot(T) transformed_connector_coords = connector_coords.dot(T) #Restore column names transformed_node_coords.columns = ['x', 'y', 'z', 'c'] transformed_connector_coords.columns = ['x', 'y', 'z', 'c'] neuron.nodes.loc[:, ['x', 'y', 'z']] = transformed_node_coords[[ 'x', 'y', 'z' ]] neuron.connectors.loc[:, ['x', 'y', 'z']] = transformed_connector_coords[[ 'x', 'y', 'z' ]] neuron.neuron_name += ' - affine transform' transformed_neurons.append(neuron) return pymaid.CatmaidNeuronList(transformed_neurons)
def setUp(self): self.rm = pymaid.CatmaidInstance(config_test.server_url, config_test.http_user, config_test.http_pw, config_test.token) self.n = pymaid.get_neuron(config_test.test_skids[0], remote_instance=self.rm)
def setUp(self): self.rm = pymaid.CatmaidInstance( config_test.server_url, config_test.http_user, config_test.http_pw, config_test.token) self.nl = pymaid.get_neuron('annotation:%s' % config_test.test_annotations[ 0], remote_instance=self.rm)
def setUp(self): self.rm = pymaid.CatmaidInstance( config_test.server_url, config_test.http_user, config_test.http_pw, config_test.token) self.n = pymaid.get_neuron(config_test.test_skids[0], remote_instance=self.rm) self.cn_table = pymaid.get_partners(config_test.test_skids[0], remote_instance=self.rm) self.nB = pymaid.get_neuron(self.cn_table.iloc[0].skeleton_id, remote_instance=self.rm) self.adj = pymaid.adjacency_matrix( self.cn_table[self.cn_table.relation == 'upstream'].iloc[:10].skeleton_id.values)
def add_skeleton_layer(x, scene): """Add skeleton as new layer to scene. Parameters ---------- x : navis.TreeNeuron | pymaid.CatmaidNeuron | int Neuron to generate a URL for. Integers are interpreted as CATMAID skeleton IDs. CatmaidNeurons will automatically be transformed to flywire coordinates. Neurons are expected to be in nanometers and will be converted to pixels. scene : dict Scene to add annotation layer to. Returns ------- modified scene : dict """ if not isinstance(scene, dict): raise TypeError(f'`scene` must be dict, got "{type(scene)}"') scene = scene.copy() if not isinstance(x, (navis.TreeNeuron, navis.NeuronList, pd.DataFrame)): x = pymaid.get_neuron(x) if isinstance(x, navis.NeuronList): if len(x) > 1: raise ValueError(f'Expected a single neuron, got {len(x)}') if isinstance(x, pymaid.CatmaidNeuron): x = xform.fafb14_to_flywire(x, coordinates='nm') if not isinstance(x, (navis.TreeNeuron, pd.DataFrame)): raise TypeError(f'Expected skeleton, got {type(x)}') if isinstance(x, navis.TreeNeuron): nodes = x.nodes else: nodes = x # Generate list of segments not_root = nodes[nodes.parent_id >= 0] loc1 = not_root[['x', 'y', 'z']].values loc2 = nodes.set_index('node_id').loc[not_root.parent_id.values, ['x', 'y', 'z']].values stack = np.dstack((loc1, loc2)) stack = np.transpose(stack, (0, 2, 1)) stack = stack / [4, 4, 40] return add_annotation_layer(stack, scene)
def get_pruned_by_hardcoded_dict(prune_params=default_prune_params, **kwargs): neurons = [] for skid in prune_params: neuron = pymaid.get_neuron(skid) if 'new root' in prune_params[skid]: neuron.reroot(prune_params[skid]['new root'], inplace=True) for prune_point in prune_params[skid]['prune points']: neuron.prune_distal_to(prune_point, inplace=True) if 'final root' in prune_params[skid]: neuron.reroot(prune_params[skid]['final root'], inplace=True) neuron.plot3d(color=None) neurons.append(neuron) upload_or_update_neurons(neurons, **kwargs) return neurons
def replace_skeleton_from_swc(skid, swc_file, remote_instance=None, fake=True): assert isinstance(skid, int) if remote_instance is None: try: remote_instance = target_project print('Performing skeleton replacement in TARGET project.') except: remote_instance = source_project print('Performing skeleton replacement in SOURCE project.') new_neuron = pymaid.from_swc(swc_file) old_neuron = pymaid.get_neuron(skid, remote_instance=remote_instance) dist = lambda old, new: sum((new.nodes[['x', 'y', 'z']].mean() - old.nodes[ ['x', 'y', 'z']].mean())**2)**0.5 print(f'Neuron to be replaced: {old_neuron.neuron_name}') print('Distance between mean coordinate of old neuron and mean' f' coordinate of new neuron: {dist(old_neuron, new_neuron):.0f}nm') nid = pymaid.get_neuron_id(skid, remote_instance=remote_instance)[str(skid)] if len(old_neuron.connectors) != 0: print('WARNING: connectors on old neuron will become unlinked' ' (i.e. they will not be linked to the new neuron).') if len(old_neuron.tags) != 0: print('WARNING: tags on old neuron will be deleted.') if fake: return False old_root_radius = old_neuron.nodes.radius[ old_neuron.nodes.parent_id.isnull()].iloc[0] new_neuron.nodes.loc[new_neuron.nodes.parent_id.isnull(), 'radius'] = old_root_radius #new_neuron.annotations = old_neuron.annotations new_neuron.neuron_name = old_neuron.neuron_name pymaid.upload_neuron( new_neuron, skeleton_id=skid, neuron_id=nid, force_id=True, #import_tags=True, #import_annotations=True, #import_connectors=import_connectors, #reuse_existing_connectors=reuse_existing_connectors, remote_instance=remote_instance)
def setUp(self): self.rm = pymaid.CatmaidInstance(config_test.server_url, config_test.http_user, config_test.http_pw, config_test.token) self.nl = pymaid.get_neuron(config_test.test_skids[0:2], remote_instance=self.rm) self.n = self.nl[0] self.n.reroot(self.n.soma) # Get some random leaf node self.leaf_id = self.n.nodes[self.n.nodes.type == 'end'].sample( 1).iloc[0].treenode_id self.slab_id = self.n.nodes[self.n.nodes.type == 'slab'].sample( 1).iloc[0].treenode_id
def delete_dummy_nodes_by_skid(skids, dummy_coords=(-1, -1), fake=True, remote_instance=None): assert remote_instance is not None, 'Must pass a remote_instance. Exiting.' remote_instance.clear_cache() neurons = pymaid.get_neuron(skids, remote_instance=remote_instance) neurons_with_dummy_nodes = [] for neuron in neurons: if len(neuron.nodes) is not 2: print( f'"{neuron.neuron_name}" doesn\'t have exactly 2 nodes. Skipping.' ) continue # TODO checking for equality between floats is bad. # Change it to difference < 0.1 or something. if not (neuron.nodes[['x', 'y']] == dummy_coords).all(axis=1).any(): print( f'"{neuron.neuron_name}" has no nodes at (x, y) = {dummy_coords}. Skipping.' ) continue neurons_with_dummy_nodes.append(neuron) neurons = pymaid.core.CatmaidNeuronList(neurons_with_dummy_nodes) if len(neurons) is 0: raise ValueError('No neurons appear to have dummy nodes.') server_responses = [] # TODO checking for equality between floats is bad. # Change it to difference < 0.1 or something is_at_dummy_coords = (neurons.nodes[['x', 'y']] == dummy_coords).all(axis=1) nodes_to_delete = neurons.nodes.node_id[is_at_dummy_coords].to_list() if not fake: server_responses.append( pymaid.delete_nodes(nodes_to_delete, 'TREENODE', remote_instance=remote_instance)) else: print(f"pymaid.delete_nodes({nodes_to_delete}," " 'TREENODE', remote_instance=remote_instance)") return server_responses
def get_translated_neurons_by_skid(skids, translation, unit='nm', pixel_size=(4, 4, 40)): if len(translation) != 3: raise ValueError('Expected translation to look like [x, y, z]' f' and have length 3 but got {translation}') if unit not in ('nm', 'pixel'): raise ValueError(f"Expected unit to be 'nm' or 'pixel' but got {unit}") if unit is 'pixel': print(f'Translation of ({translation[0]}, {translation[1]},' f' {translation[2]}) pixels requested. Using pixel size of' f' {pixel_size} nm to convert to nm. Resulting translation is' f' ({translation[0]*pixel_size[0]},' f' {translation[1]*pixel_size[1]},' f' {translation[2]*pixel_size[2]}) nm.') translation = (translation[0] * pixel_size[0], translation[1] * pixel_size[1], translation[2] * pixel_size[2]) neurons = pymaid.get_neuron(skids, remote_instance=source_project) if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) for neuron in neurons: neuron.nodes.x += translation[0] neuron.nodes.y += translation[1] neuron.nodes.z += translation[2] neuron.connectors.x += translation[0] neuron.connectors.y += translation[1] neuron.connectors.z += translation[2] neuron.neuron_name += ' - translated' return neurons
def test_treenode_tags(self): n = pymaid.get_neuron(config_test.test_skids[0]) self.assertIsInstance(pymaid.get_node_tags(n.nodes.node_id.values[0:50], node_type='TREENODE'), dict)
def test_treenode_info(self): n = pymaid.get_neuron(config_test.test_skids[0]) self.assertIsInstance(pymaid.get_treenode_info(n.nodes.node_id.values[0:50]), pd.DataFrame)
def test_skid_from_treenode(self): n = pymaid.get_neuron(config_test.test_skids[0]) self.assertIsInstance(pymaid.get_skid_from_treenode(n.nodes.iloc[0].node_id), dict)
def test_node_details(self): n = pymaid.get_neuron(config_test.test_skids[0]) self.assertIsInstance(pymaid.get_node_details(n.nodes.sample(100).node_id.values), pd.DataFrame)
def test_get_neuron(self): self.assertIsInstance(pymaid.get_neuron(config_test.test_skids, remote_instance=self.rm), pymaid.CatmaidNeuronList)
def skid_to_id(x, dataset='production', progress=True, **kwargs): """Find the flywire ID(s) corresponding to given CATMAID skeleton ID(s). This function works by: 1. Fetch supervoxels for all nodes in the CATMAID skeletons 2. Pick a random sample of ``sample`` of these supervoxels 3. Fetch the most recent root IDs for the sample supervoxels 4. Return the root ID that collectively cover 90% of the supervoxels Parameters ---------- x : int | list-like | str | TreeNeuron/List Anything that's not a TreeNeuron/List will be passed directly to ``pymaid.get_neuron``. dataset : str | CloudVolume Against which flywire dataset to query:: - "production" (current production dataset, fly_v31) - "sandbox" (i.e. fly_v26) progress : bool If True, shows progress bar. Returns ------- pandas.DataFrame Mapping of flywire IDs to skeleton IDs with confidence:: flywire_id skeleton_id confidence 0 1 """ vol = parse_volume(dataset, **kwargs) if not isinstance(x, (navis.TreeNeuron, navis.NeuronList)): x = pymaid.get_neuron(x) if isinstance(x, navis.TreeNeuron): nodes = x.nodes[['x', 'y', 'z']].copy() nodes['skeleton_id'] = x.id elif isinstance(x, navis.NeuronList): nodes = x.nodes[['x', 'y', 'z']].copy() else: raise TypeError(f'Unable to data of type "{type(x)}"') # XForm coordinates from FAFB14 to FAFB14.1 xformed = xform.fafb14_to_flywire(nodes[['x', 'y', 'z']].values, coordinates='nm') # Get the root IDs for each of these locations roots = locs_to_segments(xformed, coordinates='nm') # Drop zeros roots = roots[roots != 0] # Find unique Ids and count them unique, counts = np.unique(roots, return_counts=True) # Get sorted indices sort_ix = np.argsort(counts) # New Id is the most frequent ID new_id = unique[sort_ix[-1]] # Confidence is the difference between the top and the 2nd most frequent ID if len(unique) > 1: diff_1st_2nd = counts[sort_ix[-1]] - counts[sort_ix[-2]] conf = round(diff_1st_2nd / roots.shape[0], 2) else: conf = 1 return pd.DataFrame([[x.id, new_id, conf, x.id != new_id]], columns=['old_id', 'new_id', 'confidence', 'changed'])
import pymaid import mushroom_2to3.neurogenesis as neurogenesis # %run startup_py3.py # %run load_pn_metadata_v1.py # pn_skids = cc.get_skids_from_annos(fafb_c, [['right_calyx_PN'], ['has_bouton']], ["multiglomerular PN"]) pns_ms = neurogenesis.init_from_skid_list(fafb_c, pn_skids) import pickle path = local_path + "data/pn_bouton_clusters/" with open(path + "pns_ms.pkl", 'wb') as f: pickle.dump(pns_ms, f, -1) nl = [pymaid.get_neuron([str(i) for i in j]) for j in [pn_skids[:40], pn_skids[40:80], pn_skids[80:]]] pns_pm = nl[0] + nl[1] + nl [2] with open(path + "pns_pm.pkl", 'wb') as f: pickle.dump(pns_pm, f, -1) ca = pymaid.get_volume('MB_CA_R') with open(path + "ca.pkl", 'wb') as f: pickle.dump(ca, f, -1) df = pd.read_excel(local_path + 'data/180613-pn_subtypes.xlsx') # for loading the pickle pymaid neuronlist data path = local_path + "data/pn_bouton_clusters/"
def make_rainbow_json_by_position(annotations, filename, extract_position=None, convert_values_to_rank=False, opacity=1, **kwargs): """ extract_position can be either a lambda function that defines how to extract a position value from a CatmaidNeuron object, or can be one of the following strings: 'root_x', 'root_y' (default), 'root_z', 'mean_x', 'mean_y', 'mean_z' colormap must be a Nx3 array specifying triplets of RGB values. The smallest extracted position will get mapped to the first element of the colormap, the largest extracted position will get mapped to the last element of the colormap, and intermediate positions will take on intermediate values. """ colormap = kwargs.get('colormap', turbo_colormap_data) if extract_position == 'root_x': extract_position = lambda n: n.nodes.x[n.nodes.node_id == n.root[0] ].values[0] elif extract_position in (None, 'root_y'): extract_position = lambda n: n.nodes.y[n.nodes.node_id == n.root[0] ].values[0] elif extract_position == 'root_z': extract_position = lambda n: n.nodes.z[n.nodes.node_id == n.root[0] ].values[0] elif extract_position == 'mean_x': extract_position = lambda n: n.nodes.x.mean() elif extract_position == 'mean_y': extract_position = lambda n: n.nodes.y.mean() elif extract_position == 'mean_z': extract_position = lambda n: n.nodes.z.mean() elif extract_position in ['rand', 'random']: import random extract_position = lambda: random.random() if 'neurons' in kwargs: neurons = kwargs['neurons'] skids = neurons.skeleton_id else: try: skids = pymaid.get_skids_by_annotation( annotations, intersect=True, remote_instance=source_project) except: skids = annotations # Allows users to pass skids directly if extract_position.__code__.co_argcount > 0: # Neuron data needed # Can I avoid pulling all this neuron data if I only need the root # position? Is there a way to pull less data even if I need the nodes? neurons = pymaid.get_neuron(skids, remote_instance=source_project) if extract_position.__code__.co_argcount > 0: extracted_vals = pd.Series( {n.skeleton_id: extract_position(n) for n in neurons}) else: extracted_vals = pd.Series( {skid: extract_position() for skid in skids}) extracted_vals.sort_values(ascending=False, inplace=True) if convert_values_to_rank: extracted_vals.iloc[:] = np.arange(len(extracted_vals), 0, -1) #print(extracted_vals) max_pos = extracted_vals.iloc[0] min_pos = extracted_vals.iloc[-1] scaled_extracted_vals = (extracted_vals - min_pos) / (max_pos - min_pos) colors = [ RGB_to_catmaidhex(colormap[int(p * (len(colormap) - 1))]) for p in scaled_extracted_vals ] #print(colors) skids_to_colors = dict(zip(extracted_vals.index, colors)) filename = expand_filename(filename, datestamp=kwargs.get('datestamp', False)) write_catmaid_json(skids_to_colors, filename, default_opacity=opacity)
def get_elastictransformed_neurons_by_skid(skids, transform=None, include_connectors=True, y_coordinate_cutoff=None, **kwargs): """ Apply an arbitrary transformation to a neuron. skids: A skeleton ID or list of skeleton IDs to transform. transform: a function that takes in an Nx3 numpy array representing a list of treenodes' coordinates and returns an Nx3 numpy array representing the transformed coordinates. Defaults to using warp_points_FANC_to_template from coordinate_transforms.warp_points_between_FANC_and_template. include_connectors (bool): If True, connectors will be transformed, otherwise will not be transformed (which saves execution time). y_coordinate_cutoff (numerical): If not None, filter out treenodes and connectors with y coordinate < y_coordinate_cutoff before transforming. kwargs: left_right_flip (bool): Flips the neuron across the VNC template's midplane. Only relevant for the default transform function. """ left_right_flip = kwargs.get('left_right_flip', False) if transform is None: try: from .coordinate_transforms.warp_points_between_FANC_and_template \ import warp_points_FANC_to_template as warp except: from coordinate_transforms.warp_points_between_FANC_and_template \ import warp_points_FANC_to_template as warp transform = lambda x: warp(x, reflect=left_right_flip) y_coordinate_cutoff = 322500 # 300000 * 4.3/4. In nm print('Pulling source neuron data from catmaid') source_project.clear_cache() neurons = pymaid.get_neuron(skids, remote_instance=source_project) if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) if y_coordinate_cutoff is not None: for neuron in neurons: kept_rows = neuron.nodes.y >= y_coordinate_cutoff if neuron.n_nodes == kept_rows.sum(): # No nodes to cut off continue print(f'Applying y coordinate cutoff of {y_coordinate_cutoff}' f' to {neuron.neuron_name}') kept_node_ids = neuron.nodes[kept_rows].node_id.values # TODO double check whether this removes synapses as well. I think it does navis.subset_neuron(neuron, kept_node_ids, inplace=True) if neuron.n_skeletons > 1: print( f'{neuron.neuron_name} is fragmented. Healing before continuing.' ) navis.heal_fragmented_neuron(neuron, inplace=True) for neuron in neurons: print(f'Transforming {neuron.neuron_name}') neuron.neuron_name += ' - elastic transform' if left_right_flip: neuron.neuron_name += ' - flipped' neuron.annotations.append('left-right flipped') neuron.nodes[['x', 'y', 'z']] = transform(neuron.nodes[['x', 'y', 'z']]) if include_connectors and neuron.n_connectors > 0: neuron.connectors[['x', 'y', 'z']] = transform( neuron.connectors[['x', 'y', 'z']]) return neurons
def get_connection_list(all_pids,noi,confidence = 5): """ Input: list of all project ideas to get list for list of neurons of interest Output: list of all connections """ fullConnsList = pd.DataFrame(columns = ['connector_id', 'length', 'dist_from_root', 'neuron', 'project', 'type',]) for project in all_pids: # open an instance of CATMAID containing data https://zhencatmaid.com catmaid = pymaid.CatmaidInstance(server = 'https://zhencatmaid.com/', api_token='c48243e19b85edf37345ced8049ce5d6c5802412', project_id = project) for neurName in noi: print('Working on ' + neurName + ' in project ' + str(project)) try: catNeur = pymaid.get_neuron(neurName) except: print(neurName + " not found in project " + str(project)) continue if isinstance(catNeur, pymaid.CatmaidNeuron): catNeur = [catNeur] for neur in catNeur: if neur.n_nodes < 10: continue skid = neur.id if pymaid.find_nodes(tags=['nerve_ring_starts'],skeleton_ids=skid).empty: continue catNeurnumpy = neur.nodes[["node_id","parent_id","x","y","z"]].to_numpy() skTree = bf.build_tree(neur) nr_subtree = bf.crop_tree_nr(skTree,skid) # define and filter connections connectors = neur.connectors filt_conns = connectors[connectors.type.isin([0,1])].reset_index(drop = True) for i in range(len(nr_subtree)): if len(nr_subtree) > 1: strneurName = bf.strip_neurName(list(pymaid.get_names(skid).values())[0]) + "(" + str(i) + ")" else: strneurName = bf.strip_neurName(list(pymaid.get_names(skid).values())[0]) bl_output = bf.get_branchList(nr_subtree[i],neur) trunk = bl_output[2] connsList = pd.DataFrame(columns = ['connector_id', 'length', 'dist_from_root', 'neuron', 'project', 'type', 'inputs', 'outputs']) filt_conns.connector_id = filt_conns.connector_id.astype(str) filt_conns2 = pd.DataFrame(columns = filt_conns.columns) inputsClean = [] outputsClean = [] for connector in filt_conns.iterrows(): connector = connector[1] if connector.node_id in nr_subtree[i]: if connector.type == 1: if check_confidence(catmaid,project,connector.connector_id,skid,'input'): inputsClean.append(clean_inputs(catmaid,project,connector)) outputsClean.append(clean_outputs(catmaid,project,connector)) filt_conns2 = filt_conns2.append(connector) if connector.type == 0: if check_confidence(catmaid,project,connector.connector_id,skid,'output'): inputsClean.append(clean_inputs(catmaid,project,connector)) outputsClean.append(clean_outputs(catmaid,project,connector)) filt_conns2 = filt_conns2.append(connector) filt_conns2 = filt_conns2.reset_index(drop = True) lengthTemp = [] distTemp = [] for node in filt_conns2.node_id: lengthTemp.append(cf.get_norm_length(node,catNeurnumpy,trunk)) distTemp.append(cf.get_norm_dist(node,catNeurnumpy,trunk)) connsList.connector_id = filt_conns2.connector_id connsList.project = project connsList.neuron = strneurName connsList.length = lengthTemp connsList.dist_from_root = distTemp connsList.type = filt_conns2.type connsList.inputs = inputsClean connsList.outputs = outputsClean fullConnsList = fullConnsList.append(connsList).reset_index(drop = True) return fullConnsList
def get_volume_pruned_neurons_by_skid(skids, volume_id, mode='fele', resample=0, only_keep_largest_fragment=False, verbose=False, remote_instance=None): """ mode : 'fele' - Keep all parts of the neuron between its primary neurite's First Entry to and Last Exit from the volume. So if a segment of the primary neurite leaves and then re-enters the volume, that segment is not removed. 'strict' - All nodes outside the volume are pruned. resample : If set to a positive value, the neuron will be resampled before pruning to have treenodes placed every `resample` nanometers. If left at 0, resampling is not performed. In both cases, if a branch point is encountered before the first entry or last exit, that branch point is used as the prune point. """ if remote_instance is None: remote_instance = source_project #if exit_volume_id is None: # exit_volume_id = entry_volume_id neurons = pymaid.get_neuron(skids, remote_instance=source_project) if volume_id not in volumes: try: print(f'Pulling volume {volume_id} from project' f' {remote_instance.project_id}.') volumes[volume_id] = pymaid.get_volume( volume_id, remote_instance=remote_instance) except: print(f"Couldn't find volume {volume_id} in project_id" f" {remote_instance.project_id}! Exiting.") raise else: print(f'Loading volume {volume_id} from cache.') volume = volumes[volume_id] if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) if resample > 0: #TODO find the last node on the primary neurite and store its position neurons.resample( resample) # This throws out radius info except for root #TODO find the new node closest to the stored node and set all nodes #between that node and root to have radius 500 for neuron in neurons: if 'pruned by vol' in neuron.neuron_name: raise Exception( 'Volume pruning was requested for ' f' "{neuron.neuron_name}". You probably didn\'t mean to do' ' this since it was already pruned. Exiting.') continue print(f'Pruning neuron {neuron.neuron_name}') if mode == 'fele': """ First, find the most distal primary neurite node. Then, walk backward until either finding a node within the volume or a branch point. Prune distal to one distal to that point (so it only gets the primary neurite and not the offshoot). Then, start from the primary neurite node that's a child of the soma node, and walk forward (how?) until finding a node within the volume or a branch point. Prune proximal to that. """ nodes = neuron.nodes.set_index('node_id') # Find end of the primary neurite nodes['has_fat_child'] = False for tid in nodes.index: if nodes.at[tid, 'radius'] == PRIMARY_NEURITE_RADIUS: parent = nodes.at[tid, 'parent_id'] nodes.at[parent, 'has_fat_child'] = True is_prim_neurite_end = (~nodes['has_fat_child']) & ( nodes['radius'] == PRIMARY_NEURITE_RADIUS) prim_neurite_end = nodes.index[is_prim_neurite_end] if len(prim_neurite_end) is 0: raise ValueError(f"{neuron.neuron_name} doesn't look like a" " motor neuron. Exiting.") elif len(prim_neurite_end) is not 1: raise ValueError('Multiple primary neurite ends for' f' {neuron.neuron_name}: {prim_neurite_end}.' '\nExiting.') nodes['is_in_vol'] = navis.in_volume(nodes, volume) # Walk backwards until at a point inside the volume, or at a branch # point current_node = prim_neurite_end[0] parent_node = nodes.at[current_node, 'parent_id'] while (nodes.at[parent_node, 'type'] != 'branch' and not nodes.at[parent_node, 'is_in_vol']): current_node = parent_node if verbose: print(f'Walk back to {current_node}') parent_node = nodes.at[parent_node, 'parent_id'] if verbose: print(f'Pruning distal to {current_node}') neuron.prune_distal_to(current_node, inplace=True) # Start at the first primary neurite node downstream of root current_node = nodes.index[ (nodes.parent_id == neuron.root[0]) #& (nodes.radius == PRIMARY_NEURITE_RADIUS)][0] & (nodes.radius > 0)][0] #Walking downstream is a bit slow, but probably acceptable while (not nodes.at[current_node, 'is_in_vol'] and nodes.at[current_node, 'type'] == 'slab'): current_node = nodes.index[nodes.parent_id == current_node][0] if verbose: print(f'Walk forward to {current_node}') if not nodes.at[current_node, 'is_in_vol']: input('WARNING: Hit a branch before hitting the volume for' f' neuron {neuron.neuron_name}. This is unusual.' ' Press enter to acknowledge.') if verbose: print(f'Pruning proximal to {current_node}') neuron.prune_proximal_to(current_node, inplace=True) elif mode == 'strict': neuron.prune_by_volume(volume) #This does in-place pruning if neuron.n_skeletons > 1: if only_keep_largest_fragment: print('Neuron has multiple disconnected fragments after' ' pruning. Will only keep the largest fragment.') frags = morpho.break_fragments(neuron) neuron.nodes = frags[frags.n_nodes == max( frags.n_nodes)][0].nodes #print(neuron) #else, the neuron will get healed and print a message about being #healed during upload_neuron, so nothing needs to be done here. if mode == 'fele': neuron.annotations.append( f'pruned (first entry, last exit) by vol {volume_id}') elif mode == 'strict': neuron.annotations.append(f'pruned (strict) by vol {volume_id}') neuron.neuron_name = neuron.neuron_name + f' - pruned by vol {volume_id}' if verbose: print('\n') return neurons
#A requirement of this code is access to CATMAID. #This is for API access to the CATMAID servers and to download skeleton information server = http_user = http_pw = token = pymaid.CatmaidInstance( server, http_user, http_pw, token) #This is an integer number that is unique to your neuron of interest Neuron_1_skeleton_id_number = Neuron_1 = pymaid.get_neuron(Neuron_skeleton_id_number) Neuron_2_skeleton_id_number = Neuron_2 = pymaid.get_neuron(Neuron_2_skeleton_id_number) #This function downsamples the neuron - it removes large stretches of skeleton that do not have any branch points. #When the argument preserve_cn_treenodes = True, this preserves the treenodes where connectors (pre/postsynapses) #have been placed. Downsampling is used to reduce the computational time, as some 3D reconstructed neurons can become #very large. Neuron.downsample(1000000, preserve_cn_treenodes = True) Neuron_2.downsample(1000000, preserve_cn_treenodes = True) #Get the connectors between the two neurons of interest #When True, the directional argument will return the connectors (pre and post synapses) #from neuron A to neuron B (A-->B; in this case Neuron_1 to Neuron_2). When False, it will return all
def adj_split_axons_dendrites(all_neurons, split_tag, special_split_tags, not_split_skids): # user must login to CATMAID instance before starting t0 = time.time() # find today's date and make an output folder with that name today = date.today() today = date.strftime(today, '%Y-%m-%d') output_path = Path(f"data/processed/{today}") if not os.path.isdir(output_path): os.mkdir(output_path) print("Pulling neurons...\n") ids = all_neurons batch_size = 20 max_tries = 10 n_batches = int(np.floor(len(ids) / batch_size)) if len(ids) % n_batches > 0: n_batches += 1 print(f"Batch size: {batch_size}") print(f"Number of batches: {n_batches}") print(f"Number of neurons: {len(ids)}") print(f"Batch product: {n_batches * batch_size}\n") i = 0 currtime = time.time() nl = pymaid.get_neuron(ids[i * batch_size:(i + 1) * batch_size], with_connectors=False) print(f"{time.time() - currtime:.3f} seconds elapsed for batch {i}.") for i in range(1, n_batches): currtime = time.time() n_tries = 0 success = False while not success and n_tries < max_tries: try: nl += pymaid.get_neuron(ids[i * batch_size:(i + 1) * batch_size], with_connectors=False) success = True except ChunkedEncodingError: print(f"Failed pull on batch {i}, trying again...") n_tries += 1 print(f"{time.time() - currtime:.3f} seconds elapsed for batch {i}.") print("\nPulled all neurons.\b") print("\nPickling neurons...") currtime = time.time() with open(output_path / "neurons.pickle", "wb") as f: dump(nl, f) print(f"{time.time() - currtime:.3f} seconds elapsed to pickle.") print("Pulling split points and special split neuron ids...") currtime = time.time() ########## # double-check for issues with split tags # find neurons and nodes with split tag splits = pymaid.find_nodes(tags=split_tag) splits = splits.set_index("skeleton_id")["node_id"].squeeze() # find neurons and nodes with special split start tag special_splits_start = pymaid.find_nodes(tags=special_split_tags[0]) special_splits_start = special_splits_start.set_index( "skeleton_id")["node_id"].squeeze() # find neurons and nodes with special split end tag special_splits_end = pymaid.find_nodes(tags=special_split_tags[1]) special_splits_end = special_splits_end.set_index( "skeleton_id")["node_id"].squeeze() # every skeleton with special split start should also have a special split end special_start_ids = np.unique(special_splits_start.index) special_end_ids = np.unique(special_splits_end.index) intersect = np.intersect1d(special_start_ids, special_end_ids) union = np.union1d(special_start_ids, special_end_ids) if (len(intersect) != len(union)): print('Not all neurons with complex splits have the proper tags!') sys.exit( f'Check tags in the following skids: {list(np.setdiff1d(union, intersect))}' ) # split tag skeletons and special split tag skeletons should be mutually exclusive split_ids = list(splits.index) special_ids = list(union) if (len(np.intersect1d(split_ids, special_ids)) != 0): sys.exit( f'Splitting error! Check {problem_skids} which have a combination of {split_tag}, {special_split_tags[0]}, and {special_split_tags[1]} tags' ) print(f"{time.time() - currtime:.3f} elapsed.\n") # all splittable neurons should have split tags should_not_split = not_split_skids # unsplittable neurons defined by user should_split = list(np.setdiff1d(all_neurons, should_not_split)) not_split = list(np.setdiff1d(should_split, split_ids + special_ids)) if len(not_split) > 0: print( f"WARNING: {len(not_split)} neurons should have had split tag(s) and didn't:" ) print(not_split) sys.exit('Check the above skeleton IDs') ######## # processing data print("Getting treenode compartment types...") currtime = time.time() treenode_types = get_treenode_types(nl, splits, special_ids, output_path) print(f"{time.time() - currtime:.3f} elapsed.\n") print("Pulling connectors...\n") currtime = time.time() connectors = get_connectors(nl) print(f"{time.time() - currtime:.3f} elapsed.\n") explode_cols = ["postsynaptic_to", "postsynaptic_to_node"] index_cols = np.setdiff1d(connectors.columns, explode_cols) print("Exploding connector DataFrame...") # explode the lists within the connectors dataframe connectors = (connectors.set_index(list(index_cols)).apply( pd.Series.explode).reset_index()) # TODO figure out these nans bad_connectors = connectors[connectors.isnull().any(axis=1)] bad_connectors.to_csv(output_path / "bad_connectors.csv") # connectors = connectors[~connectors.isnull().any(axis=1)] print(f"Connectors with errors: {len(bad_connectors)}") connectors = connectors.astype({ "presynaptic_to": "Int64", "presynaptic_to_node": "Int64", "postsynaptic_to": "Int64", "postsynaptic_to_node": "Int64", }) print("Applying treenode types to connectors...") currtime = time.time() connectors["presynaptic_type"] = connectors["presynaptic_to_node"].map( treenode_types) connectors["postsynaptic_type"] = connectors["postsynaptic_to_node"].map( treenode_types) connectors["in_subgraph"] = connectors["presynaptic_to"].isin( ids) & connectors["postsynaptic_to"].isin(ids) print(f"{time.time() - currtime:.3f} elapsed.\n") meta = pd.DataFrame(index=all_neurons) print("Calculating neuron total inputs and outputs...") axon_output_map = (connectors[connectors["presynaptic_type"] == "axon"].groupby("presynaptic_to").size()) axon_input_map = (connectors[connectors["postsynaptic_type"] == "axon"].groupby("postsynaptic_to").size()) dendrite_output_map = (connectors[connectors["presynaptic_type"].isin( ["dendrite", "unsplit"])].groupby("presynaptic_to").size()) dendrite_input_map = (connectors[connectors["postsynaptic_type"].isin( ["dendrite", "unsplit"])].groupby("postsynaptic_to").size()) meta["axon_output"] = meta.index.map(axon_output_map).fillna(0.0) meta["axon_input"] = meta.index.map(axon_input_map).fillna(0.0) meta["dendrite_output"] = meta.index.map(dendrite_output_map).fillna(0.0) meta["dendrite_input"] = meta.index.map(dendrite_input_map).fillna(0.0) print() # remap the true compartment type mappings to the 4 that we usually use connectors["compartment_type"] = flatten( connectors["presynaptic_type"]) + flatten( connectors["postsynaptic_type"]) subgraph_connectors = connectors[connectors["in_subgraph"]] meta_data_dict = meta.to_dict(orient="index") full_g = connectors_to_nx_multi(subgraph_connectors, meta_data_dict) graph_types = ["aa", "ad", "da", "dd"] color_multigraphs = {} color_flat_graphs = {} for graph_type in graph_types: color_subgraph_connectors = subgraph_connectors[ subgraph_connectors["compartment_type"] == graph_type] color_g = connectors_to_nx_multi(color_subgraph_connectors, meta_data_dict) color_multigraphs[graph_type] = color_g flat_color_g = flatten_muligraph(color_g, meta_data_dict) color_flat_graphs[graph_type] = flat_color_g flat_g = flatten_muligraph(full_g, meta_data_dict) ######## # saving data print("Saving metadata as csv...") meta.to_csv(output_path / "meta_data.csv") meta.to_csv(output_path / "meta_data_unmodified.csv") print("Saving connectors as csv...") connectors.to_csv(output_path / "connectors.csv") print("Saving each flattened color graph as graphml...") for graph_type in graph_types: nx.write_graphml(color_flat_graphs[graph_type], output_path / f"G{graph_type}.graphml") nx.write_graphml(flat_g, output_path / "G.graphml") print("Saving each flattened color graph as txt edgelist...") for graph_type in graph_types: nx.write_weighted_edgelist(color_flat_graphs[graph_type], output_path / f"G{graph_type}_edgelist.txt") nx.write_weighted_edgelist(flat_g, output_path / "G_edgelist.txt") ########## # saving data as pandas DataFrame adjacencies # import graphs of axon-dendrite split data; generated by Ben Pedigo's scripts G = nx.readwrite.graphml.read_graphml(f'data/processed/{today}/G.graphml', node_type=int) Gad = nx.readwrite.graphml.read_graphml( f'data/processed/{today}/Gad.graphml', node_type=int) Gaa = nx.readwrite.graphml.read_graphml( f'data/processed/{today}/Gaa.graphml', node_type=int) Gdd = nx.readwrite.graphml.read_graphml( f'data/processed/{today}/Gdd.graphml', node_type=int) Gda = nx.readwrite.graphml.read_graphml( f'data/processed/{today}/Gda.graphml', node_type=int) print("Generating split adjacency matrices as csv...") # generate adjacency matrices adj_all = pd.DataFrame(nx.adjacency_matrix(G=G, weight='weight').todense(), columns=G.nodes, index=G.nodes) adj_ad = pd.DataFrame(nx.adjacency_matrix(G=Gad, weight='weight').todense(), columns=Gad.nodes, index=Gad.nodes) adj_aa = pd.DataFrame(nx.adjacency_matrix(G=Gaa, weight='weight').todense(), columns=Gaa.nodes, index=Gaa.nodes) adj_dd = pd.DataFrame(nx.adjacency_matrix(G=Gdd, weight='weight').todense(), columns=Gdd.nodes, index=Gdd.nodes) adj_da = pd.DataFrame(nx.adjacency_matrix(G=Gda, weight='weight').todense(), columns=Gda.nodes, index=Gda.nodes) # add back in skids with no edges (with rows/columns of 0); simplifies some later analysis def refill_adjs(adj, adj_all): skids_diff = np.setdiff1d(adj_all.index, adj.index) adj = adj.append( pd.DataFrame([[0] * adj.shape[1]] * len(skids_diff), index=skids_diff, columns=adj.columns)) # add in rows with 0s for skid in skids_diff: adj[skid] = [0] * len(adj.index) return (adj) adj_ad = refill_adjs(adj_ad, adj_all) adj_aa = refill_adjs(adj_aa, adj_all) adj_dd = refill_adjs(adj_dd, adj_all) adj_da = refill_adjs(adj_da, adj_all) #convert column names to int for easier indexing adj_all.columns = adj_all.columns.astype(int) adj_ad.columns = adj_ad.columns.astype(int) adj_aa.columns = adj_aa.columns.astype(int) adj_da.columns = adj_da.columns.astype(int) adj_dd.columns = adj_dd.columns.astype(int) # export adjacency matrices adj_all.to_csv(f'data/adj/all-all_{today}.csv') adj_ad.to_csv(f'data/adj/ad_{today}.csv') adj_aa.to_csv(f'data/adj/aa_{today}.csv') adj_dd.to_csv(f'data/adj/dd_{today}.csv') adj_da.to_csv(f'data/adj/da_{today}.csv') # import input data and export as simplified csv meta_data = pd.read_csv(f'data/processed/{today}/meta_data.csv', index_col=0) inputs = meta_data.loc[:, ['axon_input', 'dendrite_input']] outputs = meta_data.loc[:, ['axon_output', 'dendrite_output']] # exporting input data inputs.to_csv(f'data/adj/inputs_{today}.csv') outputs.to_csv(f'data/adj/outputs_{today}.csv') print() print() print("Done!") elapsed = time.time() - t0 delta = timedelta(seconds=elapsed) print("----") print(f"{delta} elapsed for whole script.") print("----") sys.stdout.close()
def get_branches(all_pids, noi, conn_data_path=None, ignore_tags=False, branch_threshold=0.05): """ Input: list of all project ids list of neurons of interest path to conn_data_per_neuron folder ignore 'not a branch' tags % of main branch for threshold to consider branch Output: """ project_data = {} on_branch_per_project = {} if conn_data_path != None: for project in all_pids: project_data[project] = pd.read_csv(conn_data_path + str(project) + '/' + str(project) + '.csv') project_data[project]['neuron'] = project_data[project][ 'neuron'].str.split('(').str[0] on_branch_per_project[project] = [] fixed_outputs = [] for sublist in project_data[project].outputs: fixed_outputs.append(literal_eval(sublist)) project_data[project]['outputs'] = fixed_outputs fullBranchList = pd.DataFrame(columns=[ 'leafnode', 'length', 'dist_from_root', 'neurName', 'project', 'n_conns' ]) for project in all_pids: # open an instance of CATMAID containing data https://zhencatmaid.com catmaid = pymaid.CatmaidInstance( server='https://zhencatmaid.com/', api_token='c48243e19b85edf37345ced8049ce5d6c5802412', project_id=project) if conn_data_path != None: curr_project = project_data[project] for neurName in noi: print('Working on ' + neurName + ' in project ' + str(project)) try: catNeur = pymaid.get_neuron(neurName) except: print(neurName + " not found in project " + str(project)) continue if isinstance(catNeur, pymaid.CatmaidNeuron): catNeur = [catNeur] for neur in catNeur: if neur.n_nodes < 10: continue skid = neur.id if pymaid.find_nodes(tags=['nerve_ring_starts'], skeleton_ids=skid).empty: continue catNeurnumpy = neur.nodes[[ "node_id", "parent_id", "x", "y", "z" ]].to_numpy() skTree = bf.build_tree(neur) nr_subtree = bf.crop_tree_nr(skTree, skid) for i in range(0, len(nr_subtree)): strneurName = bf.strip_neurName( list(pymaid.get_names(skid).values())[0]) if conn_data_path != None: connsList = curr_project.loc[ curr_project['neuron'].isin([strneurName])] else: connsList = None bl_output = bf.get_branchList(nr_subtree[i], neur, branch_threshold) branchList = bl_output[0] pathList = bl_output[1] trunk = bl_output[2] trunklen = bf.cable_length(trunk[-1], catNeurnumpy, trunk[0]) lengthTemp = [] connTemp = [] for path in pathList: lengthTemp.append( bf.cable_length(path[0], catNeurnumpy, trunk[0])) connTemp.append( sum_Conns_on_Branch(path, neur, connsList)) branchList['length'] = branchList['length'] / trunklen branchList['dist_from_root'] = [ i / trunklen for i in lengthTemp ] branchList['neurName'] = strneurName branchList['project'] = project branchList['n_conns'] = connTemp if not ignore_tags: try: branchList = branchList[ ~branchList['leafnode'].astype(float). astype(int).isin(neur.tags['not a branch'])] except: pass fullBranchList = fullBranchList.append(branchList) return fullBranchList
def plot_neurons(meta, key=None, label=None, barplot=False): if label is not None: ids = list(meta[meta[key] == label].index.values) else: ids = list(meta.index.values) ids = [int(i) for i in ids] new_ids = [] for i in ids: try: pymaid.get_neuron( i, raise_missing=True, with_connectors=False, with_tags=False ) new_ids.append(i) except: print(f"Missing neuron {i}, not plotting it.") ids = new_ids meta = meta.loc[ids] fig = plt.figure(figsize=(30, 10)) gs = plt.GridSpec(2, 3, figure=fig, wspace=0, hspace=0, height_ratios=[0.8, 0.2]) skeleton_color_dict = dict( zip(meta.index, np.vectorize(CLASS_COLOR_DICT.get)(meta["merge_class"])) ) ax = fig.add_subplot(gs[0, 0], projection="3d") pymaid.plot2d( ids, color=skeleton_color_dict, ax=ax, connectors=False, method="3d", autoscale=True, ) ax.azim = -90 ax.elev = 0 ax.dist = 5 set_axes_equal(ax) ax = fig.add_subplot(gs[0, 1], projection="3d") pymaid.plot2d( ids, color=skeleton_color_dict, ax=ax, connectors=False, method="3d", autoscale=True, ) ax.azim = 0 ax.elev = 0 ax.dist = 5 set_axes_equal(ax) ax = fig.add_subplot(gs[0, 2], projection="3d") pymaid.plot2d( ids, color=skeleton_color_dict, ax=ax, connectors=False, method="3d", autoscale=True, ) ax.azim = -90 ax.elev = 90 ax.dist = 5 set_axes_equal(ax) if barplot: ax = fig.add_subplot(gs[1, :]) temp_meta = meta[meta[key] == label] cat = temp_meta[key + "_side"].values subcat = temp_meta["merge_class"].values stacked_barplot( cat, subcat, ax=ax, color_dict=CLASS_COLOR_DICT, category_order=np.unique(cat), ) ax.get_legend().remove() # fig.suptitle(label) return fig, ax
def upload_or_update_neurons(neurons, linking_relation='', annotate_source_neuron=False, import_connectors=False, reuse_existing_connectors=True, refuse_to_update=True, verbose=False, fake=True): server_responses = [] start_day = time.strftime('%Y-%m-%d') start_time = time.strftime('%Y-%m-%d %I:%M %p') if type(neurons) is pymaid.core.CatmaidNeuron: neurons = pymaid.core.CatmaidNeuronList(neurons) # There are some pesky corner cases where updates will unintentionally create unlinked # connectors. When that occurs, the user is warned and asked to investigate manually. unlinked_connectors_start = find_unlinked_connectors( remote_instance=target_project) for source_neuron in neurons: source_project.clear_cache() target_project.clear_cache() # Check if a neuron/skeleton with this neuron's name already exists in the target project # If so, replace that neuron/skeleton's data with this neuron's data. skid_to_update = None nid_to_update = None force_id = False if linking_relation is '': linking_annotation_template = 'LINKED NEURON - skeleton id {skid} in project id {pid} on server {server}' else: linking_annotation_template = 'LINKED NEURON - {relation} skeleton id {skid} in project id {pid} on server {server}' linking_annotation_target = linking_annotation_template.format( relation=linking_relation, skid=source_neuron.skeleton_id, name=source_neuron.neuron_name, #Not used currently pid=source_project.project_id, server=source_project.server) if verbose: print("Linking annotation is: '{linking_annotation_target}'") try: linked_neuron_skid = pymaid.get_skids_by_annotation( add_escapes(linking_annotation_target), raise_not_found=False, remote_instance=target_project) except Exception as e: # There appears to be a bug in get_skids_by_annotation where it still # raises exceptions sometimes even with raise_not_found=False, so # use this block to continue through any of those cases without raising. #print(e) linked_neuron_skid = [] source_neuron.annotations = [ annot for annot in source_neuron.annotations if 'LINKED NEURON' not in annot ] if len(linked_neuron_skid) is 0: # Prepare to upload neuron as new print(f'Uploading "{source_neuron.neuron_name}" to project' f' {target_project.project_id} as a new skeleton.') source_neuron.annotations.append(linking_annotation_target) source_neuron.annotations.append( f'UPDATED FROM LINKED NEURON - {start_time}') elif len(linked_neuron_skid) is not 1: print('Found multiple neurons annotated with' f' "{linking_annotation_target}" in target project.' ' Go fix that! Skipping upload for this neuron.') else: # Prepare to update the linked neuron linked_neuron = pymaid.get_neuron(linked_neuron_skid[0], remote_instance=target_project) m = ', connectors,' if import_connectors else '' print(f'{source_neuron.neuron_name}: Found linked neuron with ' f'skeleton ID {linked_neuron.skeleton_id} in target project.' f' Updating its treenodes{m} and annotations to match the' ' source neuron.') # Check whether names match if not source_neuron.neuron_name == linked_neuron.neuron_name: user_input = input( 'WARNING: The linked neuron\'s name is' f' "{linked_neuron.neuron_name}" but was expected to be' f' "{source_neuron.neuron_name}". Continuing will rename' ' the linked neuron to the expected name. Proceed? [Y/n] ') if user_input not in ('y', 'Y'): continue # TODO # Check whether there are any nodes or connectors in the source # neuron with edition dates after the previous upload date. If not, # skip the upload and tell the user. # Check whether any edited nodes will be overwritten linked_node_details = pymaid.get_node_details( linked_neuron.nodes.node_id.values, remote_instance=target_project) is_edited = linked_node_details.edition_time != min( linked_node_details.edition_time) if is_edited.any(): edited_nodes = linked_node_details.loc[ is_edited, ['node_id', 'edition_time', 'editor']] users = pymaid.get_user_list( remote_instance=target_project).set_index('id') edited_nodes.loc[:, 'editor'] = [ users.loc[user_id, 'login'] for user_id in edited_nodes.editor ] print('WARNING: The linked neuron has been manually edited,' f' with {len(edited_nodes)} nodes modified. Those' ' changes will get thrown away if this update is allowed' ' to continue.') print(edited_nodes) user_input = input( 'OK to proceed and throw away the above changes? [Y/n] ') if user_input not in ('y', 'Y'): print(f'Skipping update for "{source_neuron.neuron_name}"') continue if refuse_to_update: print('refuse_to_update set to true. Skipping.\n') continue # This does NOT annotate the source neuron on the server, # it only appends to the object in memory source_neuron.annotations.append( f'UPDATED FROM LINKED NEURON - {start_time}') # Make sure to preserve all annotations on the target neuron. This will not # be necessary once # https://github.com/catmaid/CATMAID/issues/2042 is resolved source_neuron.annotations.extend([ a for a in linked_neuron.annotations if a not in source_neuron.annotations ]) skid_to_update = linked_neuron.skeleton_id nid_to_update = pymaid.get_neuron_id( linked_neuron.skeleton_id, remote_instance=target_project)[str(linked_neuron.skeleton_id)] force_id = True if not fake: # Actually do the upload/update: server_responses.append( pymaid.upload_neuron( source_neuron, skeleton_id=skid_to_update, neuron_id=nid_to_update, force_id=force_id, import_tags=True, import_annotations=True, import_connectors=import_connectors, reuse_existing_connectors=reuse_existing_connectors, remote_instance=target_project)) if annotate_source_neuron: try: upload_skid = server_responses[-1]['skeleton_id'] source_annotation = linking_annotation_template.format( relation=linking_relation, skid=server_responses[-1]['skeleton_id'], name=source_neuron.neuron_name, #Not used currently pid=target_project.project_id, server=target_project.server) try: server_responses[-1][ 'source_annotation'] = pymaid.add_annotations( source_neuron.skeleton_id, source_annotation, remote_instance=source_project) except: m = ('WARNING: annotate_source_neuron was requested,' ' but failed. You may not have permissions to' ' annotate the source project through the API') print(m) input('(Press enter to acknowledge and continue.)') server_responses[-1]['source_annotation'] = m except: print('WARNING: upload was not successful,' ' so could not annotate source neuron.') input('(Press enter to acknowledge and continue.)') print(f'{source_neuron.neuron_name}: Done with upload or update.') print(' ') if fake: print('fake was set to True. Set fake=False to actually run' ' upload_or_update_neurons with settings:\n' f'annotate_source_neuron={annotate_source_neuron}\n' f'import_connectors={import_connectors},\n' f'reuse_existing_connectors={reuse_existing_connectors},\n' f'refuse_to_update={refuse_to_update}') else: # There are some pesky corner cases where updates will unintentionally # create unlinked connectors. When that occurs, the user is warned and # asked to investigate manually. Note that if a human tracer is # annotating in catmaid and happens to make an unlinked connector that # exists when the following lines are run, this will throw an warning # despite there being nothing to worry about. Not much I can do there. target_project.clear_cache() unlinked_connectors_end = find_unlinked_connectors( remote_instance=target_project) newly_unlinked_connectors = set(unlinked_connectors_end).difference( set(unlinked_connectors_start)) if len(newly_unlinked_connectors) != 0: print("WARNING: This upload caused some connectors in the " "target project to become unlinked from any skeleton. " "(This can harmlessly result from deleting connectors from " "the source project, or it may indicate a bug in the code.) " "You may want to go clean up the new unlinked connectors:") print(newly_unlinked_connectors) input('(Press enter to acknowledge warning and continue.)') return server_responses