def test_fetch_neurons(client): bodyId = [ 294792184, 329566174, 329599710, 417199910, 420274150, 424379864, 425790257, 451982486, 480927537, 481268653 ] # This works but takes a long time. #neurons, roi_counts = fetch_neurons(NC()) neurons, roi_counts = fetch_neurons(NC(bodyId=bodyId)) assert len(neurons) == len(bodyId) assert set(roi_counts['bodyId']) == set(bodyId) neurons, roi_counts = fetch_neurons(NC(instance='APL_R')) assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" assert neurons.loc[0, 'type'] == "APL" assert neurons.loc[0, 'instance'] == "APL_R" neurons, roi_counts = fetch_neurons(NC(instance='APL[^ ]*', regex=True)) assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" assert neurons.loc[0, 'type'] == "APL" assert neurons.loc[0, 'instance'] == "APL_R" neurons, roi_counts = fetch_neurons(NC(type='APL.*', regex=True)) assert len(neurons) == 1, "There's only one APL neuron in the hemibrain" assert neurons.loc[0, 'type'] == "APL" assert neurons.loc[0, 'instance'] == "APL_R" neurons, roi_counts = fetch_neurons(NC(type=['.*01', '.*02'], regex=True)) assert len(neurons), "Didn't find any neurons of the given type pattern" assert all(lambda t: t.endswith('01') or t.endswith('02') for t in neurons['type']) assert any(lambda t: t.endswith('01') for t in neurons['type']) assert any(lambda t: t.endswith('02') for t in neurons['type']) neurons, roi_counts = fetch_neurons( NC(instance=['.*_L', '.*_R'], regex=True)) assert len( neurons), "Didn't find any neurons of the given instance pattern" assert all(lambda t: t.endswith('_L') or t.endswith('_R') for t in neurons['instance']) neurons, roi_counts = fetch_neurons( NC(status=['Traced', 'Orphan'], cropped=False)) assert neurons.eval('status == "Traced" or status == "Orphan"').all() assert not neurons['cropped'].any() neurons, roi_counts = fetch_neurons( NC(inputRois='AL(R)', outputRois='SNP(R)')) assert all(['AL(R)' in rois for rois in neurons['inputRois']]) assert all(['SNP(R)' in rois for rois in neurons['outputRois']]) assert sorted( roi_counts.query('roi == "AL(R)" and post > 0')['bodyId']) == sorted( neurons['bodyId']) assert sorted( roi_counts.query('roi == "SNP(R)" and pre > 0')['bodyId']) == sorted( neurons['bodyId']) neurons, roi_counts = fetch_neurons(NC(min_pre=1000, min_post=2000)) assert neurons.eval('pre >= 1000 and post >= 2000').all()
def get_all_neurons_roi_counts(): crit = NC() neuron_df, roi_counts_df = fetch_neurons(crit) neuron_df = neuron_df[[ 'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped', 'size' ]] neuron_df.to_csv('all_neurons.csv', index=False) roi_counts_df.to_csv('all_roi_counts', index=False)
def fetch_synapse_counts(bodies): try: c = neuprint_default_client() except Exception: c = Client(server, dataset) ndf, cdf = fetch_neurons(NC(bodyId=bodies, label='Segment'), client=c) return ndf.set_index('bodyId')[['pre', 'post']].rename_axis('body')
def test_fetch_synapses(client): nc = NC(type='ExR.*', regex=True, rois=['EB']) sc = SC(rois=['FB', 'LAL(R)'], primary_only=True) syn_df = fetch_synapses(nc, sc) assert set(syn_df['roi']) == {'FB', 'LAL(R)'} neuron_df, _count_df = fetch_neurons(nc) syn_df = syn_df.merge(neuron_df[['bodyId', 'type']], 'left', on='bodyId', suffixes=['_syn', '_body']) assert syn_df['type_body'].isnull().sum() == 0 assert syn_df['type_body'].apply(lambda s: s.startswith('ExR')).all()
def getBodyIDs(sourceParams, synapseParams): print('Getting neurons/synapses...') sourceNeurons = NC(**sourceParams) if not (synapseParams == None): synapseCriteria = SC(**synapseParams) synapses = fetch_synapse_bodyIDs(source_criteria=sourceNeurons, target_criteria=None, synapse_criteria=synapseCriteria) bodyIds = synapses['bodyId_post'].unique() #[0:max_number] else: neuron_df, roi_counts_df = fetch_neurons(sourceNeurons) bodyIds = neuron_df.bodyId synapses = None return bodyIds, synapses
def doAlignmentTest(cell_type, neuprint_search): # (1) Get TBar count based on atlas aligntment tbar = pd.read_csv(os.path.join(data_dir, 'hemi_2_atlas', '{}_ito_tbar.csv'.format(cell_type)), header=0).iloc[:, 1:] tbar.index = np.arange(1, 87) include_inds_ito, name_list_ito = bridge.getItoNames() tbar = tbar.loc[include_inds_ito, :] tbar.index = name_list_ito tbar = pd.DataFrame(tbar.sum(axis=1), columns=['sum']) fh, ax = plt.subplots(2, 1, figsize=(4, 2)) vmin = 1 vmax = np.nanmax(tbar.to_numpy()) sns.heatmap(np.log10(tbar.T).replace(-np.inf, 0), ax=ax[1], cmap='cividis', cbar=False, xticklabels=[bridge.displayName(x) for x in tbar.index], yticklabels=False, vmin=0, vmax=np.log10(vmax)) # ax[1].set_title('Atlas\nregistration') ax[1].tick_params(axis='both', which='major', labelsize=8) # (2) Get TBar count according to Neuprint & Janelia region tags Neur, _ = fetch_neurons(NeuronCriteria(type=neuprint_search, status='Traced', regex=True)) neuprint_rois = [bridge.ito_to_neuprint(x) for x in name_list_ito] tbar_count = np.zeros((Neur.shape[0], len(neuprint_rois))) for roi_ind, nr in enumerate(neuprint_rois): for n_ind, roiInfo in enumerate(Neur.roiInfo): if len(nr) > 1: new_tbars = np.sum([roiInfo.get(x, {'pre': 0}).get('pre', 0) for x in nr]) else: new_tbars = roiInfo.get(nr[0], {'pre': 0}).get('pre', 0) tbar_count[n_ind, roi_ind] = new_tbars tbar_neuprint = pd.DataFrame(tbar_count.sum(axis=0).T, index=name_list_ito, columns=['ct']) sns.heatmap(np.log10(tbar_neuprint.T).replace(-np.inf, 0), ax=ax[0], cmap='cividis', cbar=False, xticklabels=False, yticklabels=False, vmin=0, vmax=np.log10(vmax)) # ax[0].set_title('Neuprint') ax[0].tick_params(axis='both', which='major', labelsize=8) cb = fh.colorbar(matplotlib.cm.ScalarMappable(norm=matplotlib.colors.SymLogNorm(vmin=vmin, vmax=vmax, base=10, linthresh=0.1, linscale=1), cmap="cividis"), ax=ax, shrink=1.0, label='T-Bars') r, p = pearsonr(tbar.to_numpy().ravel(), tbar_neuprint.to_numpy().ravel()) print('r = {}'.format(r)) print(tbar.sum().sum()) print(tbar_neuprint.sum().sum()) return fh
def test_fetch_mitochondria(client): nc = NC(type='ExR.*', regex=True, rois=['EB']) mc = MC(rois=['FB', 'LAL(R)'], mitoType='dark', size=100_000, primary_only=True) mito_df = fetch_mitochondria(nc, mc) assert set(mito_df['roi']) == {'FB', 'LAL(R)'} assert (mito_df['mitoType'] == 'dark').all() assert (mito_df['size'] >= 100_000).all() neuron_df, _count_df = fetch_neurons(nc) mito_df = mito_df.merge(neuron_df[['bodyId', 'type']], 'left', on='bodyId', suffixes=['_mito', '_body']) assert mito_df['type'].isnull().sum() == 0 assert mito_df['type'].apply(lambda s: s.startswith('ExR')).all()
def main(): RESULTS_PKL_PATH = sys.argv[1] if len(sys.argv) == 3: PROCESSES = int(sys.argv[2]) else: PROCESSES = 4 # Calculate the difference in resolution between the stored mito segmentation and neuron segmenation. # If they differ, it must be by a power of 2. mito_res = fetch_info(*MITO_SEG)["Extended"]["VoxelSize"][0] assert mito_res % NEIGHBORHOOD_RES == 0 assert np.log2(mito_res / NEIGHBORHOOD_RES) == int(np.log2(mito_res / NEIGHBORHOOD_RES)), \ "This script assumes that the mito resolution and neighborhood resolution differ by a power of 2." mito_res_scale_diff = int(np.log2(mito_res // NEIGHBORHOOD_RES)) with open(RESULTS_PKL_PATH, 'rb') as f: mc_df = pickle.load(f) new_names = {col: col.replace(' ', '_') for col in mc_df.columns} new_names['result'] = 'proofreader_count' mc_df = mc_df.rename(columns=new_names) print("Evaluating mito count results") results = compute_parallel(partial(_task_results, mito_res_scale_diff), iter_batches( mc_df.drop_duplicates('neighborhood_id'), 1), total=len(mc_df), processes=PROCESSES, leave_progress=True, ordered=False) cols = [ 'neighborhood_id', 'neighborhood_origin', 'proofreader_count', 'mito_id_count', 'mito_ids', 'mito_sizes', 'num_ccs', 'mito_cc_ids', 'mito_cc_sizes', 'ng_link' ] df = pd.DataFrame(results, columns=cols) # Add columns for cell type (from neuprint) print("Fetching neuron cell types") origins_df = pd.DataFrame(df['neighborhood_origin'].tolist(), columns=[*'xyz']) df['body'] = fetch_labels_batched(*NEURON_SEG, origins_df[[*'zyx']].values, processes=8) neurons_df, _ = fetch_neurons(df['body'].unique()) neurons_df = neurons_df.rename(columns={ 'bodyId': 'body', 'type': 'body_type', 'instance': 'body_instance' }) df = df.merge(neurons_df[['body', 'body_type', 'body_instance']], 'left', on='body') df['body_type'].fillna("", inplace=True) df['body_instance'].fillna("", inplace=True) # Append roi column print("Determining ROIs") determine_point_rois(*NEURON_SEG[:2], NEUPRINT_CLIENT.primary_rois, origins_df) df['roi'] = origins_df['roi'] # Results only path = 'mito-seg-counts.pkl' print(f"Writing {path}") with open(path, 'wb') as f: pickle.dump(df, f) path = 'mito-seg-counts.tab-delimited.csv' print(f"Writing {path}") df.to_csv(path, sep='\t', header=True, index=False) # Full results (with task info columns) df = df.merge( mc_df.drop(columns=['neighborhood_origin', 'proofreader_count']), 'left', on='neighborhood_id') path = 'full-results-with-mito-seg-counts.pkl' print(f"Writing {path}") with open(path, 'wb') as f: pickle.dump(df, f) path = 'full-results-with-mito-seg-counts.tab-delimited.csv' print(f"Writing {path}") df.to_csv(path, sep='\t', header=True, index=False) print("DONE")
def find_neurons(criteria): neuron_df, roi_counts_df = fetch_neurons(criteria) return neuron_df[[ 'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped', 'size' ]]
def roistats_table(input_path, neuprint_dataset, voxel_col=None, num_ranked=5, exclude_none_roi=False, neuprint_server='neuprint.janelia.org', output_path=None): """ The RoiStats workflow produces a list of body-ROI pairs and corresponding voxel counts. This function - appends fractional columns to the absolute numbers, - adds synapse counts (from neuprint), - filters for the top-N ROIs for each body, - and pivots (reshapes) the data so that the top-5 ROIs (and all stats) can be viewed on a single row of a spreadsheet. Args: input_path: CSV file produced by the RoiStats workflow, e.g. roi-stats.csv neuprint_dataset: Name of the neuprint dataset, e.v. 'hemibrain:v1.1' voxel_col: Name of the column which contains the voxel counts, e.g. voxels_s1 num_ranked: How many ranked ROIs to include in the result columns exclude_none_roi: When computing totals (and fractions), should voxels that were not part of any ROI be excluded from the total (denomitator)? output_path: If provided, export the results to the given path. neuprint_server: Which neuprint server to use to obtain synapse counts. Returns: Table of top-N ROI stats for each body. """ from neuprint import Client, fetch_neurons c = Client(neuprint_server, neuprint_dataset) stats = pd.read_csv(input_path) if voxel_col is None: vcs = [*filter(lambda c: c.startswith('voxels'), stats.columns)] if len(vcs) != 1: raise RuntimeError("Could not auto-determine voxel_col. Please provide the column name to use.") voxel_col = vcs[0] stats = stats.rename(columns={f'{voxel_col}': f'roi_{voxel_col}'}) if exclude_none_roi: # Exclude <none> from totals so fractions are # given in terms of neuropil totals only. stats = stats.query('roi_id != 0') # Compute totals and fractions voxel_totals = stats.groupby('body')[f'roi_{voxel_col}'].sum().rename(f'total_{voxel_col}') stats = stats.merge(voxel_totals, 'left', on='body') stats['roi_voxels_frac'] = stats.eval(f'roi_{voxel_col} / total_{voxel_col}') # Drop the '<none>' ROI, since that isn't conveniently available in the synapse # data, and probably isn't of interest anyway. # (Above, the '<none>' voxels were already counted in the totals (denominators) # if necessary, but we never list '<none>' as an ROI column.) stats = stats.query('roi_id != 0').drop(columns=['roi_id']) # Fetch synapse counts from neuprint bodies = stats['body'].unique() logger.info(f"Fetching synapse counts from neuprint for {len(bodies)} bodies") neuron_batches, roi_syn_batches = [], [] for bodies in tqdm_proxy(iter_batches(bodies, 10_000)): n, r = fetch_neurons(bodies, client=c) neuron_batches.append(n) roi_syn_batches.append(r)
criteria = NC(rois=['PB', 'EB']) # Example: Select traced neurons which intersect the PB ROI with at least 100 inputs (PSDs). criteria = NC(inputRois=['PB'], min_roi_inputs=100, status='Traced', cropped=False) # ------------------------------------------ # Fetch neuron properties # ------------------------------------------ ''' Neuron properties and per-ROI synapse distributions can be obtained with fetch_neurons(). Two dataframes are returned: one for neuron properties, and one for the counts of synapses in each ROI. ''' neuron_df, roi_counts_df = fetch_neurons( criteria ) # return properties and per-ROI synapse counts for a set of neurons print(neuron_df[[ 'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped', 'size' ]]) print(roi_counts_df.query('bodyId==5813128308')) # synapse counts for ROIs # print(neuron_df.columns) # neuron_df, roi_counts_df = fetch_neurons(NC(type='MBON.*', regex=True)) # get mushroom body output neurons # ------------------------------------------ # Fetch connections # ------------------------------------------ ''' Find synaptic connection strengths between one set of neurons and another using fetch_adjacencies(). The “source” and/or “target” neurons are selected using NeuronCriteria. Additional parameters allow you to filter by connection strength or ROI. Two DataFrames are returned, for properties of pre/post synaptic neurons and per-ROI connection strengths (for each pre-post pair)
logging.info("Invalid cluster name: `%s`, skipping file", cluster_name) continue else: logging.info("cluster name: %s", cluster_name) file = open(os.path.join(cluster_dir, filename), 'r') local_df[cluster_name] = [int(line.strip()) for line in file.readlines()] file.close() local_df.drop(columns=["type", "instance"], inplace=True) logging.info("Finished constructing local neuron df:") print(local_df.head()) print() logging.info("Fetching neuron datafrom from neuprint") neuprint_df, conn_df = fetch_neurons(local_df["bodyId"]) neuprint_df.fillna(value={"type": "None", "instance": "None"}, inplace=True) print(neuprint_df.head()) print() logging.info("Merging dataframes") FB_node_df = pd.merge(local_df, neuprint_df, on="bodyId").rename(columns={"bodyId": "id", "type": "celltype"}).set_index("id") print(FB_node_df.head()) print() logging.info("Writing preprocessed node df to %s", output_node_csv) FB_node_df.to_csv(output_node_csv) logging.info("Loading edges and renaming columns from %s", edge_csv) edge_df = pd.read_csv(edge_csv).rename(columns={"bodyId_pre": "node1", "bodyId_post": "node2"})
def computeConnectivityMatrix(neuprint_client, mapping): """ Compute region connectivity matrix from neuprint tags, for various metrics. neuprint_client mapping: mapping dict to bridge hemibrain regions to atlas regions """ rois = list(mapping.keys()) rois.sort() WeakConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) MediumConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) StrongConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) Connectivity = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) WeightedSynapseNumber = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) TBars = pd.DataFrame(data=np.zeros((len(rois), len(rois))), index=rois, columns=rois) body_ids = [] # keep list of all cells connecting among regions for roi_source in rois: for roi_target in rois: sources = mapping[roi_source] targets = mapping[roi_target] weak_neurons = 0 medium_neurons = 0 strong_neurons = 0 summed_connectivity = 0 weighted_synapse_number = 0 tbars = 0 for s_ind, sour in enumerate( sources ): # this multiple sources/targets is necessary for collapsing rois based on mapping for targ in targets: Neur, Syn = fetch_neurons( NeuronCriteria( inputRois=sour, outputRois=targ, status='Traced', cropped=False)) # only take uncropped neurons outputs_in_targ = np.array([ x[targ]['pre'] for x in Neur.roiInfo ]) # neurons with Tbar output in target inputs_in_sour = np.array([ x[sour]['post'] for x in Neur.roiInfo ]) # neuron with PSD input in source n_weak = np.sum( np.logical_and(outputs_in_targ > 0, inputs_in_sour < 3)) n_medium = np.sum( np.logical_and( outputs_in_targ > 0, np.logical_and(inputs_in_sour >= 3, inputs_in_sour < 10))) n_strong = np.sum( np.logical_and(outputs_in_targ > 0, inputs_in_sour >= 10)) # Connection strength for each cell := sqrt(input PSDs in source x output tbars in target) conn_strengths = [ np.sqrt(x[targ]['pre'] * x[sour]['post']) for x in Neur.roiInfo ] # weighted synapses, for each cell going from sour -> targ: # := output tbars (presynapses) in targ * (input (post)synapses in sour)/(total (post)synapses onto that cell) weighted_synapses = [ Neur.roiInfo[x][targ]['pre'] * (Neur.roiInfo[x][sour]['post'] / Neur.loc[x, 'post']) for x in range(len(Neur)) ] new_tbars = [ Neur.roiInfo[x][targ]['pre'] for x in range(len(Neur)) ] # body_ids body_ids.append(Neur.bodyId.values) if Neur.roiInfo.shape[0] > 0: summed_connectivity += np.sum(conn_strengths) weighted_synapse_number += np.sum(weighted_synapses) weak_neurons += n_weak medium_neurons += n_medium strong_neurons += n_strong tbars += np.sum(new_tbars) WeakConnections.loc[[roi_source], [roi_target]] = weak_neurons MediumConnections.loc[[roi_source], [roi_target]] = medium_neurons StrongConnections.loc[[roi_source], [roi_target]] = strong_neurons Connectivity.loc[[roi_source], [roi_target]] = summed_connectivity WeightedSynapseNumber.loc[[roi_source], [roi_target]] = weighted_synapse_number TBars.loc[[roi_source], [roi_target]] = tbars body_ids = np.unique( np.hstack(body_ids) ) # don't double count cells that contribute to multiple connections return WeakConnections, MediumConnections, StrongConnections, Connectivity, WeightedSynapseNumber, TBars, body_ids