def test_fetch_neurons(client):
bodyId = [
294792184, 329566174, 329599710, 417199910, 420274150, 424379864,
425790257, 451982486, 480927537, 481268653
]
# This works but takes a long time.
#neurons, roi_counts = fetch_neurons(NC())
neurons, roi_counts = fetch_neurons(NC(bodyId=bodyId))
assert len(neurons) == len(bodyId)
assert set(roi_counts['bodyId']) == set(bodyId)
neurons, roi_counts = fetch_neurons(NC(instance='APL_R'))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(instance='APL[^ ]*', regex=True))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(type='APL.*', regex=True))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(type=['.*01', '.*02'], regex=True))
assert len(neurons), "Didn't find any neurons of the given type pattern"
assert all(lambda t: t.endswith('01') or t.endswith('02')
for t in neurons['type'])
assert any(lambda t: t.endswith('01') for t in neurons['type'])
assert any(lambda t: t.endswith('02') for t in neurons['type'])
neurons, roi_counts = fetch_neurons(
NC(instance=['.*_L', '.*_R'], regex=True))
assert len(
neurons), "Didn't find any neurons of the given instance pattern"
assert all(lambda t: t.endswith('_L') or t.endswith('_R')
for t in neurons['instance'])
neurons, roi_counts = fetch_neurons(
NC(status=['Traced', 'Orphan'], cropped=False))
assert neurons.eval('status == "Traced" or status == "Orphan"').all()
assert not neurons['cropped'].any()
neurons, roi_counts = fetch_neurons(
NC(inputRois='AL(R)', outputRois='SNP(R)'))
assert all(['AL(R)' in rois for rois in neurons['inputRois']])
assert all(['SNP(R)' in rois for rois in neurons['outputRois']])
assert sorted(
roi_counts.query('roi == "AL(R)" and post > 0')['bodyId']) == sorted(
neurons['bodyId'])
assert sorted(
roi_counts.query('roi == "SNP(R)" and pre > 0')['bodyId']) == sorted(
neurons['bodyId'])
neurons, roi_counts = fetch_neurons(NC(min_pre=1000, min_post=2000))
assert neurons.eval('pre >= 1000 and post >= 2000').all()
def get_all_neurons_roi_counts():
crit = NC()
neuron_df, roi_counts_df = fetch_neurons(crit)
neuron_df = neuron_df[[
'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped',
'size'
]]
neuron_df.to_csv('all_neurons.csv', index=False)
roi_counts_df.to_csv('all_roi_counts', index=False)
def fetch_synapse_counts(bodies):
try:
c = neuprint_default_client()
except Exception:
c = Client(server, dataset)
ndf, cdf = fetch_neurons(NC(bodyId=bodies, label='Segment'),
client=c)
return ndf.set_index('bodyId')[['pre', 'post']].rename_axis('body')
def test_fetch_synapses(client):
nc = NC(type='ExR.*', regex=True, rois=['EB'])
sc = SC(rois=['FB', 'LAL(R)'], primary_only=True)
syn_df = fetch_synapses(nc, sc)
assert set(syn_df['roi']) == {'FB', 'LAL(R)'}
neuron_df, _count_df = fetch_neurons(nc)
syn_df = syn_df.merge(neuron_df[['bodyId', 'type']],
'left',
on='bodyId',
suffixes=['_syn', '_body'])
assert syn_df['type_body'].isnull().sum() == 0
assert syn_df['type_body'].apply(lambda s: s.startswith('ExR')).all()
def getBodyIDs(sourceParams, synapseParams):
print('Getting neurons/synapses...')
sourceNeurons = NC(**sourceParams)
if not (synapseParams == None):
synapseCriteria = SC(**synapseParams)
synapses = fetch_synapse_bodyIDs(source_criteria=sourceNeurons,
target_criteria=None,
synapse_criteria=synapseCriteria)
bodyIds = synapses['bodyId_post'].unique() #[0:max_number]
else:
neuron_df, roi_counts_df = fetch_neurons(sourceNeurons)
bodyIds = neuron_df.bodyId
synapses = None
return bodyIds, synapses
def doAlignmentTest(cell_type, neuprint_search):
# (1) Get TBar count based on atlas aligntment
tbar = pd.read_csv(os.path.join(data_dir, 'hemi_2_atlas', '{}_ito_tbar.csv'.format(cell_type)), header=0).iloc[:, 1:]
tbar.index = np.arange(1, 87)
include_inds_ito, name_list_ito = bridge.getItoNames()
tbar = tbar.loc[include_inds_ito, :]
tbar.index = name_list_ito
tbar = pd.DataFrame(tbar.sum(axis=1), columns=['sum'])
fh, ax = plt.subplots(2, 1, figsize=(4, 2))
vmin = 1
vmax = np.nanmax(tbar.to_numpy())
sns.heatmap(np.log10(tbar.T).replace(-np.inf, 0), ax=ax[1], cmap='cividis', cbar=False, xticklabels=[bridge.displayName(x) for x in tbar.index], yticklabels=False, vmin=0, vmax=np.log10(vmax))
# ax[1].set_title('Atlas\nregistration')
ax[1].tick_params(axis='both', which='major', labelsize=8)
# (2) Get TBar count according to Neuprint & Janelia region tags
Neur, _ = fetch_neurons(NeuronCriteria(type=neuprint_search, status='Traced', regex=True))
neuprint_rois = [bridge.ito_to_neuprint(x) for x in name_list_ito]
tbar_count = np.zeros((Neur.shape[0], len(neuprint_rois)))
for roi_ind, nr in enumerate(neuprint_rois):
for n_ind, roiInfo in enumerate(Neur.roiInfo):
if len(nr) > 1:
new_tbars = np.sum([roiInfo.get(x, {'pre': 0}).get('pre', 0) for x in nr])
else:
new_tbars = roiInfo.get(nr[0], {'pre': 0}).get('pre', 0)
tbar_count[n_ind, roi_ind] = new_tbars
tbar_neuprint = pd.DataFrame(tbar_count.sum(axis=0).T, index=name_list_ito, columns=['ct'])
sns.heatmap(np.log10(tbar_neuprint.T).replace(-np.inf, 0), ax=ax[0], cmap='cividis', cbar=False, xticklabels=False, yticklabels=False, vmin=0, vmax=np.log10(vmax))
# ax[0].set_title('Neuprint')
ax[0].tick_params(axis='both', which='major', labelsize=8)
cb = fh.colorbar(matplotlib.cm.ScalarMappable(norm=matplotlib.colors.SymLogNorm(vmin=vmin, vmax=vmax, base=10, linthresh=0.1, linscale=1), cmap="cividis"), ax=ax, shrink=1.0, label='T-Bars')
r, p = pearsonr(tbar.to_numpy().ravel(), tbar_neuprint.to_numpy().ravel())
print('r = {}'.format(r))
print(tbar.sum().sum())
print(tbar_neuprint.sum().sum())
return fh
def test_fetch_mitochondria(client):
nc = NC(type='ExR.*', regex=True, rois=['EB'])
mc = MC(rois=['FB', 'LAL(R)'],
mitoType='dark',
size=100_000,
primary_only=True)
mito_df = fetch_mitochondria(nc, mc)
assert set(mito_df['roi']) == {'FB', 'LAL(R)'}
assert (mito_df['mitoType'] == 'dark').all()
assert (mito_df['size'] >= 100_000).all()
neuron_df, _count_df = fetch_neurons(nc)
mito_df = mito_df.merge(neuron_df[['bodyId', 'type']],
'left',
on='bodyId',
suffixes=['_mito', '_body'])
assert mito_df['type'].isnull().sum() == 0
assert mito_df['type'].apply(lambda s: s.startswith('ExR')).all()
def main():
RESULTS_PKL_PATH = sys.argv[1]
if len(sys.argv) == 3:
PROCESSES = int(sys.argv[2])
else:
PROCESSES = 4
# Calculate the difference in resolution between the stored mito segmentation and neuron segmenation.
# If they differ, it must be by a power of 2.
mito_res = fetch_info(*MITO_SEG)["Extended"]["VoxelSize"][0]
assert mito_res % NEIGHBORHOOD_RES == 0
assert np.log2(mito_res / NEIGHBORHOOD_RES) == int(np.log2(mito_res / NEIGHBORHOOD_RES)), \
"This script assumes that the mito resolution and neighborhood resolution differ by a power of 2."
mito_res_scale_diff = int(np.log2(mito_res // NEIGHBORHOOD_RES))
with open(RESULTS_PKL_PATH, 'rb') as f:
mc_df = pickle.load(f)
new_names = {col: col.replace(' ', '_') for col in mc_df.columns}
new_names['result'] = 'proofreader_count'
mc_df = mc_df.rename(columns=new_names)
print("Evaluating mito count results")
results = compute_parallel(partial(_task_results, mito_res_scale_diff),
iter_batches(
mc_df.drop_duplicates('neighborhood_id'),
1),
total=len(mc_df),
processes=PROCESSES,
leave_progress=True,
ordered=False)
cols = [
'neighborhood_id', 'neighborhood_origin', 'proofreader_count',
'mito_id_count', 'mito_ids', 'mito_sizes', 'num_ccs', 'mito_cc_ids',
'mito_cc_sizes', 'ng_link'
]
df = pd.DataFrame(results, columns=cols)
# Add columns for cell type (from neuprint)
print("Fetching neuron cell types")
origins_df = pd.DataFrame(df['neighborhood_origin'].tolist(),
columns=[*'xyz'])
df['body'] = fetch_labels_batched(*NEURON_SEG,
origins_df[[*'zyx']].values,
processes=8)
neurons_df, _ = fetch_neurons(df['body'].unique())
neurons_df = neurons_df.rename(columns={
'bodyId': 'body',
'type': 'body_type',
'instance': 'body_instance'
})
df = df.merge(neurons_df[['body', 'body_type', 'body_instance']],
'left',
on='body')
df['body_type'].fillna("", inplace=True)
df['body_instance'].fillna("", inplace=True)
# Append roi column
print("Determining ROIs")
determine_point_rois(*NEURON_SEG[:2], NEUPRINT_CLIENT.primary_rois,
origins_df)
df['roi'] = origins_df['roi']
# Results only
path = 'mito-seg-counts.pkl'
print(f"Writing {path}")
with open(path, 'wb') as f:
pickle.dump(df, f)
path = 'mito-seg-counts.tab-delimited.csv'
print(f"Writing {path}")
df.to_csv(path, sep='\t', header=True, index=False)
# Full results (with task info columns)
df = df.merge(
mc_df.drop(columns=['neighborhood_origin', 'proofreader_count']),
'left',
on='neighborhood_id')
path = 'full-results-with-mito-seg-counts.pkl'
print(f"Writing {path}")
with open(path, 'wb') as f:
pickle.dump(df, f)
path = 'full-results-with-mito-seg-counts.tab-delimited.csv'
print(f"Writing {path}")
df.to_csv(path, sep='\t', header=True, index=False)
print("DONE")
def find_neurons(criteria):
neuron_df, roi_counts_df = fetch_neurons(criteria)
return neuron_df[[
'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped',
'size'
]]
def roistats_table(input_path,
neuprint_dataset,
voxel_col=None,
num_ranked=5,
exclude_none_roi=False,
neuprint_server='neuprint.janelia.org',
output_path=None):
"""
The RoiStats workflow produces a list of body-ROI pairs and corresponding voxel counts.
This function
- appends fractional columns to the absolute numbers,
- adds synapse counts (from neuprint),
- filters for the top-N ROIs for each body,
- and pivots (reshapes) the data so that the top-5 ROIs (and all stats)
can be viewed on a single row of a spreadsheet.
Args:
input_path:
CSV file produced by the RoiStats workflow, e.g. roi-stats.csv
neuprint_dataset:
Name of the neuprint dataset, e.v. 'hemibrain:v1.1'
voxel_col:
Name of the column which contains the voxel counts, e.g. voxels_s1
num_ranked:
How many ranked ROIs to include in the result columns
exclude_none_roi:
When computing totals (and fractions), should voxels that were not
part of any ROI be excluded from the total (denomitator)?
output_path:
If provided, export the results to the given path.
neuprint_server:
Which neuprint server to use to obtain synapse counts.
Returns:
Table of top-N ROI stats for each body.
"""
from neuprint import Client, fetch_neurons
c = Client(neuprint_server, neuprint_dataset)
stats = pd.read_csv(input_path)
if voxel_col is None:
vcs = [*filter(lambda c: c.startswith('voxels'), stats.columns)]
if len(vcs) != 1:
raise RuntimeError("Could not auto-determine voxel_col. Please provide the column name to use.")
voxel_col = vcs[0]
stats = stats.rename(columns={f'{voxel_col}': f'roi_{voxel_col}'})
if exclude_none_roi:
# Exclude <none> from totals so fractions are
# given in terms of neuropil totals only.
stats = stats.query('roi_id != 0')
# Compute totals and fractions
voxel_totals = stats.groupby('body')[f'roi_{voxel_col}'].sum().rename(f'total_{voxel_col}')
stats = stats.merge(voxel_totals, 'left', on='body')
stats['roi_voxels_frac'] = stats.eval(f'roi_{voxel_col} / total_{voxel_col}')
# Drop the '<none>' ROI, since that isn't conveniently available in the synapse
# data, and probably isn't of interest anyway.
# (Above, the '<none>' voxels were already counted in the totals (denominators)
# if necessary, but we never list '<none>' as an ROI column.)
stats = stats.query('roi_id != 0').drop(columns=['roi_id'])
# Fetch synapse counts from neuprint
bodies = stats['body'].unique()
logger.info(f"Fetching synapse counts from neuprint for {len(bodies)} bodies")
neuron_batches, roi_syn_batches = [], []
for bodies in tqdm_proxy(iter_batches(bodies, 10_000)):
n, r = fetch_neurons(bodies, client=c)
neuron_batches.append(n)
roi_syn_batches.append(r)
criteria = NC(rois=['PB', 'EB'])
# Example: Select traced neurons which intersect the PB ROI with at least 100 inputs (PSDs).
criteria = NC(inputRois=['PB'],
min_roi_inputs=100,
status='Traced',
cropped=False)
# ------------------------------------------
# Fetch neuron properties
# ------------------------------------------
''' Neuron properties and per-ROI synapse distributions can be obtained with fetch_neurons(). Two
dataframes are returned: one for neuron properties, and one for the counts of synapses in each ROI.
'''
neuron_df, roi_counts_df = fetch_neurons(
criteria
) # return properties and per-ROI synapse counts for a set of neurons
print(neuron_df[[
'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped', 'size'
]])
print(roi_counts_df.query('bodyId==5813128308')) # synapse counts for ROIs
# print(neuron_df.columns)
# neuron_df, roi_counts_df = fetch_neurons(NC(type='MBON.*', regex=True)) # get mushroom body output neurons
# ------------------------------------------
# Fetch connections
# ------------------------------------------
''' Find synaptic connection strengths between one set of neurons and another using fetch_adjacencies().
The “source” and/or “target” neurons are selected using NeuronCriteria. Additional parameters allow you
to filter by connection strength or ROI. Two DataFrames are returned, for properties of pre/post synaptic
neurons and per-ROI connection strengths (for each pre-post pair)
logging.info("Invalid cluster name: `%s`, skipping file", cluster_name)
continue
else:
logging.info("cluster name: %s", cluster_name)
file = open(os.path.join(cluster_dir, filename), 'r')
local_df[cluster_name] = [int(line.strip()) for line in file.readlines()]
file.close()
local_df.drop(columns=["type", "instance"], inplace=True)
logging.info("Finished constructing local neuron df:")
print(local_df.head())
print()
logging.info("Fetching neuron datafrom from neuprint")
neuprint_df, conn_df = fetch_neurons(local_df["bodyId"])
neuprint_df.fillna(value={"type": "None", "instance": "None"}, inplace=True)
print(neuprint_df.head())
print()
logging.info("Merging dataframes")
FB_node_df = pd.merge(local_df, neuprint_df, on="bodyId").rename(columns={"bodyId": "id", "type": "celltype"}).set_index("id")
print(FB_node_df.head())
print()
logging.info("Writing preprocessed node df to %s", output_node_csv)
FB_node_df.to_csv(output_node_csv)
logging.info("Loading edges and renaming columns from %s", edge_csv)
edge_df = pd.read_csv(edge_csv).rename(columns={"bodyId_pre": "node1", "bodyId_post": "node2"})
def computeConnectivityMatrix(neuprint_client, mapping):
"""
Compute region connectivity matrix from neuprint tags, for various metrics.
neuprint_client
mapping: mapping dict to bridge hemibrain regions to atlas regions
"""
rois = list(mapping.keys())
rois.sort()
WeakConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
MediumConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
StrongConnections = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
Connectivity = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
WeightedSynapseNumber = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
TBars = pd.DataFrame(data=np.zeros((len(rois), len(rois))),
index=rois,
columns=rois)
body_ids = [] # keep list of all cells connecting among regions
for roi_source in rois:
for roi_target in rois:
sources = mapping[roi_source]
targets = mapping[roi_target]
weak_neurons = 0
medium_neurons = 0
strong_neurons = 0
summed_connectivity = 0
weighted_synapse_number = 0
tbars = 0
for s_ind, sour in enumerate(
sources
): # this multiple sources/targets is necessary for collapsing rois based on mapping
for targ in targets:
Neur, Syn = fetch_neurons(
NeuronCriteria(
inputRois=sour,
outputRois=targ,
status='Traced',
cropped=False)) # only take uncropped neurons
outputs_in_targ = np.array([
x[targ]['pre'] for x in Neur.roiInfo
]) # neurons with Tbar output in target
inputs_in_sour = np.array([
x[sour]['post'] for x in Neur.roiInfo
]) # neuron with PSD input in source
n_weak = np.sum(
np.logical_and(outputs_in_targ > 0,
inputs_in_sour < 3))
n_medium = np.sum(
np.logical_and(
outputs_in_targ > 0,
np.logical_and(inputs_in_sour >= 3,
inputs_in_sour < 10)))
n_strong = np.sum(
np.logical_and(outputs_in_targ > 0,
inputs_in_sour >= 10))
# Connection strength for each cell := sqrt(input PSDs in source x output tbars in target)
conn_strengths = [
np.sqrt(x[targ]['pre'] * x[sour]['post'])
for x in Neur.roiInfo
]
# weighted synapses, for each cell going from sour -> targ:
# := output tbars (presynapses) in targ * (input (post)synapses in sour)/(total (post)synapses onto that cell)
weighted_synapses = [
Neur.roiInfo[x][targ]['pre'] *
(Neur.roiInfo[x][sour]['post'] / Neur.loc[x, 'post'])
for x in range(len(Neur))
]
new_tbars = [
Neur.roiInfo[x][targ]['pre'] for x in range(len(Neur))
]
# body_ids
body_ids.append(Neur.bodyId.values)
if Neur.roiInfo.shape[0] > 0:
summed_connectivity += np.sum(conn_strengths)
weighted_synapse_number += np.sum(weighted_synapses)
weak_neurons += n_weak
medium_neurons += n_medium
strong_neurons += n_strong
tbars += np.sum(new_tbars)
WeakConnections.loc[[roi_source], [roi_target]] = weak_neurons
MediumConnections.loc[[roi_source], [roi_target]] = medium_neurons
StrongConnections.loc[[roi_source], [roi_target]] = strong_neurons
Connectivity.loc[[roi_source], [roi_target]] = summed_connectivity
WeightedSynapseNumber.loc[[roi_source],
[roi_target]] = weighted_synapse_number
TBars.loc[[roi_source], [roi_target]] = tbars
body_ids = np.unique(
np.hstack(body_ids)
) # don't double count cells that contribute to multiple connections
return WeakConnections, MediumConnections, StrongConnections, Connectivity, WeightedSynapseNumber, TBars, body_ids
