def test_fetch_adjacencies(client):
bodies = [
294792184, 329566174, 329599710, 417199910, 420274150, 424379864,
425790257, 451982486, 480927537, 481268653
]
neuron_df, roi_conn_df = fetch_adjacencies(NC(bodyId=bodies),
NC(bodyId=bodies))
# Should not include non-primary ROIs (except 'NotPrimary')
assert not ({*roi_conn_df['roi'].unique()} - {*fetch_primary_rois()} -
{'NotPrimary'})
#
# For backwards compatibility with the previous API,
# You can also pass a list of bodyIds to this function (instead of NeuronCriteria).
#
bodies = [
294792184, 329566174, 329599710, 417199910, 420274150, 424379864,
425790257, 451982486, 480927537, 481268653
]
neuron_df2, roi_conn_df2 = fetch_adjacencies(bodies, bodies)
# Should not include non-primary ROIs (except 'NotPrimary')
assert not ({*roi_conn_df2['roi'].unique()} - {*fetch_primary_rois()} -
{'NotPrimary'})
assert (neuron_df.fillna('') == neuron_df2.fillna('')).all().all()
assert (roi_conn_df == roi_conn_df2).all().all()
# What happens if results are empty
neuron_df, roi_conn_df = fetch_adjacencies(879442155, 5813027103)
assert len(neuron_df) == 0
assert len(roi_conn_df) == 0
assert neuron_df.columns.tolist() == ['bodyId', 'instance', 'type']
def pn_kc_connections(properties=None,
sum_across_rois=False,
checks=True,
**kwargs):
"""
Returns `neuron_df`, `connection_df` as `neuprint.fetch_adjacencies`, but
only for PN->KC connections.
See also keyword arguments added by the `@fetch_function` decorator.
Keywords not covered above are passed to `neuprint.fetch_adjacencies`.
"""
if properties is None:
# TODO TODO TODO replace hardcoded properties w/ something enumerated
# using a cypher query
properties = nc
# TODO TODO TODO the docs make this sound like it might only return weights
# for connections shared by ALL neurons matched. is this true? see also the
# two kwargs dealing with minimum weights
# TODO consider using rois=[<appropriate str for calyx>] to just match
# those. compare results to those from manually filtering output not
# specifying those rois.
neuron_df, conn_df = nu.fetch_adjacencies(
# TODO if i end up factoring some of this fn into something to be shared
# across fns that get connections from one type to another, maybe just
# test whether input strs have asterisk in them, and set regex=True if
# so? first check if * is in any of the types that exist in the db!!!
NC(type='.*PN.*', regex=True),
NC(type='.*KC.*', regex=True),
# Default is just ['type','instance']
properties=properties,
**kwargs)
# TODO test the number of things returned above is the same as when doing
# the equivalent CONTAINS query in the web interface
# (also check just .* as suffix, not also prefix)
if checks:
# There CAN be (pre ID, post ID) duplicates, because weights are
# reported per ROI, so need to sum across ROIs if we just want
# the total weight between two neurons, ignoring where the synapses are.
assert not conn_df.duplicated(
[c for c in conn_df.columns if c != 'weight']).any()
# TODO TODO technically only do this check if include_nonprimary=False
# (so check kwargs) (if it's True, would need to do another check)
assert (set(conn_df.roi.unique()) -
set(nu.fetch_primary_rois()) == {'NotPrimary'})
if sum_across_rois:
conn_df = conn_df.groupby(['bodyId_pre',
'bodyId_post']).weight.sum().reset_index()
# No need to call filter_nontraced_or_cropped, at least as of 2020-05-31,
# where it had no effect.
return neuron_df, conn_df
def test_fetch_neurons(client):
bodyId = [
294792184, 329566174, 329599710, 417199910, 420274150, 424379864,
425790257, 451982486, 480927537, 481268653
]
# This works but takes a long time.
#neurons, roi_counts = fetch_neurons(NC())
neurons, roi_counts = fetch_neurons(NC(bodyId=bodyId))
assert len(neurons) == len(bodyId)
assert set(roi_counts['bodyId']) == set(bodyId)
neurons, roi_counts = fetch_neurons(NC(instance='APL_R'))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(instance='APL[^ ]*', regex=True))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(type='APL.*', regex=True))
assert len(neurons) == 1, "There's only one APL neuron in the hemibrain"
assert neurons.loc[0, 'type'] == "APL"
assert neurons.loc[0, 'instance'] == "APL_R"
neurons, roi_counts = fetch_neurons(NC(type=['.*01', '.*02'], regex=True))
assert len(neurons), "Didn't find any neurons of the given type pattern"
assert all(lambda t: t.endswith('01') or t.endswith('02')
for t in neurons['type'])
assert any(lambda t: t.endswith('01') for t in neurons['type'])
assert any(lambda t: t.endswith('02') for t in neurons['type'])
neurons, roi_counts = fetch_neurons(
NC(instance=['.*_L', '.*_R'], regex=True))
assert len(
neurons), "Didn't find any neurons of the given instance pattern"
assert all(lambda t: t.endswith('_L') or t.endswith('_R')
for t in neurons['instance'])
neurons, roi_counts = fetch_neurons(
NC(status=['Traced', 'Orphan'], cropped=False))
assert neurons.eval('status == "Traced" or status == "Orphan"').all()
assert not neurons['cropped'].any()
neurons, roi_counts = fetch_neurons(
NC(inputRois='AL(R)', outputRois='SNP(R)'))
assert all(['AL(R)' in rois for rois in neurons['inputRois']])
assert all(['SNP(R)' in rois for rois in neurons['outputRois']])
assert sorted(
roi_counts.query('roi == "AL(R)" and post > 0')['bodyId']) == sorted(
neurons['bodyId'])
assert sorted(
roi_counts.query('roi == "SNP(R)" and pre > 0')['bodyId']) == sorted(
neurons['bodyId'])
neurons, roi_counts = fetch_neurons(NC(min_pre=1000, min_post=2000))
assert neurons.eval('pre >= 1000 and post >= 2000').all()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--neuprint-server', '-n', default='neuprint.janelia.org')
parser.add_argument('--dataset', '-d')
parser.add_argument('--init', '-i', choices=['groundtruth', 'random'])
parser.add_argument('--verbose', '-v', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--min-weight', '-w', default=10, type=int)
args = parser.parse_args()
c = Client(args.neuprint_server, args.dataset)
export_dir = f"{c.dataset}-w{args.min_weight}-from-{args.init}"
os.makedirs(export_dir, exist_ok=True)
# Fetch connectome (and export)
with Timer("Fetching/exporting connectome", logger):
criteria = NC(status='Traced', cropped=False, client=c)
neuron_df, roi_conn_df = fetch_adjacencies(criteria, criteria, min_total_weight=args.min_weight, export_dir=export_dir, properties=['type', 'instance'], client=c)
conn_df = roi_conn_df.groupby(['bodyId_pre', 'bodyId_post'], as_index=False)['weight'].sum()
strong_connections_df, g, nbs, partition_df = infer_hierarchy(neuron_df,
conn_df,
args.min_weight,
args.init,
args.verbose,
args.debug)
with Timer("Exporting inference results", logger):
pickle.dump(g, open(f'{export_dir}/graph.pkl', 'wb'))
pickle.dump(nbs, open(f'{export_dir}/nested-block-state.pkl', 'wb'))
pickle.dump(partition_df, open(f'{export_dir}/partition_df.pkl', 'wb'))
pickle.dump(strong_connections_df, open(f'{export_dir}/strong_connections_df.pkl', 'wb'))
logger.info("DONE")
def get_all_neurons_roi_counts():
crit = NC()
neuron_df, roi_counts_df = fetch_neurons(crit)
neuron_df = neuron_df[[
'bodyId', 'instance', 'type', 'pre', 'post', 'status', 'cropped',
'size'
]]
neuron_df.to_csv('all_neurons.csv', index=False)
roi_counts_df.to_csv('all_roi_counts', index=False)
def fetch_synapse_counts(bodies):
try:
c = neuprint_default_client()
except Exception:
c = Client(server, dataset)
ndf, cdf = fetch_neurons(NC(bodyId=bodies, label='Segment'),
client=c)
return ndf.set_index('bodyId')[['pre', 'post']].rename_axis('body')
def gui_to_criteria():
# defaults for now, will be the function to tie GUI and backend together
# body_id_in = [387023620, 387364605, 416642425]
# instance_in = "OA-VPM3"
# type_in = 'PENPEN_b(PEN2)'
input_rois_in = ['AL(R)']
output_rois_in = ['AL(L)']
criteria_in = NC(inputRois=input_rois_in, outputRois=output_rois_in)
return criteria_in
def test_fetch_synapses(client):
nc = NC(type='ExR.*', regex=True, rois=['EB'])
sc = SC(rois=['FB', 'LAL(R)'], primary_only=True)
syn_df = fetch_synapses(nc, sc)
assert set(syn_df['roi']) == {'FB', 'LAL(R)'}
neuron_df, _count_df = fetch_neurons(nc)
syn_df = syn_df.merge(neuron_df[['bodyId', 'type']],
'left',
on='bodyId',
suffixes=['_syn', '_body'])
assert syn_df['type_body'].isnull().sum() == 0
assert syn_df['type_body'].apply(lambda s: s.startswith('ExR')).all()
def getBodyIDs(sourceParams, synapseParams):
print('Getting neurons/synapses...')
sourceNeurons = NC(**sourceParams)
if not (synapseParams == None):
synapseCriteria = SC(**synapseParams)
synapses = fetch_synapse_bodyIDs(source_criteria=sourceNeurons,
target_criteria=None,
synapse_criteria=synapseCriteria)
bodyIds = synapses['bodyId_post'].unique() #[0:max_number]
else:
neuron_df, roi_counts_df = fetch_neurons(sourceNeurons)
bodyIds = neuron_df.bodyId
synapses = None
return bodyIds, synapses
def test_fetch_simple_connections(client):
bodyId = [
294792184, 329566174, 329599710, 417199910, 420274150, 424379864,
425790257, 451982486, 480927537, 481268653
]
conn_df = fetch_simple_connections(NC(bodyId=bodyId))
assert set(conn_df['bodyId_pre'].unique()) == set(bodyId)
conn_df = fetch_simple_connections(None, NC(bodyId=bodyId))
assert set(conn_df['bodyId_post'].unique()) == set(bodyId)
APL_R = 425790257
conn_df = fetch_simple_connections(NC(instance='APL_R'))
assert (conn_df['bodyId_pre'] == APL_R).all()
conn_df = fetch_simple_connections(NC(type='APL'))
assert (conn_df['bodyId_pre'] == APL_R).all()
conn_df = fetch_simple_connections(None, NC(instance='APL_R'))
assert (conn_df['bodyId_post'] == APL_R).all()
conn_df = fetch_simple_connections(None, NC(type='APL'))
assert (conn_df['bodyId_post'] == APL_R).all()
conn_df = fetch_simple_connections(NC(bodyId=APL_R), min_weight=10)
assert (conn_df['bodyId_pre'] == APL_R).all()
assert (conn_df['weight'] >= 10).all()
conn_df = fetch_simple_connections(NC(bodyId=APL_R),
min_weight=10,
properties=['somaLocation'])
assert 'somaLocation_pre' in conn_df
assert 'somaLocation_post' in conn_df
conn_df = fetch_simple_connections(NC(bodyId=APL_R),
min_weight=10,
properties=['roiInfo'])
assert 'roiInfo_pre' in conn_df
assert 'roiInfo_post' in conn_df
assert isinstance(conn_df['roiInfo_pre'].iloc[0], dict)
def test_fetch_mitochondria(client):
nc = NC(type='ExR.*', regex=True, rois=['EB'])
mc = MC(rois=['FB', 'LAL(R)'],
mitoType='dark',
size=100_000,
primary_only=True)
mito_df = fetch_mitochondria(nc, mc)
assert set(mito_df['roi']) == {'FB', 'LAL(R)'}
assert (mito_df['mitoType'] == 'dark').all()
assert (mito_df['size'] >= 100_000).all()
neuron_df, _count_df = fetch_neurons(nc)
mito_df = mito_df.merge(neuron_df[['bodyId', 'type']],
'left',
on='bodyId',
suffixes=['_mito', '_body'])
assert mito_df['type'].isnull().sum() == 0
assert mito_df['type'].apply(lambda s: s.startswith('ExR')).all()
def fetch_adj_and_merge(criteria):
default_criteria_for_test = NC(inputRois='AB(L)')
neuron_df, conn_df = fetch_adjacencies(criteria, default_criteria_for_test)
conn_df = merge_neuron_properties(neuron_df, conn_df, ['type', 'instance'])
return conn_df
fetch_roi_hierarchy(include_subprimary=True,
mark_primary=True,
format='text')
) # Shows ROI hierarchy, with primary ROIs marked with '*'
# ------------------------------------------
# Neuron Search Criteria
# ------------------------------------------
''' Specify neurons of interest by bodyId, type/instance, or via a NeuronCriteria object. With
NeuronCriteria, you can specify multiple search constraints, including the ROIs in which matched
neurons must contain synapses.
'''
# Example: Select several, specific bodies
criteria = [387023620, 387364605, 416642425]
criteria = NC(bodyId=[387023620, 387364605, 416642425])
# Example: Select bodies by exact type
criteria = 'PEN_b(PEN2)'
criteria = NC(type='PENPEN_b(PEN2)')
# Example: Select bodies by exact instance
criteria = 'PEN(PB06)_b_L4'
criteria = NC(type='PEN(PB06)_b_L4')
# Example: Select bodies by type name pattern
criteria = NC(type='PEN.*', regex=True)
# Example: Select bodies by region (input or output)
criteria = NC(rois=['PB', 'EB'])
def test_fetch_synapses_and_closest_mitochondria(client):
syn_mito_distances = fetch_synapses_and_closest_mitochondria(
NC(type='ExR2'), SC(type='pre'))
assert len(syn_mito_distances), "Shouldn't be empty!"
def test_NeuronCriteria(client):
##
## basic_exprs()
##
assert NC(bodyId=123).basic_exprs() == ["n.bodyId = 123"]
assert NC('m', bodyId=123).basic_exprs() == ["m.bodyId = 123"]
assert NC(bodyId=[123, 456]).basic_exprs() == ["n.bodyId in [123, 456]"]
assert NC(instance="foo").basic_exprs() == ["n.instance = 'foo'"]
assert NC(instance="foo",
regex=True).basic_exprs() == ["n.instance =~ 'foo'"]
assert NC(instance=["foo", "bar"]).basic_exprs() == [
"n.instance in ['foo', 'bar']"
]
assert NC(instance=["foo", "bar"],
regex=True).basic_exprs() == ["n.instance =~ '(foo)|(bar)'"]
assert NC(type="foo").basic_exprs() == ["n.type = 'foo'"]
assert NC(type="foo", regex=True).basic_exprs() == ["n.type =~ 'foo'"]
assert NC(type=["foo", "bar"]).basic_exprs() == [
"n.type in ['foo', 'bar']"
]
assert NC(type=["foo", "bar"],
regex=True).basic_exprs() == ["n.type =~ '(foo)|(bar)'"]
assert NC(status="foo").basic_exprs() == ["n.status = 'foo'"]
assert NC(status="foo",
regex=True).basic_exprs() == ["n.status = 'foo'"] # not regex
assert NC(status=["foo", "bar"]).basic_exprs() == [
"n.status in ['foo', 'bar']"
]
assert NC(status=["foo", "bar"],
regex=True).basic_exprs() == ["n.status in ['foo', 'bar']"]
assert NC(cropped=True).basic_exprs() == ["n.cropped"]
assert NC(cropped=False).basic_exprs() == [
"(NOT n.cropped OR NOT exists(n.cropped))"
]
assert NC(
inputRois=['EB', 'FB'], outputRois=['FB', 'PB'],
roi_req='all').basic_exprs() == ['(n.`EB` AND n.`FB` AND n.`PB`)']
assert NC(inputRois=['EB', 'FB'], outputRois=['FB', 'PB'],
roi_req='any').basic_exprs() == ['(n.`EB` OR n.`FB` OR n.`PB`)']
assert NC(min_pre=5).basic_exprs() == ["n.pre >= 5"]
assert NC(min_post=5).basic_exprs() == ["n.post >= 5"]
assert NC(bodyId=np.arange(1, 6)).basic_exprs() == [
"n.bodyId in n_search_bodyId"
]
##
## basic_conditions()
##
assert NC().basic_conditions() == ""
assert NC().all_conditions() == ""
assert NC.combined_conditions([NC(), NC(), NC()]) == ""
bodies = [1, 2, 3]
assert NC(bodyId=bodies).basic_conditions(
comments=False) == "n.bodyId in [1, 2, 3]"
bodies = [1, 2, 3, 4, 5]
nc = NC(bodyId=bodies)
assert nc.global_with() == dedent(f"""\
WITH {bodies} as n_search_bodyId""")
assert nc.basic_conditions(
comments=False) == dedent("n.bodyId in n_search_bodyId")
statuses = ['Traced', 'Orphan']
nc = NC(status=statuses)
assert nc.basic_conditions(comments=False) == f"n.status in {statuses}"
statuses = ['Traced', 'Orphan', 'Assign', 'Unimportant']
nc = NC(status=statuses)
assert nc.global_with() == dedent(f"""\
WITH {statuses} as n_search_status""")
assert nc.basic_conditions(comments=False) == "n.status in n_search_status"
# If None is included, then exists() should be checked.
statuses = ['Traced', 'Orphan', 'Assign', None]
nc = NC(status=statuses)
assert nc.global_with() == dedent(f"""\
WITH ['Traced', 'Orphan', 'Assign'] as n_search_status""")
assert nc.basic_conditions(comments=False) == dedent(
"n.status in n_search_status OR NOT exists(n.status)")
def get_all_connections():
connections_crit = NC()
neuron_df, conn_df = fetch_adjacencies(connections_crit, None)
print('got adjacencies!!!!!!!!!!!!!!!!!!!!!!')
conn_df.to_csv('all_connections.csv', index=False)
print('got csv file!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')