def test_members():
set_default_client(None)
assert default_client() is None
c = Client(NEUPRINT_SERVER, DATASET)
assert c.server == f'https://{NEUPRINT_SERVER}'
assert c.dataset == DATASET
assert default_client() is c
df = c.fetch_custom("MATCH (m:Meta) RETURN m.primaryRois as rois")
assert isinstance(df, pd.DataFrame)
assert df.columns == ['rois']
assert len(df) == 1
assert isinstance(df['rois'].iloc[0], list)
assert isinstance(c.fetch_available(), list)
assert isinstance(c.fetch_help(), str)
assert c.fetch_server_info() is True
assert isinstance(c.fetch_version(), str)
assert isinstance(c.fetch_database(), dict)
assert isinstance(c.fetch_datasets(), dict)
assert isinstance(c.fetch_db_version(), str)
assert isinstance(c.fetch_profile(), dict)
assert isinstance(c.fetch_token(), str)
assert isinstance(c.fetch_daily_type(), tuple)
assert isinstance(c.fetch_roi_completeness(), pd.DataFrame)
assert isinstance(c.fetch_roi_connectivity(), pd.DataFrame)
assert isinstance(c.fetch_roi_mesh('AB(R)'), bytes)
assert isinstance(c.fetch_skeleton(EXAMPLE_BODY), pd.DataFrame)
def get_body_ids_from_roi(roi,
server,
token,
hemibrain_neuron=True,
pre_threshold=0,
post_threshold=0,
total_threshold=0):
"""
Args:
roi (str): Neuropil abreviation from Neuprint
server (str): Neuprint server URL
token (str): Neuprint auth token
hemibrain_neuron (bool): Specifies whether to search
Hemibrain-Neuron of Hemibrain-Segment.
pre_threshold (int): Requires bodies meet threshold of presynaptic sites
in the ROI
post_threshold (int): Requires bodies meet threshold of postsynaptic sites
in the ROI
total_threshold (int): Requires bodies meet threshold of total synaptic sites
in the ROI
Returns:
Dataframe containing body IDs that innervate the ROI
"""
query_template = (
'MATCH (n:`{HEMIBRAIN}`)\n'
'WHERE n.{ROI}\n'
'RETURN n.bodyId AS ID, n.name AS NAME, n.roiInfo AS ROIINFO'
)
# Start Neuprint python client
client = Client(server, token)
client.fetch_version()
if hemibrain_neuron:
query = query_template.format(HEMIBRAIN='hemibrain-Neuron', ROI=roi)
else:
query = query_template.format(HEMIBRAIN='hemibrain-Segment', ROI=roi)
results = client.fetch_custom(query)
results['ROIINFO'] = results['ROIINFO'].apply(ast.literal_eval)
results['PRE'] = results['ROIINFO'].apply(lambda x: int(x[roi]['pre']))
results['POST'] = results['ROIINFO'].apply(lambda x: int(x[roi]['post']))
results = results[results['PRE'] + results['POST'] >= total_threshold]
results = results[results['PRE'] >= pre_threshold]
results = results[results['POST'] >= post_threshold]
return results[['ID', 'NAME', 'PRE', 'POST']].reset_index()
SERVER = 'emdata1.int.janelia.org:11000'
np = Client(SERVER)
# find nodes in the graph
roifilter = ""
roifilterset = set()
ignorelabels = set(["location", "timeStamp", "confidence", "type"])
connections = {}
points = {}
roirent = {}
if rois is not None and len(rois) > 0:
# traverse one ROI at a time
query = "MATCH (m :Meta:hemibrain) RETURN m.superLevelRois"
res = np.fetch_custom(query)
major_rois = set(res.iloc[0, 0])
for iter1 in range(len(rois)):
print("Processing ROI:", rois[iter1])
# warn if ROI is not a super level ROI
if rois[iter1] not in major_rois:
print("ROI not a major ROI, compartment stats might be wrong")
# calculate pins vs compute
totalcompute = 0
totalpins = 0
query = "MATCH (n :`hemibrain-Neuron`) WHERE (n.status=\"Roughly traced\" OR n.status=\"Prelim Roughly traced\" OR n.status=\"Traced\" OR n.status=\"Leaves\") AND n.`" + rois[
iter1] + "` RETURN n.roiInfo AS roiInfo, n.bodyId AS bodyid"
res = np.fetch_custom(query)
class NeuPrintExecutor(Neo4jExecutor):
"""
A NeuPrintExecutor may be used to access an existing neuPrint server.
This class converts a DotMotif motif object into a neuPrint-compatible
query. Not all neuPrint datatypes or query types are available, but this
adds complete support for DotMotif motif searches by passing raw Cypher
queries to the neuPrint server over the HTTP API.
Note that the neuPrint default timeout is quite short, and slower motif
queries may not run in time.
"""
def __init__(self, host: str, dataset: str, token: str) -> None:
"""
Create a new NeuPrintExecutor that points to a deployed neuPrint DB.
Arguments:
host (str): The host of the neuPrint server (for example,
'neuprint.janelia.org')
dataset (str): The name of the dataset to reference (for example,
'hemibrain:v1.1`)
token (str): The user's neuPrint access token. To retrieve this
token, go to https://[host]/account.
Returns:
None
"""
self._created_container = False
self.host = host
self.dataset = dataset
self.token = token
self.client = Client(host, dataset=self.dataset, token=self.token)
def run(self, cypher: str) -> pd.DataFrame:
"""
Run an arbitrary cypher command.
You should usually ignore this, and use .find() instead.
Arguments:
cypher (str): The command to run
Returns:
The result of the cypher query
"""
return self.client.fetch_custom(cypher)
def count(self, motif: dotmotif, limit=None) -> int:
"""
Count a motif in a larger graph.
Arguments:
motif (dotmotif.dotmotif): The motif to search for
Returns:
int: The count of this motif in the host graph
"""
qry = self.motif_to_cypher(motif,
count_only=True,
static_entity_labels=_DEFAULT_ENTITY_LABELS)
if limit:
qry += f" LIMIT {limit}"
res = self.client.fetch_custom(qry)
print(res)
return int(res.to_numpy())
def find(self, motif: dotmotif, limit=None) -> pd.DataFrame:
"""
Find a motif in a larger graph.
Arguments:
motif (dotmotif.dotmotif): The motif to search for
Returns:
pd.DataFrame: The results of the search
"""
qry = self.motif_to_cypher(motif,
static_entity_labels=_DEFAULT_ENTITY_LABELS)
if limit:
qry += f" LIMIT {limit}"
return self.client.fetch_custom(qry)
@staticmethod
def motif_to_cypher(motif: dotmotif,
count_only: bool = False,
static_entity_labels: dict = None) -> str:
"""
Convert a motif to neuprint-flavored Cypher.
This is currently a thin passthrough for Neo4jExecutor.motif_to_cypher.
"""
static_entity_labels = static_entity_labels or _DEFAULT_ENTITY_LABELS
return Neo4jExecutor.motif_to_cypher(motif, count_only,
static_entity_labels)
# ------------------------------------------
'''Tip: Inspect all cypher queries by enabling debug logging'''
# setup_debug_logging()
# ------------------------------------------
# Execute a custom query
# ------------------------------------------
'''This query will return all neurons in the ROI ‘AB’ that have greater than 10 pre-synaptic sites.
Results are ordered by total synaptic sites (pre+post)'''
query = """\
MATCH (n :Neuron {`AB(R)`: true})
WHERE n.pre > 10
RETURN n.bodyId AS bodyId, n.name AS name, n.pre AS numpre, n.post AS numpost
ORDER BY n.pre + n.post DESC
"""
results = c.fetch_custom(query)
print(f"Found {len(results)} results")
print(results.head())
# ------------------------------------------
# ROIs
# ------------------------------------------
'''In neuprint, each neuron is annotated with the list of regions (ROIs) it intersects, along
with the synapse counts in each. The ROIs comprise a hierarchy, with smaller ROIs nested within
larger ROIs. Furthermore, primary ROIs are guaranteed not to overlap, and they roughly tile the
entire brain (with some gaps)
'''
'''For a quick overview of the ROI hierarchy, use fetch_roi_hierarchy()'''
print(
fetch_roi_hierarchy(include_subprimary=True,
mark_primary=True,
nest.Simulate(float(exp_length))
#v = nest.GetStatus(voltmeter)[0]['events']['V_m'] # volts
s = nest.GetStatus(spikedetector)[0]['events'] # spikes
np.save(
root + '/simulations/droso_' + stimulus_odor + '_' + simulation_id +
'.npy', np.array(s, dtype=object))
ntnsubstatus("Simulation name: droso_" + stimulus_odor + "_" +
simulation_id)
#******************************************************************************#
#Run everything
if __name__ == "__main__":
# If we need to download data from Janelia do so
if not args.disable_build or not args.disable_design:
ntnstatus('Downloading neuron data from Janelia')
token = list(open(token_address))[0].replace('\n', '')
c = Client('neuprint.janelia.org', dataset=dataset, token=token)
q = """MATCH (n :Neuron )
WHERE n.instance <>'{}'
RETURN n.bodyId AS bodyId, n.type AS type
ORDER BY n.bodyId ASC
"""
results = c.fetch_custom(q)
if not args.disable_build:
build_matrix(results)
if not args.disable_design:
design_stimuli(results)
if not args.disable_simulate:
run_nest_simulation()
class Graph:
"""
Graph used to construct, store, and transform neuron connectivity graphs.
Attributes:
_client (Client): Neuprint API client.
depth (int): Depth of search to build connectivity graph.
graph (Dataframe): Rows are edges in connectivity graph.
paths (Dataframe): Rows are paths in the format N0, W_1, N_1, ..., W_n, N_n.
sources (list of int): List of source body IDs.
targets (list of int): List of target body IDs.
"""
def __init__(self, sources, targets, depth, verbose=False):
""" Initialization method.
Args:
sources (list of int): List of source body IDs.
targets (list of int): List of target body IDs.
depth (int): Depth of search to build connectivity graph.
Returns:
None
"""
self._client = None
self.verbose = verbose
self.depth = int(depth)
self.graph = pd.DataFrame(columns=['SRC', 'DES', 'WEIGHT', 'LAYER'])
self.paths = pd.DataFrame()
self.sources = list(sources)
self.targets = list(targets)
def _verbose_print(self, msg):
""" Verbocity print statement
Args:
msg (str): Message that should be printed for user
verbose (bool): If True print message
Returns:
None
"""
if self.verbose:
print(msg)
def start_client(self, server, token):
""" Creates a Neuprint API Client instance.
Args:
server (str): URL of Neuprint server.
token (str): Authentication token for Neuprint.
Returns:
None
"""
try:
self._client = Client(server, token)
except RuntimeError as error:
raise RuntimeError("Issue instantiating Neuprint client:", error)
def _query_downstream_partners(self, ids, threshold, hemibrain_neuron=True):
""" Query Neuprint for downstream partners of a list of body IDs.
Args:
ids (list of int): List of body IDs to search from.
threshold (int): Minimum connection strength between partners.
hemibrain_neuron (bool): Specifies whether to search
Hemibrain-Neuron of Hemibrain-Segment.
Returns:
Dataframe containing the connection pairs between query IDs
and downstream partners.
"""
query_template = (
'WITH {IDS} AS SRC\n'
'UNWIND SRC AS ID\n'
'MATCH (n:`{HEMIBRAIN}`)-[w:ConnectsTo]->(m:`{HEMIBRAIN}`)\n'
'WHERE n.bodyId = ID AND w.weight >= {THRESHOLD}\n'
'RETURN n.bodyId AS SRC, m.bodyId AS DES, w.weight AS WEIGHT'
)
if hemibrain_neuron:
query = query_template.format(
HEMIBRAIN='hemibrain-Neuron', IDS=list(ids), THRESHOLD=threshold
)
else:
query = query_template.format(
HEMIBRAIN='hemibrain-Segment', IDS=list(ids), THRESHOLD=threshold
)
# print(query)
results = self._client.fetch_custom(query)
return results
def _prune_graph(self, graph):
""" Removes paths that do not include target bodies
Starting from the top-most layer, remove all nodes from a layer that
are not target bodies and do not connect to a body in the previous layer.
Args:
graph (Dataframe): Rows are edges of the connectivity graph.
verbose (bool): When true print data regarding execution and
progress.
Returns:
Dataframe with edges only edges that are part of pathways to targets
"""
frontier = set(self.targets)
pruned = pd.DataFrame(columns=['SRC', 'DES', 'WEIGHT', 'LAYER'])
for layer in reversed(range(max(graph['LAYER']))):
layer_edges = graph[graph['LAYER'] == layer + 1]
lc1 = len(layer_edges)
layer_edges = layer_edges[layer_edges.DES.isin(frontier)]
pruned = pruned.append(layer_edges, ignore_index=True)
frontier = set(layer_edges.SRC.values) | frontier
self._verbose_print('Layer {LAYER}: {NUM_EDGES} edges'
.format(LAYER=layer + 1, NUM_EDGES=lc1))
self._verbose_print('Layer {LAYER} Pruned: {NUM_EDGES} edges'
.format(LAYER=layer + 1, NUM_EDGES=len(layer_edges)))
return pruned.reset_index(drop=True)
def make_graph(self, threshold=1):
""" Create connectivity graph of all downstream neurons
Use breadth first search to find all bodies within N
hops downstream of the source IDs. Does not explore
past nodes that have been visited or that are part of
target set.
Args:
threshold (int, optional): Connection weight threshold between bodies
verbose (bool, optional): Whether to print data
Returns:
Dataframe containing connectivity graph edges and connection weights.
"""
frontier = set(self.sources)
explored = set(self.targets)
graph = pd.DataFrame(columns=['SRC', 'DES', 'WEIGHT', 'LAYER'])
self._verbose_print('\nBuilding graph')
for i in range(self.depth):
self._verbose_print('Retrieving layer {LAYER} bodies'.format(LAYER=str(i+1)))
layer_bodies = self._query_downstream_partners(frontier, threshold)
layer_bodies['LAYER'] = i + 1
explored = frontier | explored
frontier = set(layer_bodies['DES'].values) - explored
graph = graph.append(layer_bodies, ignore_index=True)
self._verbose_print('Layer {LAYER}: {NUM} bodies'.format(NUM=len(frontier), LAYER=i+1))
self._verbose_print('\nPruning graph of edges that do not lead to target bodies')
self.graph = self._prune_graph(graph)
return self.graph
def graph_to_csv(self, file_name):
""" Save graph to CSV
Args:
file_name (str): desired destination file for graph data
Returns:
None
"""
self.graph.to_csv(file_name, index=False)
def paths_to_csv(self, file_name):
""" Save path to CSV
Args:
file_name (str): desired destination file for path data
Returns:
None
"""
self.paths.to_csv(file_name, index=False)
def compute_paths(self):
""" Get all paths from from source bodies to targets.
Find all permutations of edges that create pathways to target bodies.
Starting from the second deepest layer, join layer N-1 DES column
on layer N SRC column.
Returns:
"""
if self.graph.empty:
raise Exception('No graph is not defined.')
max_layer = self.graph['LAYER'].max()
right = self.graph[self.graph['LAYER'] == max_layer][['SRC', 'WEIGHT', 'DES']]
right = right.rename(
index=str,
columns={
'SRC': 'N_{LAYER}'.format(LAYER=str(max_layer - 1)),
'WEIGHT': 'W_{LAYER}'.format(LAYER=str(max_layer)),
'DES': 'N_{LAYER}'.format(LAYER=str(max_layer)),
},
)
for layer in reversed(range(max_layer - 1)):
output_layer = 'N_{LAYER}'.format(LAYER=str(layer))
connection_weight = 'W_{LAYER}'.format(LAYER=str(layer + 1))
input_layer = 'N_{LAYER}'.format(LAYER=str(layer + 1))
left = self.graph[self.graph['LAYER'] == layer + 1][
['SRC', 'WEIGHT', 'DES']
]
left = left.rename(
index=str,
columns={
'SRC': output_layer,
'WEIGHT': connection_weight,
'DES': input_layer,
},
)
right = pd.merge(left, right, on=input_layer, how='left')
self.paths = right