def __init__(self,
server,
dataset,
tag="",
neuron_pre=2,
neuron_post=10,
verify=True):
self.client = neu.Client(server, verify=verify)
self.dataset = dataset
self.tag = tag
# criteria for Segment to be a Neuron
self.neuron_pre = neuron_pre
self.neuron_post = neuron_post
def __init__(self,
neuprint_endpoint=None,
neuprint_dataset=None,
neuprint_token=None,
catmaid_endpoint=None):
if neuprint_endpoint and neuprint_dataset and neuprint_token:
self.neuprint_client = neuprint.Client(neuprint_endpoint,
dataset=neuprint_dataset,
token=neuprint_token)
elif catmaid_endpoint:
self.rm = pymaid.CatmaidInstance(catmaid_endpoint, '', '', '')
#no self?
else:
self.neuprint_client = None
self.vc = VfbConnect(neo_endpoint="http://kb.p2.virtualflybrain.org")
if not "source" in jsonSynapses:
print("JSON 'synapses' contains no 'source' key")
quit()
synapseSource = jsonSynapses["source"]
if synapseSource.startswith("http"):
i1 = synapseSource.find("://")
i2 = synapseSource.find("/?dataset=")
if i2 == -1:
server = synapseSource[i1:]
dataset = None
else:
server = synapseSource[i1 + len("://"):i2]
dataset = synapseSource[i2 + len("/?dataset="):]
dataset = dataset.replace("%3A", ":")
client = neuprint.Client(server, dataset=dataset)
client.fetch_version()
for synapseSetName, synapseSetSpec in jsonSynapses.items():
if synapseSetName == "source":
continue
if not "neuron" in synapseSetSpec:
print("Error: synapse set '{}' is missing 'neuron'\n".format(synapseSetName))
continue
body = synapseSetSpec["neuron"]
if not "type" in synapseSetSpec:
print("Error: synapse set '{}' is missing 'type'\n".format(synapseSetName))
continue
type = synapseSetSpec["type"]
# TODO maybe unconditionally monkey patch neuprint.Client on init, just to
# add a warning if manually creating clients (saying when it is / isn't
# appropriate) (maybe + kwarg to disable warning, also mentioned in
# warning)? (might need care to not cause same warning when neuprint
# itself copies code... maybe check the __file__ of the calling code
# (possible?)?
# TODO TODO TODO before any of the above, just put a line in a README.md
# explaining that users should prefer to access my client rather than
# making their own, and they should only use it for the list of fns below
# (the ones only the client can do)
# TODO TODO or maybe just monkey patch the client so it can't make the calls
# provided by the others? (unless it breaks internals, which it very well
# might)
# TODO or maybe only instantiate this inside my own fns (like in fetch_function
# wrapped fn part + other places that would need client) and never present it?
client = nu.Client('neuprint.janelia.org', 'hemibrain:v1.0.1')
# This seems to be the most up-to-date documentation on what properties are
# availble: https://neuprint.janelia.org/public/neuprintuserguide.pdf
#maybe_missing = ['primaryNeurite', 'somaLocation', 'somaRadius', 'timestamp']
# (just a hack to hide warning for now. commented above is accurate **wrt PDF**)
maybe_missing = ['somaLocation', 'somaRadius']
# These don't include the "<rois>: boolean" properties from the docs, which seem
# to only be defined when True, or only for sets where some have the True
# value...
# TODO TODO (as some comments say elsewhere) still try to replace this w/
# queries that enumerate all available properties
userguide_neuron_properties = [
'bodyId', 'cropped', 'instance', 'post', 'pre', 'roiInfo', 'size',
'status', 'statusLabel', 'type'
import neuprint as neu
import csv
import pandas as pd
import numpy as np
import ast
import os
from collections import defaultdict, OrderedDict
from multiprocess import Pool
#client = neu.Client('emdata1.int.janelia.org:13000', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NTc5NzcyMTR9.uoMuun4AQ82VI7qUjO7f0G5CKOUX4KqAIF89CkQN4do')
client = neu.Client(
'https://neuprint.janelia.org',
dataset='hemibrain:v1.1',
token=
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NTc5NzcyMTR9.uoMuun4AQ82VI7qUjO7f0G5CKOUX4KqAIF89CkQN4do'
)
import chart_studio.plotly as py
import plotly.graph_objects as go
import collections
import operator
from copy import deepcopy
import seaborn as sns
GFInputs = [
1102761868, 454347237, 5812983519, 829162792, 829155062, 642262407,
5813022764, 642263196, 1815947603, 1254331599, 1102412245, 1876993474,
2126296267, 1506382974, 1288650927, 1072059176, 5813068048, 1722483751,
1073428001, 1381730321, 1594562431, 1443450601, 1722177539, 1691885573,
1412761053, 1504850262, 1630332999, 1507139129, 5813065458, 1753751888,
1602704738, 1631019730, 1628291400, 1691310419, 1384712714, 1037972384,
def find_violations(neuprint_addr, dataset, roilist):
client = neu.Client('https://emdata1.int.janelia.org:11000')
# neuron extractor query (restrict to .*raced for now)
extractquery = f"MATCH (n :`{dataset}-Neuron`) WHERE n.status=~'.*raced' {{}} RETURN n.bodyId as bodyid"
# get roi violation set (default all)
checkrois = None
if roilist is not None and len(roilist) > 0:
checkrois = set(roilist)
# restrict neurons to these ROIs
roistring = "AND ("
for roi in roilist:
roistring += f"n.{roi} OR "
roistring = roistring[0:-4]
roistring += ")"
extractquery = extractquery.format(roistring)
else:
roiquery = f"MATCH (m :Meta) WHERE m.dataset=\"{dataset}\" RETURN m.superLevelRois AS rois"
roires = client.fetch_custom(roiquery)
checkrois = set(roires["rois"].iloc[0])
extractquery = extractquery.format("")
# extract working set
candbodies_res = client.fetch_custom(extractquery)
candbodies = candbodies_res["bodyid"].tolist()
candbodies_set = set(candbodies)
# query db X neurons at a time (restrict to traced neurons for the targets)
# (connections can be one way since all we care about connections only within the body set
# (note: m cannot be restricted to body set up front because the calls are run in minibatches)
outputsquery = f"WITH {{}} AS TARGETS MATCH(x :`{dataset}-ConnectionSet`)-\
[:From]->(n :`{dataset}-Neuron`) WHERE n.bodyId in TARGETS WITH collect(x) as\
csets, n UNWIND csets as cset MATCH (cset)-[:To]->(m :`{dataset}-Neuron`) where\
(m.status=~'.*raced') RETURN n.bodyId AS body1, cset.roiInfo AS info, m.bodyId AS body2"
total_conns = 0
total_splits = 0
total_msplits = 0
candidate_connections = []
print(f"Candidate bodies: {len(candbodies)}")
# find candidate split bodies (might as well compute
# the number of split neurons as well)
for iter1 in range(0, len(candbodies), 50):
#for iter1 in range(1100, len(candbodies), 50):
#if iter1 == 1200:
# break
print(f"fetch batch: {iter1}")
currlist = candbodies[iter1:iter1 + 50]
restrictedquery = outputsquery.format(currlist)
res = client.fetch_custom(restrictedquery)
for idx, row in res.iterrows():
if (row["body1"] == row["body2"]) or (row["body2"]
not in candbodies_set):
continue
roiconns = json.loads(row["info"])
roiconns_keys = set(roiconns.keys())
inter_rois = checkrois.intersection(roiconns_keys)
if len(inter_rois) > 0:
total_conns += 1
if len(inter_rois) > 1:
total_splits += 1
if len(inter_rois) > 2:
total_msplits += 1
roi2count = {}
for roi in inter_rois:
roi2count[roi] = roiconns[roi]["post"]
candidate_connections.append(
(row["body1"], row["body2"], roi2count))
print(
f"Total connection pairs examined: {total_conns}\nTotal split across ROIs: {total_splits}\nTotal multi-splits across ROIs: {total_msplits}"
)
# distance threshold
pixel_threshold = 400
# global stats
roipairs_numvio = 0
roipairs_wgtvio = 0
real_roi_splits = 0
roi_scores = {}
roi_bodies = {}
for connpair in candidate_connections:
(body1, body2, roi2count) = connpair
locquery = f"MATCH (n :`{dataset}-Neuron` {{bodyId: {body1}}})<-[:From]-(x :ConnectionSet)-[:To]->(m :`{dataset}-Neuron` {{bodyId: {body2}}}) MATCH (x)-[:Contains]->(s :PostSyn) RETURN s AS synapse"
resloc = client.fetch_custom(locquery)
# check each location
roipsd = {}
for idx, row in resloc.iterrows():
synapse = row["synapse"]
for (roi, count) in roi2count.items():
if roi in synapse:
roipsd[roi] = synapse["location"]["coordinates"]
# look at all pairs of ROIs
roipsd2 = list(roipsd.items())
for iter1 in range(0, len(roipsd2)):
for iter2 in range(iter1 + 1, len(roipsd2)):
roi1, loc1 = roipsd2[iter1]
roi2, loc2 = roipsd2[iter2]
loc1 = numpy.array(loc1)
loc2 = numpy.array(loc2)
diff = (loc1 - loc2)**2
sumsq = diff.sum()
dist = sumsq**(1 / 2)
if dist < pixel_threshold:
roipairs_numvio += 1
roiwgt = min(roi2count[roi1], roi2count[roi2])
roipairs_wgtvio += roiwgt
if roi1 > roi2:
roi1, roi2 = roi2, roi1
if (roi1, roi2) not in roi_scores:
roi_scores[(roi1, roi2)] = 0
roi_bodies[roi1 + "-" + roi2] = []
roi_scores[(roi1, roi2)] += roiwgt
roi_bodies[roi1 + "-" + roi2].append(
(roiwgt, body1, body2))
else:
real_roi_splits += 1
# print stats
print(
f"Total violations: {roipairs_numvio}\nWeighted violations: {roipairs_wgtvio}\nNum real ROI splits: {real_roi_splits}\nBad ROI boundaries: {len(roi_scores)}"
)
# sort roi scores and print out worst pairs as just commma separated; dump roi_bodies to file
sorted_rois = list(zip(roi_scores.values(), roi_scores.keys()))
sorted_rois.sort()
sorted_rois.reverse()
print(sorted_rois)
fout = open("connviolations.json", 'w')
for key, bodies in roi_bodies.items():
bodies.sort()
bodies.reverse()
fout.write(json.dumps(roi_bodies))
fout.close()
return
NeuPrint queries that are relevant to a target body ID and a target ROI.
They have different "hop" amounts - number of interneurons between target and input
"""
import neuprint as neu
import pandas as pd
import numpy as np
from collections import defaultdict
import statistics
# 13000 server
# client = neu.Client('emdata1.int.janelia.org:13000/?dataset=hemibrain', token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjM3NzA2MTR9.xNL4c8vg8tN4KKrLUiHEgy_9Wlf4tDoGCXEKlLs8lLU')
# test server
client = neu.Client(
'neuprint-test.janelia.org',
dataset='hemibrain:v1.0.1',
token=
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUtGMDVuRFRUOHBhUHFCT3dMNk5nblZHVENIR2xGWEp0QS9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjQ5ODIyODh9.UjUHbKZQpReYOShvtKMzZWxJE-saJ4FWN6iD9hAcZaQ'
)
'''
These MAIN functions should be run by user and are dependent on other functions in the file
'''
# main functions
def twoHopMain():
print(
'\nUse the answers in parentheses to answer, otherwise will not run properly'
'\nExample Circuit: x -> y -> target\n')
target = input('\nEnter the targeted body ID: ')
ROI = input('\nEnter the targeted ROI: ')
ROI = str(ROI)
def load_roi_label_volume(server,
uuid,
rois_or_neuprint,
box_s5=[None, None],
export_path=None,
export_labelmap=None):
"""
Fetch several ROIs from DVID and combine them into a single label volume or mask.
The label values in the returned volume correspond to the order in which the ROI
names were passed in, starting at label 1.
This function is essentially a convenience function around fetch_combined_roi_volume(),
but in this case it will optionally auto-fetch the ROI list, and auto-export the volume.
Args:
server:
DVID server
uuid:
DVID uuid
rois_or_neuprint:
Either a list of ROIs or a neuprint server from which to obtain the roi list.
box_s5:
If you want to restrict the ROIs to a particular subregion,
you may pass your own bounding box (at scale 5).
Alternatively, you may pass the name of a segmentation
instance from DVID whose bounding box will be used.
export_path:
If you want the ROI volume to be exported to disk,
provide a path name ending with .npy or .h5.
export_labelmap:
If you want the ROI volume to be exported to a DVID labelmap instance,
Provide the instance name, or a tuple of (server, uuid, instance).
Returns:
(roi_vol, roi_box), containing the fetched label volume and the
bounding box it corresponds to, in DVID scale-5 coordinates.
Note:
If you have a list of (full-res) points to extract from the returned volume,
pass a DataFrame with columns ['z','y','x'] to the following function.
If you already downloaded the roi_vol (above), provide it.
Otherwise, leave out those args and it will be fetched first.
Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int).
>>> from neuclease.dvid import determine_point_rois
>>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box)
"""
if isinstance(box_s5, str):
# Assume that this is a segmentation instance whose dimensions should be used
# Fetch the maximum extents of the segmentation,
# and rescale it for scale-5.
seg_box = fetch_volume_box(server, uuid, box_s5)
box_s5 = round_box(seg_box, (2**5), 'out') // 2**5
box_s5[0] = (0, 0, 0)
if export_labelmap:
assert isinstance(box_s5, np.ndarray)
assert not (box_s5 % 64).any(), \
("If exporting to a labelmap instance, please supply "
"an explicit box and make sure it is block-aligned.")
if isinstance(rois_or_neuprint, (str, neuprint.Client)):
if isinstance(rois_or_neuprint, str):
npclient = neuprint.Client(rois_or_neuprint)
else:
npclient = rois_or_neuprint
# Fetch ROI names from neuprint
q = "MATCH (m: Meta) RETURN m.superLevelRois as rois"
rois = npclient.fetch_custom(q)['rois'].iloc[0]
rois = sorted(rois)
# # Remove '.*ACA' ROIs. Apparently there is some
# # problem with them. (They overlap with other ROIs.)
# rois = [*filter(lambda r: 'ACA' not in r, rois)]
else:
assert isinstance(rois_or_neuprint, collections.abc.Iterable)
rois = rois_or_neuprint
# Fetch each ROI and write it into a volume
with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger):
roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume(
server, uuid, rois, box_zyx=box_s5)
if len(overlap_stats) > 0:
logger.warn(
f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}"
)
# Export to npy/h5py for external use
if export_path:
with Timer(f"Exporting to {export_path}", logger):
if export_path.endswith('.npy'):
np.save(export_path, roi_vol)
elif export_path.endswith('.h5'):
with h5py.File(export_path, 'w') as f:
f.create_dataset('rois_scale_5', data=roi_vol, chunks=True)
if export_labelmap:
if isinstance(export_labelmap, str):
export_labelmap = (server, uuid, export_labelmap)
assert len(export_labelmap) == 3
with Timer(f"Exporting to {export_labelmap[2]}", logger):
if export_labelmap[2] not in fetch_repo_instances(
server, uuid, 'labelmap'):
create_labelmap_instance(
*export_labelmap, voxel_size=8 * (2**5),
max_scale=6) # FIXME: hard-coded voxel size
# It's really important to use this block shape.
# See https://github.com/janelia-flyem/dvid/issues/342
boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True)
for box in tqdm_proxy(boxes):
block = extract_subvol(roi_vol, box - roi_box[0])
post_labelmap_voxels(*export_labelmap,
box[0],
block,
scale=0,
downres=True)
return roi_vol, roi_box, rois
