def __init__(self, server, dataset, tag="", neuron_pre=2, neuron_post=10, verify=True): self.client = neu.Client(server, verify=verify) self.dataset = dataset self.tag = tag # criteria for Segment to be a Neuron self.neuron_pre = neuron_pre self.neuron_post = neuron_post
def __init__(self, neuprint_endpoint=None, neuprint_dataset=None, neuprint_token=None, catmaid_endpoint=None): if neuprint_endpoint and neuprint_dataset and neuprint_token: self.neuprint_client = neuprint.Client(neuprint_endpoint, dataset=neuprint_dataset, token=neuprint_token) elif catmaid_endpoint: self.rm = pymaid.CatmaidInstance(catmaid_endpoint, '', '', '') #no self? else: self.neuprint_client = None self.vc = VfbConnect(neo_endpoint="http://kb.p2.virtualflybrain.org")
if not "source" in jsonSynapses: print("JSON 'synapses' contains no 'source' key") quit() synapseSource = jsonSynapses["source"] if synapseSource.startswith("http"): i1 = synapseSource.find("://") i2 = synapseSource.find("/?dataset=") if i2 == -1: server = synapseSource[i1:] dataset = None else: server = synapseSource[i1 + len("://"):i2] dataset = synapseSource[i2 + len("/?dataset="):] dataset = dataset.replace("%3A", ":") client = neuprint.Client(server, dataset=dataset) client.fetch_version() for synapseSetName, synapseSetSpec in jsonSynapses.items(): if synapseSetName == "source": continue if not "neuron" in synapseSetSpec: print("Error: synapse set '{}' is missing 'neuron'\n".format(synapseSetName)) continue body = synapseSetSpec["neuron"] if not "type" in synapseSetSpec: print("Error: synapse set '{}' is missing 'type'\n".format(synapseSetName)) continue type = synapseSetSpec["type"]
# TODO maybe unconditionally monkey patch neuprint.Client on init, just to # add a warning if manually creating clients (saying when it is / isn't # appropriate) (maybe + kwarg to disable warning, also mentioned in # warning)? (might need care to not cause same warning when neuprint # itself copies code... maybe check the __file__ of the calling code # (possible?)? # TODO TODO TODO before any of the above, just put a line in a README.md # explaining that users should prefer to access my client rather than # making their own, and they should only use it for the list of fns below # (the ones only the client can do) # TODO TODO or maybe just monkey patch the client so it can't make the calls # provided by the others? (unless it breaks internals, which it very well # might) # TODO or maybe only instantiate this inside my own fns (like in fetch_function # wrapped fn part + other places that would need client) and never present it? client = nu.Client('neuprint.janelia.org', 'hemibrain:v1.0.1') # This seems to be the most up-to-date documentation on what properties are # availble: https://neuprint.janelia.org/public/neuprintuserguide.pdf #maybe_missing = ['primaryNeurite', 'somaLocation', 'somaRadius', 'timestamp'] # (just a hack to hide warning for now. commented above is accurate **wrt PDF**) maybe_missing = ['somaLocation', 'somaRadius'] # These don't include the "<rois>: boolean" properties from the docs, which seem # to only be defined when True, or only for sets where some have the True # value... # TODO TODO (as some comments say elsewhere) still try to replace this w/ # queries that enumerate all available properties userguide_neuron_properties = [ 'bodyId', 'cropped', 'instance', 'post', 'pre', 'roiInfo', 'size', 'status', 'statusLabel', 'type'
import neuprint as neu import csv import pandas as pd import numpy as np import ast import os from collections import defaultdict, OrderedDict from multiprocess import Pool #client = neu.Client('emdata1.int.janelia.org:13000', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NTc5NzcyMTR9.uoMuun4AQ82VI7qUjO7f0G5CKOUX4KqAIF89CkQN4do') client = neu.Client( 'https://neuprint.janelia.org', dataset='hemibrain:v1.1', token= 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NTc5NzcyMTR9.uoMuun4AQ82VI7qUjO7f0G5CKOUX4KqAIF89CkQN4do' ) import chart_studio.plotly as py import plotly.graph_objects as go import collections import operator from copy import deepcopy import seaborn as sns GFInputs = [ 1102761868, 454347237, 5812983519, 829162792, 829155062, 642262407, 5813022764, 642263196, 1815947603, 1254331599, 1102412245, 1876993474, 2126296267, 1506382974, 1288650927, 1072059176, 5813068048, 1722483751, 1073428001, 1381730321, 1594562431, 1443450601, 1722177539, 1691885573, 1412761053, 1504850262, 1630332999, 1507139129, 5813065458, 1753751888, 1602704738, 1631019730, 1628291400, 1691310419, 1384712714, 1037972384,
def find_violations(neuprint_addr, dataset, roilist): client = neu.Client('https://emdata1.int.janelia.org:11000') # neuron extractor query (restrict to .*raced for now) extractquery = f"MATCH (n :`{dataset}-Neuron`) WHERE n.status=~'.*raced' {{}} RETURN n.bodyId as bodyid" # get roi violation set (default all) checkrois = None if roilist is not None and len(roilist) > 0: checkrois = set(roilist) # restrict neurons to these ROIs roistring = "AND (" for roi in roilist: roistring += f"n.{roi} OR " roistring = roistring[0:-4] roistring += ")" extractquery = extractquery.format(roistring) else: roiquery = f"MATCH (m :Meta) WHERE m.dataset=\"{dataset}\" RETURN m.superLevelRois AS rois" roires = client.fetch_custom(roiquery) checkrois = set(roires["rois"].iloc[0]) extractquery = extractquery.format("") # extract working set candbodies_res = client.fetch_custom(extractquery) candbodies = candbodies_res["bodyid"].tolist() candbodies_set = set(candbodies) # query db X neurons at a time (restrict to traced neurons for the targets) # (connections can be one way since all we care about connections only within the body set # (note: m cannot be restricted to body set up front because the calls are run in minibatches) outputsquery = f"WITH {{}} AS TARGETS MATCH(x :`{dataset}-ConnectionSet`)-\ [:From]->(n :`{dataset}-Neuron`) WHERE n.bodyId in TARGETS WITH collect(x) as\ csets, n UNWIND csets as cset MATCH (cset)-[:To]->(m :`{dataset}-Neuron`) where\ (m.status=~'.*raced') RETURN n.bodyId AS body1, cset.roiInfo AS info, m.bodyId AS body2" total_conns = 0 total_splits = 0 total_msplits = 0 candidate_connections = [] print(f"Candidate bodies: {len(candbodies)}") # find candidate split bodies (might as well compute # the number of split neurons as well) for iter1 in range(0, len(candbodies), 50): #for iter1 in range(1100, len(candbodies), 50): #if iter1 == 1200: # break print(f"fetch batch: {iter1}") currlist = candbodies[iter1:iter1 + 50] restrictedquery = outputsquery.format(currlist) res = client.fetch_custom(restrictedquery) for idx, row in res.iterrows(): if (row["body1"] == row["body2"]) or (row["body2"] not in candbodies_set): continue roiconns = json.loads(row["info"]) roiconns_keys = set(roiconns.keys()) inter_rois = checkrois.intersection(roiconns_keys) if len(inter_rois) > 0: total_conns += 1 if len(inter_rois) > 1: total_splits += 1 if len(inter_rois) > 2: total_msplits += 1 roi2count = {} for roi in inter_rois: roi2count[roi] = roiconns[roi]["post"] candidate_connections.append( (row["body1"], row["body2"], roi2count)) print( f"Total connection pairs examined: {total_conns}\nTotal split across ROIs: {total_splits}\nTotal multi-splits across ROIs: {total_msplits}" ) # distance threshold pixel_threshold = 400 # global stats roipairs_numvio = 0 roipairs_wgtvio = 0 real_roi_splits = 0 roi_scores = {} roi_bodies = {} for connpair in candidate_connections: (body1, body2, roi2count) = connpair locquery = f"MATCH (n :`{dataset}-Neuron` {{bodyId: {body1}}})<-[:From]-(x :ConnectionSet)-[:To]->(m :`{dataset}-Neuron` {{bodyId: {body2}}}) MATCH (x)-[:Contains]->(s :PostSyn) RETURN s AS synapse" resloc = client.fetch_custom(locquery) # check each location roipsd = {} for idx, row in resloc.iterrows(): synapse = row["synapse"] for (roi, count) in roi2count.items(): if roi in synapse: roipsd[roi] = synapse["location"]["coordinates"] # look at all pairs of ROIs roipsd2 = list(roipsd.items()) for iter1 in range(0, len(roipsd2)): for iter2 in range(iter1 + 1, len(roipsd2)): roi1, loc1 = roipsd2[iter1] roi2, loc2 = roipsd2[iter2] loc1 = numpy.array(loc1) loc2 = numpy.array(loc2) diff = (loc1 - loc2)**2 sumsq = diff.sum() dist = sumsq**(1 / 2) if dist < pixel_threshold: roipairs_numvio += 1 roiwgt = min(roi2count[roi1], roi2count[roi2]) roipairs_wgtvio += roiwgt if roi1 > roi2: roi1, roi2 = roi2, roi1 if (roi1, roi2) not in roi_scores: roi_scores[(roi1, roi2)] = 0 roi_bodies[roi1 + "-" + roi2] = [] roi_scores[(roi1, roi2)] += roiwgt roi_bodies[roi1 + "-" + roi2].append( (roiwgt, body1, body2)) else: real_roi_splits += 1 # print stats print( f"Total violations: {roipairs_numvio}\nWeighted violations: {roipairs_wgtvio}\nNum real ROI splits: {real_roi_splits}\nBad ROI boundaries: {len(roi_scores)}" ) # sort roi scores and print out worst pairs as just commma separated; dump roi_bodies to file sorted_rois = list(zip(roi_scores.values(), roi_scores.keys())) sorted_rois.sort() sorted_rois.reverse() print(sorted_rois) fout = open("connviolations.json", 'w') for key, bodies in roi_bodies.items(): bodies.sort() bodies.reverse() fout.write(json.dumps(roi_bodies)) fout.close() return
NeuPrint queries that are relevant to a target body ID and a target ROI. They have different "hop" amounts - number of interneurons between target and input """ import neuprint as neu import pandas as pd import numpy as np from collections import defaultdict import statistics # 13000 server # client = neu.Client('emdata1.int.janelia.org:13000/?dataset=hemibrain', token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUNIaTNyZWcwN21uZHVkQ2RpWVRkeFJGWGdONmlnMENOZy9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjM3NzA2MTR9.xNL4c8vg8tN4KKrLUiHEgy_9Wlf4tDoGCXEKlLs8lLU') # test server client = neu.Client( 'neuprint-test.janelia.org', dataset='hemibrain:v1.0.1', token= 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImVtaWx5dGVuc2hhd0BnbWFpbC5jb20iLCJsZXZlbCI6InJlYWR3cml0ZSIsImltYWdlLXVybCI6Imh0dHBzOi8vbGg0Lmdvb2dsZXVzZXJjb250ZW50LmNvbS8tdmhYaGxhYjFxcFEvQUFBQUFBQUFBQUkvQUFBQUFBQUFBQUEvQUtGMDVuRFRUOHBhUHFCT3dMNk5nblZHVENIR2xGWEp0QS9waG90by5qcGc_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjQ5ODIyODh9.UjUHbKZQpReYOShvtKMzZWxJE-saJ4FWN6iD9hAcZaQ' ) ''' These MAIN functions should be run by user and are dependent on other functions in the file ''' # main functions def twoHopMain(): print( '\nUse the answers in parentheses to answer, otherwise will not run properly' '\nExample Circuit: x -> y -> target\n') target = input('\nEnter the targeted body ID: ') ROI = input('\nEnter the targeted ROI: ') ROI = str(ROI)
def load_roi_label_volume(server, uuid, rois_or_neuprint, box_s5=[None, None], export_path=None, export_labelmap=None): """ Fetch several ROIs from DVID and combine them into a single label volume or mask. The label values in the returned volume correspond to the order in which the ROI names were passed in, starting at label 1. This function is essentially a convenience function around fetch_combined_roi_volume(), but in this case it will optionally auto-fetch the ROI list, and auto-export the volume. Args: server: DVID server uuid: DVID uuid rois_or_neuprint: Either a list of ROIs or a neuprint server from which to obtain the roi list. box_s5: If you want to restrict the ROIs to a particular subregion, you may pass your own bounding box (at scale 5). Alternatively, you may pass the name of a segmentation instance from DVID whose bounding box will be used. export_path: If you want the ROI volume to be exported to disk, provide a path name ending with .npy or .h5. export_labelmap: If you want the ROI volume to be exported to a DVID labelmap instance, Provide the instance name, or a tuple of (server, uuid, instance). Returns: (roi_vol, roi_box), containing the fetched label volume and the bounding box it corresponds to, in DVID scale-5 coordinates. Note: If you have a list of (full-res) points to extract from the returned volume, pass a DataFrame with columns ['z','y','x'] to the following function. If you already downloaded the roi_vol (above), provide it. Otherwise, leave out those args and it will be fetched first. Adds columns to the input DF (in-place) for 'roi' (str) and 'roi_label' (int). >>> from neuclease.dvid import determine_point_rois >>> determine_point_rois(*master, rois, point_df, roi_vol, roi_box) """ if isinstance(box_s5, str): # Assume that this is a segmentation instance whose dimensions should be used # Fetch the maximum extents of the segmentation, # and rescale it for scale-5. seg_box = fetch_volume_box(server, uuid, box_s5) box_s5 = round_box(seg_box, (2**5), 'out') // 2**5 box_s5[0] = (0, 0, 0) if export_labelmap: assert isinstance(box_s5, np.ndarray) assert not (box_s5 % 64).any(), \ ("If exporting to a labelmap instance, please supply " "an explicit box and make sure it is block-aligned.") if isinstance(rois_or_neuprint, (str, neuprint.Client)): if isinstance(rois_or_neuprint, str): npclient = neuprint.Client(rois_or_neuprint) else: npclient = rois_or_neuprint # Fetch ROI names from neuprint q = "MATCH (m: Meta) RETURN m.superLevelRois as rois" rois = npclient.fetch_custom(q)['rois'].iloc[0] rois = sorted(rois) # # Remove '.*ACA' ROIs. Apparently there is some # # problem with them. (They overlap with other ROIs.) # rois = [*filter(lambda r: 'ACA' not in r, rois)] else: assert isinstance(rois_or_neuprint, collections.abc.Iterable) rois = rois_or_neuprint # Fetch each ROI and write it into a volume with Timer(f"Fetching combined ROI volume for {len(rois)} ROIs", logger): roi_vol, roi_box, overlap_stats = fetch_combined_roi_volume( server, uuid, rois, box_zyx=box_s5) if len(overlap_stats) > 0: logger.warn( f"Some ROIs overlap! Here's an incomplete list of overlapping pairs:\n{overlap_stats}" ) # Export to npy/h5py for external use if export_path: with Timer(f"Exporting to {export_path}", logger): if export_path.endswith('.npy'): np.save(export_path, roi_vol) elif export_path.endswith('.h5'): with h5py.File(export_path, 'w') as f: f.create_dataset('rois_scale_5', data=roi_vol, chunks=True) if export_labelmap: if isinstance(export_labelmap, str): export_labelmap = (server, uuid, export_labelmap) assert len(export_labelmap) == 3 with Timer(f"Exporting to {export_labelmap[2]}", logger): if export_labelmap[2] not in fetch_repo_instances( server, uuid, 'labelmap'): create_labelmap_instance( *export_labelmap, voxel_size=8 * (2**5), max_scale=6) # FIXME: hard-coded voxel size # It's really important to use this block shape. # See https://github.com/janelia-flyem/dvid/issues/342 boxes = boxes_from_grid(roi_box, (256, 256, 256), clipped=True) for box in tqdm_proxy(boxes): block = extract_subvol(roi_vol, box - roi_box[0]) post_labelmap_voxels(*export_labelmap, box[0], block, scale=0, downres=True) return roi_vol, roi_box, rois