def test_fetch_skeleton():
c = Client(NEUPRINT_SERVER, DATASET)
orig_df = c.fetch_skeleton(5813027016, False)
healed_df = c.fetch_skeleton(5813027016, True)
assert len(orig_df) == len(healed_df)
assert (healed_df['link'] == -1).sum() == 1
assert healed_df['link'].iloc[0] == -1
def test_broken_members():
"""
These endpoints are listed in the neuprintHTTP API,
but don't seem to work.
"""
c = Client(NEUPRINT_SERVER, DATASET)
# Broken. neuprint returns error 500
assert isinstance(c.fetch_instances(), list)
def get_body_ids_from_roi(roi,
server,
token,
hemibrain_neuron=True,
pre_threshold=0,
post_threshold=0,
total_threshold=0):
"""
Args:
roi (str): Neuropil abreviation from Neuprint
server (str): Neuprint server URL
token (str): Neuprint auth token
hemibrain_neuron (bool): Specifies whether to search
Hemibrain-Neuron of Hemibrain-Segment.
pre_threshold (int): Requires bodies meet threshold of presynaptic sites
in the ROI
post_threshold (int): Requires bodies meet threshold of postsynaptic sites
in the ROI
total_threshold (int): Requires bodies meet threshold of total synaptic sites
in the ROI
Returns:
Dataframe containing body IDs that innervate the ROI
"""
query_template = (
'MATCH (n:`{HEMIBRAIN}`)\n'
'WHERE n.{ROI}\n'
'RETURN n.bodyId AS ID, n.name AS NAME, n.roiInfo AS ROIINFO'
)
# Start Neuprint python client
client = Client(server, token)
client.fetch_version()
if hemibrain_neuron:
query = query_template.format(HEMIBRAIN='hemibrain-Neuron', ROI=roi)
else:
query = query_template.format(HEMIBRAIN='hemibrain-Segment', ROI=roi)
results = client.fetch_custom(query)
results['ROIINFO'] = results['ROIINFO'].apply(ast.literal_eval)
results['PRE'] = results['ROIINFO'].apply(lambda x: int(x[roi]['pre']))
results['POST'] = results['ROIINFO'].apply(lambda x: int(x[roi]['post']))
results = results[results['PRE'] + results['POST'] >= total_threshold]
results = results[results['PRE'] >= pre_threshold]
results = results[results['POST'] >= post_threshold]
return results[['ID', 'NAME', 'PRE', 'POST']].reset_index()
def start_client(self, server, token):
""" Creates a Neuprint API Client instance.
Args:
server (str): URL of Neuprint server.
token (str): Authentication token for Neuprint.
Returns:
None
"""
try:
self._client = Client(server, token)
except RuntimeError as error:
raise RuntimeError("Issue instantiating Neuprint client:", error)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--neuprint-server', '-n', default='neuprint.janelia.org')
parser.add_argument('--dataset', '-d')
parser.add_argument('--init', '-i', choices=['groundtruth', 'random'])
parser.add_argument('--verbose', '-v', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--min-weight', '-w', default=10, type=int)
args = parser.parse_args()
c = Client(args.neuprint_server, args.dataset)
export_dir = f"{c.dataset}-w{args.min_weight}-from-{args.init}"
os.makedirs(export_dir, exist_ok=True)
# Fetch connectome (and export)
with Timer("Fetching/exporting connectome", logger):
criteria = NC(status='Traced', cropped=False, client=c)
neuron_df, roi_conn_df = fetch_adjacencies(criteria, criteria, min_total_weight=args.min_weight, export_dir=export_dir, properties=['type', 'instance'], client=c)
conn_df = roi_conn_df.groupby(['bodyId_pre', 'bodyId_post'], as_index=False)['weight'].sum()
strong_connections_df, g, nbs, partition_df = infer_hierarchy(neuron_df,
conn_df,
args.min_weight,
args.init,
args.verbose,
args.debug)
with Timer("Exporting inference results", logger):
pickle.dump(g, open(f'{export_dir}/graph.pkl', 'wb'))
pickle.dump(nbs, open(f'{export_dir}/nested-block-state.pkl', 'wb'))
pickle.dump(partition_df, open(f'{export_dir}/partition_df.pkl', 'wb'))
pickle.dump(strong_connections_df, open(f'{export_dir}/strong_connections_df.pkl', 'wb'))
logger.info("DONE")
def fetch_rois_from_metadata(**kwargs):
'''
DESCRIPTION MISSING
'''
if 'client' not in kwargs.keys():
kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version,
conf.api_token)
metadata = fetch_meta(client=kwargs['client'])
#print(metadata.keys())
g = fetch_roi_hierarchy(mark_primary=False,
include_subprimary=True,
format='nx',
client=kwargs['client'])
nonhierarchy_rois = metadata['nonHierarchicalROIs']
primary_rois = metadata['primaryRois']
all_rois = list(g.nodes)
toplevel_rois = []
for i, s in enumerate(g.successors('hemibrain')):
toplevel_rois += [s]
return all_rois, primary_rois, nonhierarchy_rois, toplevel_rois
def fetch_synapse_counts(bodies):
try:
c = neuprint_default_client()
except Exception:
c = Client(server, dataset)
ndf, cdf = fetch_neurons(NC(bodyId=bodies, label='Segment'),
client=c)
return ndf.set_index('bodyId')[['pre', 'post']].rename_axis('body')
def fetch_adjacency(criteria=None,
prefix='noncropped_traced',
force_download=False,
neuprint=conf.enable_neuprint,
adjpath=None,
**kwargs):
'''
simple neuprint.fetch_adjacencies wrapper.
Checks whether datasets were already downloaded and loads them accordingly.
By default func loads traced and noncropped neurons.
'''
assert not force_download or neuprint, 'no neuprint; cannot download dataset'
#compose adjpath
if adjpath is None:
if not isinstance(kwargs['rois'], list):
kwargs['rois'] = [kwargs['rois']]
datadir = conf.datasets_dir
postfix = '_' + '.'.join(
kwargs['rois']) if 'rois' in kwargs.keys() else ''
adjpath = os.path.join(datadir, prefix + postfix)
roipath = os.path.join(adjpath, conf.roi_connections_file)
neurpath = os.path.join(adjpath, conf.neurons_file)
files_exist = os.path.exists(adjpath) and os.path.exists(
roipath) and os.path.exists(neurpath)
assert files_exist or neuprint, 'no neuprint; cannot find dataset (and no way of fetching)'
if neuprint:
if 'client' not in kwargs.keys():
kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version,
conf.api_token)
if criteria == None:
criteria = NeuronCriteria(status='Traced',
cropped=False,
client=kwargs['client'])
print('dataset in adjpath=', adjpath)
if os.path.exists(adjpath) and os.path.exists(roipath) and os.path.exists(
neurpath) and not force_download:
print('dataset already downloaded')
adj = pd.read_csv(roipath)
neurons = pd.read_csv(neurpath)
else:
print('downloading dataset')
print(criteria)
neurons, adj = fetch_adjacencies(sources=criteria,
targets=criteria,
export_dir=adjpath,
**kwargs)
return neurons, adj
def test_inject_client():
c = Client(NEUPRINT_SERVER, DATASET)
c2 = Client(NEUPRINT_SERVER, DATASET)
set_default_client(c)
@inject_client
def f(*, client):
return client
# Uses default client unless client was specified
assert f() is c
assert f(client=c2) is c2
with pytest.raises(AssertionError):
# Wrong signature -- asserts
@inject_client
def f2(client):
pass
def __init__(self, host: str, dataset: str, token: str) -> None:
"""
Create a new NeuPrintExecutor that points to a deployed neuPrint DB.
Arguments:
host (str): The host of the neuPrint server (for example,
'neuprint.janelia.org')
dataset (str): The name of the dataset to reference (for example,
'hemibrain:v1.1`)
token (str): The user's neuPrint access token. To retrieve this
token, go to https://[host]/account.
Returns:
None
"""
self._created_container = False
self.host = host
self.dataset = dataset
self.token = token
self.client = Client(host, dataset=self.dataset, token=self.token)
def __init__(self, server, dataset, tag="", neuron_pre=2, neuron_post=10, verify=True):
self.dataset = dataset
self.tag = tag
if tag:
self.full_dataset = f"{dataset}:{tag}"
else:
self.full_dataset = dataset
self.client = Client(server, self.full_dataset, verify=verify)
# criteria for Segment to be a Neuron
self.neuron_pre = neuron_pre
self.neuron_post = neuron_post
def __get_instance(self):
if None in [self.server, self.neuprint_dataset, self.token]:
if os.path.exists(self.credentials):
with open(self.credentials) as fp:
config = configparser.ConfigParser()
config.readfp(fp)
if self.server is None:
self.server = config.get("Credentials", "server")
if self.neuprint_dataset is None:
self.neuprint_dataset = config.get(
"Credentials", "dataset")
if self.token is None:
self.token = config.get("Credentials", "token")
client = Client(self.server,
dataset=self.neuprint_dataset,
token=self.token)
return client
def fetch_toplevel_roi_datasets(**kwargs):
'''
DESCRIPTION MISSING
'''
if 'client' not in kwargs.keys():
kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version,
conf.api_token)
_, _, _, toplevel_rois = fetch_rois_from_metadata(client=client)
empty_rois = []
for roi in toplevel_rois:
try:
#print(roi)
fetch_adjacency(rois=[roi], **kwargs)
except KeyError:
print('download problem, skipping')
empty_rois += [roi]
continue
return empty_rois
def fetch_CX_datasets(**kwargs):
'''
DESCRIPTION MISSING
'''
if 'client' not in kwargs.keys():
kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version,
conf.api_token)
CX_rois = [['CX'], ['PB', 'NO', 'FB', 'EB', 'AB(L)', 'AB(R)']]
empty_rois = []
for roi in CX_rois:
try:
#print(roi)
fetch_adjacency(rois=roi, **kwargs)
except KeyError:
print('download problem, skipping')
empty_rois += [roi]
continue
return empty_rois
def test_keyvalue():
# TODO:
# What is an appropriate key/value to test with?
c = Client(NEUPRINT_SERVER, DATASET)
c.post_raw_keyvalue(instance, key, b'test-test-test')
c.fetch_raw_keyvalue(instance, key)
from neuprint import Client
# Please put the API token into the environment variable NEUPRINT_APPLICATION_CREDENTIALS
client = Client('neuprint.janelia.org', dataset='hemibrain:v1.1')
client.fetch_version()
usage: python createpartition.py config.yaml
"""
configfile = sys.argv[1]
data = json.load(open(configfile))
# only partition within rois (if provided)
rois = data["roiFilter"]
threshold = data["connectionthreshold"]
if threshold is None:
threshold = 6
SERVER = 'emdata1.int.janelia.org:11000'
np = Client(SERVER)
# find nodes in the graph
roifilter = ""
roifilterset = set()
ignorelabels = set(["location", "timeStamp", "confidence", "type"])
connections = {}
points = {}
roirent = {}
if rois is not None and len(rois) > 0:
# traverse one ROI at a time
query = "MATCH (m :Meta:hemibrain) RETURN m.superLevelRois"
res = np.fetch_custom(query)
major_rois = set(res.iloc[0, 0])
mito_ids = mito_seg[tuple(p_mito_xyz.transpose())]
mito_ids[np.isinf(point_distances)] = 0
# Transpose back to C-order
p_mito_zyx = p_mito_xyz[:, ::-1]
return mito_ids, point_distances, p_mito_zyx, crossed_gaps
if __name__ == "__main__":
from neuprint import Client
from neuclease.dvid import fetch_label, fetch_supervoxels
from neuclease import configure_default_logging
configure_default_logging()
c = Client('neuprint.janelia.org', 'hemibrain:v1.2')
# body = 519046655
# tbars = fetch_synapses(body, SC(rois='FB', type='pre', primary_only=True))
# tbars = tbars.iloc[:10]
# EXPORT_DEBUG_VOLUMES = True
# body = 295474876
# tbars = fetch_synapses(body, SC(type='pre', primary_only=True))
# selections = (tbars[[*'xyz']] == (24721, 21717, 22518)).all(axis=1)
# print(selections.sum())
# tbars = tbars.loc[selections]
# EXPORT_DEBUG_VOLUMES = True
# body = 1002848124
# tbars = fetch_synapses(body, SC(type='pre', primary_only=True))
#!/usr/bin/env python
import sys
import os
import h5py
import neuprint
from neuprint import Client, queries, SegmentCriteria
from extract_labels_from_volume import extract_labels_from_volume
import numpy as np
c = Client(
'neuprint.janelia.org',
dataset='hemibrain:v1.1',
token=
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFuZHJldy5saW5Ac2hlZmZpZWxkLmFjLnVrIiwibGV2ZWwiOiJub2F1dGgiLCJpbWFnZS11cmwiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUF1RTdtQmp2LXlGaUlDWjdFQ3AwUHhzYmNVSHNDclppTU96eWtHbnFSemk_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjE0MzE3Mzh9.kfurqTReeqmGsFHQdkKKGhTcaw3JR-FnpxCpMpS7uOA'
)
print(c.fetch_version())
print("Loading volume")
with h5py.File('hemibrain-v1.1-primary-roi-segmentation.h5', 'r') as f:
roi_vol_scale_5 = f['volume-256nm'][:]
KCs = SegmentCriteria(type="^KC.*", regex=True)
KCneurons, KCroicounts = neuprint.queries.fetch_neurons(KCs)
KCids = KCneurons['bodyId']
KCidsarray = np.asarray(KCids)
np.savetxt("KCids_v1.1.csv", KCidsarray, delimiter=",", fmt="%u")
for i in range(0, len(KCids)):
print(f"Fetching skeleton for KC {i}")
# imports
from neuprint import Client
from neuprint import fetch_adjacencies, merge_neuron_properties, NeuronCriteria as NC
from neuprint import fetch_neurons
# Client creation - option to receive user' own TOKEN
TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6Im5vYWhoYXJlbEBtYWlsLnRhdS5h' \
'Yy5pbCIsImxldmVsIjoibm9hdXRoIiwiaW1hZ2UtdXJsIjoiaHR0cHM6Ly9saDYuZ29vZ2xldXNlc' \
'mNvbnRlbnQuY29tLy13dGxuQlZqSkE2OC9BQUFBQUFBQUFBSS9BQUFBQUFBQUFBQS9BQUtXSkpObG' \
'J4UVc5NXdRd19oZlhxcVM1ZFJ4MTNvXzR3L3Bob3RvLmpwZz9zej01MD9zej01MCIsImV4cCI6MTc' \
'2NjU5NTAzMn0.4At5bDlTYiVWawEUmO2uoyBn2u7eu6-UJwJbuvFCz90'
c = Client(server='neuprint.janelia.org',
dataset='hemibrain:v1.0.1',
token=TOKEN)
def gui_to_criteria():
# defaults for now, will be the function to tie GUI and backend together
# body_id_in = [387023620, 387364605, 416642425]
# instance_in = "OA-VPM3"
# type_in = 'PENPEN_b(PEN2)'
input_rois_in = ['AL(R)']
output_rois_in = ['AL(L)']
criteria_in = NC(inputRois=input_rois_in, outputRois=output_rois_in)
return criteria_in
def roistats_table(input_path,
neuprint_dataset,
voxel_col=None,
num_ranked=5,
exclude_none_roi=False,
neuprint_server='neuprint.janelia.org',
output_path=None):
"""
The RoiStats workflow produces a list of body-ROI pairs and corresponding voxel counts.
This function
- appends fractional columns to the absolute numbers,
- adds synapse counts (from neuprint),
- filters for the top-N ROIs for each body,
- and pivots (reshapes) the data so that the top-5 ROIs (and all stats)
can be viewed on a single row of a spreadsheet.
Args:
input_path:
CSV file produced by the RoiStats workflow, e.g. roi-stats.csv
neuprint_dataset:
Name of the neuprint dataset, e.v. 'hemibrain:v1.1'
voxel_col:
Name of the column which contains the voxel counts, e.g. voxels_s1
num_ranked:
How many ranked ROIs to include in the result columns
exclude_none_roi:
When computing totals (and fractions), should voxels that were not
part of any ROI be excluded from the total (denomitator)?
output_path:
If provided, export the results to the given path.
neuprint_server:
Which neuprint server to use to obtain synapse counts.
Returns:
Table of top-N ROI stats for each body.
"""
from neuprint import Client, fetch_neurons
c = Client(neuprint_server, neuprint_dataset)
stats = pd.read_csv(input_path)
if voxel_col is None:
vcs = [*filter(lambda c: c.startswith('voxels'), stats.columns)]
if len(vcs) != 1:
raise RuntimeError("Could not auto-determine voxel_col. Please provide the column name to use.")
voxel_col = vcs[0]
stats = stats.rename(columns={f'{voxel_col}': f'roi_{voxel_col}'})
if exclude_none_roi:
# Exclude <none> from totals so fractions are
# given in terms of neuropil totals only.
stats = stats.query('roi_id != 0')
# Compute totals and fractions
voxel_totals = stats.groupby('body')[f'roi_{voxel_col}'].sum().rename(f'total_{voxel_col}')
stats = stats.merge(voxel_totals, 'left', on='body')
stats['roi_voxels_frac'] = stats.eval(f'roi_{voxel_col} / total_{voxel_col}')
# Drop the '<none>' ROI, since that isn't conveniently available in the synapse
# data, and probably isn't of interest anyway.
# (Above, the '<none>' voxels were already counted in the totals (denominators)
# if necessary, but we never list '<none>' as an ROI column.)
stats = stats.query('roi_id != 0').drop(columns=['roi_id'])
# Fetch synapse counts from neuprint
bodies = stats['body'].unique()
logger.info(f"Fetching synapse counts from neuprint for {len(bodies)} bodies")
neuron_batches, roi_syn_batches = [], []
for bodies in tqdm_proxy(iter_batches(bodies, 10_000)):
n, r = fetch_neurons(bodies, client=c)
neuron_batches.append(n)
roi_syn_batches.append(r)
'inputRois': ['AL(R)'],
'outputRois': ['CA(R)'],
'status': 'Traced',
'cropped': False
},
'synapseParams': None,
'max_neurons': 50
}
}
# Setup client
print('Creating client...')
with open('../authToken', 'r') as file:
authToken = file.read()
c = Client(server='neuprint.janelia.org',
dataset='hemibrain:v1.0.1',
token=authToken)
# Print regions (primary ROIs marked with '*')
# print(fetch_roi_hierarchy(include_subprimary=True, mark_primary=True, format='text'))
# Function to get body IDs
def getBodyIDs(sourceParams, synapseParams):
print('Getting neurons/synapses...')
sourceNeurons = NC(**sourceParams)
if not (synapseParams == None):
synapseCriteria = SC(**synapseParams)
synapses = fetch_synapse_bodyIDs(source_criteria=sourceNeurons,
target_criteria=None,
synapse_criteria=synapseCriteria)
}
# Unfortunately, this is not stored in the task!
RADIUS = 250
MITO_SVS = ('http://emdata3.int.janelia.org:8900',
'62f6394a18d4490c93892fbd9f1416b5', 'masked-mito-cc'
) # for visualization
MITO_SEG = ('http://emdata3.int.janelia.org:8900',
'd31b64ac81444923a0319961736a6c31', 'masked-mito-cc')
# Ideally, we would have listed this in the assignments, indicating
# which segmentation was used to generate the neighborhoods.
NEURON_SEG = ('http://emdata4.int.janelia.org:8900',
'20631f94c3f446d7864bc55bf515706e', 'segmentation')
NEUPRINT_CLIENT = Client('neuprint.janelia.org', 'hemibrain:v1.1')
# Mito size threshold (in terms of 8nm voxels)
# Mito segments smaller than this weren't assigned
# to any body and weren't loaded into neuprint,
# so mitos smaller than this should not be included in our counts here.
MIN_MITO_SIZE = 10_000
# Some of our mito segmentations are stored at 8nm resolution, some at 16nm,
# but we assume the neighborhood resolution is full-scale (8nm)
NEIGHBORHOOD_RES = 8
# It doesn't make sense to do this analysis at scale 0,
# since the mitos were generated at scale-1 anyway.
ANALYSIS_SCALE = 1
class NeuPrintExecutor(Neo4jExecutor):
"""
A NeuPrintExecutor may be used to access an existing neuPrint server.
This class converts a DotMotif motif object into a neuPrint-compatible
query. Not all neuPrint datatypes or query types are available, but this
adds complete support for DotMotif motif searches by passing raw Cypher
queries to the neuPrint server over the HTTP API.
Note that the neuPrint default timeout is quite short, and slower motif
queries may not run in time.
"""
def __init__(self, host: str, dataset: str, token: str) -> None:
"""
Create a new NeuPrintExecutor that points to a deployed neuPrint DB.
Arguments:
host (str): The host of the neuPrint server (for example,
'neuprint.janelia.org')
dataset (str): The name of the dataset to reference (for example,
'hemibrain:v1.1`)
token (str): The user's neuPrint access token. To retrieve this
token, go to https://[host]/account.
Returns:
None
"""
self._created_container = False
self.host = host
self.dataset = dataset
self.token = token
self.client = Client(host, dataset=self.dataset, token=self.token)
def run(self, cypher: str) -> pd.DataFrame:
"""
Run an arbitrary cypher command.
You should usually ignore this, and use .find() instead.
Arguments:
cypher (str): The command to run
Returns:
The result of the cypher query
"""
return self.client.fetch_custom(cypher)
def count(self, motif: dotmotif, limit=None) -> int:
"""
Count a motif in a larger graph.
Arguments:
motif (dotmotif.dotmotif): The motif to search for
Returns:
int: The count of this motif in the host graph
"""
qry = self.motif_to_cypher(motif,
count_only=True,
static_entity_labels=_DEFAULT_ENTITY_LABELS)
if limit:
qry += f" LIMIT {limit}"
res = self.client.fetch_custom(qry)
print(res)
return int(res.to_numpy())
def find(self, motif: dotmotif, limit=None) -> pd.DataFrame:
"""
Find a motif in a larger graph.
Arguments:
motif (dotmotif.dotmotif): The motif to search for
Returns:
pd.DataFrame: The results of the search
"""
qry = self.motif_to_cypher(motif,
static_entity_labels=_DEFAULT_ENTITY_LABELS)
if limit:
qry += f" LIMIT {limit}"
return self.client.fetch_custom(qry)
@staticmethod
def motif_to_cypher(motif: dotmotif,
count_only: bool = False,
static_entity_labels: dict = None) -> str:
"""
Convert a motif to neuprint-flavored Cypher.
This is currently a thin passthrough for Neo4jExecutor.motif_to_cypher.
"""
static_entity_labels = static_entity_labels or _DEFAULT_ENTITY_LABELS
return Neo4jExecutor.motif_to_cypher(motif, count_only,
static_entity_labels)
def client():
c = Client(NEUPRINT_SERVER, DATASET)
set_default_client(c)
assert default_client() is c
return c
ax2.set_xscale('log')
ax2.spines['right'].set_visible(True)
figS2_7.savefig(os.path.join(analysis_dir, 'figpanels', 'figS2_7.svg'), format='svg', transparent=True, dpi=save_dpi)
# %% Supp: AC+FC vs. completeness, distance
include_inds_ito, name_list_ito = bridge.getItoNames()
response_filepaths = glob.glob(os.path.join(data_dir, 'ito_responses') + '/' + '*.pkl')
Functional_Matrix, cmats_z = functional_connectivity.getCmat(response_filepaths, include_inds_ito, name_list_ito)
Structural_Matrix = anatomical_connectivity.getAtlasConnectivity(include_inds_ito, name_list_ito, 'ito')
Structural_Matrix = (Structural_Matrix + Structural_Matrix.T) / 2 # symmetrize
# start client
token = bridge.getUserConfiguration()['token']
neuprint_client = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=token)
# Atlas roi completeness measures
roi_completeness = anatomical_connectivity.getRoiCompleteness(neuprint_client, name_list_ito)
CompletenessMatrix = pd.DataFrame(data=np.outer(roi_completeness['frac_post'], roi_completeness['frac_pre']), index=roi_completeness.index, columns=roi_completeness.index)
CompletenessMatrix = (CompletenessMatrix + CompletenessMatrix.T) / 2 # symmetrize to compare with symmetrized SC
# upper triangle
fc = Functional_Matrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)]
sc = Structural_Matrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)]
compl = CompletenessMatrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)]
figS2_8, ax = plt.subplots(1, 2, figsize=(6.5, 3))
ax[0].plot(compl, sc, 'k.', alpha=1.0, rasterized=True)
r, p = plotting.addLinearFit(ax[0], compl, sc, alpha=1.0)
ax[0].set_xlabel('Completeness')
def test_members():
set_default_client(None)
assert default_client() is None
c = Client(NEUPRINT_SERVER, DATASET)
assert c.server == f'https://{NEUPRINT_SERVER}'
assert c.dataset == DATASET
assert default_client() is c
df = c.fetch_custom("MATCH (m:Meta) RETURN m.primaryRois as rois")
assert isinstance(df, pd.DataFrame)
assert df.columns == ['rois']
assert len(df) == 1
assert isinstance(df['rois'].iloc[0], list)
assert isinstance(c.fetch_available(), list)
assert isinstance(c.fetch_help(), str)
assert c.fetch_server_info() is True
assert isinstance(c.fetch_version(), str)
assert isinstance(c.fetch_database(), dict)
assert isinstance(c.fetch_datasets(), dict)
assert isinstance(c.fetch_db_version(), str)
assert isinstance(c.fetch_profile(), dict)
assert isinstance(c.fetch_token(), str)
assert isinstance(c.fetch_daily_type(), tuple)
assert isinstance(c.fetch_roi_completeness(), pd.DataFrame)
assert isinstance(c.fetch_roi_connectivity(), pd.DataFrame)
assert isinstance(c.fetch_roi_mesh('AB(R)'), bytes)
assert isinstance(c.fetch_skeleton(EXAMPLE_BODY), pd.DataFrame)
print("The current version of data is v1.2")
print("The data is provided by Janelia. To get further information, please visit: https://neuprint.janelia.org/")
print("This program for FlyEM data acuisition from neuprint is developed by Ching-Che Charng (Jerry).\n Current version is beta version.\nIf you encounter any problem, please contact: [email protected]\n")
# Token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImNoYXJuZ2NoaW5nY2hlQGxvbGFiLW50aHUub3JnIiwibGV2ZWwiOiJub2F1dGgiLCJpbWFnZS11cmwiOiJodHRwczovL2xoNC5nb29nbGV1c2VyY29udGVudC5jb20vLTNQdU9Xb0RVQUxNL0FBQUFBQUFBQUFJL0FBQUFBQUFBQUFBL0FDSGkzcmRFZ2NhSDJlVzNSbVJfc2ZuM1F2RUNVdWRBQlEvcGhvdG8uanBnP3N6PTUwP3N6PTUwIiwiZXhwIjoxNzYwODgwNDE4fQ.haDl_9vFGuJRxP8UAJIKaHocYtPYwhIcrrotq4VJSpQ"
token_file_name="token_for_check_connection.txt"
if not os.path.isfile(token_file_name):
check_token=0
while check_token==0:
Token = input(
"The token is not correct. Please type it again."
)
if len(Token)>10:
try:
c = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=Token)
with open(token_file_name, "wt")as ff:
ff.writelines(Token)
check_token=1
except:
pass
else:
with open(token_file_name,"rt")as ff:
for line in ff:
if line.find("\n")!=-1:
line=line[:-1]
Token=line
break
c = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=Token)
c.fetch_version()
def get_neuprint_client(hemibrain_version="v1.1", auth_token_file="notebooks/nx_graph_utils/flybrain.auth"):
"""Load a NeuPrint `Client` object connected to the hemibrain database"""
auth_token = next(open(auth_token_file)).strip()
np_client = Client('neuprint.janelia.org', dataset='hemibrain:' + hemibrain_version, token=auth_token)
return np_client
def setup_dataset(dataset, published):
""" Insert or update a data set in Mongo
Keyword arguments:
dataset: data set
published: True=public, False=private
Returns:
last_uid: last UID assigned
action: what to do with bodies in this data set (ignore, insert, or update)
"""
LOGGER.info("Initializing Client for %s %s", ARG.SERVER, dataset)
npc = Client(ARG.SERVER, dataset=dataset)
set_default_client(npc)
result = fetch_meta(client=npc)
if ':' in dataset:
name, version = dataset.split(':')
version = version.replace('v', '')
else:
name = dataset
version = ''
coll = DBM.emDataSet
check = coll.find_one({
"name": name,
"version": version,
"published": published
})
action = 'ignore'
if not check:
if result['dataset'] not in GENDER:
LOGGER.error("%s does not have a gender defined",
result['dataset'])
sys.exit(-1)
payload = {
"class": "org.janelia.model.domain.flyem.EMDataSet",
"ownerKey": "group:flyem",
"readers": ["group:flyem"],
"writers": ["group:flyem"],
"name": result['dataset'],
"version": version,
"gender": GENDER[result['dataset']],
"creationDate": to_datetime(result['lastDatabaseEdit']),
"updatedDate": to_datetime(result['lastDatabaseEdit']),
"active": True,
"published": published
}
if published:
payload['readers'] = ["group:flyem", "group:workstation_users"]
last_uid = generate_uid()
payload['_id'] = last_uid
LOGGER.debug(payload)
if ARG.WRITE:
post_id = coll.insert_one(payload).inserted_id
else:
post_id = last_uid
if post_id != last_uid:
LOGGER.critical("Could not insert to Mongo with requested _id")
sys.exit(-1)
LOGGER.info("Inserted data set %s (UID: %s, datetime: %s)", dataset,
post_id, result['lastDatabaseEdit'])
action = 'insert'
else:
LOGGER.info("%s already exists in Mongo (UID: %s)", dataset,
check['_id'])
last_uid = check['_id']
neuprint_dt = to_datetime(result['lastDatabaseEdit'])
if neuprint_dt > check['updatedDate'] or ARG.FORCE:
LOGGER.warning("Update required for %s (last changed %s)", dataset,
result['lastDatabaseEdit'])
payload = {"updatedDate": datetime.now(), "active": True}
if ARG.WRITE:
coll.update_one({"_id": check['_id']}, {"$set": payload})
action = 'update'
else:
LOGGER.info("No update required for %s (last changed %s)", dataset,
check['updatedDate'])
return last_uid, action