def test_fetch_skeleton(): c = Client(NEUPRINT_SERVER, DATASET) orig_df = c.fetch_skeleton(5813027016, False) healed_df = c.fetch_skeleton(5813027016, True) assert len(orig_df) == len(healed_df) assert (healed_df['link'] == -1).sum() == 1 assert healed_df['link'].iloc[0] == -1
def test_broken_members(): """ These endpoints are listed in the neuprintHTTP API, but don't seem to work. """ c = Client(NEUPRINT_SERVER, DATASET) # Broken. neuprint returns error 500 assert isinstance(c.fetch_instances(), list)
def get_body_ids_from_roi(roi, server, token, hemibrain_neuron=True, pre_threshold=0, post_threshold=0, total_threshold=0): """ Args: roi (str): Neuropil abreviation from Neuprint server (str): Neuprint server URL token (str): Neuprint auth token hemibrain_neuron (bool): Specifies whether to search Hemibrain-Neuron of Hemibrain-Segment. pre_threshold (int): Requires bodies meet threshold of presynaptic sites in the ROI post_threshold (int): Requires bodies meet threshold of postsynaptic sites in the ROI total_threshold (int): Requires bodies meet threshold of total synaptic sites in the ROI Returns: Dataframe containing body IDs that innervate the ROI """ query_template = ( 'MATCH (n:`{HEMIBRAIN}`)\n' 'WHERE n.{ROI}\n' 'RETURN n.bodyId AS ID, n.name AS NAME, n.roiInfo AS ROIINFO' ) # Start Neuprint python client client = Client(server, token) client.fetch_version() if hemibrain_neuron: query = query_template.format(HEMIBRAIN='hemibrain-Neuron', ROI=roi) else: query = query_template.format(HEMIBRAIN='hemibrain-Segment', ROI=roi) results = client.fetch_custom(query) results['ROIINFO'] = results['ROIINFO'].apply(ast.literal_eval) results['PRE'] = results['ROIINFO'].apply(lambda x: int(x[roi]['pre'])) results['POST'] = results['ROIINFO'].apply(lambda x: int(x[roi]['post'])) results = results[results['PRE'] + results['POST'] >= total_threshold] results = results[results['PRE'] >= pre_threshold] results = results[results['POST'] >= post_threshold] return results[['ID', 'NAME', 'PRE', 'POST']].reset_index()
def start_client(self, server, token): """ Creates a Neuprint API Client instance. Args: server (str): URL of Neuprint server. token (str): Authentication token for Neuprint. Returns: None """ try: self._client = Client(server, token) except RuntimeError as error: raise RuntimeError("Issue instantiating Neuprint client:", error)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--neuprint-server', '-n', default='neuprint.janelia.org') parser.add_argument('--dataset', '-d') parser.add_argument('--init', '-i', choices=['groundtruth', 'random']) parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--debug', action='store_true') parser.add_argument('--min-weight', '-w', default=10, type=int) args = parser.parse_args() c = Client(args.neuprint_server, args.dataset) export_dir = f"{c.dataset}-w{args.min_weight}-from-{args.init}" os.makedirs(export_dir, exist_ok=True) # Fetch connectome (and export) with Timer("Fetching/exporting connectome", logger): criteria = NC(status='Traced', cropped=False, client=c) neuron_df, roi_conn_df = fetch_adjacencies(criteria, criteria, min_total_weight=args.min_weight, export_dir=export_dir, properties=['type', 'instance'], client=c) conn_df = roi_conn_df.groupby(['bodyId_pre', 'bodyId_post'], as_index=False)['weight'].sum() strong_connections_df, g, nbs, partition_df = infer_hierarchy(neuron_df, conn_df, args.min_weight, args.init, args.verbose, args.debug) with Timer("Exporting inference results", logger): pickle.dump(g, open(f'{export_dir}/graph.pkl', 'wb')) pickle.dump(nbs, open(f'{export_dir}/nested-block-state.pkl', 'wb')) pickle.dump(partition_df, open(f'{export_dir}/partition_df.pkl', 'wb')) pickle.dump(strong_connections_df, open(f'{export_dir}/strong_connections_df.pkl', 'wb')) logger.info("DONE")
def fetch_rois_from_metadata(**kwargs): ''' DESCRIPTION MISSING ''' if 'client' not in kwargs.keys(): kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version, conf.api_token) metadata = fetch_meta(client=kwargs['client']) #print(metadata.keys()) g = fetch_roi_hierarchy(mark_primary=False, include_subprimary=True, format='nx', client=kwargs['client']) nonhierarchy_rois = metadata['nonHierarchicalROIs'] primary_rois = metadata['primaryRois'] all_rois = list(g.nodes) toplevel_rois = [] for i, s in enumerate(g.successors('hemibrain')): toplevel_rois += [s] return all_rois, primary_rois, nonhierarchy_rois, toplevel_rois
def fetch_synapse_counts(bodies): try: c = neuprint_default_client() except Exception: c = Client(server, dataset) ndf, cdf = fetch_neurons(NC(bodyId=bodies, label='Segment'), client=c) return ndf.set_index('bodyId')[['pre', 'post']].rename_axis('body')
def fetch_adjacency(criteria=None, prefix='noncropped_traced', force_download=False, neuprint=conf.enable_neuprint, adjpath=None, **kwargs): ''' simple neuprint.fetch_adjacencies wrapper. Checks whether datasets were already downloaded and loads them accordingly. By default func loads traced and noncropped neurons. ''' assert not force_download or neuprint, 'no neuprint; cannot download dataset' #compose adjpath if adjpath is None: if not isinstance(kwargs['rois'], list): kwargs['rois'] = [kwargs['rois']] datadir = conf.datasets_dir postfix = '_' + '.'.join( kwargs['rois']) if 'rois' in kwargs.keys() else '' adjpath = os.path.join(datadir, prefix + postfix) roipath = os.path.join(adjpath, conf.roi_connections_file) neurpath = os.path.join(adjpath, conf.neurons_file) files_exist = os.path.exists(adjpath) and os.path.exists( roipath) and os.path.exists(neurpath) assert files_exist or neuprint, 'no neuprint; cannot find dataset (and no way of fetching)' if neuprint: if 'client' not in kwargs.keys(): kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version, conf.api_token) if criteria == None: criteria = NeuronCriteria(status='Traced', cropped=False, client=kwargs['client']) print('dataset in adjpath=', adjpath) if os.path.exists(adjpath) and os.path.exists(roipath) and os.path.exists( neurpath) and not force_download: print('dataset already downloaded') adj = pd.read_csv(roipath) neurons = pd.read_csv(neurpath) else: print('downloading dataset') print(criteria) neurons, adj = fetch_adjacencies(sources=criteria, targets=criteria, export_dir=adjpath, **kwargs) return neurons, adj
def test_inject_client(): c = Client(NEUPRINT_SERVER, DATASET) c2 = Client(NEUPRINT_SERVER, DATASET) set_default_client(c) @inject_client def f(*, client): return client # Uses default client unless client was specified assert f() is c assert f(client=c2) is c2 with pytest.raises(AssertionError): # Wrong signature -- asserts @inject_client def f2(client): pass
def __init__(self, host: str, dataset: str, token: str) -> None: """ Create a new NeuPrintExecutor that points to a deployed neuPrint DB. Arguments: host (str): The host of the neuPrint server (for example, 'neuprint.janelia.org') dataset (str): The name of the dataset to reference (for example, 'hemibrain:v1.1`) token (str): The user's neuPrint access token. To retrieve this token, go to https://[host]/account. Returns: None """ self._created_container = False self.host = host self.dataset = dataset self.token = token self.client = Client(host, dataset=self.dataset, token=self.token)
def __init__(self, server, dataset, tag="", neuron_pre=2, neuron_post=10, verify=True): self.dataset = dataset self.tag = tag if tag: self.full_dataset = f"{dataset}:{tag}" else: self.full_dataset = dataset self.client = Client(server, self.full_dataset, verify=verify) # criteria for Segment to be a Neuron self.neuron_pre = neuron_pre self.neuron_post = neuron_post
def __get_instance(self): if None in [self.server, self.neuprint_dataset, self.token]: if os.path.exists(self.credentials): with open(self.credentials) as fp: config = configparser.ConfigParser() config.readfp(fp) if self.server is None: self.server = config.get("Credentials", "server") if self.neuprint_dataset is None: self.neuprint_dataset = config.get( "Credentials", "dataset") if self.token is None: self.token = config.get("Credentials", "token") client = Client(self.server, dataset=self.neuprint_dataset, token=self.token) return client
def fetch_toplevel_roi_datasets(**kwargs): ''' DESCRIPTION MISSING ''' if 'client' not in kwargs.keys(): kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version, conf.api_token) _, _, _, toplevel_rois = fetch_rois_from_metadata(client=client) empty_rois = [] for roi in toplevel_rois: try: #print(roi) fetch_adjacency(rois=[roi], **kwargs) except KeyError: print('download problem, skipping') empty_rois += [roi] continue return empty_rois
def fetch_CX_datasets(**kwargs): ''' DESCRIPTION MISSING ''' if 'client' not in kwargs.keys(): kwargs['client'] = Client(conf.neuprint_URL, conf.dataset_version, conf.api_token) CX_rois = [['CX'], ['PB', 'NO', 'FB', 'EB', 'AB(L)', 'AB(R)']] empty_rois = [] for roi in CX_rois: try: #print(roi) fetch_adjacency(rois=roi, **kwargs) except KeyError: print('download problem, skipping') empty_rois += [roi] continue return empty_rois
def test_keyvalue(): # TODO: # What is an appropriate key/value to test with? c = Client(NEUPRINT_SERVER, DATASET) c.post_raw_keyvalue(instance, key, b'test-test-test') c.fetch_raw_keyvalue(instance, key)
from neuprint import Client # Please put the API token into the environment variable NEUPRINT_APPLICATION_CREDENTIALS client = Client('neuprint.janelia.org', dataset='hemibrain:v1.1') client.fetch_version()
usage: python createpartition.py config.yaml """ configfile = sys.argv[1] data = json.load(open(configfile)) # only partition within rois (if provided) rois = data["roiFilter"] threshold = data["connectionthreshold"] if threshold is None: threshold = 6 SERVER = 'emdata1.int.janelia.org:11000' np = Client(SERVER) # find nodes in the graph roifilter = "" roifilterset = set() ignorelabels = set(["location", "timeStamp", "confidence", "type"]) connections = {} points = {} roirent = {} if rois is not None and len(rois) > 0: # traverse one ROI at a time query = "MATCH (m :Meta:hemibrain) RETURN m.superLevelRois" res = np.fetch_custom(query) major_rois = set(res.iloc[0, 0])
mito_ids = mito_seg[tuple(p_mito_xyz.transpose())] mito_ids[np.isinf(point_distances)] = 0 # Transpose back to C-order p_mito_zyx = p_mito_xyz[:, ::-1] return mito_ids, point_distances, p_mito_zyx, crossed_gaps if __name__ == "__main__": from neuprint import Client from neuclease.dvid import fetch_label, fetch_supervoxels from neuclease import configure_default_logging configure_default_logging() c = Client('neuprint.janelia.org', 'hemibrain:v1.2') # body = 519046655 # tbars = fetch_synapses(body, SC(rois='FB', type='pre', primary_only=True)) # tbars = tbars.iloc[:10] # EXPORT_DEBUG_VOLUMES = True # body = 295474876 # tbars = fetch_synapses(body, SC(type='pre', primary_only=True)) # selections = (tbars[[*'xyz']] == (24721, 21717, 22518)).all(axis=1) # print(selections.sum()) # tbars = tbars.loc[selections] # EXPORT_DEBUG_VOLUMES = True # body = 1002848124 # tbars = fetch_synapses(body, SC(type='pre', primary_only=True))
#!/usr/bin/env python import sys import os import h5py import neuprint from neuprint import Client, queries, SegmentCriteria from extract_labels_from_volume import extract_labels_from_volume import numpy as np c = Client( 'neuprint.janelia.org', dataset='hemibrain:v1.1', token= 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImFuZHJldy5saW5Ac2hlZmZpZWxkLmFjLnVrIiwibGV2ZWwiOiJub2F1dGgiLCJpbWFnZS11cmwiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUF1RTdtQmp2LXlGaUlDWjdFQ3AwUHhzYmNVSHNDclppTU96eWtHbnFSemk_c3o9NTA_c3o9NTAiLCJleHAiOjE3NjE0MzE3Mzh9.kfurqTReeqmGsFHQdkKKGhTcaw3JR-FnpxCpMpS7uOA' ) print(c.fetch_version()) print("Loading volume") with h5py.File('hemibrain-v1.1-primary-roi-segmentation.h5', 'r') as f: roi_vol_scale_5 = f['volume-256nm'][:] KCs = SegmentCriteria(type="^KC.*", regex=True) KCneurons, KCroicounts = neuprint.queries.fetch_neurons(KCs) KCids = KCneurons['bodyId'] KCidsarray = np.asarray(KCids) np.savetxt("KCids_v1.1.csv", KCidsarray, delimiter=",", fmt="%u") for i in range(0, len(KCids)): print(f"Fetching skeleton for KC {i}")
# imports from neuprint import Client from neuprint import fetch_adjacencies, merge_neuron_properties, NeuronCriteria as NC from neuprint import fetch_neurons # Client creation - option to receive user' own TOKEN TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6Im5vYWhoYXJlbEBtYWlsLnRhdS5h' \ 'Yy5pbCIsImxldmVsIjoibm9hdXRoIiwiaW1hZ2UtdXJsIjoiaHR0cHM6Ly9saDYuZ29vZ2xldXNlc' \ 'mNvbnRlbnQuY29tLy13dGxuQlZqSkE2OC9BQUFBQUFBQUFBSS9BQUFBQUFBQUFBQS9BQUtXSkpObG' \ 'J4UVc5NXdRd19oZlhxcVM1ZFJ4MTNvXzR3L3Bob3RvLmpwZz9zej01MD9zej01MCIsImV4cCI6MTc' \ '2NjU5NTAzMn0.4At5bDlTYiVWawEUmO2uoyBn2u7eu6-UJwJbuvFCz90' c = Client(server='neuprint.janelia.org', dataset='hemibrain:v1.0.1', token=TOKEN) def gui_to_criteria(): # defaults for now, will be the function to tie GUI and backend together # body_id_in = [387023620, 387364605, 416642425] # instance_in = "OA-VPM3" # type_in = 'PENPEN_b(PEN2)' input_rois_in = ['AL(R)'] output_rois_in = ['AL(L)'] criteria_in = NC(inputRois=input_rois_in, outputRois=output_rois_in) return criteria_in
def roistats_table(input_path, neuprint_dataset, voxel_col=None, num_ranked=5, exclude_none_roi=False, neuprint_server='neuprint.janelia.org', output_path=None): """ The RoiStats workflow produces a list of body-ROI pairs and corresponding voxel counts. This function - appends fractional columns to the absolute numbers, - adds synapse counts (from neuprint), - filters for the top-N ROIs for each body, - and pivots (reshapes) the data so that the top-5 ROIs (and all stats) can be viewed on a single row of a spreadsheet. Args: input_path: CSV file produced by the RoiStats workflow, e.g. roi-stats.csv neuprint_dataset: Name of the neuprint dataset, e.v. 'hemibrain:v1.1' voxel_col: Name of the column which contains the voxel counts, e.g. voxels_s1 num_ranked: How many ranked ROIs to include in the result columns exclude_none_roi: When computing totals (and fractions), should voxels that were not part of any ROI be excluded from the total (denomitator)? output_path: If provided, export the results to the given path. neuprint_server: Which neuprint server to use to obtain synapse counts. Returns: Table of top-N ROI stats for each body. """ from neuprint import Client, fetch_neurons c = Client(neuprint_server, neuprint_dataset) stats = pd.read_csv(input_path) if voxel_col is None: vcs = [*filter(lambda c: c.startswith('voxels'), stats.columns)] if len(vcs) != 1: raise RuntimeError("Could not auto-determine voxel_col. Please provide the column name to use.") voxel_col = vcs[0] stats = stats.rename(columns={f'{voxel_col}': f'roi_{voxel_col}'}) if exclude_none_roi: # Exclude <none> from totals so fractions are # given in terms of neuropil totals only. stats = stats.query('roi_id != 0') # Compute totals and fractions voxel_totals = stats.groupby('body')[f'roi_{voxel_col}'].sum().rename(f'total_{voxel_col}') stats = stats.merge(voxel_totals, 'left', on='body') stats['roi_voxels_frac'] = stats.eval(f'roi_{voxel_col} / total_{voxel_col}') # Drop the '<none>' ROI, since that isn't conveniently available in the synapse # data, and probably isn't of interest anyway. # (Above, the '<none>' voxels were already counted in the totals (denominators) # if necessary, but we never list '<none>' as an ROI column.) stats = stats.query('roi_id != 0').drop(columns=['roi_id']) # Fetch synapse counts from neuprint bodies = stats['body'].unique() logger.info(f"Fetching synapse counts from neuprint for {len(bodies)} bodies") neuron_batches, roi_syn_batches = [], [] for bodies in tqdm_proxy(iter_batches(bodies, 10_000)): n, r = fetch_neurons(bodies, client=c) neuron_batches.append(n) roi_syn_batches.append(r)
'inputRois': ['AL(R)'], 'outputRois': ['CA(R)'], 'status': 'Traced', 'cropped': False }, 'synapseParams': None, 'max_neurons': 50 } } # Setup client print('Creating client...') with open('../authToken', 'r') as file: authToken = file.read() c = Client(server='neuprint.janelia.org', dataset='hemibrain:v1.0.1', token=authToken) # Print regions (primary ROIs marked with '*') # print(fetch_roi_hierarchy(include_subprimary=True, mark_primary=True, format='text')) # Function to get body IDs def getBodyIDs(sourceParams, synapseParams): print('Getting neurons/synapses...') sourceNeurons = NC(**sourceParams) if not (synapseParams == None): synapseCriteria = SC(**synapseParams) synapses = fetch_synapse_bodyIDs(source_criteria=sourceNeurons, target_criteria=None, synapse_criteria=synapseCriteria)
} # Unfortunately, this is not stored in the task! RADIUS = 250 MITO_SVS = ('http://emdata3.int.janelia.org:8900', '62f6394a18d4490c93892fbd9f1416b5', 'masked-mito-cc' ) # for visualization MITO_SEG = ('http://emdata3.int.janelia.org:8900', 'd31b64ac81444923a0319961736a6c31', 'masked-mito-cc') # Ideally, we would have listed this in the assignments, indicating # which segmentation was used to generate the neighborhoods. NEURON_SEG = ('http://emdata4.int.janelia.org:8900', '20631f94c3f446d7864bc55bf515706e', 'segmentation') NEUPRINT_CLIENT = Client('neuprint.janelia.org', 'hemibrain:v1.1') # Mito size threshold (in terms of 8nm voxels) # Mito segments smaller than this weren't assigned # to any body and weren't loaded into neuprint, # so mitos smaller than this should not be included in our counts here. MIN_MITO_SIZE = 10_000 # Some of our mito segmentations are stored at 8nm resolution, some at 16nm, # but we assume the neighborhood resolution is full-scale (8nm) NEIGHBORHOOD_RES = 8 # It doesn't make sense to do this analysis at scale 0, # since the mitos were generated at scale-1 anyway. ANALYSIS_SCALE = 1
class NeuPrintExecutor(Neo4jExecutor): """ A NeuPrintExecutor may be used to access an existing neuPrint server. This class converts a DotMotif motif object into a neuPrint-compatible query. Not all neuPrint datatypes or query types are available, but this adds complete support for DotMotif motif searches by passing raw Cypher queries to the neuPrint server over the HTTP API. Note that the neuPrint default timeout is quite short, and slower motif queries may not run in time. """ def __init__(self, host: str, dataset: str, token: str) -> None: """ Create a new NeuPrintExecutor that points to a deployed neuPrint DB. Arguments: host (str): The host of the neuPrint server (for example, 'neuprint.janelia.org') dataset (str): The name of the dataset to reference (for example, 'hemibrain:v1.1`) token (str): The user's neuPrint access token. To retrieve this token, go to https://[host]/account. Returns: None """ self._created_container = False self.host = host self.dataset = dataset self.token = token self.client = Client(host, dataset=self.dataset, token=self.token) def run(self, cypher: str) -> pd.DataFrame: """ Run an arbitrary cypher command. You should usually ignore this, and use .find() instead. Arguments: cypher (str): The command to run Returns: The result of the cypher query """ return self.client.fetch_custom(cypher) def count(self, motif: dotmotif, limit=None) -> int: """ Count a motif in a larger graph. Arguments: motif (dotmotif.dotmotif): The motif to search for Returns: int: The count of this motif in the host graph """ qry = self.motif_to_cypher(motif, count_only=True, static_entity_labels=_DEFAULT_ENTITY_LABELS) if limit: qry += f" LIMIT {limit}" res = self.client.fetch_custom(qry) print(res) return int(res.to_numpy()) def find(self, motif: dotmotif, limit=None) -> pd.DataFrame: """ Find a motif in a larger graph. Arguments: motif (dotmotif.dotmotif): The motif to search for Returns: pd.DataFrame: The results of the search """ qry = self.motif_to_cypher(motif, static_entity_labels=_DEFAULT_ENTITY_LABELS) if limit: qry += f" LIMIT {limit}" return self.client.fetch_custom(qry) @staticmethod def motif_to_cypher(motif: dotmotif, count_only: bool = False, static_entity_labels: dict = None) -> str: """ Convert a motif to neuprint-flavored Cypher. This is currently a thin passthrough for Neo4jExecutor.motif_to_cypher. """ static_entity_labels = static_entity_labels or _DEFAULT_ENTITY_LABELS return Neo4jExecutor.motif_to_cypher(motif, count_only, static_entity_labels)
def client(): c = Client(NEUPRINT_SERVER, DATASET) set_default_client(c) assert default_client() is c return c
ax2.set_xscale('log') ax2.spines['right'].set_visible(True) figS2_7.savefig(os.path.join(analysis_dir, 'figpanels', 'figS2_7.svg'), format='svg', transparent=True, dpi=save_dpi) # %% Supp: AC+FC vs. completeness, distance include_inds_ito, name_list_ito = bridge.getItoNames() response_filepaths = glob.glob(os.path.join(data_dir, 'ito_responses') + '/' + '*.pkl') Functional_Matrix, cmats_z = functional_connectivity.getCmat(response_filepaths, include_inds_ito, name_list_ito) Structural_Matrix = anatomical_connectivity.getAtlasConnectivity(include_inds_ito, name_list_ito, 'ito') Structural_Matrix = (Structural_Matrix + Structural_Matrix.T) / 2 # symmetrize # start client token = bridge.getUserConfiguration()['token'] neuprint_client = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=token) # Atlas roi completeness measures roi_completeness = anatomical_connectivity.getRoiCompleteness(neuprint_client, name_list_ito) CompletenessMatrix = pd.DataFrame(data=np.outer(roi_completeness['frac_post'], roi_completeness['frac_pre']), index=roi_completeness.index, columns=roi_completeness.index) CompletenessMatrix = (CompletenessMatrix + CompletenessMatrix.T) / 2 # symmetrize to compare with symmetrized SC # upper triangle fc = Functional_Matrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)] sc = Structural_Matrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)] compl = CompletenessMatrix.to_numpy()[np.triu_indices(len(name_list_ito), k=1)] figS2_8, ax = plt.subplots(1, 2, figsize=(6.5, 3)) ax[0].plot(compl, sc, 'k.', alpha=1.0, rasterized=True) r, p = plotting.addLinearFit(ax[0], compl, sc, alpha=1.0) ax[0].set_xlabel('Completeness')
def test_members(): set_default_client(None) assert default_client() is None c = Client(NEUPRINT_SERVER, DATASET) assert c.server == f'https://{NEUPRINT_SERVER}' assert c.dataset == DATASET assert default_client() is c df = c.fetch_custom("MATCH (m:Meta) RETURN m.primaryRois as rois") assert isinstance(df, pd.DataFrame) assert df.columns == ['rois'] assert len(df) == 1 assert isinstance(df['rois'].iloc[0], list) assert isinstance(c.fetch_available(), list) assert isinstance(c.fetch_help(), str) assert c.fetch_server_info() is True assert isinstance(c.fetch_version(), str) assert isinstance(c.fetch_database(), dict) assert isinstance(c.fetch_datasets(), dict) assert isinstance(c.fetch_db_version(), str) assert isinstance(c.fetch_profile(), dict) assert isinstance(c.fetch_token(), str) assert isinstance(c.fetch_daily_type(), tuple) assert isinstance(c.fetch_roi_completeness(), pd.DataFrame) assert isinstance(c.fetch_roi_connectivity(), pd.DataFrame) assert isinstance(c.fetch_roi_mesh('AB(R)'), bytes) assert isinstance(c.fetch_skeleton(EXAMPLE_BODY), pd.DataFrame)
print("The current version of data is v1.2") print("The data is provided by Janelia. To get further information, please visit: https://neuprint.janelia.org/") print("This program for FlyEM data acuisition from neuprint is developed by Ching-Che Charng (Jerry).\n Current version is beta version.\nIf you encounter any problem, please contact: [email protected]\n") # Token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6ImNoYXJuZ2NoaW5nY2hlQGxvbGFiLW50aHUub3JnIiwibGV2ZWwiOiJub2F1dGgiLCJpbWFnZS11cmwiOiJodHRwczovL2xoNC5nb29nbGV1c2VyY29udGVudC5jb20vLTNQdU9Xb0RVQUxNL0FBQUFBQUFBQUFJL0FBQUFBQUFBQUFBL0FDSGkzcmRFZ2NhSDJlVzNSbVJfc2ZuM1F2RUNVdWRBQlEvcGhvdG8uanBnP3N6PTUwP3N6PTUwIiwiZXhwIjoxNzYwODgwNDE4fQ.haDl_9vFGuJRxP8UAJIKaHocYtPYwhIcrrotq4VJSpQ" token_file_name="token_for_check_connection.txt" if not os.path.isfile(token_file_name): check_token=0 while check_token==0: Token = input( "The token is not correct. Please type it again." ) if len(Token)>10: try: c = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=Token) with open(token_file_name, "wt")as ff: ff.writelines(Token) check_token=1 except: pass else: with open(token_file_name,"rt")as ff: for line in ff: if line.find("\n")!=-1: line=line[:-1] Token=line break c = Client('neuprint.janelia.org', dataset='hemibrain:v1.2', token=Token) c.fetch_version()
def get_neuprint_client(hemibrain_version="v1.1", auth_token_file="notebooks/nx_graph_utils/flybrain.auth"): """Load a NeuPrint `Client` object connected to the hemibrain database""" auth_token = next(open(auth_token_file)).strip() np_client = Client('neuprint.janelia.org', dataset='hemibrain:' + hemibrain_version, token=auth_token) return np_client
def setup_dataset(dataset, published): """ Insert or update a data set in Mongo Keyword arguments: dataset: data set published: True=public, False=private Returns: last_uid: last UID assigned action: what to do with bodies in this data set (ignore, insert, or update) """ LOGGER.info("Initializing Client for %s %s", ARG.SERVER, dataset) npc = Client(ARG.SERVER, dataset=dataset) set_default_client(npc) result = fetch_meta(client=npc) if ':' in dataset: name, version = dataset.split(':') version = version.replace('v', '') else: name = dataset version = '' coll = DBM.emDataSet check = coll.find_one({ "name": name, "version": version, "published": published }) action = 'ignore' if not check: if result['dataset'] not in GENDER: LOGGER.error("%s does not have a gender defined", result['dataset']) sys.exit(-1) payload = { "class": "org.janelia.model.domain.flyem.EMDataSet", "ownerKey": "group:flyem", "readers": ["group:flyem"], "writers": ["group:flyem"], "name": result['dataset'], "version": version, "gender": GENDER[result['dataset']], "creationDate": to_datetime(result['lastDatabaseEdit']), "updatedDate": to_datetime(result['lastDatabaseEdit']), "active": True, "published": published } if published: payload['readers'] = ["group:flyem", "group:workstation_users"] last_uid = generate_uid() payload['_id'] = last_uid LOGGER.debug(payload) if ARG.WRITE: post_id = coll.insert_one(payload).inserted_id else: post_id = last_uid if post_id != last_uid: LOGGER.critical("Could not insert to Mongo with requested _id") sys.exit(-1) LOGGER.info("Inserted data set %s (UID: %s, datetime: %s)", dataset, post_id, result['lastDatabaseEdit']) action = 'insert' else: LOGGER.info("%s already exists in Mongo (UID: %s)", dataset, check['_id']) last_uid = check['_id'] neuprint_dt = to_datetime(result['lastDatabaseEdit']) if neuprint_dt > check['updatedDate'] or ARG.FORCE: LOGGER.warning("Update required for %s (last changed %s)", dataset, result['lastDatabaseEdit']) payload = {"updatedDate": datetime.now(), "active": True} if ARG.WRITE: coll.update_one({"_id": check['_id']}, {"$set": payload}) action = 'update' else: LOGGER.info("No update required for %s (last changed %s)", dataset, check['updatedDate']) return last_uid, action