def main(output_dir, resolution, retain_transform_data, zoom,
parallel_downloads, parallel_processors):
mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
mcc.get_annotation_volume()
mcc.get_reference_space()
experiments = [e['id'] for e in mcc.get_experiments(dataframe=False)]
experiments = sorted(list(set(experiments)))
try:
os.makedirs(f'{output_dir}/data/input')
for e in experiments:
os.makedirs(f'{output_dir}/data/input/{e}')
except FileExistsError:
for e in experiments:
if os.path.isdir(f'{output_dir}/data/input/{e}'):
shutil.rmtree(f'{output_dir}/data/input/{e}')
os.makedirs(f'{output_dir}/data/input/{e}')
downloads = [
subprocess.Popen([
"python", "./displacement_data_downloader.py", f"-o{output_dir}",
f"-r{resolution}", f"-n{i}"
]) for i in range(parallel_downloads)
]
processes = [
subprocess.Popen([
"python", "./segmentation_data_builder.py", f"-o{output_dir}",
f"-r{resolution}", f"-c{len(experiments)}", f"-z{zoom}"
] + (["-t"] if retain_transform_data else []) + [f"-n{i}"])
for i in range(parallel_processors)
]
exit_codes = [p.wait() for p in processes + downloads]
def get_all_experiments(connectivity_dir):
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
from allensdk.api.queries.mouse_connectivity_api import MouseConnectivityApi
mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json',
resolution=MouseConnectivityApi.VOXEL_RESOLUTION_100_MICRONS)
experiments = mcc.get_experiments(dataframe=False)
return experiments, mcc
class CellProcessor(DirWatcher):
experiment_fields_to_save = [
'id',
'gender',
'injection_structures',
'injection_volume',
'injection_x',
'injection_y',
'injection_z',
'product_id',
'specimen_name',
'strain',
'structure_abbrev',
'structure_id',
'structure_name',
'transgenic_line',
'transgenic_line_id',
'primary_injection_structure'
]
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, annotate,
_processor_number):
super().__init__(input_dir, process_dir, output_dir, f'cell-processor-{_processor_number}')
self.annotate = annotate
self.structure_ids = structs
self.brain_seg_data_dir = structure_map_dir
self.source_dir = input_dir
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json',
resolution=25)
self.experiments = {int(e['id']): e for e in self.mcc.get_experiments(dataframe=False)}
def process_item(self, item, directory):
item = int(item)
experiment = ExperimentCellsProcessor(self.mcc,
item,
directory,
self.brain_seg_data_dir,
self.structure_ids,
self.experiment_fields_to_save,
self.experiments[item],
self.logger)
experiment.process()
if self.annotate:
annotator = ExperimentDataAnnotator(int(item), directory, self.logger)
annotator.process()
def on_process_error(self, item, exception):
retval = super().on_process_error(item, exception)
self.logger.error(f"Error occurred during processing", exc_info=True)
if type(exception) in [urllib.error.HTTPError, OSError, ValueError,
urllib.error.URLError, socket.gaierror, MemoryError]:
return False
else:
return retval
import allensdk
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# Create an instance of the class and assign it to a variable, mcc
mcc = MouseConnectivityCache(
manifest_file='connectivity/mouse_connectivity_manifest.json')
print(mcc)
# ## Download experimental metadata by transgenic line
# Now that we have our instance of the mouse connectivity cache, we can start downloading our experimental metadata. To do this, we will call `get_experiments()` on our connectivity instance. We'll use the argument `dataframe=True` to automatically make this a dataframe when it is created.
# In[3]:
# Gather all the experiments with transgenic as well as wildtype mice
mouse_exp_df = mcc.get_experiments(dataframe=True)
mouse_exp_df.head()
# This gives us metadata on all the expereiments in the dataset. Alternatively, you can specify within the method wether you would like to filter certain experiments by `transgenic_line`. Let's take a look at what trangenic lines are available to us.
# In[4]:
transgenic_lines = mouse_exp_df['transgenic_line'].unique()
print(transgenic_lines)
# Let's start by creating a dataframe that only contains experiments with the first three Cre lines in the list above *(Penk-IRES2-Cre-neo, Gabrr3-Cre_KC112, Hdc-Cre_IM1)*. You can change the Cre lines by changing the values in the list assigned to `transgenic_lines`. Remember to copy the Cre line of interest exactly, including the single quotes. We'll then use this list in the argument `cre = ` in our call to `get_experiments`.
# In[5]:
# Choose your Cre lines
transgenic_lines = ['Penk-IRES2-Cre-neo', 'Gabrr3-Cre_KC112', 'Hdc-Cre_IM1']
def test_notebook(fn_temp_dir):
# coding: utf-8
# ## Mouse Connectivity
#
# This notebook demonstrates how to access and manipulate data in the Allen Mouse Brain Connectivity Atlas. The `MouseConnectivityCache` AllenSDK class provides methods for downloading metadata about experiments, including their viral injection site and the mouse's transgenic line. You can request information either as a Pandas DataFrame or a simple list of dictionaries.
#
# An important feature of the `MouseConnectivityCache` is how it stores and retrieves data for you. By default, it will create (or read) a manifest file that keeps track of where various connectivity atlas data are stored. If you request something that has not already been downloaded, it will download it and store it in a well known location.
#
# Download this notebook in .ipynb format <a href='mouse_connectivity.ipynb'>here</a>.
# In[1]:
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
mcc = MouseConnectivityCache(manifest_file='connectivity/mouse_connectivity_manifest.json')
# open up a list of all of the experiments
all_experiments = mcc.get_experiments(dataframe=True)
print("%d total experiments" % len(all_experiments))
# take a look at what we know about an experiment with a primary motor injection
all_experiments.loc[122642490]
# `MouseConnectivityCache` has a method for retrieving the adult mouse structure tree as an `StructureTree` class instance. This is a wrapper around a list of dictionaries, where each dictionary describes a structure. It is principally useful for looking up structures by their properties.
# In[2]:
# pandas for nice tables
import pandas as pd
# grab the StructureTree instance
structure_tree = mcc.get_structure_tree()
# get info on some structures
structures = structure_tree.get_structures_by_name(['Primary visual area', 'Hypothalamus'])
pd.DataFrame(structures)
# As a convenience, structures are grouped in to named collections called "structure sets". These sets can be used to quickly gather a useful subset of structures from the tree. The criteria used to define structure sets are eclectic; a structure set might list:
#
# * structures that were used in a particular project.
# * structures that coarsely partition the brain.
# * structures that bear functional similarity.
#
# or something else entirely. To view all of the available structure sets along with their descriptions, follow this [link](http://api.brain-map.org/api/v2/data/StructureSet/query.json). To see only structure sets relevant to the adult mouse brain, use the StructureTree:
# In[3]:
from allensdk.api.queries.ontologies_api import OntologiesApi
oapi = OntologiesApi()
# get the ids of all the structure sets in the tree
structure_set_ids = structure_tree.get_structure_sets()
# query the API for information on those structure sets
pd.DataFrame(oapi.get_structure_sets(structure_set_ids))
# On the connectivity atlas web site, you'll see that we show most of our data at a fairly coarse structure level. We did this by creating a structure set of ~300 structures, which we call the "summary structures". We can use the structure tree to get all of the structures in this set:
# In[4]:
# From the above table, "Mouse Connectivity - Summary" has id 167587189
summary_structures = structure_tree.get_structures_by_set_id([167587189])
pd.DataFrame(summary_structures)
# This is how you can filter experiments by transgenic line:
# In[5]:
# fetch the experiments that have injections in the isocortex of cre-positive mice
isocortex = structure_tree.get_structures_by_name(['Isocortex'])[0]
cre_cortical_experiments = mcc.get_experiments(cre=True,
injection_structure_ids=[isocortex['id']])
print("%d cre cortical experiments" % len(cre_cortical_experiments))
# same as before, but restrict the cre line
rbp4_cortical_experiments = mcc.get_experiments(cre=[ 'Rbp4-Cre_KL100' ],
injection_structure_ids=[isocortex['id']])
print("%d Rbp4 cortical experiments" % len(rbp4_cortical_experiments))
# ## Structure Signal Unionization
#
# The ProjectionStructureUnionizes API data tells you how much signal there was in a given structure and experiment. It contains the density of projecting signal, volume of projecting signal, and other information. `MouseConnectivityCache` provides methods for querying and storing this data.
# In[6]:
# find wild-type injections into primary visual area
visp = structure_tree.get_structures_by_acronym(['VISp'])[0]
visp_experiments = mcc.get_experiments(cre=False,
injection_structure_ids=[visp['id']])
print("%d VISp experiments" % len(visp_experiments))
structure_unionizes = mcc.get_structure_unionizes([ e['id'] for e in visp_experiments ],
is_injection=False,
structure_ids=[isocortex['id']],
include_descendants=True)
print("%d VISp non-injection, cortical structure unionizes" % len(structure_unionizes))
# In[7]:
structure_unionizes.head()
# This is a rather large table, even for a relatively small number of experiments. You can filter it down to a smaller list of structures like this.
# In[8]:
dense_unionizes = structure_unionizes[ structure_unionizes.projection_density > .5 ]
large_unionizes = dense_unionizes[ dense_unionizes.volume > .5 ]
large_structures = pd.DataFrame(structure_tree.nodes(large_unionizes.structure_id))
print("%d large, dense, cortical, non-injection unionizes, %d structures" % ( len(large_unionizes), len(large_structures) ))
print(large_structures.name)
large_unionizes
# ## Generating a Projection Matrix
# The `MouseConnectivityCache` class provides a helper method for converting ProjectionStructureUnionize records for a set of experiments and structures into a matrix. This code snippet demonstrates how to make a matrix of projection density values in auditory sub-structures for cre-negative VISp experiments.
# In[9]:
import numpy as np
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
visp_experiment_ids = [ e['id'] for e in visp_experiments ]
ctx_children = structure_tree.child_ids( [isocortex['id']] )[0]
pm = mcc.get_projection_matrix(experiment_ids = visp_experiment_ids,
projection_structure_ids = ctx_children,
hemisphere_ids= [2], # right hemisphere, ipsilateral
parameter = 'projection_density')
row_labels = pm['rows'] # these are just experiment ids
column_labels = [ c['label'] for c in pm['columns'] ]
matrix = pm['matrix']
fig, ax = plt.subplots(figsize=(15,15))
heatmap = ax.pcolor(matrix, cmap=plt.cm.afmhot)
# put the major ticks at the middle of each cell
ax.set_xticks(np.arange(matrix.shape[1])+0.5, minor=False)
ax.set_yticks(np.arange(matrix.shape[0])+0.5, minor=False)
ax.set_xlim([0, matrix.shape[1]])
ax.set_ylim([0, matrix.shape[0]])
# want a more natural, table-like display
ax.invert_yaxis()
ax.xaxis.tick_top()
ax.set_xticklabels(column_labels, minor=False)
ax.set_yticklabels(row_labels, minor=False)
# ## Manipulating Grid Data
#
# The `MouseConnectivityCache` class also helps you download and open every experiment's projection grid data volume. By default it will download 25um volumes, but you could also download data at other resolutions if you prefer (10um, 50um, 100um).
#
# This demonstrates how you can load the projection density for a particular experiment. It also shows how to download the template volume to which all grid data is registered. Voxels in that template have been structurally annotated by neuroanatomists and the resulting labels stored in a separate annotation volume image.
# In[10]:
# we'll take this experiment - an injection into the primary somatosensory - as an example
experiment_id = 181599674
# In[11]:
# projection density: number of projecting pixels / voxel volume
pd, pd_info = mcc.get_projection_density(experiment_id)
# injection density: number of projecting pixels in injection site / voxel volume
ind, ind_info = mcc.get_injection_density(experiment_id)
# injection fraction: number of pixels in injection site / voxel volume
inf, inf_info = mcc.get_injection_fraction(experiment_id)
# data mask:
# binary mask indicating which voxels contain valid data
dm, dm_info = mcc.get_data_mask(experiment_id)
template, template_info = mcc.get_template_volume()
annot, annot_info = mcc.get_annotation_volume()
# in addition to the annotation volume, you can get binary masks for individual structures
# in this case, we'll get one for the isocortex
cortex_mask, cm_info = mcc.get_structure_mask(315)
print(pd_info)
print(pd.shape, template.shape, annot.shape)
# Once you have these loaded, you can use matplotlib see what they look like.
# In[12]:
# compute the maximum intensity projection (along the anterior-posterior axis) of the projection data
pd_mip = pd.max(axis=0)
ind_mip = ind.max(axis=0)
inf_mip = inf.max(axis=0)
# show that slice of all volumes side-by-side
f, pr_axes = plt.subplots(1, 3, figsize=(15, 6))
pr_axes[0].imshow(pd_mip, cmap='hot', aspect='equal')
pr_axes[0].set_title("projection density MaxIP")
pr_axes[1].imshow(ind_mip, cmap='hot', aspect='equal')
pr_axes[1].set_title("injection density MaxIP")
pr_axes[2].imshow(inf_mip, cmap='hot', aspect='equal')
pr_axes[2].set_title("injection fraction MaxIP")
# In[13]:
# Look at a slice from the average template and annotation volumes
# pick a slice to show
slice_idx = 264
f, ccf_axes = plt.subplots(1, 3, figsize=(15, 6))
ccf_axes[0].imshow(template[slice_idx,:,:], cmap='gray', aspect='equal', vmin=template.min(), vmax=template.max())
ccf_axes[0].set_title("registration template")
ccf_axes[1].imshow(annot[slice_idx,:,:], cmap='gray', aspect='equal', vmin=0, vmax=2000)
ccf_axes[1].set_title("annotation volume")
ccf_axes[2].imshow(cortex_mask[slice_idx,:,:], cmap='gray', aspect='equal', vmin=0, vmax=1)
ccf_axes[2].set_title("isocortex mask")
# On occasion the TissueCyte microscope fails to acquire a tile. In this case the data from that tile should not be used for analysis. The data mask associated with each experiment can be used to determine which portions of the grid data came from correctly acquired tiles.
#
# In this experiment, a missed tile can be seen in the data mask as a dark warped square. The values in the mask exist within [0, 1], describing the fraction of each voxel that was correctly acquired
# In[14]:
f, data_mask_axis = plt.subplots(figsize=(5, 6))
data_mask_axis.imshow(dm[81, :, :], cmap='hot', aspect='equal', vmin=0, vmax=1)
data_mask_axis.set_title('data mask')
#Import MouseConnectivityCache and convert it to mcc from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache mcc = MouseConnectivityCache(manifest_file='connectivity/mouse_connectivity_manifest.json') print(mcc) # ## Download experimental metadata by trangenic line # Now that we have our instance of the mouse connectivity cache, we can start downloading our experimental metadata. To do this, we will call `get_experiments()` on our connectivity instance. # In[3]: #Gather all the experiments with Cre and WildType all_experiments = mcc.get_experiments(dataframe=True) mouse_exp_df = pd.DataFrame(all_experiments) mouse_exp_df.head() # This gives us metadata on all the expereiments in the dataset. Alternatively, you can specify within the method wether you would like to filter certain experiments by `transgenic_line`. Let's take a look at what trangenic lines are availabe to us. # In[4]: trangenic_lines = mouse_exp_df['transgenic_line'].unique() trangenic_lines # In[5]:
#===============================================================================
# example 1
#===============================================================================
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# tell the cache class what resolution (in microns) of data you want to download
mcc = MouseConnectivityCache(resolution=25)
# use the structure tree class to get information about the isocortex structure
structure_tree = mcc.get_structure_tree()
isocortex_id = structure_tree.get_structures_by_name(['Isocortex'])[0]['id']
# a list of dictionaries containing metadata for non-Cre experiments
experiments = mcc.get_experiments(file_name='non_cre.json',
injection_structure_ids=[isocortex_id])
# download the projection density volume for one of the experiments
pd = mcc.get_projection_density(experiments[0]['id'])
#===============================================================================
# example 2
#===============================================================================
import nrrd
file_name = 'experiment_644250774/projection_density_25.nrrd'
data_array, metadata = nrrd.read(file_name)
#===============================================================================
# example 1
#===============================================================================
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# tell the cache class what resolution (in microns) of data you want to download
mcc = MouseConnectivityCache(resolution=25)
# use the structure tree class to get information about the isocortex structure
structure_tree = mcc.get_structure_tree()
isocortex_id = structure_tree.get_structures_by_name(['Isocortex'])[0]['id']
# a list of dictionaries containing metadata for non-Cre experiments
experiments = mcc.get_experiments(file_name='non_cre.json',
injection_structure_ids=[isocortex_id])
# download the projection density volume for one of the experiments
pd = mcc.get_projection_density(experiments[0]['id'])
#===============================================================================
# example 2
#===============================================================================
import nrrd
file_name = 'mouse_connectivity/experiment_644250774/projection_density_25.nrrd'
data_array, metadata = nrrd.read(file_name)
def xrange(r):
return range(r)
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
mcc = MouseConnectivityCache(
manifest_file='connectivity/mouse_connectivity_manifest.json')
# open up a list of all of the experiments
all_experiments = mcc.get_experiments(dataframe=True)
print("{} total experiments".format(len(all_experiments)))
# take a look at what we know about an experiment with a primary motor injection
print(all_experiments.loc[122642490])
# grab the StructureTree instance
structure_tree = mcc.get_structure_tree()
print(structure_tree)
from allensdk.api.queries.ontologies_api import OntologiesApi
oapi = OntologiesApi()
# get the ids of all the structure sets in the tree
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
manifest_file = os.path.join(drive_path, "manifest.json")
# Start processing data
mcc = MouseConnectivityCache(manifest_file=manifest_file,
resolution=resolution_um)
ontology = mcc.get_ontology()
# Injection structure of interest
isocortex = ontology['Isocortex']
# open up a pandas dataframe of all of the experiments
experiments = mcc.get_experiments(dataframe=True,
injection_structure_ids=\
[isocortex['id'].values[0]],
cre=False)
print "%d total experiments" % len(experiments)
view_paths_file = h5py.File(view_paths_fn, 'r')
view_lut = view_paths_file['view lookup'][:]
view_paths = view_paths_file['paths'][:]
view_paths_file.close()
# Compute size of each path to convert path averages to sums
norm_lut = np.zeros(view_lut.shape, dtype=int)
ind = np.where(view_lut != no_data)
ind = zip(ind[0], ind[1])
for curr_ind in ind:
curr_path_id = view_lut[curr_ind]
curr_path = view_paths[curr_path_id, :]
def main(args):
starttime = datetime.now()
# parse in args
parser = parsefn()
lbl, trans, projmet = parse_inputs(parser, args)
[cutoff, miracl_home, annot_csv, exclude] = initialize()
mcc = MouseConnectivityCache(
manifest_file=
'%s/connect/connectivity_exps/mouse_connectivity_manifest.json' %
miracl_home)
# Load all injection experiments from Allen api
all_experiments = mcc.get_experiments(dataframe=True)
# Filter to only wild-type strain
if trans:
print("\n Searching for experiments with the %s mouse line" % trans)
projexps = all_experiments[all_experiments['transgenic-line'] == "%s" %
trans]
else:
print(
"\n Searching for experiments with the wild-type (C57BL/6J) strain"
)
# wild-type
projexps = all_experiments[all_experiments['strain'] == "C57BL/6J"]
# no transgenic mice
projexps = projexps[projexps['transgenic-line'] == ""]
# ---------------
# Check if labels have injection exps
print(
"\n Checking if labels have injection experiments in the connectivity search"
)
inj_exp = projexps[projexps['structure-abbrev'] == lbl].id.index[0]
while inj_exp is None:
pid = annot_csv.parent_structure_id[annot_csv.id == lbl].values[0]
inj_exp = projexps[projexps['structure-id'] == pid].id.index[0]
# ---------------
# Get projection density
projd = getprojden(mcc, inj_exp)
# lbl_abrv = annot_csv[annot_csv['id'] == lbl]['acronym'].values[0]
# lbl_abrv = lbl_abrv[1:] # drop 1st char
print(
"\n Downloading projection density volume for experiment %d of lbl %s"
% (inj_exp, lbl))
outpd = '%s_exp%s_projection_density_image.nii.gz' % (lbl, inj_exp)
outtif = '%s_exp%s_projection_density_image.tif' % (lbl, inj_exp)
# outind = '%s_injection_density.nii.gz' % experiment_id
# outdm = '%s_binary_mask.nii.gz' % experiment_id
vx = 0.025
savenii(projd, vx, outpd)
savetiff(projd, outtif)
# orient
with add_paths():
call(["c3d", "%s" % outpd, "-orient", "ASR", "-o", "%s" % outpd])
# savenii(ind, vx, outind)
# savenii(dm, vx, outdm)
# ---------------
# Get connectivity graph
# query structure connectivity from Allen API
print(
"\n Quering structural connectivity of injection labels in the Allen API & sorting by %s"
% projmet)
[all_connect_ids, all_norm_proj] = query_connect(inj_exp, cutoff, exclude,
mcc, projmet)
# save csv
export_connect_abv = saveconncsv(all_connect_ids, annot_csv, lbl, inj_exp,
projmet)
# compute & save proj map
exportprojmap(all_norm_proj, export_connect_abv, lbl, inj_exp, projmet)
print(
"\n Downloading connectivity graph & projection map done in %s ... Have a good day!\n"
% (datetime.now() - starttime))
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
from allensdk.api.queries.ontologies_api import OntologiesApi
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import warnings
warnings.filterwarnings('ignore')
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
mcc = MouseConnectivityCache()
# open up a list of all of the experiments
all_experiments = mcc.get_experiments(dataframe=True)
print("%d total experiments" % len(all_experiments))
all_experiments.loc[122642490]
# grab the StructureTree instance
structure_tree = mcc.get_structure_tree()
# get info on some structures
structures = structure_tree.get_structures_by_acronym(['PAG'
]) #Periaqueductal gray
df = pd.DataFrame(structures)
print(df['structure_set_ids'][0])
oapi = OntologiesApi()
# get the ids of all the structure sets in the tree
structure_set_ids = structure_tree.get_structure_sets()
# query the API for information on those structure sets
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # tell the cache class what resolution (in microns) of data you want to download mcc = MouseConnectivityCache(resolution=25) # use the ontology class to get the id of the isocortex structure ontology = mcc.get_ontology() isocortex = ontology["Isocortex"] # a list of dictionaries containing metadata for non-Cre experiments experiments = mcc.get_experiments(injection_structure_ids=isocortex["id"]) # download the projection density volume for one of the experiments pd = mcc.get_projection_density(experiments[0]["id"])
print ""
print "please provide a working directory & output file name, e.g."
print ""
print " build_projection_datasets.py data/ pms.pickle"
print ""
sys.exit(1)
import os
os.chdir(sys.argv[1])
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
from allensdk.api.queries.ontologies_api import OntologiesApi
mcc = MouseConnectivityCache(manifest_file="mcc_manifest.json")
all_experiments = mcc.get_experiments(dataframe=True)
ontology = mcc.get_ontology()
summary_structures = OntologiesApi().get_structures(structure_set_names="Mouse Connectivity - Summary")
summary_structure_ids = [s["id"] for s in summary_structures]
print "build dict of injection structure id to experiment list"
ist2e = {}
for eid in all_experiments.index:
for ist in all_experiments.ix[eid]["injection-structures"]:
isti = ist["id"]
if isti not in ist2e:
ist2e[isti] = []
ist2e[isti].append(eid)
# Need to:
# source ~/virtualenv/py2a/bin/activate
# then run
# python test2.py
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
mcc = MouseConnectivityCache(
manifest_file='connectivity/mouse_connectivity_manifest.json')
# open up a list of all of the experiments
all_experiments = mcc.get_experiments(dataframe=True)
# take a look at what we know about an experiment with a primary motor injection
#all_experiments.loc[122642490]
# grab the StructureTree instance
structure_tree = mcc.get_structure_tree()
from allensdk.api.queries.ontologies_api import OntologiesApi
oapi = OntologiesApi()
# get the ids of all the structure sets in the tree
structure_set_ids = structure_tree.get_structure_sets()
# query the API for information on those structure sets
class ABA(Paths):
"""
This class handles interaction with the Allen Brain Atlas datasets and APIs to get structure trees,
experimental metadata and results, tractography data etc.
"""
# useful vars for analysis
excluded_regions = ["fiber tracts"]
resolution = 25
def __init__(self, projection_metric = "projection_energy", base_dir=None, **kwargs):
"""
Set up file paths and Allen SDKs
:param base_dir: path to directory to use for saving data (default value None)
:param path_fiprojection_metricle: - str, metric to quantify the strength of projections from the Allen Connectome. (default value 'projection_energy')
:param kwargs: can be used to pass path to individual data folders. See brainrender/Utils/paths_manager.py
"""
Paths.__init__(self, base_dir=base_dir, **kwargs)
self.projection_metric = projection_metric
# get mouse connectivity cache and structure tree
self.mcc = MouseConnectivityCache(manifest_file=os.path.join(self.mouse_connectivity_cache, "manifest.json"))
self.structure_tree = self.mcc.get_structure_tree()
# get ontologies API and brain structures sets
self.oapi = OntologiesApi()
self.get_structures_sets()
# get reference space
self.space = ReferenceSpaceApi()
self.spacecache = ReferenceSpaceCache(
manifest=os.path.join(self.annotated_volume, "manifest.json"), # downloaded files are stored relative to here
resolution=self.resolution,
reference_space_key="annotation/ccf_2017" # use the latest version of the CCF
)
self.annotated_volume, _ = self.spacecache.get_annotation_volume()
# mouse connectivity API [used for tractography]
self.mca = MouseConnectivityApi()
# Get tree search api
self.tree_search = TreeSearchApi()
# Get some metadata about experiments
self.all_experiments = self.mcc.get_experiments(dataframe=True)
self.strains = sorted([x for x in set(self.all_experiments.strain) if x is not None])
self.transgenic_lines = sorted(set([x for x in set(self.all_experiments.transgenic_line) if x is not None]))
####### GET EXPERIMENTS DATA
def get_structures_sets(self):
"""
Get the Allen's structure sets.
"""
summary_structures = self.structure_tree.get_structures_by_set_id([167587189]) # main summary structures
summary_structures = [s for s in summary_structures if s["acronym"] not in self.excluded_regions]
self.structures = pd.DataFrame(summary_structures)
# Other structures sets
try:
all_sets = pd.DataFrame(self.oapi.get_structure_sets())
except:
print("Could not retrieve data, possibly because there is no internet connection.")
else:
sets = ["Summary structures of the pons", "Summary structures of the thalamus",
"Summary structures of the hypothalamus", "List of structures for ABA Fine Structure Search",
"Structures representing the major divisions of the mouse brain", "Summary structures of the midbrain", "Structures whose surfaces are represented by a precomputed mesh"]
self.other_sets = {}
for set_name in sets:
set_id = all_sets.loc[all_sets.description == set_name].id.values[0]
self.other_sets[set_name] = pd.DataFrame(self.structure_tree.get_structures_by_set_id([set_id]))
self.all_avaliable_meshes = sorted(self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].acronym.values)
def print_structures_list_to_text(self):
"""
Saves the name of every brain structure for which a 3d mesh (.obj file) is available in a text file.
"""
s = self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].sort_values('acronym')
with open('all_regions.txt', 'w') as o:
for acr, name in zip(s.acronym.values, s['name'].values):
o.write("({}) -- {}\n".format(acr, name))
def load_all_experiments(self, cre=False):
"""
This function downloads all the experimental data from the MouseConnectivityCache and saves the unionized results
as pickled pandas dataframes. The process is slow, but the ammount of disk space necessary to save the data is small,
so it's worth downloading all the experiments at once to speed up subsequent analysis.
:param cre: Bool - data from either wild time or cre mice lines (Default value = False)
"""
if not cre: raise NotImplementedError("Only works for wild type sorry")
# Downloads all experiments from allen brain atlas and saves the results as an easy to read pkl file
for acronym in self.structures.acronym.values:
print("Fetching experiments for : {}".format(acronym))
structure = self.structure_tree.get_structures_by_acronym([acronym])[0]
experiments = self.mcc.get_experiments(cre=cre, injection_structure_ids=[structure['id']])
print(" found {} experiments".format(len(experiments)))
try:
structure_unionizes = self.mcc.get_structure_unionizes([e['id'] for e in experiments],
is_injection=False,
structure_ids=self.structures.id.values,
include_descendants=False)
except: pass
structure_unionizes.to_pickle(os.path.join(self.output_data, "{}.pkl".format(acronym)))
def print_structures(self):
"""
Prints the name of every structure in the structure tree to the console.
"""
acronyms, names = self.structures.acronym.values, self.structures['name'].values
sort_idx = np.argsort(acronyms)
acronyms, names = acronyms[sort_idx], names[sort_idx]
[print("({}) - {}".format(a, n)) for a,n in zip(acronyms, names)]
def experiments_source_search(self, SOI, *args, source=True, **kwargs):
"""
Returns data about experiments whose injection was in the SOI, structure of interest
:param SOI: str, structure of interest. Acronym of structure to use as seed for teh search
:param *args:
:param source: (Default value = True)
:param **kwargs:
"""
"""
list of possible kwargs
injection_structures : list of integers or strings
Integer Structure.id or String Structure.acronym.
target_domain : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
injection_hemisphere : string, optional
'right' or 'left', Defaults to both hemispheres.
target_hemisphere : string, optional
'right' or 'left', Defaults to both hemispheres.
transgenic_lines : list of integers or strings, optional
Integer TransgenicLine.id or String TransgenicLine.name. Specify ID 0 to exclude all TransgenicLines.
injection_domain : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
primary_structure_only : boolean, optional
product_ids : list of integers, optional
Integer Product.id
start_row : integer, optional
For paging purposes. Defaults to 0.
num_rows : integer, optional
For paging purposes. Defaults to 2000.
"""
transgenic_id = kwargs.pop('transgenic_id', 0) # id = 0 means use only wild type
primary_structure_only = kwargs.pop('primary_structure_only', True)
if not isinstance(SOI, list): SOI = [SOI]
if source:
injection_structures=SOI
target_domain = None
else:
injection_structures = None
target_domain = SOI
return pd.DataFrame(self.mca.experiment_source_search(injection_structures=injection_structures,
target_domain = target_domain,
transgenic_lines=transgenic_id,
primary_structure_only=primary_structure_only))
def experiments_target_search(self, *args, **kwargs):
"""
:param *args:
:param **kwargs:
"""
return self.experiments_source_search(*args, source=False, **kwargs)
def fetch_experiments_data(self, experiments_id, *args, average_experiments=False, **kwargs):
"""
Get data and metadata for expeirments in the Allen Mouse Connectome project.
:param experiments_id: int, list, np.ndarray with ID of experiments whose data need to be fetched
:param *args:
:param average_experiments: (Default value = False)
:param **kwargs:
"""
if isinstance(experiments_id, np.ndarray):
experiments_id = [int(x) for x in experiments_id]
elif not isinstance(experiments_id, list):
experiments_id = [experiments_id]
if [x for x in experiments_id if not isinstance(x, int)]:
raise ValueError("Invalid experiments_id argument: {}".format(experiments_id))
default_structures_ids = self.structures.id.values
is_injection = kwargs.pop('is_injection', False) # Include only structures that are not injection
structure_ids = kwargs.pop('structure_ids', default_structures_ids) # Pass IDs of structures of interest
hemisphere_ids= kwargs.pop('hemisphere_ids', None) # 1 left, 2 right, 3 both
if not average_experiments:
return pd.DataFrame(self.mca.get_structure_unionizes(experiments_id,
is_injection = is_injection,
structure_ids = structure_ids,
hemisphere_ids = hemisphere_ids))
else:
raise NotImplementedError("Need to find a way to average across experiments")
unionized = pd.DataFrame(self.mca.get_structure_unionizes(experiments_id,
is_injection = is_injection,
structure_ids = structure_ids,
hemisphere_ids = hemisphere_ids))
for regionid in list(set(unionized.structure_id)):
region_avg = unionized.loc[unionized.structure_id == regionid].mean(axis=1)
####### ANALYSIS ON EXPERIMENTAL DATA
def analyze_efferents(self, ROI, projection_metric = None):
"""
Loads the experiments on ROI and looks at average statistics of efferent projections
:param ROI: str, acronym of brain region of interest
:param projection_metric: if None, the default projection metric is used, otherwise pass a string with metric to use (Default value = None)
"""
if projection_metric is None:
projection_metric = self.projection_metric
experiment_data = pd.read_pickle(os.path.join(self.output_data, "{}.pkl".format(ROI)))
experiment_data = experiment_data.loc[experiment_data.volume > self.volume_threshold]
# Loop over all structures and get the injection density
results = {"left":[], "right":[], "both":[], "id":[], "acronym":[], "name":[]}
for target in self.structures.id.values:
target_acronym = self.structures.loc[self.structures.id == target].acronym.values[0]
target_name = self.structures.loc[self.structures.id == target].name.values[0]
exp_target = experiment_data.loc[experiment_data.structure_id == target]
exp_target_hemi = self.hemispheres(exp_target.loc[exp_target.hemisphere_id == 1],
exp_target.loc[exp_target.hemisphere_id == 2],
exp_target.loc[exp_target.hemisphere_id == 3])
proj_energy = self.hemispheres(np.nanmean(exp_target_hemi.left[projection_metric].values),
np.nanmean(exp_target_hemi.right[projection_metric].values),
np.nanmean(exp_target_hemi.both[projection_metric].values)
)
for hemi in self.hemispheres_names:
results[hemi].append(proj_energy._asdict()[hemi])
results["id"].append(target)
results["acronym"].append(target_acronym)
results["name"].append(target_name)
results = pd.DataFrame.from_dict(results).sort_values("right", na_position = "first")
return results
def analyze_afferents(self, ROI, projection_metric = None):
"""[Loads the experiments on ROI and looks at average statistics of afferent projections]
:param ROI: str, acronym of region of itnerest
:param projection_metric: if None, the default projection metric is used, otherwise pass a string with metric to use (Default value = None)
"""
if projection_metric is None:
projection_metric = self.projection_metric
ROI_id = self.structure_tree.get_structures_by_acronym([ROI])[0]["id"]
# Loop over all strctures and get projection towards SOI
results = {"left":[], "right":[], "both":[], "id":[], "acronym":[], "name":[]}
for origin in self.structures.id.values:
origin_acronym = self.structures.loc[self.structures.id == origin].acronym.values[0]
origin_name = self.structures.loc[self.structures.id == origin].name.values[0]
experiment_data = pd.read_pickle(os.path.join(self.output_data, "{}.pkl".format(origin_acronym)))
experiment_data = experiment_data.loc[experiment_data.volume > self.volume_threshold]
exp_target = experiment_data.loc[experiment_data.structure_id == SOI_id]
exp_target_hemi = self.hemispheres(exp_target.loc[exp_target.hemisphere_id == 1], exp_target.loc[exp_target.hemisphere_id == 2], exp_target.loc[exp_target.hemisphere_id == 3])
proj_energy = self.hemispheres(np.nanmean(exp_target_hemi.left[projection_metric].values),
np.nanmean(exp_target_hemi.right[projection_metric].values),
np.nanmean(exp_target_hemi.both[projection_metric].values)
)
for hemi in self.hemispheres_names:
results[hemi].append(proj_energy._asdict()[hemi])
results["id"].append(origin)
results["acronym"].append(origin_acronym)
results["name"].append(origin_name)
results = pd.DataFrame.from_dict(results).sort_values("right", na_position = "first")
return results
####### GET TRACTOGRAPHY AND SPATIAL DATA
def get_projection_tracts_to_target(self, p0=None, **kwargs):
"""
Gets tractography data for all experiments whose projections reach the brain region or location of iterest.
:param p0: list of 3 floats with XYZ coordinates of point to be used as seed (Default value = None)
:param **kwargs:
"""
# check args
if p0 is None:
raise ValueError("Please pass coordinates")
elif isinstance(p0, np.ndarray):
p0 = list(p0)
elif not isinstance(p0, (list, tuple)):
raise ValueError("Invalid argument passed (p0): {}".format(p0))
tract = self.mca.experiment_spatial_search(seed_point=p0, **kwargs)
if isinstance(tract, str):
raise ValueError('Something went wrong with query, query error message:\n{}'.format(tract))
else:
return tract
### OPERATIONS ON STRUCTURE TREES
def get_structure_ancestors(self, regions, ancestors=True, descendants=False):
"""
Get's the ancestors of the region(s) passed as arguments
:param regions: str, list of str with acronums of regions of interest
:param ancestors: if True, returns the ancestors of the region (Default value = True)
:param descendants: if True, returns the descendants of the region (Default value = False)
"""
if not isinstance(regions, list):
struct_id = self.structure_tree.get_structures_by_acronym([regions])[0]['id']
return pd.DataFrame(self.tree_search.get_tree('Structure', struct_id, ancestors=ancestors, descendants=descendants))
else:
ancestors = []
for region in regions:
struct_id = self.structure_tree.get_structures_by_acronym([region])[0]['id']
ancestors.append(pd.DataFrame(self.tree_search.get_tree('Structure', struct_id, ancestors=ancestors, descendants=descendants)))
return ancestors
def get_structure_descendants(self, regions):
return self.get_structure_ancestors(regions, ancestors=False, descendants=True)
def get_structure_from_coordinates(self, p0):
"""
Given a point in the Allen Mouse Brain reference space, returns the brain region that the point is in.
:param p0: list of floats with XYZ coordinates.
"""
voxel = np.round(np.array(p0) / self.resolution).astype(int)
try:
structure_id = self.annotated_volume[voxel[0], voxel[1], voxel[2]]
except:
return None
# Each voxel in the annotation volume is annotated as specifically as possible
structure = self.structure_tree.get_structures_by_id([structure_id])[0]
return structure
class ExperimentImagesDownloader(DirWatcher):
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir,
structs, connectivity_dir, _processor_number,
brightness_threshold, strains):
super().__init__(input_dir, process_dir, output_dir,
f'experiment-images-downloader-{_processor_number}')
self.brightness_threshold = brightness_threshold
self.structs = ast.literal_eval(structs)
self.segmentation_dir = structure_map_dir
self.mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
struct_tree = self.mcc.get_structure_tree()
structure_ids = [
i for sublist in struct_tree.descendant_ids(self.structs)
for i in sublist
]
self.structure_ids = set(structure_ids)
self.image_api = ImageDownloadApi()
self.bbox_dilation_kernel = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (14, 14))
exps = self.mcc.get_experiments(dataframe=True)
items = []
for s, gs in strains.items():
strain_items = []
for g in gs:
strain_items += [
sorted(exps[(exps.strain == s)
& (exps.gender == g)].id.tolist())
]
if strain_items:
min_len = min([len(i) for i in strain_items])
strain_items = [i[:min_len] for i in strain_items]
items += [str(i) for j in zip(*strain_items) for i in j]
self.initial_items = [i for i in items if i in self.initial_items] + [
i for i in self.initial_items if i not in items
]
def on_process_error(self, item, exception):
retval = super().on_process_error(item, exception)
self.logger.error(f"Error occurred during processing", exc_info=True)
if any(
map(lambda x: issubclass(type(exception), x), [
urllib.error.HTTPError, OSError, ValueError,
http.client.error
])):
return False
else:
return retval
def process_item(self, item, directory):
experiment_id = int(item)
retries = 0
images = []
while True:
try:
time.sleep(2**(retries // 2))
images = self.image_api.section_image_query(experiment_id)
break
except simplejson.errors.JSONDecodeError as e:
if retries > 10:
raise e
else:
self.logger.info(f"Exception invoking image API, retrying")
retries += 1
continue
images = {i['section_number']: i for i in images}
segmentation = np.load(
f'{self.segmentation_dir}/{item}/{item}-sections.npz')['arr_0']
mask = np.isin(segmentation, list(self.structure_ids))
locs = np.where(mask)
sections = [
s for s in sorted(np.unique(locs[2]).tolist()) if s in images
]
bboxes = {
section: self.extract_bounding_boxes(mask[:, :, section])
for section in sections
}
valid_sections = list(filter(lambda s: bboxes[s], sections))
self.logger.info(
f"Experiment {experiment_id}, evaluating brightness...")
brightness = self.calculate_brightness(directory, experiment_id,
images, valid_sections, mask)
if brightness < self.brightness_threshold:
self.logger.info(
f"Experiment {experiment_id}: brightness less than required minimum, removing..."
)
return False
with open(f'{directory}/bboxes.pickle', 'wb') as f:
pickle.dump(bboxes, f)
for section in valid_sections:
self.process_section(directory, experiment_id, images, section,
bboxes[section])
def calculate_brightness(self, directory, experiment_id, images,
valid_sections, mask):
pixels = np.zeros_like(mask, dtype=np.uint8)
for section in valid_sections:
self.download_snapshot(experiment_id, section, images[section],
directory)
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
image = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
pixels[:, :, section] = image[:mask.shape[0], :mask.shape[1]]
return np.median(pixels[mask != 0])
def process_section(self, directory, experiment_id, images, section,
bboxes):
self.logger.info(
f"Experiment {experiment_id}, downloading section {section}...")
self.download_snapshot(experiment_id, section, images[section],
directory)
for bbox in bboxes:
self.download_fullres(experiment_id, section, bbox,
images[section], directory)
def extract_bounding_boxes(self, mask, area_threshold=0):
bboxes = self.get_bounding_boxes(mask)
bbmask = np.zeros_like(mask, dtype=np.uint8)
for bbox in bboxes:
cv2.rectangle(bbmask, *bbox.corners(), color=1, thickness=-1)
bbmask = cv2.dilate(bbmask, self.bbox_dilation_kernel)
bboxes = [
bbox for bbox in self.get_bounding_boxes(bbmask)
if bbox.area() > area_threshold
]
return bboxes
@staticmethod
def get_bounding_boxes(mask):
contours, hierarchy = cv2.findContours(mask.astype(np.uint8),
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[-2:]
rects = []
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
if w > 5 and h > 5:
rects.append(Rect(x=x, y=y, w=w, h=h))
return rects
def download_fullres(self, experiment_id, section, bbox, image_desc,
directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={8}&&filter=range&filterVals=0,534,0,1006,0,4095'
x, y, w, h = bbox.scale(64)
url += f'&top={y}&left={x}&width={w}&height={h}'
filename = f'{directory}/full-{experiment_id}-{section}-{x}_{y}_{w}_{h}.jpg'
for retries in range(3):
fname, _, downloaded = self.retrieve_url(filename, url)
if downloaded:
try:
image = Image.open(fname)
image.load()
break
except OSError or FileNotFoundError as e:
os.remove(fname)
if retries == 2:
raise e
else:
self.logger.info(
f"Corrupted file {fname}, re-downloading {filename}"
)
else:
self.logger.info(
f"Cached version of {filename} used, skipping verification"
)
return filename
def download_snapshot(self, experiment_id, section, image_desc, directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={2}&&filter=range&filterVals=0,534,0,1006,0,4095'
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
filename, _, _ = self.retrieve_url(filename, url)
return filename
def download_brightness_snapshot(self, experiment_id, section, image_desc,
directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={2}&&filter=range&filterVals=0,534,0,1006,0,4095'
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
filename, _, _ = self.retrieve_url(filename, url)
return filename
def retrieve_url(self, filename, url, retries=10):
if os.path.isfile(filename):
self.logger.info(f"File {filename} already downloaded")
return filename, None, False
backoff = 0
urllib.request.urlcleanup()
while True:
try:
time.sleep(2**backoff)
fname, msg = urllib.request.urlretrieve(
url, filename=f'{filename}.partial')
os.replace(fname, filename)
return filename, msg, True
except http.client.HTTPException or OSError or urllib.error.HTTPError as e:
backoff += 1
retries -= 1
if retries > 0:
self.logger.info(
f"Transient error downloading {url}, "
f"retrying ({retries} retries left) ...",
exc_info=True)
continue
else:
self.logger.exception(
f"Retry count exceeded or permanent error for {url} ({filename}), exiting..."
)
raise e
def main(args):
# parse in args
parser = parsefn()
inmask, num_out_lbl = parse_inputs(parser, args)
print("\n Reading input mask (ROI) and Allen annotations")
[cutoff, maxannot, miracl_home, annot_csv, atlas_lbls,
exclude] = initialize()
# Get 'histogram' of masked labels
print(
"\n Getting histogram of included labels in mask & sorting by volume")
masked_lbls = gethist(miracl_home, inmask)
# Setup Allen connect jason cache file
mcc = MouseConnectivityCache(
manifest_file=
'%s/connect/connectivity_exps/mouse_connectivity_manifest.json' %
miracl_home)
# Load all injection experiments from Allen api
all_experiments = mcc.get_experiments(dataframe=True)
# Filter to only wild-type strain
projexps = all_experiments[all_experiments['strain'] == "C57BL/6J"]
# no transgenic mice
projexps = projexps[projexps['transgenic-line'] == ""]
# excluding background/negatives
masked_lbls = exlude_maj_lbls(masked_lbls, exclude)
masked_lbls = masked_lbls[(masked_lbls > 0) & (masked_lbls < maxannot)]
# Check if labels have injection exps
print(
"\n Checking if labels have injection experiments in the connectivity search"
)
inj_exps = check_inj_exp(masked_lbls, projexps)
# get parent labels for ones w/out inj exp
print("\n Getting parent labels for labels without injection experiments")
uniq_lbls = get_parentlbl(inj_exps, masked_lbls, annot_csv, exclude)
# check all labels have inj exps
print("\n Checking that all parent labels have injection exps")
inj_exps = check_inj_exp(uniq_lbls, projexps)
while len(inj_exps) != sum(inj_exps):
uniq_lbls = get_parentlbl(inj_exps, uniq_lbls, annot_csv, exclude)
inj_exps = check_inj_exp(uniq_lbls, projexps)
# Restrict to n labels
uniq_lbls = uniq_lbls[0:num_out_lbl]
# query structure connectivity from Allen API
print(
"\n Querying structural connectivity of injection labels in the Allen API & sorting by projection volume"
)
[all_connect_ids, all_norm_proj] = query_connect(uniq_lbls, projexps,
cutoff, exclude, mcc)
# ---------------
print(
"\n Excluding larger 'parent' labels (with graph depth < 5 and graph order < 6)"
)
# right injection sites
uniq_lbls += 20000
# exclude primary injection if found as a target regions (mutually exclusive)
filconn = [
np.delete(all_connect_ids[t],
np.where(np.in1d(all_connect_ids[t], uniq_lbls)))
for t in range(len(all_connect_ids))
]
# exclude labels not included in atlas annotations
lblinatl = [np.in1d(filconn[i], atlas_lbls) for i in range(len(filconn))]
atlfilconn = [
np.delete(filconn[i], np.where(lblinatl[i] == False))
for i in range(len(filconn))
]
conn_ids = [
np.hstack((uniq_lbls[i], atlfilconn[i])) for i in range(num_out_lbl)
]
# ---------------
# save csv
[export_connect_abv, dic] = saveconncsv(conn_ids, annot_csv, num_out_lbl)
# compute & save proj map
names = exportprojmap(all_norm_proj, num_out_lbl, export_connect_abv)
# compute & save connectivity matrix
[heatmap, targ] = exportheatmap(num_out_lbl, conn_ids, all_norm_proj,
uniq_lbls, export_connect_abv, dic, names)
# compute & save connectivity graph
createconnectogram(num_out_lbl, heatmap, annot_csv, uniq_lbls, targ, dic)
class ExperimentsSelector(widgets.VBox):
def __init__(self, available_brains=None):
self.mcc = MouseConnectivityCache(
manifest_file=
f'../mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.messages = widgets.Output()
self.set_available_brains(available_brains)
self.filter = {
col: widgets.SelectMultiple(description=col,
options=self.get_column_options(
self.experiments[col]))
for col in ['gender', 'strain', 'transgenic_line', 'id']
}
for c, f in self.filter.items():
f.observe(
lambda change, col=c: self.selection_changed(change, col))
self.change_handler = None
self.selection_changed({'name': 'value'}, 'gender')
super().__init__(
(widgets.HBox(list(self.filter.values())), self.messages))
def reset(self):
for v in self.filter.values():
v.value = []
@staticmethod
def get_column_options(df):
return sorted(df.unique().tolist())
def set_available_brains(self, available_brains):
self.experiments = self.mcc.get_experiments(dataframe=True)
self.experiments.at[self.experiments.strain.isin([None]),
'strain'] = '<none>'
self.experiments.at[self.experiments.transgenic_line.isin([None]),
'transgenic_line'] = '<none>'
if available_brains is not None:
self.experiments = self.experiments[self.experiments.id.isin(
[int(e) for e in available_brains])]
self.messages.clear_output()
with self.messages:
display(Markdown(f'Selected {len(self.get_selection())} brains'))
def get_filter_value(self, col):
selection = self.filter[col].value
if selection is None or not selection:
selection = self.filter[col].options
return selection
def selection_changed(self, change, col):
if change['name'] == 'value':
self.messages.clear_output()
with self.messages:
display(
Markdown(f'Selected {len(self.get_selection())} brains'))
selection = self.get_filter_value(col)
if col == 'gender':
oldval = self.filter['strain'].value
self.filter['strain'].options = self.get_column_options(
self.experiments[self.experiments.gender.isin(
selection)].strain)
oldval = [
v for v in oldval if v in self.filter['strain'].options
]
self.filter['strain'].value = oldval
self.selection_changed({'name': 'value'}, 'strain')
elif col == 'strain':
oldval = self.filter['transgenic_line'].value
self.filter[
'transgenic_line'].options = self.get_column_options(
self.experiments[self.experiments.gender.isin(
self.get_filter_value('gender'))
& self.experiments.strain.isin(
selection)].transgenic_line)
oldval = [
v for v in oldval
if v in self.filter['transgenic_line'].options
]
self.filter['transgenic_line'].value = oldval
self.selection_changed({'name': 'value'}, 'transgenic_line')
elif col == 'transgenic_line':
oldval = self.filter['id'].value
self.filter['id'].options = self.get_column_options(
self.experiments[
self.experiments.gender.isin(
self.get_filter_value('gender'))
& self.experiments.strain.isin(
self.get_filter_value('strain'))
& self.experiments.transgenic_line.isin(selection)].id)
oldval = [v for v in oldval if v in self.filter['id'].options]
self.filter['id'].value = oldval
if self.change_handler is not None:
self.change_handler(change, col)
def get_selection(self):
selection = {col: sel.value for col, sel in self.filter.items()}
query_string = ' and '.join(
[f'{col} == {val}' for col, val in selection.items() if val])
if query_string:
selection = self.experiments.query(query_string)
else:
selection = self.experiments
return set(selection.id.unique().tolist())
def get_selection_label(self):
label = '.'.join([
f'({",".join([str(v) for v in val.value])})'
for val in self.filter.values() if val.value
])
if label == '':
label = '<all>'
return label
def on_selection_change(self, handler):
self.change_handler = handler
