def __init__(self, input_dir):
self.mcc = MouseConnectivityCache(
manifest_file=
f'../mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.input_dir = input_dir
self.experiments_selector = ExperimentsSelector()
self.histograms_output = widgets.Output()
self.section_combo = widgets.Dropdown(options=[],
value=None,
description='Choose section:')
self.refresh_button = widgets.Button(
description='Refresh experiment list')
self.refresh_button.layout.width = 'auto'
self.refresh_button.on_click(lambda b: self.refresh)
self.display_button = widgets.Button(description='Go!')
self.display_button.on_click(self.display_experiment)
self.clear_button = widgets.Button(description='Clear plots')
self.clear_button.on_click(
lambda b: self.histograms_output.clear_output())
self.experiments_selector.on_selection_change(
lambda col, change: self.refresh())
super().__init__(
(widgets.HBox((self.experiments_selector, self.refresh_button)),
widgets.HBox((self.section_combo, self.display_button,
self.clear_button)), self.histograms_output))
self.refresh()
self.on_select_experiment([])
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir,
parent_structs, connectivity_dir, _processor_number,
cell_model, crop_size, border_size, device, threshold):
super().__init__(input_dir, process_dir, output_dir,
f'experiment-images-predictor-{_processor_number}')
self.threshold = threshold
self.device = device
if device == 'cuda:':
self.device += str(_processor_number)
self.border_size = border_size
self.crop_size = crop_size
self.cell_model = cell_model
self.parent_structs = parent_structs
self.segmentation_dir = structure_map_dir
self.mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
struct_tree = self.mcc.get_structure_tree()
structure_ids = [
i for sublist in struct_tree.descendant_ids(self.parent_structs)
for i in sublist
]
self.structure_ids = set(structure_ids)
self.bbox_dilation_kernel = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (14, 14))
self.init_model()
def __init__(self):
self.onto_id2parent_regions = {}
# only find parents up to these major brain regions
big_reg_acros = [
'Isocortex', 'OLF', 'STR', 'PAL', 'TH', 'HY', 'MB', 'PAL', 'MY',
'CB', 'HPF', 'CTXsp'
]
mcc = MouseConnectivityCache()
onto = mcc.get_ontology()
df = onto.df
struct_ids = df.id
for struct_id in struct_ids:
structure_path_str = onto[struct_id].structure_id_path.item()
region_path = structure_path_str.split('/')
parent_list = []
for r in reversed(region_path):
if r:
parent_list.append(r)
tdf = df[df.id == int(r)]
acronym = tdf.acronym.item()
if acronym in big_reg_acros:
break
self.onto_id2parent_regions[struct_id] = parent_list
def __init__(self, data_dir):
self.data_frames = DataFramesHolder(data_dir)
self.available_brains = [
e for e in os.listdir(data_dir) if e.isdigit()
]
self.data_template = self.data_frames[self.available_brains[0]]
self.messages = widgets.Output()
self.mcc = MouseConnectivityCache(
manifest_file=
f'mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.structure_tree = self.mcc.get_structure_tree()
self.aggregates = get_struct_aggregates(
set(self.get_column_options(self.data_template['structure_id'])))
self.structure_selector = StructureTree(ids=[
s['acronym'] for s in self.structure_tree.get_structures_by_id(
list(self.aggregates.keys()))
],
multiple_selection=True)
self.parameter_selector = widgets.Dropdown(
description="Parameter", options=['coverage', 'area', 'perimeter'])
self.change_handler = None
super().__init__(
(widgets.HBox([self.structure_selector,
self.parameter_selector]), self.messages))
def load_ontologies():
"""Loads all of the ontologies into a nice dictionary data structure"""
# a massive dictionary containing key : dictionary mappings between HBP ontology id's and .obo ontology terms
big_onto = {}
mcc = MouseConnectivityCache()
aba_onto = mcc.get_ontology()
file_name_list = [f for f in glob.glob(onto_root + "*.robo")]
file_name_list.extend([f for f in glob.glob(onto_root + "*.obo")])
for fn in file_name_list:
for o in oboparser.parse(fn):
if 'synonym' in o:
for s in o['synonym']:
if "BROAD ACRONYM" in s:
acro = re.search("\w+", s).group()
o['acronym'] = acro
if 'id' in o:
big_onto[o['id']] = o
for k in big_onto.keys():
if 'ABA' in k:
new_o = big_onto[k]
aba_id = int(k[11:])
new_o['acronym'] = aba_onto[aba_id]['acronym'].item()
big_onto[k] = new_o
return big_onto
def load_ontologies():
"""Loads all of the ontologies into a nice dictionary data structure"""
# a massive dictionary containing key : dictionary mappings between HBP ontology id's and .obo ontology terms
big_onto = {}
mcc = MouseConnectivityCache()
aba_onto = mcc.get_ontology()
file_name_list = [f for f in glob.glob(onto_root + "*.robo")]
file_name_list.extend([f for f in glob.glob(onto_root + "*.obo")])
for fn in file_name_list:
for o in oboparser.parse(fn):
if 'synonym' in o:
for s in o['synonym']:
if "BROAD ACRONYM" in s:
acro = re.search("\w+", s).group()
o['acronym'] = acro
if 'id' in o:
big_onto[o['id']] = o
for k in big_onto.keys():
if 'ABA' in k:
new_o = big_onto[k]
aba_id = int(k[11:])
new_o['acronym'] = aba_onto[aba_id]['acronym'].item()
big_onto[k] = new_o
return big_onto
def __init__(self):
# get mouse connectivity cache and structure tree
self.mcc = MouseConnectivityCache(manifest_file=os.path.join(
self.mouse_connectivity_cache, "manifest.json"))
self.structure_tree = self.mcc.get_structure_tree()
# get ontologies API and brain structures sets
self.oapi = OntologiesApi()
# get reference space
self.space = ReferenceSpaceApi()
self.spacecache = ReferenceSpaceCache(
manifest=os.path.join(
self.annotated_volume_fld, "manifest.json"
), # downloaded files are stored relative to here
resolution=int(self.resolution[0]),
reference_space_key=
"annotation/ccf_2017", # use the latest version of the CCF
)
self.annotated_volume, _ = self.spacecache.get_annotation_volume()
# mouse connectivity API [used for tractography]
self.mca = MouseConnectivityApi()
# Get tree search api
self.tree_search = TreeSearchApi()
def test_validate_structure_ids(inp, fails):
if fails:
with pytest.raises(ValueError) as exc:
MouseConnectivityCache.validate_structure_ids(inp)
else:
out = MouseConnectivityCache.validate_structure_ids(inp)
assert (out == [int(i) for i in inp])
def get_all_experiments(connectivity_dir):
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
from allensdk.api.queries.mouse_connectivity_api import MouseConnectivityApi
mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json',
resolution=MouseConnectivityApi.VOXEL_RESOLUTION_100_MICRONS)
experiments = mcc.get_experiments(dataframe=False)
return experiments, mcc
def test_validate_structure_ids(inp, fails):
if fails:
with pytest.raises(ValueError) as exc:
MouseConnectivityCache.validate_structure_ids(inp)
else:
out = MouseConnectivityCache.validate_structure_ids(inp)
assert( out == [ int(i) for i in inp ] )
def __init__(self, experiment_id, directory, structdata_dir, connectivity_dir, logger):
self.logger = logger
self.structdata_dir = structdata_dir
self.directory = directory
self.experiment_id = experiment_id
mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json')
tree = mcc.get_structure_tree()
self.structures = [f'Field CA{i}' for i in range(1, 4)]
self.pyramidal_layers = {s: self.get_pyramidal_layer(tree, s) for s in self.structures}
def get_anatomical_info(self):
manifest_path = './connectivity/mouse_connectivity_manifest.json'
mcc = MouseConnectivityCache(resolution=self.resolution, manifest_file=manifest_path)
tree = mcc.get_structure_tree()
annotation, _ = mcc.get_annotation_volume()
self.areas_in_blob = list(set([tree.get_structures_by_id([i])[0]['acronym'] for i in annotation[np.where(self.mask>0)] if i>0]))
self.cuboid_center_areas = [tree.get_structures_by_id([i])[0]['acronym'] for i in annotation[tuple(self.cuboid_points.T)] if i>0]
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, annotate,
_processor_number):
super().__init__(input_dir, process_dir, output_dir, f'cell-processor-{_processor_number}')
self.annotate = annotate
self.structure_ids = structs
self.brain_seg_data_dir = structure_map_dir
self.source_dir = input_dir
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json',
resolution=25)
self.experiments = {int(e['id']): e for e in self.mcc.get_experiments(dataframe=False)}
def __init__(self, ids, multiple_selection):
self.ids = ids
self.mcc = MouseConnectivityCache(
manifest_file=
f'../mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.structure_tree = self.mcc.get_structure_tree()
node = self.fill_node('grey')
while len(node.nodes) < 2:
node = node.nodes[0]
super().__init__(nodes=[node], multiple_selection=multiple_selection)
def __init__(self, output_dir, number, resolution):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/input', 'data/dl', 'data/ready']
], f'downloader-{number}')
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.mcc.get_annotation_volume()
self.mcc.get_reference_space()
def __init__(self,
projection_metric="projection_energy",
base_dir=None,
**kwargs):
"""
Set up file paths and Allen SDKs
:param base_dir: path to directory to use for saving data (default value None)
:param path_fiprojection_metricle: - str, metric to quantify the strength of projections from the Allen Connectome. (default value 'projection_energy')
:param kwargs: can be used to pass path to individual data folders. See brainrender/Utils/paths_manager.py
"""
Paths.__init__(self, base_dir=base_dir, **kwargs)
self.projection_metric = projection_metric
# get mouse connectivity cache and structure tree
self.mcc = MouseConnectivityCache(manifest_file=os.path.join(
self.mouse_connectivity_cache, "manifest.json"))
self.structure_tree = self.mcc.get_structure_tree()
# get ontologies API and brain structures sets
self.oapi = OntologiesApi()
self.get_structures_sets()
# get reference space
self.space = ReferenceSpaceApi()
self.spacecache = ReferenceSpaceCache(
manifest=os.path.join(
self.annotated_volume, "manifest.json"
), # downloaded files are stored relative to here
resolution=self.resolution,
reference_space_key=
"annotation/ccf_2017" # use the latest version of the CCF
)
self.annotated_volume, _ = self.spacecache.get_annotation_volume()
# mouse connectivity API [used for tractography]
self.mca = MouseConnectivityApi()
# Get tree search api
self.tree_search = TreeSearchApi()
# Get some metadata about experiments
self.all_experiments = self.mcc.get_experiments(dataframe=True)
self.strains = sorted(
[x for x in set(self.all_experiments.strain) if x is not None])
self.transgenic_lines = sorted(
set([
x for x in set(self.all_experiments.transgenic_line)
if x is not None
]))
def main():
# Find the path to an existing ontology document that was installed through the Brain Explorer interface
path = r'C:\Users\jenniferwh\AppData\Local\Allen Institute\Brain Explorer 2\Atlases\Allen Mouse Brain Common Coordinate Framework'
dat = pd.read_csv(os.path.join(path, 'ontology_v2.csv'))
mcc = MouseConnectivityCache(
manifest_file='/connectivity/mouse_connectivity_manifest.json')
st = mcc.get_structure_tree()
ia_map = st.get_id_acronym_map()
jsonfile = r'/Users/jenniferwh/Dropbox (Allen Institute)/Diamond_collaboration/data_files/consensus_structures.json'
with open(jsonfile, 'r') as file:
consensus_structures = json.load(file)
age = str(args.age) + 'monthTG'
print(age)
tau_structures = consensus_structures[age]
tau_structure_ids = [ia_map[structure] for structure in tau_structures]
'''
for structure in thal_structure_ids:
if structure not in ss:
children = st.descendant_ids([structure])[0]
child_ss = [structure_id for structure_id in children if structure_id in ss]
ss += [structure]
thal_structure_ids+=child_ss
'''
colormap = cm.Reds
all_structures = dat['abbreviation'].unique()
all_structures = [
structure for structure in all_structures
if structure in ia_map.keys()
]
all_structures.remove('root')
all_structure_ids = [ia_map[structure] for structure in all_structures]
structure_vals = dict()
for structure in all_structure_ids:
if structure in tau_structure_ids:
structure_vals[structure] = 0.9
else:
structure_vals[structure] = 0
rgb_vals = structure_vals.copy()
for key in all_structure_ids:
rgb_vals[key] = tuple(
[255 * i for i in colormap(structure_vals[key])[:3]])
dat.loc[dat['database_id'] == key, 'red'] = int(rgb_vals[key][0])
dat.loc[dat['database_id'] == key, 'green'] = int(rgb_vals[key][1])
dat.loc[dat['database_id'] == key, 'blue'] = int(rgb_vals[key][2])
rgb_vals[0] = (0, 0, 0)
dat.loc[dat['abbreviation'] == 'root', 'parent'] = 0.
dat['parent'] = [int(value) for value in dat['parent']]
dat.to_csv(os.path.join(path, 'ontology_v2.csv'), index=False)
def prepare_input(self, input_dir, connectivity_dir, structure_map_dir,
**kwargs):
mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
mcc.get_structure_tree()
experiments = os.listdir(structure_map_dir)
try:
os.makedirs(input_dir)
for e in experiments:
os.makedirs(f'{input_dir}/{e}')
except FileExistsError:
pass
def main():
mcc = MouseConnectivityCache(resolution=10)
rsp = mcc.get_reference_space()
# structures come from an rma query like this:
# http://api.brain-map.org/api/v2/data/query.json?q=model::Structure,rma::criteria,[graph_id$in1]
# which does the same thing as using structure_graph_download, but without nesting
# the graph id is 1
structures = rsp.structure_tree.nodes()
ontology_ids = [s['id'] for s in structures]
# the annotation comes from: http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/annotation/ccf_2017/annotation_10.nrrd
annotation_ids = np.unique(rsp.annotation)
class StructureTree(ipytree.Tree):
def __init__(self, ids, multiple_selection):
self.ids = ids
self.mcc = MouseConnectivityCache(
manifest_file=
f'../mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.structure_tree = self.mcc.get_structure_tree()
node = self.fill_node('grey')
while len(node.nodes) < 2:
node = node.nodes[0]
super().__init__(nodes=[node], multiple_selection=multiple_selection)
def fill_node(self, start_node):
start_struct = self.structure_tree.get_structures_by_acronym(
[start_node])[0]
children = [
self.fill_node(c['acronym']) for c in self.structure_tree.children(
[self.structure_tree.get_id_acronym_map()[start_node]])[0]
]
children = [c for c in children if c is not None]
if not children and start_struct['acronym'] not in self.ids:
return None
node = StructureTreeNode(start_struct['acronym'],
start_struct['acronym'], children)
if start_struct['acronym'] not in self.ids:
node.disabled = True
return node
def mcc():
mcc = MouseConnectivityCache(
resolution=MouseConnectivityApi.VOXEL_RESOLUTION_100_MICRONS,
manifest_file='mcc_manifest.json')
mcc.api.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
return mcc
def main(output_dir, resolution, retain_transform_data, zoom,
parallel_downloads, parallel_processors):
mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
mcc.get_annotation_volume()
mcc.get_reference_space()
experiments = [e['id'] for e in mcc.get_experiments(dataframe=False)]
experiments = sorted(list(set(experiments)))
try:
os.makedirs(f'{output_dir}/data/input')
for e in experiments:
os.makedirs(f'{output_dir}/data/input/{e}')
except FileExistsError:
for e in experiments:
if os.path.isdir(f'{output_dir}/data/input/{e}'):
shutil.rmtree(f'{output_dir}/data/input/{e}')
os.makedirs(f'{output_dir}/data/input/{e}')
downloads = [
subprocess.Popen([
"python", "./displacement_data_downloader.py", f"-o{output_dir}",
f"-r{resolution}", f"-n{i}"
]) for i in range(parallel_downloads)
]
processes = [
subprocess.Popen([
"python", "./segmentation_data_builder.py", f"-o{output_dir}",
f"-r{resolution}", f"-c{len(experiments)}", f"-z{zoom}"
] + (["-t"] if retain_transform_data else []) + [f"-n{i}"])
for i in range(parallel_processors)
]
exit_codes = [p.wait() for p in processes + downloads]
class CellProcessor(DirWatcher):
experiment_fields_to_save = [
'id',
'gender',
'injection_structures',
'injection_volume',
'injection_x',
'injection_y',
'injection_z',
'product_id',
'specimen_name',
'strain',
'structure_abbrev',
'structure_id',
'structure_name',
'transgenic_line',
'transgenic_line_id',
'primary_injection_structure'
]
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, annotate,
_processor_number):
super().__init__(input_dir, process_dir, output_dir, f'cell-processor-{_processor_number}')
self.annotate = annotate
self.structure_ids = structs
self.brain_seg_data_dir = structure_map_dir
self.source_dir = input_dir
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json',
resolution=25)
self.experiments = {int(e['id']): e for e in self.mcc.get_experiments(dataframe=False)}
def process_item(self, item, directory):
item = int(item)
experiment = ExperimentCellsProcessor(self.mcc,
item,
directory,
self.brain_seg_data_dir,
self.structure_ids,
self.experiment_fields_to_save,
self.experiments[item],
self.logger)
experiment.process()
if self.annotate:
annotator = ExperimentDataAnnotator(int(item), directory, self.logger)
annotator.process()
def on_process_error(self, item, exception):
retval = super().on_process_error(item, exception)
self.logger.error(f"Error occurred during processing", exc_info=True)
if type(exception) in [urllib.error.HTTPError, OSError, ValueError,
urllib.error.URLError, socket.gaierror, MemoryError]:
return False
else:
return retval
def __init__(self, projection_metric="projection_energy", paths_file=None):
""" path_file {[str]} -- [Path to a YAML file specifying paths to data folders, to replace default paths] (default: {None}) """
Paths.__init__(self, paths_file=paths_file)
self.projection_metric = projection_metric
# get mouse connectivity cache and structure tree
self.mcc = MouseConnectivityCache(manifest_file=os.path.join(
self.mouse_connectivity_cache, "manifest.json"))
self.structure_tree = self.mcc.get_structure_tree()
# get ontologies API and brain structures sets
self.oapi = OntologiesApi()
self.get_structures_sets()
# get reference space
self.space = ReferenceSpaceApi()
self.spacecache = ReferenceSpaceCache(
manifest=os.path.join(
"Data/ABA", "manifest.json"
), # downloaded files are stored relative to here
resolution=self.resolution,
reference_space_key=
"annotation/ccf_2017" # use the latest version of the CCF
)
self.annotated_volume, _ = self.spacecache.get_annotation_volume()
# mouse connectivity API [used for tractography]
self.mca = MouseConnectivityApi()
# Get tree search api
self.tree_search = TreeSearchApi()
# Get some metadata about experiments
self.all_experiments = self.mcc.get_experiments(dataframe=True)
self.strains = sorted(
[x for x in set(self.all_experiments.strain) if x is not None])
self.transgenic_lines = sorted(
set([
x for x in set(self.all_experiments.transgenic_line)
if x is not None
]))
def get_neuron_region(neuron_ob):
""" Given a neuron object, return a dictionary describing its major brain region from the Allen Mouse Atlas
:param neuron_ob: A neuroelectro.models Neuron object
:return: returns a dictionary of region attributes
"""
regs = neuron_ob.regions.all()
mcc = MouseConnectivityCache()
onto = mcc.get_ontology()
if regs:
r = regs[0]
structure_path_str = onto[r.allenid].structure_id_path.item()
region_dict = get_major_brain_region(structure_path_str, onto)
return region_dict
else:
return None
# if region_dict:
# print (n.name, region_dict['region_name'])
# else:
# print 'No region found for %s' % n.name
def __init__(self, output_dir, resolution, retain_transform_data, zoom,
number, count):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/ready', 'data/proc', 'data/result']
], f'segmentation-data-builder-{number}')
self.count = count
self.zoom = zoom
self.retain_transform_data = retain_transform_data
self.resolution = resolution
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.anno, self.meta = self.mcc.get_annotation_volume()
self.rsp = self.mcc.get_reference_space()
self.rsp.remove_unassigned(
) # This removes ids that are not in this particular reference space
def __init__(self, available_brains=None):
self.mcc = MouseConnectivityCache(
manifest_file=
f'../mouse_connectivity/mouse_connectivity_manifest.json',
resolution=25)
self.messages = widgets.Output()
self.set_available_brains(available_brains)
self.filter = {
col: widgets.SelectMultiple(description=col,
options=self.get_column_options(
self.experiments[col]))
for col in ['gender', 'strain', 'transgenic_line', 'id']
}
for c, f in self.filter.items():
f.observe(
lambda change, col=c: self.selection_changed(change, col))
self.change_handler = None
self.selection_changed({'name': 'value'}, 'gender')
super().__init__(
(widgets.HBox(list(self.filter.values())), self.messages))
def __init__(self, base_dir=None, **kwargs):
"""
Set up file paths and Allen SDKs
:param base_dir: path to directory to use for saving data (default value None)
:param kwargs: can be used to pass path to individual data folders. See brainrender/Utils/paths_manager.py
"""
Atlas.__init__(self, base_dir=base_dir, **kwargs)
self.meshes_folder = self.mouse_meshes # where the .obj mesh for each region is saved
# get mouse connectivity cache and structure tree
self.mcc = MouseConnectivityCache(manifest_file=os.path.join(self.mouse_connectivity_cache, "manifest.json"))
self.structure_tree = self.mcc.get_structure_tree()
# get ontologies API and brain structures sets
self.oapi = OntologiesApi()
self.get_structures_sets()
# get reference space
self.space = ReferenceSpaceApi()
self.spacecache = ReferenceSpaceCache(
manifest=os.path.join(self.annotated_volume_fld, "manifest.json"), # downloaded files are stored relative to here
resolution=self.resolution,
reference_space_key="annotation/ccf_2017" # use the latest version of the CCF
)
self.annotated_volume, _ = self.spacecache.get_annotation_volume()
# mouse connectivity API [used for tractography]
self.mca = MouseConnectivityApi()
# Get tree search api
self.tree_search = TreeSearchApi()
# Store all regions metadata [If there's internet connection]
if self.other_sets is not None:
self.regions = self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].sort_values('acronym')
self.region_acronyms = list(self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].sort_values(
'acronym').acronym.values)
def __init__(self):
self.onto_id2parent_regions = {}
# only find parents up to these major brain regions
big_reg_acros = ['Isocortex', 'OLF', 'STR', 'PAL', 'TH', 'HY', 'MB', 'PAL', 'MY', 'CB', 'HPF', 'CTXsp']
mcc = MouseConnectivityCache()
onto = mcc.get_ontology()
df = onto.df
struct_ids = df.id
for struct_id in struct_ids:
structure_path_str = onto[struct_id].structure_id_path.item()
region_path = structure_path_str.split('/')
parent_list = []
for r in reversed(region_path):
if r:
parent_list.append(r)
tdf = df[df.id == int(r)]
acronym = tdf.acronym.item()
if acronym in big_reg_acros:
break
self.onto_id2parent_regions[struct_id] = parent_list
class SegmentationDataBuilder(DirWatcher):
def __init__(self, output_dir, resolution, retain_transform_data, zoom,
number, count):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/ready', 'data/proc', 'data/result']
], f'segmentation-data-builder-{number}')
self.count = count
self.zoom = zoom
self.retain_transform_data = retain_transform_data
self.resolution = resolution
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.anno, self.meta = self.mcc.get_annotation_volume()
self.rsp = self.mcc.get_reference_space()
self.rsp.remove_unassigned(
) # This removes ids that are not in this particular reference space
def process_item(self, item, directory):
item = int(item)
experiment = ExperimentSectionData(
self.mcc,
item,
directory,
self.anno,
self.meta,
self.rsp,
self.logger,
zoom=self.zoom,
remove_transform_data=not self.retain_transform_data)
experiment.create_section_data()
experiment.cleanup()
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir,
structs, connectivity_dir, _processor_number,
brightness_threshold, strains):
super().__init__(input_dir, process_dir, output_dir,
f'experiment-images-downloader-{_processor_number}')
self.brightness_threshold = brightness_threshold
self.structs = ast.literal_eval(structs)
self.segmentation_dir = structure_map_dir
self.mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
struct_tree = self.mcc.get_structure_tree()
structure_ids = [
i for sublist in struct_tree.descendant_ids(self.structs)
for i in sublist
]
self.structure_ids = set(structure_ids)
self.image_api = ImageDownloadApi()
self.bbox_dilation_kernel = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (14, 14))
exps = self.mcc.get_experiments(dataframe=True)
items = []
for s, gs in strains.items():
strain_items = []
for g in gs:
strain_items += [
sorted(exps[(exps.strain == s)
& (exps.gender == g)].id.tolist())
]
if strain_items:
min_len = min([len(i) for i in strain_items])
strain_items = [i[:min_len] for i in strain_items]
items += [str(i) for j in zip(*strain_items) for i in j]
self.initial_items = [i for i in items if i in self.initial_items] + [
i for i in self.initial_items if i not in items
]
class SegmentationDataDownloader(DirWatcher):
def __init__(self, output_dir, number, resolution):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/input', 'data/dl', 'data/ready']
], f'downloader-{number}')
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.mcc.get_annotation_volume()
self.mcc.get_reference_space()
def process_item(self, item, directory):
self.logger.info(f"Download displacement data for {item}")
self.mcc.get_deformation_field(item,
header_path=f'{directory}/dfmfld.mhd',
voxel_path=f'{directory}/dfmfld.raw')
self.mcc.get_affine_parameters(item,
direction='trv',
file_name=f'{directory}/aff_param.txt')
def get_mcc(res=50, manifest_file=manifest_file):
'''
Fetches MouseConnectivityCache object to interact with Allen data. Defaults
to setting manifest file to user's home directory.
Parameters
__________
res : int
Sets voxel size for Allen data. Must be 100, 50, 25, or 10.
Default is 50.
manifest_file : str
Sets location of manifest file
Returns
_______
MouseConnectivityCache object.
'''
if res not in [100, 50, 25, 10]:
raise ValueError('Res must be 100, 50, 25, or 10')
return MouseConnectivityCache(resolution=res, manifest_file=manifest_file)
def main(args):
# parse in args
parser = parsefn()
inmask, num_out_lbl = parse_inputs(parser, args)
print("\n Reading input mask (ROI) and Allen annotations")
[cutoff, maxannot, miracl_home, annot_csv, atlas_lbls,
exclude] = initialize()
# Get 'histogram' of masked labels
print(
"\n Getting histogram of included labels in mask & sorting by volume")
masked_lbls = gethist(miracl_home, inmask)
# Setup Allen connect jason cache file
mcc = MouseConnectivityCache(
manifest_file=
'%s/connect/connectivity_exps/mouse_connectivity_manifest.json' %
miracl_home)
# Load all injection experiments from Allen api
all_experiments = mcc.get_experiments(dataframe=True)
# Filter to only wild-type strain
projexps = all_experiments[all_experiments['strain'] == "C57BL/6J"]
# no transgenic mice
projexps = projexps[projexps['transgenic-line'] == ""]
# excluding background/negatives
masked_lbls = exlude_maj_lbls(masked_lbls, exclude)
masked_lbls = masked_lbls[(masked_lbls > 0) & (masked_lbls < maxannot)]
# Check if labels have injection exps
print(
"\n Checking if labels have injection experiments in the connectivity search"
)
inj_exps = check_inj_exp(masked_lbls, projexps)
# get parent labels for ones w/out inj exp
print("\n Getting parent labels for labels without injection experiments")
uniq_lbls = get_parentlbl(inj_exps, masked_lbls, annot_csv, exclude)
# check all labels have inj exps
print("\n Checking that all parent labels have injection exps")
inj_exps = check_inj_exp(uniq_lbls, projexps)
while len(inj_exps) != sum(inj_exps):
uniq_lbls = get_parentlbl(inj_exps, uniq_lbls, annot_csv, exclude)
inj_exps = check_inj_exp(uniq_lbls, projexps)
# Restrict to n labels
uniq_lbls = uniq_lbls[0:num_out_lbl]
# query structure connectivity from Allen API
print(
"\n Querying structural connectivity of injection labels in the Allen API & sorting by projection volume"
)
[all_connect_ids, all_norm_proj] = query_connect(uniq_lbls, projexps,
cutoff, exclude, mcc)
# ---------------
print(
"\n Excluding larger 'parent' labels (with graph depth < 5 and graph order < 6)"
)
# right injection sites
uniq_lbls += 20000
# exclude primary injection if found as a target regions (mutually exclusive)
filconn = [
np.delete(all_connect_ids[t],
np.where(np.in1d(all_connect_ids[t], uniq_lbls)))
for t in range(len(all_connect_ids))
]
# exclude labels not included in atlas annotations
lblinatl = [np.in1d(filconn[i], atlas_lbls) for i in range(len(filconn))]
atlfilconn = [
np.delete(filconn[i], np.where(lblinatl[i] == False))
for i in range(len(filconn))
]
conn_ids = [
np.hstack((uniq_lbls[i], atlfilconn[i])) for i in range(num_out_lbl)
]
# ---------------
# save csv
[export_connect_abv, dic] = saveconncsv(conn_ids, annot_csv, num_out_lbl)
# compute & save proj map
names = exportprojmap(all_norm_proj, num_out_lbl, export_connect_abv)
# compute & save connectivity matrix
[heatmap, targ] = exportheatmap(num_out_lbl, conn_ids, all_norm_proj,
uniq_lbls, export_connect_abv, dic, names)
# compute & save connectivity graph
createconnectogram(num_out_lbl, heatmap, annot_csv, uniq_lbls, targ, dic)
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# tell the cache class what resolution (in microns) of data you want to download
mcc = MouseConnectivityCache(resolution=25)
# use the ontology class to get the id of the isocortex structure
ontology = mcc.get_ontology('ontology.csv')
isocortex = ontology['Isocortex']
# a list of dictionaries containing metadata for non-Cre experiments
experiments = mcc.get_experiments(file_name='non_cre.json',
injection_structure_ids=isocortex['id'])
# download the projection density volume for one of the experiments
pd = mcc.get_projection_density(experiments[0]['id'])
#===============================================================================
# example 1
#===============================================================================
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# tell the cache class what resolution (in microns) of data you want to download
mcc = MouseConnectivityCache(resolution=25)
# use the structure tree class to get information about the isocortex structure
structure_tree = mcc.get_structure_tree()
isocortex_id = structure_tree.get_structures_by_name(['Isocortex'])[0]['id']
# a list of dictionaries containing metadata for non-Cre experiments
experiments = mcc.get_experiments(file_name='non_cre.json',
injection_structure_ids=[isocortex_id])
# download the projection density volume for one of the experiments
pd = mcc.get_projection_density(experiments[0]['id'])
#===============================================================================
# example 2
#===============================================================================
import nrrd
file_name = 'mouse_connectivity/experiment_644250774/projection_density_25.nrrd'
data_array, metadata = nrrd.read(file_name)
'''
@author wronk
Script to calculate the volume of the mouse brain from the common coordinate
frame developed by ABI. This requires the connectivity dataset and the
allensdk.
Output from connectivity dataset (as of Sept 2015) is below.
'''
import numpy as np
from allensdk.core.mouse_connectivity_cache import (MouseConnectivityCache as
MCC)
# load in nrdd data
mcc = MCC(manifest_file='/Volumes/Brain2015/connectivity/manifest.json')
annot, annot_info = mcc.get_annotation_volume()
# get all voxels that are non-zeros
brain_voxels = annot.flatten() > 0
sum_vox = np.sum(brain_voxels)
print 'Found %d voxels' % (sum_vox)
print 'Density is %0.3f percent' % (100 * sum_vox / len(brain_voxels))
# Get size of each box, make sure they're in cubes
width_dims = []
for di in range(3):
width_dims.append(annot_info['space directions'][di][di])
assert len(set(width_dims)) == 1
if len(sys.argv) < 3:
print ""
print "please provide a working directory & output file name, e.g."
print ""
print " build_projection_datasets.py data/ pms.pickle"
print ""
sys.exit(1)
import os
os.chdir(sys.argv[1])
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
from allensdk.api.queries.ontologies_api import OntologiesApi
mcc = MouseConnectivityCache(manifest_file="mcc_manifest.json")
all_experiments = mcc.get_experiments(dataframe=True)
ontology = mcc.get_ontology()
summary_structures = OntologiesApi().get_structures(structure_set_names="Mouse Connectivity - Summary")
summary_structure_ids = [s["id"] for s in summary_structures]
print "build dict of injection structure id to experiment list"
ist2e = {}
for eid in all_experiments.index:
for ist in all_experiments.ix[eid]["injection-structures"]:
isti = ist["id"]
if isti not in ist2e:
ist2e[isti] = []
ist2e[isti].append(eid)
def launch(self, resolution, weighting, inf_vox_thresh, vol_thresh):
resolution = int(resolution)
weighting = int(weighting)
inf_vox_thresh = float(inf_vox_thresh)
vol_thresh = float(vol_thresh)
project = dao.get_project_by_id(self.current_project_id)
manifest_file = self.file_handler.get_allen_mouse_cache_folder(project.name)
manifest_file = os.path.join(manifest_file, "mouse_connectivity_manifest.json")
cache = MouseConnectivityCache(resolution=resolution, manifest_file=manifest_file)
# the method creates a dictionary with information about which experiments need to be downloaded
ist2e = DictionaireBuilder(cache, False)
# the method downloads experiments necessary to build the connectivity
projmaps = DownloadAndConstructMatrix(cache, weighting, ist2e, False)
# the method cleans the file projmaps in 4 steps
projmaps = pmsCleaner(projmaps)
Vol, annot_info = cache.get_annotation_volume()
ontology = cache.get_ontology()
# the method includes in the parcellation only brain regions whose volume is greater than vol_thresh
projmaps = AreasVolumeTreshold(projmaps, vol_thresh, resolution, Vol, ontology)
# the method includes in the parcellation only brain regions where at least one injection experiment had infected more than N voxel (where N is inf_vox_thresh)
projmaps = AreasVoxelTreshold(cache, projmaps, inf_vox_thresh, Vol, ontology)
# the method creates file order and keyord that will be the link between the SC order and the id key in the Allen database
[order, key_ord] = CreateFileOrder(projmaps, ontology)
# the method builds the Structural Connectivity (SC) matrix
SC = ConstructingSC(projmaps, order, key_ord)
# the method returns the coordinate of the centres and the name of the brain areas in the selected parcellation
[centres, names] = Construct_centres(ontology, order, key_ord, Vol)
# the method returns the tract lengths between the brain areas in the selected parcellation
tract_lengths = ConstructTractLengths(centres)
# the method associated the parent and the grandparents to the child in the selected parcellation with the biggest volume
[unique_parents, unique_grandparents] = ParentsAndGrandParentsFinder(order, key_ord, Vol, ontology)
# the method returns a volume indexed between 0 and N-1, with N=tot brain areas in the parcellation. -1=background and areas that are not in the parcellation
Vol_parcel = MouseBrainVisualizer(Vol, order, key_ord, unique_parents, unique_grandparents, ontology, projmaps)
# results: Connectivity, Volume & RegionVolumeMapping
# Connectivity
result_connectivity = Connectivity(storage_path=self.storage_path)
result_connectivity.centres = centres
result_connectivity.region_labels = names
result_connectivity.weights = SC
result_connectivity.tract_lengths = tract_lengths
# Volume
result_volume = Volume(storage_path=self.storage_path)
result_volume.origin = [[0.0, 0.0, 0.0]]
result_volume.voxel_size = [resolution, resolution, resolution]
# result_volume.voxel_unit= micron
# Region Volume Mapping
result_rvm = RegionVolumeMapping(storage_path=self.storage_path)
result_rvm.volume = result_volume
result_rvm.array_data = Vol_parcel
result_rvm.connectivity = result_connectivity
result_rvm.title = "Volume mouse brain "
result_rvm.dimensions_labels = ["X", "Y", "Z"]
return [result_connectivity, result_rvm, result_volume]
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # tell the cache class what resolution (in microns) of data you want to download mcc = MouseConnectivityCache(resolution=25) # use the ontology class to get the id of the isocortex structure ontology = mcc.get_ontology() isocortex = ontology["Isocortex"] # a list of dictionaries containing metadata for non-Cre experiments experiments = mcc.get_experiments(injection_structure_ids=isocortex["id"]) # download the projection density volume for one of the experiments pd = mcc.get_projection_density(experiments[0]["id"])
for exp in range(len(experiments)):
pd = mca.download_projection_density('example.nrrd', experiments[exp]['id'], resolution=25)
print(type(pd))
pdb.set_trace()
# The manifest file is a simple JSON file that keeps track of all of
# the data that has already been downloaded onto the hard drives.
# If you supply a relative path, it is assumed to be relative to your
# current working directory.
mcc = MouseConnectivityCache(manifest_file='connectivity/manifest.json')
# open up a list of all of the experiments
all_experiments = mcc.get_experiments(dataframe=True)
from allensdk.api.queries.biophysical_perisomatic_api import \
BiophysicalPerisomaticApi
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import os
from allensdk.core.nwb_data_set import NwbDataSet
from IPython import __main__
def launch(self, resolution, weighting, inj_f_thresh, vol_thresh):
resolution = int(resolution)
weighting = int(weighting)
inj_f_thresh = float(inj_f_thresh)/100.
vol_thresh = float(vol_thresh)
project = dao.get_project_by_id(self.current_project_id)
manifest_file = self.file_handler.get_allen_mouse_cache_folder(project.name)
manifest_file = os.path.join(manifest_file, 'mouse_connectivity_manifest.json')
cache = MouseConnectivityCache(resolution=resolution, manifest_file=manifest_file)
# the method creates a dictionary with information about which experiments need to be downloaded
ist2e = dictionary_builder(cache, False)
# the method downloads experiments necessary to build the connectivity
projmaps = download_an_construct_matrix(cache, weighting, ist2e, False)
# the method cleans the file projmaps in 4 steps
projmaps = pms_cleaner(projmaps)
# download from the AllenSDK the annotation volume, the template volume
vol, annot_info = cache.get_annotation_volume()
template, template_info = cache.get_template_volume()
# rotate template in the TVB 3D reference:
template = rotate_reference(template)
# grab the StructureTree instance
structure_tree = cache.get_structure_tree()
# the method includes in the parcellation only brain regions whose volume is greater than vol_thresh
projmaps = areas_volume_threshold(cache, projmaps, vol_thresh, resolution)
# the method exclude from the experimental dataset
# those exps where the injected fraction of pixel in the injection site is lower than than the inj_f_thr
projmaps = infected_threshold(cache, projmaps, inj_f_thresh)
# the method creates file order and keyword that will be the link between the SC order and the
# id key in the Allen database
[order, key_ord] = create_file_order(projmaps, structure_tree)
# the method builds the Structural Connectivity (SC) matrix
structural_conn = construct_structural_conn(projmaps, order, key_ord)
# the method returns the coordinate of the centres and the name of the brain areas in the selected parcellation
[centres, names] = construct_centres(cache, order, key_ord)
# the method returns the tract lengths between the brain areas in the selected parcellation
tract_lengths = construct_tract_lengths(centres)
# the method associated the parent and the grandparents to the child in the selected parcellation with
# the biggest volume
[unique_parents, unique_grandparents] = parents_and_grandparents_finder(cache, order, key_ord, structure_tree)
# the method returns a volume indexed between 0 and N-1, with N=tot brain areas in the parcellation.
# -1=background and areas that are not in the parcellation
vol_parcel = mouse_brain_visualizer(vol, order, key_ord, unique_parents, unique_grandparents,
structure_tree, projmaps)
# results: Connectivity, Volume & RegionVolumeMapping
# Connectivity
result_connectivity = Connectivity(storage_path=self.storage_path)
result_connectivity.centres = centres
result_connectivity.region_labels = names
result_connectivity.weights = structural_conn
result_connectivity.tract_lengths = tract_lengths
# Volume
result_volume = Volume(storage_path=self.storage_path)
result_volume.origin = [[0.0, 0.0, 0.0]]
result_volume.voxel_size = [resolution, resolution, resolution]
# result_volume.voxel_unit= micron
# Region Volume Mapping
result_rvm = RegionVolumeMapping(storage_path=self.storage_path)
result_rvm.volume = result_volume
result_rvm.array_data = vol_parcel
result_rvm.connectivity = result_connectivity
result_rvm.title = "Volume mouse brain "
result_rvm.dimensions_labels = ["X", "Y", "Z"]
# Volume template
result_template = StructuralMRI(storage_path=self.storage_path)
result_template.array_data = template
result_template.weighting = 'T1'
result_template.volume = result_volume
return [result_connectivity, result_volume, result_rvm, result_template]
