def __init__(self, input_dir): self.mcc = MouseConnectivityCache( manifest_file= f'../mouse_connectivity/mouse_connectivity_manifest.json', resolution=25) self.input_dir = input_dir self.experiments_selector = ExperimentsSelector() self.histograms_output = widgets.Output() self.section_combo = widgets.Dropdown(options=[], value=None, description='Choose section:') self.refresh_button = widgets.Button( description='Refresh experiment list') self.refresh_button.layout.width = 'auto' self.refresh_button.on_click(lambda b: self.refresh) self.display_button = widgets.Button(description='Go!') self.display_button.on_click(self.display_experiment) self.clear_button = widgets.Button(description='Clear plots') self.clear_button.on_click( lambda b: self.histograms_output.clear_output()) self.experiments_selector.on_selection_change( lambda col, change: self.refresh()) super().__init__( (widgets.HBox((self.experiments_selector, self.refresh_button)), widgets.HBox((self.section_combo, self.display_button, self.clear_button)), self.histograms_output)) self.refresh() self.on_select_experiment([])
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, parent_structs, connectivity_dir, _processor_number, cell_model, crop_size, border_size, device, threshold): super().__init__(input_dir, process_dir, output_dir, f'experiment-images-predictor-{_processor_number}') self.threshold = threshold self.device = device if device == 'cuda:': self.device += str(_processor_number) self.border_size = border_size self.crop_size = crop_size self.cell_model = cell_model self.parent_structs = parent_structs self.segmentation_dir = structure_map_dir self.mcc = MouseConnectivityCache( manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json' ) struct_tree = self.mcc.get_structure_tree() structure_ids = [ i for sublist in struct_tree.descendant_ids(self.parent_structs) for i in sublist ] self.structure_ids = set(structure_ids) self.bbox_dilation_kernel = cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (14, 14)) self.init_model()
def __init__(self): self.onto_id2parent_regions = {} # only find parents up to these major brain regions big_reg_acros = [ 'Isocortex', 'OLF', 'STR', 'PAL', 'TH', 'HY', 'MB', 'PAL', 'MY', 'CB', 'HPF', 'CTXsp' ] mcc = MouseConnectivityCache() onto = mcc.get_ontology() df = onto.df struct_ids = df.id for struct_id in struct_ids: structure_path_str = onto[struct_id].structure_id_path.item() region_path = structure_path_str.split('/') parent_list = [] for r in reversed(region_path): if r: parent_list.append(r) tdf = df[df.id == int(r)] acronym = tdf.acronym.item() if acronym in big_reg_acros: break self.onto_id2parent_regions[struct_id] = parent_list
def __init__(self, data_dir): self.data_frames = DataFramesHolder(data_dir) self.available_brains = [ e for e in os.listdir(data_dir) if e.isdigit() ] self.data_template = self.data_frames[self.available_brains[0]] self.messages = widgets.Output() self.mcc = MouseConnectivityCache( manifest_file= f'mouse_connectivity/mouse_connectivity_manifest.json', resolution=25) self.structure_tree = self.mcc.get_structure_tree() self.aggregates = get_struct_aggregates( set(self.get_column_options(self.data_template['structure_id']))) self.structure_selector = StructureTree(ids=[ s['acronym'] for s in self.structure_tree.get_structures_by_id( list(self.aggregates.keys())) ], multiple_selection=True) self.parameter_selector = widgets.Dropdown( description="Parameter", options=['coverage', 'area', 'perimeter']) self.change_handler = None super().__init__( (widgets.HBox([self.structure_selector, self.parameter_selector]), self.messages))
def load_ontologies(): """Loads all of the ontologies into a nice dictionary data structure""" # a massive dictionary containing key : dictionary mappings between HBP ontology id's and .obo ontology terms big_onto = {} mcc = MouseConnectivityCache() aba_onto = mcc.get_ontology() file_name_list = [f for f in glob.glob(onto_root + "*.robo")] file_name_list.extend([f for f in glob.glob(onto_root + "*.obo")]) for fn in file_name_list: for o in oboparser.parse(fn): if 'synonym' in o: for s in o['synonym']: if "BROAD ACRONYM" in s: acro = re.search("\w+", s).group() o['acronym'] = acro if 'id' in o: big_onto[o['id']] = o for k in big_onto.keys(): if 'ABA' in k: new_o = big_onto[k] aba_id = int(k[11:]) new_o['acronym'] = aba_onto[aba_id]['acronym'].item() big_onto[k] = new_o return big_onto
def __init__(self): # get mouse connectivity cache and structure tree self.mcc = MouseConnectivityCache(manifest_file=os.path.join( self.mouse_connectivity_cache, "manifest.json")) self.structure_tree = self.mcc.get_structure_tree() # get ontologies API and brain structures sets self.oapi = OntologiesApi() # get reference space self.space = ReferenceSpaceApi() self.spacecache = ReferenceSpaceCache( manifest=os.path.join( self.annotated_volume_fld, "manifest.json" ), # downloaded files are stored relative to here resolution=int(self.resolution[0]), reference_space_key= "annotation/ccf_2017", # use the latest version of the CCF ) self.annotated_volume, _ = self.spacecache.get_annotation_volume() # mouse connectivity API [used for tractography] self.mca = MouseConnectivityApi() # Get tree search api self.tree_search = TreeSearchApi()
def test_validate_structure_ids(inp, fails): if fails: with pytest.raises(ValueError) as exc: MouseConnectivityCache.validate_structure_ids(inp) else: out = MouseConnectivityCache.validate_structure_ids(inp) assert (out == [int(i) for i in inp])
def get_all_experiments(connectivity_dir): from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache from allensdk.api.queries.mouse_connectivity_api import MouseConnectivityApi mcc = MouseConnectivityCache( manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json', resolution=MouseConnectivityApi.VOXEL_RESOLUTION_100_MICRONS) experiments = mcc.get_experiments(dataframe=False) return experiments, mcc
def test_validate_structure_ids(inp, fails): if fails: with pytest.raises(ValueError) as exc: MouseConnectivityCache.validate_structure_ids(inp) else: out = MouseConnectivityCache.validate_structure_ids(inp) assert( out == [ int(i) for i in inp ] )
def __init__(self, experiment_id, directory, structdata_dir, connectivity_dir, logger): self.logger = logger self.structdata_dir = structdata_dir self.directory = directory self.experiment_id = experiment_id mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json') tree = mcc.get_structure_tree() self.structures = [f'Field CA{i}' for i in range(1, 4)] self.pyramidal_layers = {s: self.get_pyramidal_layer(tree, s) for s in self.structures}
def get_anatomical_info(self): manifest_path = './connectivity/mouse_connectivity_manifest.json' mcc = MouseConnectivityCache(resolution=self.resolution, manifest_file=manifest_path) tree = mcc.get_structure_tree() annotation, _ = mcc.get_annotation_volume() self.areas_in_blob = list(set([tree.get_structures_by_id([i])[0]['acronym'] for i in annotation[np.where(self.mask>0)] if i>0])) self.cuboid_center_areas = [tree.get_structures_by_id([i])[0]['acronym'] for i in annotation[tuple(self.cuboid_points.T)] if i>0]
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, annotate, _processor_number): super().__init__(input_dir, process_dir, output_dir, f'cell-processor-{_processor_number}') self.annotate = annotate self.structure_ids = structs self.brain_seg_data_dir = structure_map_dir self.source_dir = input_dir self.output_dir = output_dir self.mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json', resolution=25) self.experiments = {int(e['id']): e for e in self.mcc.get_experiments(dataframe=False)}
def __init__(self, ids, multiple_selection): self.ids = ids self.mcc = MouseConnectivityCache( manifest_file= f'../mouse_connectivity/mouse_connectivity_manifest.json', resolution=25) self.structure_tree = self.mcc.get_structure_tree() node = self.fill_node('grey') while len(node.nodes) < 2: node = node.nodes[0] super().__init__(nodes=[node], multiple_selection=multiple_selection)
def __init__(self, output_dir, number, resolution): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/input', 'data/dl', 'data/ready'] ], f'downloader-{number}') self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.mcc.get_annotation_volume() self.mcc.get_reference_space()
def __init__(self, projection_metric="projection_energy", base_dir=None, **kwargs): """ Set up file paths and Allen SDKs :param base_dir: path to directory to use for saving data (default value None) :param path_fiprojection_metricle: - str, metric to quantify the strength of projections from the Allen Connectome. (default value 'projection_energy') :param kwargs: can be used to pass path to individual data folders. See brainrender/Utils/paths_manager.py """ Paths.__init__(self, base_dir=base_dir, **kwargs) self.projection_metric = projection_metric # get mouse connectivity cache and structure tree self.mcc = MouseConnectivityCache(manifest_file=os.path.join( self.mouse_connectivity_cache, "manifest.json")) self.structure_tree = self.mcc.get_structure_tree() # get ontologies API and brain structures sets self.oapi = OntologiesApi() self.get_structures_sets() # get reference space self.space = ReferenceSpaceApi() self.spacecache = ReferenceSpaceCache( manifest=os.path.join( self.annotated_volume, "manifest.json" ), # downloaded files are stored relative to here resolution=self.resolution, reference_space_key= "annotation/ccf_2017" # use the latest version of the CCF ) self.annotated_volume, _ = self.spacecache.get_annotation_volume() # mouse connectivity API [used for tractography] self.mca = MouseConnectivityApi() # Get tree search api self.tree_search = TreeSearchApi() # Get some metadata about experiments self.all_experiments = self.mcc.get_experiments(dataframe=True) self.strains = sorted( [x for x in set(self.all_experiments.strain) if x is not None]) self.transgenic_lines = sorted( set([ x for x in set(self.all_experiments.transgenic_line) if x is not None ]))
def main(): # Find the path to an existing ontology document that was installed through the Brain Explorer interface path = r'C:\Users\jenniferwh\AppData\Local\Allen Institute\Brain Explorer 2\Atlases\Allen Mouse Brain Common Coordinate Framework' dat = pd.read_csv(os.path.join(path, 'ontology_v2.csv')) mcc = MouseConnectivityCache( manifest_file='/connectivity/mouse_connectivity_manifest.json') st = mcc.get_structure_tree() ia_map = st.get_id_acronym_map() jsonfile = r'/Users/jenniferwh/Dropbox (Allen Institute)/Diamond_collaboration/data_files/consensus_structures.json' with open(jsonfile, 'r') as file: consensus_structures = json.load(file) age = str(args.age) + 'monthTG' print(age) tau_structures = consensus_structures[age] tau_structure_ids = [ia_map[structure] for structure in tau_structures] ''' for structure in thal_structure_ids: if structure not in ss: children = st.descendant_ids([structure])[0] child_ss = [structure_id for structure_id in children if structure_id in ss] ss += [structure] thal_structure_ids+=child_ss ''' colormap = cm.Reds all_structures = dat['abbreviation'].unique() all_structures = [ structure for structure in all_structures if structure in ia_map.keys() ] all_structures.remove('root') all_structure_ids = [ia_map[structure] for structure in all_structures] structure_vals = dict() for structure in all_structure_ids: if structure in tau_structure_ids: structure_vals[structure] = 0.9 else: structure_vals[structure] = 0 rgb_vals = structure_vals.copy() for key in all_structure_ids: rgb_vals[key] = tuple( [255 * i for i in colormap(structure_vals[key])[:3]]) dat.loc[dat['database_id'] == key, 'red'] = int(rgb_vals[key][0]) dat.loc[dat['database_id'] == key, 'green'] = int(rgb_vals[key][1]) dat.loc[dat['database_id'] == key, 'blue'] = int(rgb_vals[key][2]) rgb_vals[0] = (0, 0, 0) dat.loc[dat['abbreviation'] == 'root', 'parent'] = 0. dat['parent'] = [int(value) for value in dat['parent']] dat.to_csv(os.path.join(path, 'ontology_v2.csv'), index=False)
def prepare_input(self, input_dir, connectivity_dir, structure_map_dir, **kwargs): mcc = MouseConnectivityCache( manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json' ) mcc.get_structure_tree() experiments = os.listdir(structure_map_dir) try: os.makedirs(input_dir) for e in experiments: os.makedirs(f'{input_dir}/{e}') except FileExistsError: pass
def main(): mcc = MouseConnectivityCache(resolution=10) rsp = mcc.get_reference_space() # structures come from an rma query like this: # http://api.brain-map.org/api/v2/data/query.json?q=model::Structure,rma::criteria,[graph_id$in1] # which does the same thing as using structure_graph_download, but without nesting # the graph id is 1 structures = rsp.structure_tree.nodes() ontology_ids = [s['id'] for s in structures] # the annotation comes from: http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/annotation/ccf_2017/annotation_10.nrrd annotation_ids = np.unique(rsp.annotation)
class StructureTree(ipytree.Tree): def __init__(self, ids, multiple_selection): self.ids = ids self.mcc = MouseConnectivityCache( manifest_file= f'../mouse_connectivity/mouse_connectivity_manifest.json', resolution=25) self.structure_tree = self.mcc.get_structure_tree() node = self.fill_node('grey') while len(node.nodes) < 2: node = node.nodes[0] super().__init__(nodes=[node], multiple_selection=multiple_selection) def fill_node(self, start_node): start_struct = self.structure_tree.get_structures_by_acronym( [start_node])[0] children = [ self.fill_node(c['acronym']) for c in self.structure_tree.children( [self.structure_tree.get_id_acronym_map()[start_node]])[0] ] children = [c for c in children if c is not None] if not children and start_struct['acronym'] not in self.ids: return None node = StructureTreeNode(start_struct['acronym'], start_struct['acronym'], children) if start_struct['acronym'] not in self.ids: node.disabled = True return node
def mcc(): mcc = MouseConnectivityCache( resolution=MouseConnectivityApi.VOXEL_RESOLUTION_100_MICRONS, manifest_file='mcc_manifest.json') mcc.api.retrieve_file_over_http = \ MagicMock(name='retrieve_file_over_http') return mcc
def main(output_dir, resolution, retain_transform_data, zoom, parallel_downloads, parallel_processors): mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) mcc.get_annotation_volume() mcc.get_reference_space() experiments = [e['id'] for e in mcc.get_experiments(dataframe=False)] experiments = sorted(list(set(experiments))) try: os.makedirs(f'{output_dir}/data/input') for e in experiments: os.makedirs(f'{output_dir}/data/input/{e}') except FileExistsError: for e in experiments: if os.path.isdir(f'{output_dir}/data/input/{e}'): shutil.rmtree(f'{output_dir}/data/input/{e}') os.makedirs(f'{output_dir}/data/input/{e}') downloads = [ subprocess.Popen([ "python", "./displacement_data_downloader.py", f"-o{output_dir}", f"-r{resolution}", f"-n{i}" ]) for i in range(parallel_downloads) ] processes = [ subprocess.Popen([ "python", "./segmentation_data_builder.py", f"-o{output_dir}", f"-r{resolution}", f"-c{len(experiments)}", f"-z{zoom}" ] + (["-t"] if retain_transform_data else []) + [f"-n{i}"]) for i in range(parallel_processors) ] exit_codes = [p.wait() for p in processes + downloads]
class CellProcessor(DirWatcher): experiment_fields_to_save = [ 'id', 'gender', 'injection_structures', 'injection_volume', 'injection_x', 'injection_y', 'injection_z', 'product_id', 'specimen_name', 'strain', 'structure_abbrev', 'structure_id', 'structure_name', 'transgenic_line', 'transgenic_line_id', 'primary_injection_structure' ] def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, annotate, _processor_number): super().__init__(input_dir, process_dir, output_dir, f'cell-processor-{_processor_number}') self.annotate = annotate self.structure_ids = structs self.brain_seg_data_dir = structure_map_dir self.source_dir = input_dir self.output_dir = output_dir self.mcc = MouseConnectivityCache(manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json', resolution=25) self.experiments = {int(e['id']): e for e in self.mcc.get_experiments(dataframe=False)} def process_item(self, item, directory): item = int(item) experiment = ExperimentCellsProcessor(self.mcc, item, directory, self.brain_seg_data_dir, self.structure_ids, self.experiment_fields_to_save, self.experiments[item], self.logger) experiment.process() if self.annotate: annotator = ExperimentDataAnnotator(int(item), directory, self.logger) annotator.process() def on_process_error(self, item, exception): retval = super().on_process_error(item, exception) self.logger.error(f"Error occurred during processing", exc_info=True) if type(exception) in [urllib.error.HTTPError, OSError, ValueError, urllib.error.URLError, socket.gaierror, MemoryError]: return False else: return retval
def __init__(self, projection_metric="projection_energy", paths_file=None): """ path_file {[str]} -- [Path to a YAML file specifying paths to data folders, to replace default paths] (default: {None}) """ Paths.__init__(self, paths_file=paths_file) self.projection_metric = projection_metric # get mouse connectivity cache and structure tree self.mcc = MouseConnectivityCache(manifest_file=os.path.join( self.mouse_connectivity_cache, "manifest.json")) self.structure_tree = self.mcc.get_structure_tree() # get ontologies API and brain structures sets self.oapi = OntologiesApi() self.get_structures_sets() # get reference space self.space = ReferenceSpaceApi() self.spacecache = ReferenceSpaceCache( manifest=os.path.join( "Data/ABA", "manifest.json" ), # downloaded files are stored relative to here resolution=self.resolution, reference_space_key= "annotation/ccf_2017" # use the latest version of the CCF ) self.annotated_volume, _ = self.spacecache.get_annotation_volume() # mouse connectivity API [used for tractography] self.mca = MouseConnectivityApi() # Get tree search api self.tree_search = TreeSearchApi() # Get some metadata about experiments self.all_experiments = self.mcc.get_experiments(dataframe=True) self.strains = sorted( [x for x in set(self.all_experiments.strain) if x is not None]) self.transgenic_lines = sorted( set([ x for x in set(self.all_experiments.transgenic_line) if x is not None ]))
def get_neuron_region(neuron_ob): """ Given a neuron object, return a dictionary describing its major brain region from the Allen Mouse Atlas :param neuron_ob: A neuroelectro.models Neuron object :return: returns a dictionary of region attributes """ regs = neuron_ob.regions.all() mcc = MouseConnectivityCache() onto = mcc.get_ontology() if regs: r = regs[0] structure_path_str = onto[r.allenid].structure_id_path.item() region_dict = get_major_brain_region(structure_path_str, onto) return region_dict else: return None # if region_dict: # print (n.name, region_dict['region_name']) # else: # print 'No region found for %s' % n.name
def __init__(self, output_dir, resolution, retain_transform_data, zoom, number, count): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/ready', 'data/proc', 'data/result'] ], f'segmentation-data-builder-{number}') self.count = count self.zoom = zoom self.retain_transform_data = retain_transform_data self.resolution = resolution self.output_dir = output_dir self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.anno, self.meta = self.mcc.get_annotation_volume() self.rsp = self.mcc.get_reference_space() self.rsp.remove_unassigned( ) # This removes ids that are not in this particular reference space
def __init__(self, available_brains=None): self.mcc = MouseConnectivityCache( manifest_file= f'../mouse_connectivity/mouse_connectivity_manifest.json', resolution=25) self.messages = widgets.Output() self.set_available_brains(available_brains) self.filter = { col: widgets.SelectMultiple(description=col, options=self.get_column_options( self.experiments[col])) for col in ['gender', 'strain', 'transgenic_line', 'id'] } for c, f in self.filter.items(): f.observe( lambda change, col=c: self.selection_changed(change, col)) self.change_handler = None self.selection_changed({'name': 'value'}, 'gender') super().__init__( (widgets.HBox(list(self.filter.values())), self.messages))
def __init__(self, base_dir=None, **kwargs): """ Set up file paths and Allen SDKs :param base_dir: path to directory to use for saving data (default value None) :param kwargs: can be used to pass path to individual data folders. See brainrender/Utils/paths_manager.py """ Atlas.__init__(self, base_dir=base_dir, **kwargs) self.meshes_folder = self.mouse_meshes # where the .obj mesh for each region is saved # get mouse connectivity cache and structure tree self.mcc = MouseConnectivityCache(manifest_file=os.path.join(self.mouse_connectivity_cache, "manifest.json")) self.structure_tree = self.mcc.get_structure_tree() # get ontologies API and brain structures sets self.oapi = OntologiesApi() self.get_structures_sets() # get reference space self.space = ReferenceSpaceApi() self.spacecache = ReferenceSpaceCache( manifest=os.path.join(self.annotated_volume_fld, "manifest.json"), # downloaded files are stored relative to here resolution=self.resolution, reference_space_key="annotation/ccf_2017" # use the latest version of the CCF ) self.annotated_volume, _ = self.spacecache.get_annotation_volume() # mouse connectivity API [used for tractography] self.mca = MouseConnectivityApi() # Get tree search api self.tree_search = TreeSearchApi() # Store all regions metadata [If there's internet connection] if self.other_sets is not None: self.regions = self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].sort_values('acronym') self.region_acronyms = list(self.other_sets["Structures whose surfaces are represented by a precomputed mesh"].sort_values( 'acronym').acronym.values)
def __init__(self): self.onto_id2parent_regions = {} # only find parents up to these major brain regions big_reg_acros = ['Isocortex', 'OLF', 'STR', 'PAL', 'TH', 'HY', 'MB', 'PAL', 'MY', 'CB', 'HPF', 'CTXsp'] mcc = MouseConnectivityCache() onto = mcc.get_ontology() df = onto.df struct_ids = df.id for struct_id in struct_ids: structure_path_str = onto[struct_id].structure_id_path.item() region_path = structure_path_str.split('/') parent_list = [] for r in reversed(region_path): if r: parent_list.append(r) tdf = df[df.id == int(r)] acronym = tdf.acronym.item() if acronym in big_reg_acros: break self.onto_id2parent_regions[struct_id] = parent_list
class SegmentationDataBuilder(DirWatcher): def __init__(self, output_dir, resolution, retain_transform_data, zoom, number, count): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/ready', 'data/proc', 'data/result'] ], f'segmentation-data-builder-{number}') self.count = count self.zoom = zoom self.retain_transform_data = retain_transform_data self.resolution = resolution self.output_dir = output_dir self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.anno, self.meta = self.mcc.get_annotation_volume() self.rsp = self.mcc.get_reference_space() self.rsp.remove_unassigned( ) # This removes ids that are not in this particular reference space def process_item(self, item, directory): item = int(item) experiment = ExperimentSectionData( self.mcc, item, directory, self.anno, self.meta, self.rsp, self.logger, zoom=self.zoom, remove_transform_data=not self.retain_transform_data) experiment.create_section_data() experiment.cleanup()
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir, structs, connectivity_dir, _processor_number, brightness_threshold, strains): super().__init__(input_dir, process_dir, output_dir, f'experiment-images-downloader-{_processor_number}') self.brightness_threshold = brightness_threshold self.structs = ast.literal_eval(structs) self.segmentation_dir = structure_map_dir self.mcc = MouseConnectivityCache( manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json' ) struct_tree = self.mcc.get_structure_tree() structure_ids = [ i for sublist in struct_tree.descendant_ids(self.structs) for i in sublist ] self.structure_ids = set(structure_ids) self.image_api = ImageDownloadApi() self.bbox_dilation_kernel = cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (14, 14)) exps = self.mcc.get_experiments(dataframe=True) items = [] for s, gs in strains.items(): strain_items = [] for g in gs: strain_items += [ sorted(exps[(exps.strain == s) & (exps.gender == g)].id.tolist()) ] if strain_items: min_len = min([len(i) for i in strain_items]) strain_items = [i[:min_len] for i in strain_items] items += [str(i) for j in zip(*strain_items) for i in j] self.initial_items = [i for i in items if i in self.initial_items] + [ i for i in self.initial_items if i not in items ]
class SegmentationDataDownloader(DirWatcher): def __init__(self, output_dir, number, resolution): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/input', 'data/dl', 'data/ready'] ], f'downloader-{number}') self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.mcc.get_annotation_volume() self.mcc.get_reference_space() def process_item(self, item, directory): self.logger.info(f"Download displacement data for {item}") self.mcc.get_deformation_field(item, header_path=f'{directory}/dfmfld.mhd', voxel_path=f'{directory}/dfmfld.raw') self.mcc.get_affine_parameters(item, direction='trv', file_name=f'{directory}/aff_param.txt')
def get_mcc(res=50, manifest_file=manifest_file): ''' Fetches MouseConnectivityCache object to interact with Allen data. Defaults to setting manifest file to user's home directory. Parameters __________ res : int Sets voxel size for Allen data. Must be 100, 50, 25, or 10. Default is 50. manifest_file : str Sets location of manifest file Returns _______ MouseConnectivityCache object. ''' if res not in [100, 50, 25, 10]: raise ValueError('Res must be 100, 50, 25, or 10') return MouseConnectivityCache(resolution=res, manifest_file=manifest_file)
def main(args): # parse in args parser = parsefn() inmask, num_out_lbl = parse_inputs(parser, args) print("\n Reading input mask (ROI) and Allen annotations") [cutoff, maxannot, miracl_home, annot_csv, atlas_lbls, exclude] = initialize() # Get 'histogram' of masked labels print( "\n Getting histogram of included labels in mask & sorting by volume") masked_lbls = gethist(miracl_home, inmask) # Setup Allen connect jason cache file mcc = MouseConnectivityCache( manifest_file= '%s/connect/connectivity_exps/mouse_connectivity_manifest.json' % miracl_home) # Load all injection experiments from Allen api all_experiments = mcc.get_experiments(dataframe=True) # Filter to only wild-type strain projexps = all_experiments[all_experiments['strain'] == "C57BL/6J"] # no transgenic mice projexps = projexps[projexps['transgenic-line'] == ""] # excluding background/negatives masked_lbls = exlude_maj_lbls(masked_lbls, exclude) masked_lbls = masked_lbls[(masked_lbls > 0) & (masked_lbls < maxannot)] # Check if labels have injection exps print( "\n Checking if labels have injection experiments in the connectivity search" ) inj_exps = check_inj_exp(masked_lbls, projexps) # get parent labels for ones w/out inj exp print("\n Getting parent labels for labels without injection experiments") uniq_lbls = get_parentlbl(inj_exps, masked_lbls, annot_csv, exclude) # check all labels have inj exps print("\n Checking that all parent labels have injection exps") inj_exps = check_inj_exp(uniq_lbls, projexps) while len(inj_exps) != sum(inj_exps): uniq_lbls = get_parentlbl(inj_exps, uniq_lbls, annot_csv, exclude) inj_exps = check_inj_exp(uniq_lbls, projexps) # Restrict to n labels uniq_lbls = uniq_lbls[0:num_out_lbl] # query structure connectivity from Allen API print( "\n Querying structural connectivity of injection labels in the Allen API & sorting by projection volume" ) [all_connect_ids, all_norm_proj] = query_connect(uniq_lbls, projexps, cutoff, exclude, mcc) # --------------- print( "\n Excluding larger 'parent' labels (with graph depth < 5 and graph order < 6)" ) # right injection sites uniq_lbls += 20000 # exclude primary injection if found as a target regions (mutually exclusive) filconn = [ np.delete(all_connect_ids[t], np.where(np.in1d(all_connect_ids[t], uniq_lbls))) for t in range(len(all_connect_ids)) ] # exclude labels not included in atlas annotations lblinatl = [np.in1d(filconn[i], atlas_lbls) for i in range(len(filconn))] atlfilconn = [ np.delete(filconn[i], np.where(lblinatl[i] == False)) for i in range(len(filconn)) ] conn_ids = [ np.hstack((uniq_lbls[i], atlfilconn[i])) for i in range(num_out_lbl) ] # --------------- # save csv [export_connect_abv, dic] = saveconncsv(conn_ids, annot_csv, num_out_lbl) # compute & save proj map names = exportprojmap(all_norm_proj, num_out_lbl, export_connect_abv) # compute & save connectivity matrix [heatmap, targ] = exportheatmap(num_out_lbl, conn_ids, all_norm_proj, uniq_lbls, export_connect_abv, dic, names) # compute & save connectivity graph createconnectogram(num_out_lbl, heatmap, annot_csv, uniq_lbls, targ, dic)
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # tell the cache class what resolution (in microns) of data you want to download mcc = MouseConnectivityCache(resolution=25) # use the ontology class to get the id of the isocortex structure ontology = mcc.get_ontology('ontology.csv') isocortex = ontology['Isocortex'] # a list of dictionaries containing metadata for non-Cre experiments experiments = mcc.get_experiments(file_name='non_cre.json', injection_structure_ids=isocortex['id']) # download the projection density volume for one of the experiments pd = mcc.get_projection_density(experiments[0]['id'])
#=============================================================================== # example 1 #=============================================================================== from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # tell the cache class what resolution (in microns) of data you want to download mcc = MouseConnectivityCache(resolution=25) # use the structure tree class to get information about the isocortex structure structure_tree = mcc.get_structure_tree() isocortex_id = structure_tree.get_structures_by_name(['Isocortex'])[0]['id'] # a list of dictionaries containing metadata for non-Cre experiments experiments = mcc.get_experiments(file_name='non_cre.json', injection_structure_ids=[isocortex_id]) # download the projection density volume for one of the experiments pd = mcc.get_projection_density(experiments[0]['id']) #=============================================================================== # example 2 #=============================================================================== import nrrd file_name = 'mouse_connectivity/experiment_644250774/projection_density_25.nrrd' data_array, metadata = nrrd.read(file_name)
''' @author wronk Script to calculate the volume of the mouse brain from the common coordinate frame developed by ABI. This requires the connectivity dataset and the allensdk. Output from connectivity dataset (as of Sept 2015) is below. ''' import numpy as np from allensdk.core.mouse_connectivity_cache import (MouseConnectivityCache as MCC) # load in nrdd data mcc = MCC(manifest_file='/Volumes/Brain2015/connectivity/manifest.json') annot, annot_info = mcc.get_annotation_volume() # get all voxels that are non-zeros brain_voxels = annot.flatten() > 0 sum_vox = np.sum(brain_voxels) print 'Found %d voxels' % (sum_vox) print 'Density is %0.3f percent' % (100 * sum_vox / len(brain_voxels)) # Get size of each box, make sure they're in cubes width_dims = [] for di in range(3): width_dims.append(annot_info['space directions'][di][di]) assert len(set(width_dims)) == 1
if len(sys.argv) < 3: print "" print "please provide a working directory & output file name, e.g." print "" print " build_projection_datasets.py data/ pms.pickle" print "" sys.exit(1) import os os.chdir(sys.argv[1]) from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache from allensdk.api.queries.ontologies_api import OntologiesApi mcc = MouseConnectivityCache(manifest_file="mcc_manifest.json") all_experiments = mcc.get_experiments(dataframe=True) ontology = mcc.get_ontology() summary_structures = OntologiesApi().get_structures(structure_set_names="Mouse Connectivity - Summary") summary_structure_ids = [s["id"] for s in summary_structures] print "build dict of injection structure id to experiment list" ist2e = {} for eid in all_experiments.index: for ist in all_experiments.ix[eid]["injection-structures"]: isti = ist["id"] if isti not in ist2e: ist2e[isti] = [] ist2e[isti].append(eid)
def launch(self, resolution, weighting, inf_vox_thresh, vol_thresh): resolution = int(resolution) weighting = int(weighting) inf_vox_thresh = float(inf_vox_thresh) vol_thresh = float(vol_thresh) project = dao.get_project_by_id(self.current_project_id) manifest_file = self.file_handler.get_allen_mouse_cache_folder(project.name) manifest_file = os.path.join(manifest_file, "mouse_connectivity_manifest.json") cache = MouseConnectivityCache(resolution=resolution, manifest_file=manifest_file) # the method creates a dictionary with information about which experiments need to be downloaded ist2e = DictionaireBuilder(cache, False) # the method downloads experiments necessary to build the connectivity projmaps = DownloadAndConstructMatrix(cache, weighting, ist2e, False) # the method cleans the file projmaps in 4 steps projmaps = pmsCleaner(projmaps) Vol, annot_info = cache.get_annotation_volume() ontology = cache.get_ontology() # the method includes in the parcellation only brain regions whose volume is greater than vol_thresh projmaps = AreasVolumeTreshold(projmaps, vol_thresh, resolution, Vol, ontology) # the method includes in the parcellation only brain regions where at least one injection experiment had infected more than N voxel (where N is inf_vox_thresh) projmaps = AreasVoxelTreshold(cache, projmaps, inf_vox_thresh, Vol, ontology) # the method creates file order and keyord that will be the link between the SC order and the id key in the Allen database [order, key_ord] = CreateFileOrder(projmaps, ontology) # the method builds the Structural Connectivity (SC) matrix SC = ConstructingSC(projmaps, order, key_ord) # the method returns the coordinate of the centres and the name of the brain areas in the selected parcellation [centres, names] = Construct_centres(ontology, order, key_ord, Vol) # the method returns the tract lengths between the brain areas in the selected parcellation tract_lengths = ConstructTractLengths(centres) # the method associated the parent and the grandparents to the child in the selected parcellation with the biggest volume [unique_parents, unique_grandparents] = ParentsAndGrandParentsFinder(order, key_ord, Vol, ontology) # the method returns a volume indexed between 0 and N-1, with N=tot brain areas in the parcellation. -1=background and areas that are not in the parcellation Vol_parcel = MouseBrainVisualizer(Vol, order, key_ord, unique_parents, unique_grandparents, ontology, projmaps) # results: Connectivity, Volume & RegionVolumeMapping # Connectivity result_connectivity = Connectivity(storage_path=self.storage_path) result_connectivity.centres = centres result_connectivity.region_labels = names result_connectivity.weights = SC result_connectivity.tract_lengths = tract_lengths # Volume result_volume = Volume(storage_path=self.storage_path) result_volume.origin = [[0.0, 0.0, 0.0]] result_volume.voxel_size = [resolution, resolution, resolution] # result_volume.voxel_unit= micron # Region Volume Mapping result_rvm = RegionVolumeMapping(storage_path=self.storage_path) result_rvm.volume = result_volume result_rvm.array_data = Vol_parcel result_rvm.connectivity = result_connectivity result_rvm.title = "Volume mouse brain " result_rvm.dimensions_labels = ["X", "Y", "Z"] return [result_connectivity, result_rvm, result_volume]
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # tell the cache class what resolution (in microns) of data you want to download mcc = MouseConnectivityCache(resolution=25) # use the ontology class to get the id of the isocortex structure ontology = mcc.get_ontology() isocortex = ontology["Isocortex"] # a list of dictionaries containing metadata for non-Cre experiments experiments = mcc.get_experiments(injection_structure_ids=isocortex["id"]) # download the projection density volume for one of the experiments pd = mcc.get_projection_density(experiments[0]["id"])
for exp in range(len(experiments)): pd = mca.download_projection_density('example.nrrd', experiments[exp]['id'], resolution=25) print(type(pd)) pdb.set_trace() # The manifest file is a simple JSON file that keeps track of all of # the data that has already been downloaded onto the hard drives. # If you supply a relative path, it is assumed to be relative to your # current working directory. mcc = MouseConnectivityCache(manifest_file='connectivity/manifest.json') # open up a list of all of the experiments all_experiments = mcc.get_experiments(dataframe=True) from allensdk.api.queries.biophysical_perisomatic_api import \ BiophysicalPerisomaticApi from allensdk.api.queries.cell_types_api import CellTypesApi import allensdk.core.swc as swc import os from allensdk.core.nwb_data_set import NwbDataSet from IPython import __main__
def launch(self, resolution, weighting, inj_f_thresh, vol_thresh): resolution = int(resolution) weighting = int(weighting) inj_f_thresh = float(inj_f_thresh)/100. vol_thresh = float(vol_thresh) project = dao.get_project_by_id(self.current_project_id) manifest_file = self.file_handler.get_allen_mouse_cache_folder(project.name) manifest_file = os.path.join(manifest_file, 'mouse_connectivity_manifest.json') cache = MouseConnectivityCache(resolution=resolution, manifest_file=manifest_file) # the method creates a dictionary with information about which experiments need to be downloaded ist2e = dictionary_builder(cache, False) # the method downloads experiments necessary to build the connectivity projmaps = download_an_construct_matrix(cache, weighting, ist2e, False) # the method cleans the file projmaps in 4 steps projmaps = pms_cleaner(projmaps) # download from the AllenSDK the annotation volume, the template volume vol, annot_info = cache.get_annotation_volume() template, template_info = cache.get_template_volume() # rotate template in the TVB 3D reference: template = rotate_reference(template) # grab the StructureTree instance structure_tree = cache.get_structure_tree() # the method includes in the parcellation only brain regions whose volume is greater than vol_thresh projmaps = areas_volume_threshold(cache, projmaps, vol_thresh, resolution) # the method exclude from the experimental dataset # those exps where the injected fraction of pixel in the injection site is lower than than the inj_f_thr projmaps = infected_threshold(cache, projmaps, inj_f_thresh) # the method creates file order and keyword that will be the link between the SC order and the # id key in the Allen database [order, key_ord] = create_file_order(projmaps, structure_tree) # the method builds the Structural Connectivity (SC) matrix structural_conn = construct_structural_conn(projmaps, order, key_ord) # the method returns the coordinate of the centres and the name of the brain areas in the selected parcellation [centres, names] = construct_centres(cache, order, key_ord) # the method returns the tract lengths between the brain areas in the selected parcellation tract_lengths = construct_tract_lengths(centres) # the method associated the parent and the grandparents to the child in the selected parcellation with # the biggest volume [unique_parents, unique_grandparents] = parents_and_grandparents_finder(cache, order, key_ord, structure_tree) # the method returns a volume indexed between 0 and N-1, with N=tot brain areas in the parcellation. # -1=background and areas that are not in the parcellation vol_parcel = mouse_brain_visualizer(vol, order, key_ord, unique_parents, unique_grandparents, structure_tree, projmaps) # results: Connectivity, Volume & RegionVolumeMapping # Connectivity result_connectivity = Connectivity(storage_path=self.storage_path) result_connectivity.centres = centres result_connectivity.region_labels = names result_connectivity.weights = structural_conn result_connectivity.tract_lengths = tract_lengths # Volume result_volume = Volume(storage_path=self.storage_path) result_volume.origin = [[0.0, 0.0, 0.0]] result_volume.voxel_size = [resolution, resolution, resolution] # result_volume.voxel_unit= micron # Region Volume Mapping result_rvm = RegionVolumeMapping(storage_path=self.storage_path) result_rvm.volume = result_volume result_rvm.array_data = vol_parcel result_rvm.connectivity = result_connectivity result_rvm.title = "Volume mouse brain " result_rvm.dimensions_labels = ["X", "Y", "Z"] # Volume template result_template = StructuralMRI(storage_path=self.storage_path) result_template.array_data = template result_template.weighting = 'T1' result_template.volume = result_volume return [result_connectivity, result_volume, result_rvm, result_template]