def allen_get_raw_annotation(annotation_dir, version='annotation/ccf_2017', resolution=10): import nrrd from allensdk.api.queries.mouse_connectivity_api import MouseConnectivityApi annotation_path = pjoin( annotation_dir, 'annotation_{0}_{1}.nrrd'.format(version, resolution)) if not os.path.isdir(annotation_dir): os.makedirs(annotation_dir) if not os.path.isfile(annotation_path): mcapi = MouseConnectivityApi() mcapi.download_annotation_volume(version, resolution, annotation_path) annotation, meta = nrrd.read(annotation_path) return annotation, meta
class MouseConnectivityCache(Cache): """ Cache class for storing and accessing data related to the adult mouse Connectivity Atlas. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- resolution: int Resolution of grid data to be downloaded when accessing projection volume, the annotation volume, and the annotation volume. Must be one of (10, 25, 50, 100). Default is 25. api: MouseConnectivityApi instance Used internally to make API queries. Parameters ---------- resolution: int Resolution of grid data to be downloaded when accessing projection volume, the annotation volume, and the annotation volume. Must be one of (10, 25, 50, 100). Default is 25. cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_projection_density(file_name='file.nrrd')). manifest_file: string File name of the manifest to be read. Default is "mouse_connectivity_manifest.json". """ ANNOTATION_KEY = 'ANNOTATION' TEMPLATE_KEY = 'TEMPLATE' PROJECTION_DENSITY_KEY = 'PROJECTION_DENSITY' INJECTION_DENSITY_KEY = 'INJECTION_DENSITY' INJECTION_FRACTION_KEY = 'INJECTION_FRACTION' DATA_MASK_KEY = 'DATA_MASK' STRUCTURE_UNIONIZES_KEY = 'STRUCTURE_UNIONIZES' EXPERIMENTS_KEY = 'EXPERIMENTS' STRUCTURES_KEY = 'STRUCTURES' STRUCTURE_MASK_KEY = 'STRUCTURE_MASK' def __init__(self, resolution=25, cache=True, manifest_file='mouse_connectivity_manifest.json', base_uri=None): super(MouseConnectivityCache, self).__init__(manifest=manifest_file, cache=cache) self.resolution = resolution self.api = MouseConnectivityApi(base_uri=base_uri) def get_annotation_volume(self, file_name=None): """ Read the annotation volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the annotation volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.ANNOTATION_KEY, self.resolution) if file_name is None: raise Exception( "No save file name provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_mkdir(os.path.dirname(file_name)) annotation, info = self.api.download_annotation_volume( self.resolution, file_name) return annotation, info def get_template_volume(self, file_name=None): """ Read the template volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.TEMPLATE_KEY, self.resolution) if file_name is None: raise Exception("No save file provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_mkdir(os.path.dirname(file_name)) annotation, info = self.api.download_template_volume( self.resolution, file_name) return annotation, info def get_projection_density(self, experiment_id, file_name=None): """ Read a projection density volume for a single experiment. Download it first if it doesn't exist. Projection density is the proportion of of projecting pixels in a grid voxel in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.PROJECTION_DENSITY_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_mkdir(os.path.dirname(file_name)) self.api.download_projection_density(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_injection_density(self, experiment_id, file_name=None): """ Read an injection density volume for a single experiment. Download it first if it doesn't exist. Injection density is the proportion of projecting pixels in a grid voxel only including pixels that are part of the injection site in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.INJECTION_DENSITY_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_mkdir(os.path.dirname(file_name)) self.api.download_injection_density(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_injection_fraction(self, experiment_id, file_name=None): """ Read an injection fraction volume for a single experiment. Download it first if it doesn't exist. Injection fraction is the proportion of pixels in the injection site in a grid voxel in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.INJECTION_FRACTION_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_mkdir(os.path.dirname(file_name)) self.api.download_injection_fraction(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_data_mask(self, experiment_id, file_name=None): """ Read a data mask volume for a single experiment. Download it first if it doesn't exist. Data mask is a binary mask of voxels that have valid data. Only use valid data in analysis! Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.DATA_MASK_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_mkdir(os.path.dirname(file_name)) self.api.download_data_mask(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_ontology(self, file_name=None): """ Read the list of adult mouse structures and return an Ontology instance. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ return Ontology(self.get_structures(file_name)) def get_structures(self, file_name=None): """ Read the list of adult mouse structures and return a Pandas DataFrame. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURES_KEY) if os.path.exists(file_name): structures = pd.DataFrame.from_csv(file_name) else: structures = OntologiesApi().get_structures(1) structures = pd.DataFrame(structures) if self.cache: Manifest.safe_mkdir(os.path.dirname(file_name)) structures.to_csv(file_name) structures.set_index(['id'], inplace=True, drop=False) return structures def get_experiments(self, dataframe=False, file_name=None, cre=None, injection_structure_ids=None): """ Read a list of experiments that match certain criteria. If caching is enabled, this will save the whole (unfiltered) list of experiments to a file. Parameters ---------- dataframe: boolean Return the list of experiments as a Pandas DataFrame. If False, return a list of dictionaries. Default False. file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY) if os.path.exists(file_name): experiments = json_utilities.read(file_name) else: experiments = self.api.experiment_source_search( injection_structures='root') # removing these elements because they are specific to a particular resolution for e in experiments: del e['num-voxels'] del e['injection-volume'] del e['sum'] del e['name'] if self.cache: Manifest.safe_mkdir(os.path.dirname(file_name)) json_utilities.write(file_name, experiments) # filter the read/downloaded list of experiments experiments = self.filter_experiments(experiments, cre, injection_structure_ids) if dataframe: experiments = pd.DataFrame(experiments) experiments.set_index(['id'], inplace=True, drop=False) return experiments def filter_experiments(self, experiments, cre=None, injection_structure_ids=None): """ Take a list of experiments and filter them by cre status and injection structure. Parameters ---------- cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ if cre == True: experiments = [e for e in experiments if e['transgenic-line']] elif cre == False: experiments = [e for e in experiments if not e['transgenic-line']] elif cre is not None: experiments = [ e for e in experiments if e['transgenic-line'] in cre ] if injection_structure_ids is not None: descendant_ids = self.get_ontology().get_descendant_ids( injection_structure_ids) experiments = [ e for e in experiments if e['structure-id'] in descendant_ids ] return experiments def get_experiment_structure_unionizes(self, experiment_id, file_name=None, is_injection=None, structure_ids=None, hemisphere_ids=None): """ Retrieve the structure unionize data for a specific experiment. Filter by structure, injection status, and hemisphere. Parameters ---------- experiment_id: int ID of the experiment of interest. Corresponds to section_data_set_id in the API. file_name: string File name to save/read the experiments list. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records that are inside a specific set of structures. If None, return all records. Default None. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_UNIONIZES_KEY, experiment_id) if os.path.exists(file_name): unionizes = pd.DataFrame.from_csv(file_name) else: unionizes = self.api.get_structure_unionizes([experiment_id]) unionizes = pd.DataFrame(unionizes) # rename section_data_set_id column to experiment_id unionizes.columns = [ 'experiment_id' if c == 'section_data_set_id' else c for c in unionizes.columns ] if self.cache: Manifest.safe_mkdir(os.path.dirname(file_name)) unionizes.to_csv(file_name) return self.filter_structure_unionizes(unionizes, is_injection, structure_ids, hemisphere_ids) def filter_structure_unionizes(self, unionizes, is_injection=None, structure_ids=None, hemisphere_ids=None): """ Take a list of unionzes and return a subset of records filtered by injection status, structure, and hemisphere. Parameters ---------- is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records that are inside a specific set of structures. If None, return all records. Default None. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ if is_injection is not None: unionizes = unionizes[unionizes.is_injection == is_injection] if structure_ids is not None: descendant_ids = self.get_ontology().get_descendant_ids( structure_ids) unionizes = unionizes[unionizes['structure_id'].isin( descendant_ids)] if hemisphere_ids is not None: unionizes = unionizes[unionizes['hemisphere_id'].isin( hemisphere_ids)] return unionizes def get_structure_unionizes(self, experiment_ids, is_injection=None, structure_ids=None, hemisphere_ids=None): """ Get structure unionizes for a set of experiment IDs. Filter the results by injection status, structure, and hemisphere. Parameters ---------- experiment_ids: list List of experiment IDs. Corresponds to section_data_set_id in the API. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records that are inside a specific set of structures. If None, return all records. Default None. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ unionizes = [ self.get_experiment_structure_unionizes( eid, is_injection=is_injection, structure_ids=structure_ids, hemisphere_ids=hemisphere_ids) for eid in experiment_ids ] return pd.concat(unionizes, ignore_index=True) def get_projection_matrix(self, experiment_ids, projection_structure_ids, hemisphere_ids=None, parameter='projection_volume', dataframe=False): unionizes = self.get_structure_unionizes(experiment_ids, is_injection=False, hemisphere_ids=hemisphere_ids) unionizes = unionizes[unionizes.structure_id.isin( projection_structure_ids)] projection_structure_ids = set( unionizes['structure_id'].values.tolist()) hemisphere_ids = set(unionizes['hemisphere_id'].values.tolist()) nrows = len(experiment_ids) ncolumns = len(projection_structure_ids) * len(hemisphere_ids) matrix = np.empty((nrows, ncolumns)) matrix[:] = np.NAN row_lookup = {} for idx, e in enumerate(experiment_ids): row_lookup[e] = idx column_lookup = {} columns = [] cidx = 0 hlabel = {1: '-L', 2: '-R', 3: ''} o = self.get_ontology() for hid in hemisphere_ids: for sid in projection_structure_ids: column_lookup[(hid, sid)] = cidx label = o[sid].iloc[0]['acronym'] + hlabel[hid] columns.append({ 'hemisphere_id': hid, 'structure_id': sid, 'label': label }) cidx += 1 for _, row in unionizes.iterrows(): ridx = row_lookup[row['experiment_id']] k = (row['hemisphere_id'], row['structure_id']) cidx = column_lookup[k] matrix[ridx, cidx] = row[parameter] if dataframe: all_experiments = self.get_experiments(dataframe=True) rows_df = all_experiments.loc[experiment_ids] cols_df = pd.DataFrame(columns) return {'matrix': matrix, 'rows': rows_df, 'columns': cols_df} else: return { 'matrix': matrix, 'rows': experiment_ids, 'columns': columns } def get_structure_mask(self, structure_id, file_name=None, annotation_file_name=None): """ Read a 3D numpy array shaped like the annotation volume that has non-zero values where voxels belong to a particular structure. This will take care of identifying substructures. Parameters ---------- structure_id: int ID of a structure. file_name: string File name to store the structure mask. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. annotation_file_name: string File name to store the annotation volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_MASK_KEY, structure_id) if os.path.exists(file_name): return nrrd.read(file_name) else: ont = self.get_ontology() structure_ids = ont.get_descendant_ids([structure_id]) annotation, _ = self.get_annotation_volume(annotation_file_name) mask = self.make_structure_mask(structure_ids, annotation) if self.cache: Manifest.safe_mkdir(os.path.dirname(file_name)) nrrd.write(file_name, mask) return mask, None def make_structure_mask(self, structure_ids, annotation): """ Look at an annotation volume and identify voxels that have values in a list of structure ids. Parameters ---------- structure_ids: list List of IDs to look for in the annotation volume annotation: np.ndarray Numpy array filled with IDs. """ m = np.zeros(annotation.shape, dtype=np.uint8) for _, sid in enumerate(structure_ids): m[annotation == sid] = 1 return m def build_manifest(self, file_name): """ Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. """ manifest_builder = ManifestBuilder() manifest_builder.add_path('BASEDIR', '.') manifest_builder.add_path(self.EXPERIMENTS_KEY, 'experiments.json', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.STRUCTURES_KEY, 'structures.csv', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.STRUCTURE_UNIONIZES_KEY, 'experiment_%d/structure_unionizes.csv', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.ANNOTATION_KEY, 'annotation_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.TEMPLATE_KEY, 'average_template_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.INJECTION_DENSITY_KEY, 'experiment_%d/injection_density_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.INJECTION_FRACTION_KEY, 'experiment_%d/injection_fraction_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.DATA_MASK_KEY, 'experiment_%d/data_mask_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.PROJECTION_DENSITY_KEY, 'experiment_%d/projection_density_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.add_path(self.STRUCTURE_MASK_KEY, 'structure_masks/structure_%d.nrrd', parent_key='BASEDIR', typename='file') manifest_builder.write_json_file(file_name)
class MouseConnectivityCache(Cache): """ Cache class for storing and accessing data related to the adult mouse Connectivity Atlas. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- resolution: int Resolution of grid data to be downloaded when accessing projection volume, the annotation volume, and the annotation volume. Must be one of (10, 25, 50, 100). Default is 25. api: MouseConnectivityApi instance Used internally to make API queries. Parameters ---------- resolution: int Resolution of grid data to be downloaded when accessing projection volume, the annotation volume, and the annotation volume. Must be one of (10, 25, 50, 100). Default is 25. ccf_version: string Desired version of the Common Coordinate Framework. This affects the annotation volume (get_annotation_volume) and structure masks (get_structure_mask). Must be one of (MouseConnectivityApi.CCF_2015, MouseConnectivityApi.CCF_2016). Default: MouseConnectivityApi.CCF_2016 cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_projection_density(file_name='file.nrrd')). manifest_file: string File name of the manifest to be read. Default is "mouse_connectivity_manifest.json". """ CCF_VERSION_KEY = "CCF_VERSION" ANNOTATION_KEY = "ANNOTATION" TEMPLATE_KEY = "TEMPLATE" PROJECTION_DENSITY_KEY = "PROJECTION_DENSITY" INJECTION_DENSITY_KEY = "INJECTION_DENSITY" INJECTION_FRACTION_KEY = "INJECTION_FRACTION" DATA_MASK_KEY = "DATA_MASK" STRUCTURE_UNIONIZES_KEY = "STRUCTURE_UNIONIZES" EXPERIMENTS_KEY = "EXPERIMENTS" STRUCTURES_KEY = "STRUCTURES" STRUCTURE_MASK_KEY = "STRUCTURE_MASK" def __init__( self, resolution=None, cache=True, manifest_file="mouse_connectivity_manifest.json", ccf_version=None, base_uri=None, ): super(MouseConnectivityCache, self).__init__(manifest=manifest_file, cache=cache) if resolution is None: self.resolution = MouseConnectivityApi.VOXEL_RESOLUTION_25_MICRONS else: self.resolution = resolution self.api = MouseConnectivityApi(base_uri=base_uri) if ccf_version is None: ccf_version = MouseConnectivityApi.CCF_VERSION_DEFAULT self.ccf_version = ccf_version def get_annotation_volume(self, file_name=None): """ Read the annotation volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the annotation volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.ANNOTATION_KEY, self.ccf_version, self.resolution) if file_name is None: raise Exception("No save file name provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_make_parent_dirs(file_name) annotation, info = self.api.download_annotation_volume(self.ccf_version, self.resolution, file_name) return annotation, info def get_template_volume(self, file_name=None): """ Read the template volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.TEMPLATE_KEY, self.resolution) if file_name is None: raise Exception("No save file provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_make_parent_dirs(file_name) annotation, info = self.api.download_template_volume(self.resolution, file_name) return annotation, info def get_projection_density(self, experiment_id, file_name=None): """ Read a projection density volume for a single experiment. Download it first if it doesn't exist. Projection density is the proportion of of projecting pixels in a grid voxel in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.PROJECTION_DENSITY_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_projection_density(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_injection_density(self, experiment_id, file_name=None): """ Read an injection density volume for a single experiment. Download it first if it doesn't exist. Injection density is the proportion of projecting pixels in a grid voxel only including pixels that are part of the injection site in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.INJECTION_DENSITY_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_injection_density(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_injection_fraction(self, experiment_id, file_name=None): """ Read an injection fraction volume for a single experiment. Download it first if it doesn't exist. Injection fraction is the proportion of pixels in the injection site in a grid voxel in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.INJECTION_FRACTION_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_injection_fraction(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_data_mask(self, experiment_id, file_name=None): """ Read a data mask volume for a single experiment. Download it first if it doesn't exist. Data mask is a binary mask of voxels that have valid data. Only use valid data in analysis! Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.DATA_MASK_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_data_mask(file_name, experiment_id, self.resolution) return nrrd.read(file_name) def get_ontology(self, file_name=None): """ Read the list of adult mouse structures and return an Ontology instance. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ return Ontology(self.get_structures(file_name)) def get_structures(self, file_name=None): """ Read the list of adult mouse structures and return a Pandas DataFrame. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURES_KEY) if os.path.exists(file_name): structures = pd.DataFrame.from_csv(file_name) else: structures = OntologiesApi(base_uri=self.api.api_url).get_structures(1) structures = pd.DataFrame(structures) if self.cache: Manifest.safe_make_parent_dirs(file_name) structures.to_csv(file_name) structures.set_index(["id"], inplace=True, drop=False) return structures def get_experiments(self, dataframe=False, file_name=None, cre=None, injection_structure_ids=None): """ Read a list of experiments that match certain criteria. If caching is enabled, this will save the whole (unfiltered) list of experiments to a file. Parameters ---------- dataframe: boolean Return the list of experiments as a Pandas DataFrame. If False, return a list of dictionaries. Default False. file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY) if os.path.exists(file_name): experiments = json_utilities.read(file_name) else: experiments = self.api.experiment_source_search(injection_structures="root") # removing these elements because they are specific to a particular # resolution for e in experiments: del e["num-voxels"] del e["injection-volume"] del e["sum"] del e["name"] if self.cache: Manifest.safe_make_parent_dirs(file_name) json_utilities.write(file_name, experiments) # filter the read/downloaded list of experiments experiments = self.filter_experiments(experiments, cre, injection_structure_ids) if dataframe: experiments = pd.DataFrame(experiments) experiments.set_index(["id"], inplace=True, drop=False) return experiments def filter_experiments(self, experiments, cre=None, injection_structure_ids=None): """ Take a list of experiments and filter them by cre status and injection structure. Parameters ---------- cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ if cre is True: experiments = [e for e in experiments if e["transgenic-line"]] elif cre is False: experiments = [e for e in experiments if not e["transgenic-line"]] elif cre is not None: experiments = [e for e in experiments if e["transgenic-line"] in cre] if injection_structure_ids is not None: descendant_ids = self.get_ontology().get_descendant_ids(injection_structure_ids) experiments = [e for e in experiments if e["structure-id"] in descendant_ids] return experiments def get_experiment_structure_unionizes( self, experiment_id, file_name=None, is_injection=None, structure_ids=None, include_descendants=False, hemisphere_ids=None, ): """ Retrieve the structure unionize data for a specific experiment. Filter by structure, injection status, and hemisphere. Parameters ---------- experiment_id: int ID of the experiment of interest. Corresponds to section_data_set_id in the API. file_name: string File name to save/read the experiments list. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records for a specific set of structures. If None, return all records. Default None. include_descendants: boolean Include all descendant records for specified structures. Default False. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_UNIONIZES_KEY, experiment_id) if os.path.exists(file_name): unionizes = pd.DataFrame.from_csv(file_name) else: unionizes = self.api.get_structure_unionizes([experiment_id]) unionizes = pd.DataFrame(unionizes) # rename section_data_set_id column to experiment_id unionizes.columns = ["experiment_id" if c == "section_data_set_id" else c for c in unionizes.columns] if self.cache: Manifest.safe_make_parent_dirs(file_name) unionizes.to_csv(file_name) return self.filter_structure_unionizes( unionizes, is_injection, structure_ids, include_descendants, hemisphere_ids ) def filter_structure_unionizes( self, unionizes, is_injection=None, structure_ids=None, include_descendants=False, hemisphere_ids=None ): """ Take a list of unionzes and return a subset of records filtered by injection status, structure, and hemisphere. Parameters ---------- is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records for a set of structures. If None, return all records. Default None. include_descendants: boolean Include all descendant records for specified structures. Default False. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ if is_injection is not None: unionizes = unionizes[unionizes.is_injection == is_injection] if structure_ids is not None: if include_descendants: structure_ids = self.get_ontology().get_descendant_ids(structure_ids) else: structure_ids = set(structure_ids) unionizes = unionizes[unionizes["structure_id"].isin(structure_ids)] if hemisphere_ids is not None: unionizes = unionizes[unionizes["hemisphere_id"].isin(hemisphere_ids)] return unionizes def get_structure_unionizes( self, experiment_ids, is_injection=None, structure_ids=None, include_descendants=False, hemisphere_ids=None ): """ Get structure unionizes for a set of experiment IDs. Filter the results by injection status, structure, and hemisphere. Parameters ---------- experiment_ids: list List of experiment IDs. Corresponds to section_data_set_id in the API. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records for a specific set of structures. If None, return all records. Default None. include_descendants: boolean Include all descendant records for specified structures. Default False. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ unionizes = [ self.get_experiment_structure_unionizes( eid, is_injection=is_injection, structure_ids=structure_ids, include_descendants=include_descendants, hemisphere_ids=hemisphere_ids, ) for eid in experiment_ids ] return pd.concat(unionizes, ignore_index=True) def get_projection_matrix( self, experiment_ids, projection_structure_ids, hemisphere_ids=None, parameter="projection_volume", dataframe=False, ): unionizes = self.get_structure_unionizes( experiment_ids, is_injection=False, structure_ids=projection_structure_ids, include_descendants=False, hemisphere_ids=hemisphere_ids, ) hemisphere_ids = set(unionizes["hemisphere_id"].values.tolist()) nrows = len(experiment_ids) ncolumns = len(projection_structure_ids) * len(hemisphere_ids) matrix = np.empty((nrows, ncolumns)) matrix[:] = np.NAN row_lookup = {} for idx, e in enumerate(experiment_ids): row_lookup[e] = idx column_lookup = {} columns = [] cidx = 0 hlabel = {1: "-L", 2: "-R", 3: ""} o = self.get_ontology() for hid in hemisphere_ids: for sid in projection_structure_ids: column_lookup[(hid, sid)] = cidx label = o[sid].iloc[0]["acronym"] + hlabel[hid] columns.append({"hemisphere_id": hid, "structure_id": sid, "label": label}) cidx += 1 for _, row in unionizes.iterrows(): ridx = row_lookup[row["experiment_id"]] k = (row["hemisphere_id"], row["structure_id"]) cidx = column_lookup[k] matrix[ridx, cidx] = row[parameter] if dataframe: all_experiments = self.get_experiments(dataframe=True) rows_df = all_experiments.loc[experiment_ids] cols_df = pd.DataFrame(columns) return {"matrix": matrix, "rows": rows_df, "columns": cols_df} else: return {"matrix": matrix, "rows": experiment_ids, "columns": columns} def get_structure_mask(self, structure_id, file_name=None, annotation_file_name=None): """ Read a 3D numpy array shaped like the annotation volume that has non-zero values where voxels belong to a particular structure. This will take care of identifying substructures. Parameters ---------- structure_id: int ID of a structure. file_name: string File name to store the structure mask. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. annotation_file_name: string File name to store the annotation volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_MASK_KEY, structure_id) if os.path.exists(file_name): return nrrd.read(file_name) else: ont = self.get_ontology() structure_ids = ont.get_descendant_ids([structure_id]) annotation, _ = self.get_annotation_volume(annotation_file_name) mask = self.make_structure_mask(structure_ids, annotation) if self.cache: Manifest.safe_make_parent_dirs(file_name) nrrd.write(file_name, mask) return mask, None def make_structure_mask(self, structure_ids, annotation): """ Look at an annotation volume and identify voxels that have values in a list of structure ids. Parameters ---------- structure_ids: list List of IDs to look for in the annotation volume annotation: np.ndarray Numpy array filled with IDs. """ m = np.zeros(annotation.shape, dtype=np.uint8) for _, sid in enumerate(structure_ids): m[annotation == sid] = 1 return m def build_manifest(self, file_name): """ Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. """ manifest_builder = ManifestBuilder() manifest_builder.add_path("BASEDIR", ".") manifest_builder.add_path(self.EXPERIMENTS_KEY, "experiments.json", parent_key="BASEDIR", typename="file") manifest_builder.add_path(self.STRUCTURES_KEY, "structures.csv", parent_key="BASEDIR", typename="file") manifest_builder.add_path( self.STRUCTURE_UNIONIZES_KEY, "experiment_%d/structure_unionizes.csv", parent_key="BASEDIR", typename="file" ) manifest_builder.add_path(self.CCF_VERSION_KEY, "%s", parent_key="BASEDIR", typename="dir") manifest_builder.add_path( self.ANNOTATION_KEY, "annotation_%d.nrrd", parent_key=self.CCF_VERSION_KEY, typename="file" ) manifest_builder.add_path(self.TEMPLATE_KEY, "average_template_%d.nrrd", parent_key="BASEDIR", typename="file") manifest_builder.add_path( self.INJECTION_DENSITY_KEY, "experiment_%d/injection_density_%d.nrrd", parent_key="BASEDIR", typename="file" ) manifest_builder.add_path( self.INJECTION_FRACTION_KEY, "experiment_%d/injection_fraction_%d.nrrd", parent_key="BASEDIR", typename="file", ) manifest_builder.add_path( self.DATA_MASK_KEY, "experiment_%d/data_mask_%d.nrrd", parent_key="BASEDIR", typename="file" ) manifest_builder.add_path( self.PROJECTION_DENSITY_KEY, "experiment_%d/projection_density_%d.nrrd", parent_key="BASEDIR", typename="file", ) manifest_builder.add_path( self.STRUCTURE_MASK_KEY, "structure_masks/structure_%d.nrrd", parent_key="BASEDIR", typename="file" ) manifest_builder.write_json_file(file_name)
tree = StructureTree(structure_graph) # Example: # tree.get_structures_by_name(['Dorsal auditory area']) # The annotation download writes a file, so we will need somwhere to put it annotation_dir = os.path.dirname(structIDSource) Manifest.safe_mkdir(annotation_dir) annotation_path = os.path.join(annotation_dir, 'annotation.nrrd') #------------------------------------------------------------------------------- # Use the connectivity API: mcapi = MouseConnectivityApi() # The name of the latest ccf version (a string): annotation_version = mcapi.CCF_VERSION_DEFAULT if not os.path.exists(annotation_path): mcapi.download_annotation_volume(annotation_version, resolution, annotation_path) annotation, meta = nrrd.read(annotation_path) # Build a reference space from a StructureTree and annotation volume, the third argument is # the resolution of the space in microns rsp = ReferenceSpace(tree, annotation, [resolution, resolution, resolution]) #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # So now we're ready to go through structures, and extract their coordinates structureID_df = pd.read_csv(structIDSource) structureIDs = structureID_df['ids'].to_numpy() print("Retrieved %u structures from %s..." % (len(structureIDs), structIDSource)) # A complete mask for one structure
def test_notebook(fn_temp_dir): # coding: utf-8 # # Reference Space # # This notebook contains example code demonstrating the use of the StructureTree and ReferenceSpace classes. These classes provide methods for interacting with the 3d spaces to which Allen Institute data and atlases are registered. # # Unlike the AllenSDK cache classes, StructureTree and ReferenceSpace operate entirely in memory. We recommend using json files to store text and nrrd files to store volumetric images. # # The MouseConnectivityCache class has methods for downloading, storing, and constructing StructureTrees and ReferenceSpaces. Please see [here](https://alleninstitute.github.io/AllenSDK/_static/examples/nb/mouse_connectivity.html) for examples. # ## Constructing a StructureTree # # A StructureTree object is a wrapper around a structure graph - a list of dictionaries documenting brain structures and their containment relationships. To build a structure tree, you will first need to obtain a structure graph. # # For a list of atlases and corresponding structure graph ids, see [here](http://help.brain-map.org/display/api/Atlas+Drawings+and+Ontologies). # In[1]: from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.core.structure_tree import StructureTree oapi = OntologiesApi() structure_graph = oapi.get_structures_with_sets( [1]) # 1 is the id of the adult mouse structure graph # This removes some unused fields returned by the query structure_graph = StructureTree.clean_structures(structure_graph) tree = StructureTree(structure_graph) # In[2]: # now let's take a look at a structure tree.get_structures_by_name(['Dorsal auditory area']) # The fields are: # * acronym: a shortened name for the structure # * rgb_triplet: each structure is assigned a consistent color for visualizations # * graph_id: the structure graph to which this structure belongs # * graph_order: each structure is assigned a consistent position in the flattened graph # * id: a unique integer identifier # * name: the full name of the structure # * structure_id_path: traces a path from the root node of the tree to this structure # * structure_set_ids: the structure belongs to these predefined groups # ## Using a StructureTree # In[3]: # get a structure's parent tree.parent([1011]) # In[4]: # get a dictionary mapping structure ids to names name_map = tree.get_name_map() name_map[247] # In[5]: # ask whether one structure is contained within another strida = 385 stridb = 247 is_desc = '' if tree.structure_descends_from(385, 247) else ' not' print('{0} is{1} in {2}'.format(name_map[strida], is_desc, name_map[stridb])) # In[6]: # build a custom map that looks up acronyms by ids # the syntax here is just a pair of node-wise functions. # The first one returns keys while the second one returns values acronym_map = tree.value_map(lambda x: x['id'], lambda y: y['acronym']) print(acronym_map[385]) # ## Downloading an annotation volume # # This code snippet will download and store a nrrd file containing the Allen Common Coordinate Framework annotation. We have requested an annotation with 25-micron isometric spacing. The orientation of this space is: # * Anterior -> Posterior # * Superior -> Inferior # * Left -> Right # This is the no-frills way to download an annotation volume. See the <a href='_static/examples/nb/mouse_connectivity.html#Manipulating-Grid-Data'>mouse connectivity</a> examples if you want to properly cache the downloaded data. # In[7]: import os import nrrd from allensdk.api.queries.mouse_connectivity_api import MouseConnectivityApi from allensdk.config.manifest import Manifest # the annotation download writes a file, so we will need somwhere to put it annotation_dir = 'annotation' Manifest.safe_mkdir(annotation_dir) annotation_path = os.path.join(annotation_dir, 'annotation.nrrd') mcapi = MouseConnectivityApi() mcapi.download_annotation_volume('annotation/ccf_2016', 25, annotation_path) annotation, meta = nrrd.read(annotation_path) # ## Constructing a ReferenceSpace # In[8]: from allensdk.core.reference_space import ReferenceSpace # build a reference space from a StructureTree and annotation volume, the third argument is # the resolution of the space in microns rsp = ReferenceSpace(tree, annotation, [25, 25, 25]) # ## Using a ReferenceSpace # #### making structure masks # # The simplest use of a Reference space is to build binary indicator masks for structures or groups of structures. # In[9]: # A complete mask for one structure whole_cortex_mask = rsp.make_structure_mask([315]) # view in coronal section # What if you want a mask for a whole collection of ontologically disparate structures? Just pass more structure ids to make_structure_masks: # In[10]: # This gets all of the structures targeted by the Allen Brain Observatory project brain_observatory_structures = rsp.structure_tree.get_structures_by_set_id( [514166994]) brain_observatory_ids = [st['id'] for st in brain_observatory_structures] brain_observatory_mask = rsp.make_structure_mask(brain_observatory_ids) # view in horizontal section # You can also make and store a number of structure_masks at once: # In[11]: import functools # Define a wrapper function that will control the mask generation. # This one checks for a nrrd file in the specified base directory # and builds/writes the mask only if one does not exist mask_writer = functools.partial(ReferenceSpace.check_and_write, annotation_dir) # many_structure_masks is a generator - nothing has actrually been run yet mask_generator = rsp.many_structure_masks([385, 1097], mask_writer) # consume the resulting iterator to make and write the masks for structure_id in mask_generator: print('made mask for structure {0}.'.format(structure_id)) os.listdir(annotation_dir) # #### Removing unassigned structures # A structure graph may contain structures that are not used in a particular reference space. Having these around can complicate use of the reference space, so we generally want to remove them. # # We'll try this using "Somatosensory areas, layer 6a" as a test case. In the 2016 ccf space, this structure is unused in favor of finer distinctions (e.g. "Primary somatosensory area, barrel field, layer 6a"). # In[12]: # Double-check the voxel counts no_voxel_id = rsp.structure_tree.get_structures_by_name( ['Somatosensory areas, layer 6a'])[0]['id'] print('voxel count for structure {0}: {1}'.format( no_voxel_id, rsp.total_voxel_map[no_voxel_id])) # remove unassigned structures from the ReferenceSpace's StructureTree rsp.remove_unassigned() # check the structure tree no_voxel_id in rsp.structure_tree.node_ids() # #### View a slice from the annotation # In[13]: import numpy as np # #### Downsample the space # # If you want an annotation at a resolution we don't provide, you can make one with the downsample method. # In[14]: import warnings target_resolution = [75, 75, 75] # in some versions of scipy, scipy.ndimage.zoom raises a helpful but distracting # warning about the method used to truncate integers. warnings.simplefilter('ignore') sf_rsp = rsp.downsample(target_resolution) # re-enable warnings warnings.simplefilter('default') print(rsp.annotation.shape) print(sf_rsp.annotation.shape)
structure_graph = oapi.get_structures_with_sets([graph_id]) # This removes some unused fields returned by the query structure_graph = StructureTree.clean_structures(structure_graph) tree = StructureTree(structure_graph) # the annotation download writes a file, so we will need somwhere to put it annotation_dir = 'E:\\Histology\\allen_rsp' annotation_path = os.path.join(annotation_dir, 'annotation_10.nrrd') # this is a string which contains the name of the latest ccf version annotation_version = MouseConnectivityApi.CCF_VERSION_DEFAULT mcapi = MouseConnectivityApi() #Next line commented because the annotation volume is already downloaded mcapi.download_annotation_volume(annotation_version, 10, annotation_path) annotation, meta = nrrd.read(annotation_path) swapped_ann = np.swapaxes(annotation, 1, 2) swapped_ann = swapped_ann[:, :, :: -1] #Revert the z axis so the 0 is the ventral part rsp = ReferenceSpace(tree, swapped_ann, [10, 10, 10]) root_path = "E:\\Histology\\brain_structures_half_not_close_10\\" ##Here comes the obj creation for struct in structure_graph[:1]: path_parent = "" for parent_id in struct["structure_id_path"][:-1]: name_parent = tree.get_structures_by_id([parent_id])[0]["acronym"]