def __init__(self,
              cache=True,
              manifest_file='brain_observatory_manifest.json',
              base_uri=None):
     super(BrainObservatoryCache, self).__init__(manifest=manifest_file,
                                                 cache=cache)
     self.api = BrainObservatoryApi(base_uri=base_uri)
def bo_api_save_ophys():
    bo = BrainObservatoryApi('http://testwarehouse:9000')
    bo.json_msg_query = \
        MagicMock(name='json_msg_query',
                  return_value=[{'download_link': '/url/path/to/file'}])

    return bo
def bo_api_save_ophys():
    bo = BrainObservatoryApi('http://testwarehouse:9000')
    bo.json_msg_query = \
        MagicMock(name='json_msg_query',
                  return_value=[ {'download_link': '/url/path/to/file' } ])
    
    return bo
示例#4
0
def cell_specimen_table(md_temp_dir):
    # download a zipped version of the cell specimen table for filter tests
    # as it is orders of magnitude faster
    api = BrainObservatoryApi()
    zipped = os.path.join(md_temp_dir, "cell_specimens.zip")
    api.retrieve_file_over_http(CELL_SPECIMEN_ZIP_URL, zipped)
    df = pd.read_csv(ZipFile(zipped).open("cell_metrics.csv"),
                     true_values="t", false_values="f")
    js = json.loads(df.to_json(orient="records"))
    table_file = os.path.join(md_temp_dir, "cell_specimens.json")
    with open(table_file, "w") as f:
        json.dump(js, f, indent=1)
    return table_file
def cell_specimen_table(tmpdir_factory):
    # download a zipped version of the cell specimen table for filter tests
    # as it is orders of magnitude faster
    api = BrainObservatoryApi()
    data_dir = str(tmpdir_factory.mktemp("data"))
    zipped = os.path.join("cell_specimens.zip")
    api.retrieve_file_over_http(CELL_SPECIMEN_ZIP_URL, zipped)
    df = pd.read_csv(ZipFile(zipped).open("cell_metrics.csv"),
                     true_values="t", false_values="f")
    js = json.loads(df.to_json(orient="records"))
    table_file = os.path.join(data_dir, "cell_specimens.json")
    with open(table_file, "w") as f:
        json.dump(js, f, indent=1)
    return table_file
    def __init__(self,
                 cache=True,
                 manifest_file='brain_observatory_manifest.json',
                 base_uri=None,
                 api=None):
        super(BrainObservatoryCache,
              self).__init__(manifest=manifest_file,
                             cache=cache,
                             version=self.MANIFEST_VERSION)

        if api is None:
            self.api = BrainObservatoryApi(base_uri=base_uri)
        else:
            self.api = api
    def __init__(self,
                 cache=True,
                 manifest_file=None,
                 base_uri=None,
                 api=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('brain_observatory')

        super(BrainObservatoryCache,
              self).__init__(manifest=manifest_file,
                             cache=cache,
                             version=self.MANIFEST_VERSION)

        if api is None:
            self.api = BrainObservatoryApi(base_uri=base_uri)
        else:
            self.api = api
    def __init__(self, cache=True, manifest_file=None, base_uri=None, api=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('brain_observatory')

        super(BrainObservatoryCache, self).__init__(
            manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)

        if api is None:
            self.api = BrainObservatoryApi(base_uri=base_uri)
        else:
            self.api = api
def bo_api():
    bo = BrainObservatoryApi('http://testwarehouse:9000')
    bo.json_msg_query = MagicMock(name='json_msg_query')

    return bo
示例#10
0
def api():
    boi = BrainObservatoryApi()

    return boi
示例#11
0
 def __init__(self, cache=True, manifest_file='brain_observatory_manifest.json', base_uri=None):
     super(BrainObservatoryCache, self).__init__(
         manifest=manifest_file, cache=cache)
     self.api = BrainObservatoryApi(base_uri=base_uri)
示例#12
0
class BrainObservatoryCache(Cache):
    """
    Cache class for storing and accessing data from the Brain Observatory.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: BrainObservatoryApi instance
        The object used for making API queries related to the Brain
        Observatory.

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ophys_experiment_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "brain_observatory_manifest.json".
    """

    EXPERIMENT_CONTAINERS_KEY = 'EXPERIMENT_CONTAINERS'
    EXPERIMENTS_KEY = 'EXPERIMENTS'
    CELL_SPECIMENS_KEY = 'CELL_SPECIMENS'
    EXPERIMENT_DATA_KEY = 'EXPERIMENT_DATA'
    STIMULUS_MAPPINGS_KEY = 'STIMULUS_MAPPINGS'

    def __init__(self, cache=True, manifest_file='brain_observatory_manifest.json', base_uri=None):
        super(BrainObservatoryCache, self).__init__(
            manifest=manifest_file, cache=cache)
        self.api = BrainObservatoryApi(base_uri=base_uri)

    def get_all_targeted_structures(self):
        """ Return a list of all targeted structures in the data set. """
        containers = self.get_experiment_containers(simple=False)
        targeted_structures = set(
            [c['targeted_structure']['acronym'] for c in containers])
        return sorted(list(targeted_structures))

    def get_all_cre_lines(self):
        """ Return a list of all cre driver lines in the data set. """
        containers = self.get_experiment_containers(simple=False)
        cre_lines = set([_find_specimen_cre_line(c['specimen'])
                         for c in containers])
        return sorted(list(cre_lines))

    def get_all_reporter_lines(self):
        """ Return a list of all reporter lines in the data set. """
        containers = self.get_experiment_containers(simple=False)
        reporter_lines = set([_find_specimen_reporter_line(c['specimen'])
                              for c in containers])
        return sorted(list(reporter_lines))

    def get_all_imaging_depths(self):
        """ Return a list of all imaging depths in the data set. """
        containers = self.get_experiment_containers(simple=False)
        imaging_depths = set([c['imaging_depth'] for c in containers])
        return sorted(list(imaging_depths))

    def get_all_session_types(self):
        """ Return a list of all stimulus sessions in the data set. """
        exps = self.get_ophys_experiments(simple=False)
        names = set([exp['stimulus_name'] for exp in exps])
        return sorted(list(names))

    def get_all_stimuli(self):
        """ Return a list of all stimuli in the data set. """
        return sorted(list(stim_info.all_stimuli()))

    def get_experiment_containers(self, file_name=None,
                                  ids=None,
                                  targeted_structures=None,
                                  imaging_depths=None,
                                  cre_lines=None,
                                  transgenic_lines=None,
                                  simple=True):
        """ Get a list of experiment containers matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the experiment containers.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")

        file_name = self.get_cache_path(
            file_name, self.EXPERIMENT_CONTAINERS_KEY)

        if os.path.exists(file_name):
            containers = ju.read(file_name)
        else:
            containers = self.api.get_experiment_containers()

            if self.cache:
                ju.write(file_name, containers)

        transgenic_lines = _merge_transgenic_lines(cre_lines, transgenic_lines)

        containers = self.api.filter_experiment_containers(containers, ids=ids,
                                                           targeted_structures=targeted_structures,
                                                           imaging_depths=imaging_depths,
                                                           transgenic_lines=transgenic_lines)

        if simple:
            containers = [{
                'id': c['id'],
                'imaging_depth': c['imaging_depth'],
                'targeted_structure': c['targeted_structure']['acronym'],
                'cre_line': _find_specimen_cre_line(c['specimen']),
                'reporter_line': _find_specimen_reporter_line(c['specimen']),
                'age_days': c['specimen']['donor']['age']['days'],
                'donor_name': c['specimen']['donor']['external_donor_name'],
                'specimen_name': c['specimen']['name']
            } for c in containers]

        return containers

    def get_ophys_experiments(self, file_name=None,
                              ids=None,
                              experiment_container_ids=None,
                              targeted_structures=None,
                              imaging_depths=None,
                              cre_lines=None,
                              transgenic_lines=None,
                              stimuli=None,
                              session_types=None,
                              simple=True):
        """ Get a list of ophys experiments matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the ophys experiments.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of ophys experiment ids.

        experiment_container_ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        stimuli: list
            List of stimulus names.  Must be in the list returned by
            BrainObservatoryCache.get_all_stimuli().

        session_types: list
            List of stimulus session type names.  Must be in the list returned by
            BrainObservatoryCache.get_all_session_types().

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")
        _assert_not_string(stimuli, "stimuli")
        _assert_not_string(session_types, "session_types")

        file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)

        if os.path.exists(file_name):
            exps = ju.read(file_name)
        else:
            exps = self.api.get_ophys_experiments()

            if self.cache:
                ju.write(file_name, exps)

        transgenic_lines = _merge_transgenic_lines(cre_lines, transgenic_lines)

        exps = self.api.filter_ophys_experiments(exps,
                                                 ids=ids,
                                                 experiment_container_ids=experiment_container_ids,
                                                 targeted_structures=targeted_structures,
                                                 imaging_depths=imaging_depths,
                                                 transgenic_lines=transgenic_lines,
                                                 stimuli=stimuli,
                                                 session_types=session_types)

        if simple:
            exps = [{
                'id': e['id'],
                'imaging_depth': e['imaging_depth'],
                'targeted_structure': e['targeted_structure']['acronym'],
                'cre_line': _find_specimen_cre_line(e['specimen']),
                'reporter_line': _find_specimen_reporter_line(e['specimen']),
                'age_days': e['specimen']['donor']['age']['days'],
                'experiment_container_id': e['experiment_container_id'],
                'session_type': e['stimulus_name'],
                'donor_name': e['specimen']['donor']['external_donor_name'],
                'specimen_name': e['specimen']['name']
            } for e in exps]
        return exps

    def _get_stimulus_mappings(self, file_name=None):
        """ Returns a mapping of which metrics are related to which stimuli. Internal use only. """

        file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)

        if os.path.exists(file_name):
            mappings = ju.read(file_name)
        else:
            mappings = self.api.get_stimulus_mappings()

            if self.cache:
                ju.write(file_name, mappings)

        return mappings

    def get_cell_specimens(self,
                           file_name=None,
                           ids=None,
                           experiment_container_ids=None,
                           simple=True,
                           filters=None):
        """ Return cell specimens that have certain properies.

        Parameters
        ----------
        file_name: string
            File name to save/read the cell specimens.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of cell specimen ids.

        experiment_container_ids: list
            List of experiment container ids.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        filters: list of dicts
            List of filter dictionaries.  The Allen Brain Observatory web site can 
            generate filters in this format to reproduce a filtered set of cells
            found there.  To see what these look like, visit 
            http://observatory.brain-map.org/visualcoding, perform a cell search
            and apply some filters (e.g. find cells in a particular area), then 
            click the "view these cells in the AllenSDK" link on the bottom-left
            of the search results page.  This will take you to a page that contains
            a code sample you can use to apply those same filters via this argument.
            For more detail on the filter syntax, see BrainObservatoryApi.dataframe_query.
            

        Returns
        -------
        list of dictionaries
        """

        file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)

        if os.path.exists(file_name):
            cell_specimens = ju.read(file_name)
        else:
            cell_specimens = self.api.get_cell_metrics()

            if self.cache:
                ju.write(file_name, cell_specimens)

        cell_specimens = self.api.filter_cell_specimens(cell_specimens,
                                                        ids=ids,
                                                        experiment_container_ids=experiment_container_ids,
                                                        filters=filters)

        # drop the thumbnail columns
        if simple:
            mappings = self._get_stimulus_mappings()
            thumbnails = [m['item'] for m in mappings if m[
                'item_type'] == 'T' and m['level'] == 'R']
            for cs in cell_specimens:
                for t in thumbnails:
                    del cs[t]

        return cell_specimens

    def get_ophys_experiment_data(self, ophys_experiment_id, file_name=None):
        """ Download the NWB file for an ophys_experiment (if it hasn't already been
        downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: integer
            id of the ophys_experiment to retrieve

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        file_name = self.get_cache_path(
            file_name, self.EXPERIMENT_DATA_KEY, ophys_experiment_id)

        if not os.path.exists(file_name):
            self.api.save_ophys_experiment_data(ophys_experiment_id, file_name)

        return BrainObservatoryNwbDataSet(file_name)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()

        mb.add_path('BASEDIR', '.')
        mb.add_path(self.EXPERIMENT_CONTAINERS_KEY,
                    'experiment_containers.json', typename='file', parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENTS_KEY, 'ophys_experiments.json',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENT_DATA_KEY, 'ophys_experiment_data/%d.nwb',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.CELL_SPECIMENS_KEY, 'cell_specimens.json',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.STIMULUS_MAPPINGS_KEY, 'stimulus_mappings.json',
                    typename='file', parent_key='BASEDIR')

        mb.write_json_file(file_name)
示例#13
0
def bo_api():
    endpoint = os.environ[
        'TEST_API_ENDPOINT'] if 'TEST_API_ENDPOINT' in os.environ else 'http://twarehouse-backup'
    return BrainObservatoryApi(endpoint)
class BrainObservatoryCache(Cache):
    """
    Cache class for storing and accessing data from the Brain Observatory.
    By default, this class will cache any downloaded metadata or files in 
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------
    
    api: BrainObservatoryApi instance
        The object used for making API queries related to the Brain

 Observatory.

    Parameters
    ----------
    
    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ophys_experiment_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "brain_observatory_manifest.json".
    """

    EXPERIMENT_CONTAINERS_KEY = 'EXPERIMENT_CONTAINERS'
    EXPERIMENTS_KEY = 'EXPERIMENTS'
    CELL_SPECIMENS_KEY = 'CELL_SPECIMENS'
    EXPERIMENT_DATA_KEY = 'EXPERIMENT_DATA'
    STIMULUS_MAPPINGS_KEY = 'STIMULUS_MAPPINGS'

    def __init__(self,
                 cache=True,
                 manifest_file='brain_observatory_manifest.json',
                 base_uri=None):
        super(BrainObservatoryCache, self).__init__(manifest=manifest_file,
                                                    cache=cache)
        self.api = BrainObservatoryApi(base_uri=base_uri)

    def get_all_targeted_structures(self):
        """ Return a list of all targeted structures in the data set. """
        containers = self.get_experiment_containers(simple=False)
        targeted_structures = set(
            [c['targeted_structure']['acronym'] for c in containers])
        return sorted(list(targeted_structures))

    def get_all_cre_lines(self):
        """ Return a list of all cre driver lines in the data set. """
        containers = self.get_experiment_containers(simple=False)
        cre_lines = set(
            [_find_specimen_cre_line(c['specimen']) for c in containers])
        return sorted(list(cre_lines))

    def get_all_imaging_depths(self):
        """ Return a list of all imaging depths in the data set. """
        containers = self.get_experiment_containers(simple=False)
        imaging_depths = set([c['imaging_depth'] for c in containers])
        return sorted(list(imaging_depths))

    def get_all_session_types(self):
        """ Return a list of all stimulus sessions in the data set. """
        exps = self.get_ophys_experiments()
        names = set([exp['stimulus_name'] for exp in exps])
        return sorted(list(names))

    def get_all_stimuli(self):
        """ Return a list of all stimuli in the data set. """
        return sorted(list(stim_info.all_stimuli()))

    def get_experiment_containers(self,
                                  file_name=None,
                                  ids=None,
                                  targeted_structures=None,
                                  imaging_depths=None,
                                  cre_lines=None,
                                  simple=True):
        """ Get a list of experiment containers matching certain criteria.
        
        Parameters
        ----------
        file_name: string
            File name to save/read the experiment containers.  If file_name is None, 
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of experiment container ids.  

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by 
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by 
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by 
            BrainObservatoryCache.get_all_cre_lines().

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        file_name = self.get_cache_path(file_name,
                                        self.EXPERIMENT_CONTAINERS_KEY)

        if os.path.exists(file_name):
            containers = ju.read(file_name)
        else:
            containers = self.api.get_experiment_containers()

            if self.cache:
                ju.write(file_name, containers)

        containers = self.api.filter_experiment_containers(
            containers,
            ids=ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            transgenic_lines=cre_lines)

        if simple:
            containers = [{
                'id':
                c['id'],
                'imaging_depth':
                c['imaging_depth'],
                'targeted_structure':
                c['targeted_structure']['acronym'],
                'cre_line':
                _find_specimen_cre_line(c['specimen']),
                'age_days':
                c['specimen']['donor']['age']['days']
            } for c in containers]

        return containers

    def get_ophys_experiments(self,
                              file_name=None,
                              ids=None,
                              experiment_container_ids=None,
                              targeted_structures=None,
                              imaging_depths=None,
                              cre_lines=None,
                              stimuli=None,
                              session_types=None,
                              simple=True):
        """ Get a list of ophys experiments matching certain criteria.
        
        Parameters
        ----------
        file_name: string
            File name to save/read the ophys experiments.  If file_name is None, 
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of ophys experiment ids.  

        experiment_container_ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by 
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by 
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by 
            BrainObservatoryCache.get_all_cre_lines().

        stimuli: list
            List of stimulus names.  Must be in the list returned by 
            BrainObservatoryCache.get_all_stimuli().

        session_types: list
            List of stimulus session type names.  Must be in the list returned by 
            BrainObservatoryCache.get_all_session_types().

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)

        if os.path.exists(file_name):
            exps = ju.read(file_name)
        else:
            exps = self.api.get_ophys_experiments()

            if self.cache:
                ju.write(file_name, exps)

        exps = self.api.filter_ophys_experiments(
            exps,
            ids=ids,
            experiment_container_ids=experiment_container_ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            transgenic_lines=cre_lines,
            stimuli=stimuli,
            session_types=session_types)

        if simple:
            exps = [{
                'id': e['id'],
                'imaging_depth': e['imaging_depth'],
                'targeted_structure': e['targeted_structure']['acronym'],
                'cre_line': _find_specimen_cre_line(e['specimen']),
                'age_days': e['specimen']['donor']['age']['days'],
                'experiment_container_id': e['experiment_container_id'],
                'session_type': e['stimulus_name']
            } for e in exps]
        return exps

    def _get_stimulus_mappings(self, file_name=None):
        """ Returns a mapping of which metrics are related to which stimuli. Internal use only. """

        file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)

        if os.path.exists(file_name):
            mappings = ju.read(file_name)
        else:
            mappings = self.api.get_stimulus_mappings()

            if self.cache:
                ju.write(file_name, mappings)

        return mappings

    def get_cell_specimens(self,
                           file_name=None,
                           ids=None,
                           experiment_container_ids=None,
                           simple=True):
        """ Return cell specimens that have certain properies.
        
        Parameters
        ----------
        file_name: string
            File name to save/read the cell specimens.  If file_name is None, 
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of cell specimen ids.  

        experiment_container_ids: list
            List of experiment container ids.
            
        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.
            
        Returns
        -------
        list of dictionaries
        """

        file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)

        if os.path.exists(file_name):
            cell_specimens = ju.read(file_name)
        else:
            cell_specimens = self.api.get_cell_metrics()

            if self.cache:
                ju.write(file_name, cell_specimens)

        cell_specimens = self.api.filter_cell_specimens(
            cell_specimens,
            ids=ids,
            experiment_container_ids=experiment_container_ids)

        # drop the thumbnail columns
        if simple:
            mappings = self._get_stimulus_mappings()
            thumbnails = [
                m['item'] for m in mappings
                if m['item_type'] == 'T' and m['level'] == 'R'
            ]
            for cs in cell_specimens:
                for t in thumbnails:
                    del cs[t]

        return cell_specimens

    def get_ophys_experiment_data(self, ophys_experiment_id, file_name=None):
        """ Download the NWB file for an ophys_experiment (if it hasn't already been
        downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None, 
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.        

        ophys_experiment_id: integer
            id of the ophys_experiment to retrieve
            
        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        file_name = self.get_cache_path(file_name, self.EXPERIMENT_DATA_KEY,
                                        ophys_experiment_id)

        if not os.path.exists(file_name):
            self.api.save_ophys_experiment_data(ophys_experiment_id, file_name)

        return BrainObservatoryNwbDataSet(file_name)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.
        
        Parameters
        ----------
        
        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()

        mb.add_path('BASEDIR', '.')
        mb.add_path(self.EXPERIMENT_CONTAINERS_KEY,
                    'experiment_containers.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENTS_KEY,
                    'ophys_experiments.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENT_DATA_KEY,
                    'ophys_experiment_data/%d.nwb',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.CELL_SPECIMENS_KEY,
                    'cell_specimens.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.STIMULUS_MAPPINGS_KEY,
                    'stimulus_mappings.json',
                    typename='file',
                    parent_key='BASEDIR')

        mb.write_json_file(file_name)
class BrainObservatoryCache(Cache):
    """
    Cache class for storing and accessing data from the Brain Observatory.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: BrainObservatoryApi instance
        The object used for making API queries related to the Brain
        Observatory.

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ophys_experiment_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "brain_observatory_manifest.json".
    """

    EXPERIMENT_CONTAINERS_KEY = 'EXPERIMENT_CONTAINERS'
    EXPERIMENTS_KEY = 'EXPERIMENTS'
    CELL_SPECIMENS_KEY = 'CELL_SPECIMENS'
    EXPERIMENT_DATA_KEY = 'EXPERIMENT_DATA'
    ANALYSIS_DATA_KEY = 'ANALYSIS_DATA'
    EVENTS_DATA_KEY = 'EVENTS_DATA'
    STIMULUS_MAPPINGS_KEY = 'STIMULUS_MAPPINGS'
    MANIFEST_VERSION='1.2'

    def __init__(self, cache=True, manifest_file=None, base_uri=None, api=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('brain_observatory')

        super(BrainObservatoryCache, self).__init__(
            manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)

        if api is None:
            self.api = BrainObservatoryApi(base_uri=base_uri)
        else:
            self.api = api

    def get_all_targeted_structures(self):
        """ Return a list of all targeted structures in the data set. """
        containers = self.get_experiment_containers(simple=False)
        targeted_structures = set(
            [c['targeted_structure']['acronym'] for c in containers])
        return sorted(list(targeted_structures))

    def get_all_cre_lines(self):
        """ Return a list of all cre driver lines in the data set. """
        containers = self.get_experiment_containers(simple=True)
        cre_lines = set([c['cre_line'] for c in containers])
        return sorted(list(cre_lines))

    def get_all_reporter_lines(self):
        """ Return a list of all reporter lines in the data set. """
        containers = self.get_experiment_containers(simple=True)
        reporter_lines = set([c['reporter_line'] for c in containers])
        return sorted(list(reporter_lines))

    def get_all_imaging_depths(self):
        """ Return a list of all imaging depths in the data set. """
        containers = self.get_experiment_containers(simple=True)
        imaging_depths = set([c['imaging_depth'] for c in containers])
        return sorted(list(imaging_depths))

    def get_all_session_types(self):
        """ Return a list of all stimulus sessions in the data set. """
        exps = self.get_ophys_experiments(simple=False)
        names = set([exp['stimulus_name'] for exp in exps])
        return sorted(list(names))

    def get_all_stimuli(self):
        """ Return a list of all stimuli in the data set. """
        return sorted(list(stim_info.all_stimuli()))

    def get_experiment_containers(self, file_name=None,
                                  ids=None,
                                  targeted_structures=None,
                                  imaging_depths=None,
                                  cre_lines=None,
                                  reporter_lines=None,
                                  transgenic_lines=None,
                                  include_failed=False,
                                  simple=True):
        """ Get a list of experiment containers matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the experiment containers.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        reporter_lines: list
            List of reporter lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_reporter_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        include_failed: boolean
            Whether or not to include failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(reporter_lines, "reporter_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")

        file_name = self.get_cache_path(
            file_name, self.EXPERIMENT_CONTAINERS_KEY)

        containers = self.api.get_experiment_containers(path=file_name,
                                                        strategy='lazy',
                                                        **Cache.cache_json())

        containers = self.api.filter_experiment_containers(containers, ids=ids,
                                                           targeted_structures=targeted_structures,
                                                           imaging_depths=imaging_depths,
                                                           cre_lines=cre_lines,
                                                           reporter_lines=reporter_lines,
                                                           transgenic_lines=transgenic_lines,
                                                           include_failed=include_failed,
                                                           simple=simple)

        return containers

    def get_ophys_experiment_stimuli(self, experiment_id):
        """ For a single experiment, return the list of stimuli present in that experiment. """
        exps = self.get_ophys_experiments(ids=[experiment_id])

        if len(exps) == 0:
            return None

        return stim_info.stimuli_in_session(exps[0]['session_type'])

    def get_ophys_experiments(self, file_name=None,
                              ids=None,
                              experiment_container_ids=None,
                              targeted_structures=None,
                              imaging_depths=None,
                              cre_lines=None,
                              reporter_lines=None,
                              transgenic_lines=None,
                              stimuli=None,
                              session_types=None,
                              cell_specimen_ids=None,
                              include_failed=False,
                              require_eye_tracking=False,
                              simple=True):
        """ Get a list of ophys experiments matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the ophys experiments.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of ophys experiment ids.

        experiment_container_ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().
        
        reporter_lines: list
            List of reporter lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_reporter_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        stimuli: list
            List of stimulus names.  Must be in the list returned by
            BrainObservatoryCache.get_all_stimuli().

        session_types: list
            List of stimulus session type names.  Must be in the list returned by
            BrainObservatoryCache.get_all_session_types().

        cell_specimen_ids: list
            Only include experiments that contain cells with these ids.

        include_failed: boolean
            Whether or not to include experiments from failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        require_eye_tracking: boolean
            If True, only return experiments that have eye tracking results. Default: False.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(reporter_lines, "reporter_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")
        _assert_not_string(stimuli, "stimuli")
        _assert_not_string(session_types, "session_types")

        file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)

        exps = self.api.get_ophys_experiments(path=file_name,
                                              strategy='lazy',
                                              **Cache.cache_json())

        if cell_specimen_ids is not None:
            cells = self.get_cell_specimens(ids=cell_specimen_ids)
            cell_container_ids = set([cell['experiment_container_id'] for cell in cells])
            if experiment_container_ids is not None:
                experiment_container_ids = list(set(experiment_container_ids) - cell_container_ids)
            else:
                experiment_container_ids = list(cell_container_ids)

        exps = self.api.filter_ophys_experiments(exps,
                                                 ids=ids,
                                                 experiment_container_ids=experiment_container_ids,
                                                 targeted_structures=targeted_structures,
                                                 imaging_depths=imaging_depths,
                                                 cre_lines=cre_lines,
                                                 reporter_lines=reporter_lines,
                                                 transgenic_lines=transgenic_lines,
                                                 stimuli=stimuli,
                                                 session_types=session_types,
                                                 include_failed=include_failed,
                                                 require_eye_tracking=require_eye_tracking,
                                                 simple=simple)

        return exps

    def _get_stimulus_mappings(self, file_name=None):
        """ Returns a mapping of which metrics are related to which stimuli. Internal use only. """

        file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)

        mappings = self.api.get_stimulus_mappings(path=file_name,
                                                  strategy='lazy',
                                                  **Cache.cache_json())

        return mappings

    def get_cell_specimens(self,
                           file_name=None,
                           ids=None,
                           experiment_container_ids=None,
                           include_failed=False,
                           simple=True,
                           filters=None):
        """ Return cell specimens that have certain properies.

        Parameters
        ----------
        file_name: string
            File name to save/read the cell specimens.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of cell specimen ids.

        experiment_container_ids: list
            List of experiment container ids.

        include_failed: bool
            Whether to include cells from failed experiment containers

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        filters: list of dicts
            List of filter dictionaries.  The Allen Brain Observatory web site can
            generate filters in this format to reproduce a filtered set of cells
            found there.  To see what these look like, visit
            http://observatory.brain-map.org/visualcoding, perform a cell search
            and apply some filters (e.g. find cells in a particular area), then
            click the "view these cells in the AllenSDK" link on the bottom-left
            of the search results page.  This will take you to a page that contains
            a code sample you can use to apply those same filters via this argument.
            For more detail on the filter syntax, see BrainObservatoryApi.dataframe_query.


        Returns
        -------
        list of dictionaries
        """

        file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)

        cell_specimens = self.api.get_cell_metrics(path=file_name,
                                                   strategy='lazy',
                                                   pre= lambda x: [y for y in x],
                                                   **Cache.cache_json())

        cell_specimens = self.api.filter_cell_specimens(cell_specimens,
                                                        ids=ids,
                                                        experiment_container_ids=experiment_container_ids,
                                                        include_failed=include_failed,
                                                        filters=filters)

        # drop the thumbnail columns
        if simple:
            mappings = self._get_stimulus_mappings()
            thumbnails = [m['item'] for m in mappings if m[
                'item_type'] == 'T' and m['level'] == 'R']
            for cs in cell_specimens:
                for t in thumbnails:
                    del cs[t]

        return cell_specimens

    def get_ophys_experiment_data(self, ophys_experiment_id, file_name=None):
        """ Download the NWB file for an ophys_experiment (if it hasn't already been
        downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: integer
            id of the ophys_experiment to retrieve

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        file_name = self.get_cache_path(
            file_name, self.EXPERIMENT_DATA_KEY, ophys_experiment_id)

        self.api.save_ophys_experiment_data(ophys_experiment_id, file_name, strategy='lazy')

        return BrainObservatoryNwbDataSet(file_name)

    def get_ophys_experiment_analysis(self, ophys_experiment_id, stimulus_type, file_name=None):
        """ Download the h5 analysis file for a stimulus set, for a particular ophys_experiment 
        (if it hasn't already been downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: int
            id of the ophys_experiment to retrieve

        stimulus_name: str
            stimulus type; should be an element of self.list_stimuli()

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        data_set = self.get_ophys_experiment_data(ophys_experiment_id, file_name=None)
        session_type = data_set.get_session_type()

        if not stimulus_type in stim_info.SESSION_STIMULUS_MAP[session_type]:
            raise RuntimeError('Stimulus %s not available session type: %s' % (stimulus_type, stim_info.SESSION_STIMULUS_MAP[stimulus_type]))

        # Use manifest to figure out where to cache the file:
        file_name = self.get_cache_path(file_name, self.ANALYSIS_DATA_KEY, ophys_experiment_id, session_type)

        # Cache the analsis file from an RMA query:
        self.api.save_ophys_experiment_analysis_data(ophys_experiment_id, file_name, strategy='lazy')

        # Get the analysis class from ANALYSIS_CLASS_DICT, and build from the static method:
        if stimulus_type in stim_info.LOCALLY_SPARSE_NOISE_STIMULUS_TYPES+stim_info.NATURAL_MOVIE_STIMULUS_TYPES:
            return ANALYSIS_CLASS_DICT[stimulus_type].from_analysis_file(data_set, file_name, stimulus_type)
        else:
            return ANALYSIS_CLASS_DICT[stimulus_type].from_analysis_file(data_set, file_name)

    def get_ophys_experiment_events(self, ophys_experiment_id, file_name=None):
        """ Download the npz events file for an ophys_experiment if it hasn't
        already been downloaded and return the events array.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.
        ophys_experiment_id: int
            id of the ophys_experiment to retrieve events for
        Returns
        -------
        events: numpy.ndarray
            [N_cells,N_times] array of events.
        """
        file_name = self.get_cache_path(
            file_name, self.EVENTS_DATA_KEY, ophys_experiment_id)

        self.api.save_ophys_experiment_event_data(ophys_experiment_id, file_name, strategy='lazy')

        return np.load(file_name, allow_pickle=False)["ev"]

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()
        mb.set_version(self.MANIFEST_VERSION)
        mb.add_path('BASEDIR', '.')
        mb.add_path(self.EXPERIMENT_CONTAINERS_KEY,
                    'experiment_containers.json', typename='file', parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENTS_KEY, 'ophys_experiments.json',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENT_DATA_KEY, 'ophys_experiment_data/%d.nwb',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.ANALYSIS_DATA_KEY, 'ophys_experiment_analysis/%d_%s_analysis.h5',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EVENTS_DATA_KEY, 'ophys_experiment_events/%d_events.npz',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.CELL_SPECIMENS_KEY, 'cell_specimens.json',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.STIMULUS_MAPPINGS_KEY, 'stimulus_mappings.json',
                    typename='file', parent_key='BASEDIR')

        mb.write_json_file(file_name)
def bo_api():
    bo = BrainObservatoryApi('http://testwarehouse:9000')
    bo.json_msg_query = MagicMock(name='json_msg_query')
    
    return bo
示例#17
0
class BrainObservatoryCache(Cache):
    """
    Cache class for storing and accessing data from the Brain Observatory.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: BrainObservatoryApi instance
        The object used for making API queries related to the Brain
        Observatory.

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ophys_experiment_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "brain_observatory_manifest.json".
    """

    EXPERIMENT_CONTAINERS_KEY = 'EXPERIMENT_CONTAINERS'
    EXPERIMENTS_KEY = 'EXPERIMENTS'
    CELL_SPECIMENS_KEY = 'CELL_SPECIMENS'
    EXPERIMENT_DATA_KEY = 'EXPERIMENT_DATA'
    STIMULUS_MAPPINGS_KEY = 'STIMULUS_MAPPINGS'
    MANIFEST_VERSION = None

    def __init__(self,
                 cache=True,
                 manifest_file='brain_observatory_manifest.json',
                 base_uri=None):
        super(BrainObservatoryCache,
              self).__init__(manifest=manifest_file,
                             cache=cache,
                             version=self.MANIFEST_VERSION)
        self.api = BrainObservatoryApi(base_uri=base_uri)

    def get_all_targeted_structures(self):
        """ Return a list of all targeted structures in the data set. """
        containers = self.get_experiment_containers(simple=False)
        targeted_structures = set(
            [c['targeted_structure']['acronym'] for c in containers])
        return sorted(list(targeted_structures))

    def get_all_cre_lines(self):
        """ Return a list of all cre driver lines in the data set. """
        containers = self.get_experiment_containers(simple=False)
        cre_lines = set(
            [_find_specimen_cre_line(c['specimen']) for c in containers])
        return sorted(list(cre_lines))

    def get_all_reporter_lines(self):
        """ Return a list of all reporter lines in the data set. """
        containers = self.get_experiment_containers(simple=False)
        reporter_lines = set(
            [_find_specimen_reporter_line(c['specimen']) for c in containers])
        return sorted(list(reporter_lines))

    def get_all_imaging_depths(self):
        """ Return a list of all imaging depths in the data set. """
        containers = self.get_experiment_containers(simple=False)
        imaging_depths = set([c['imaging_depth'] for c in containers])
        return sorted(list(imaging_depths))

    def get_all_session_types(self):
        """ Return a list of all stimulus sessions in the data set. """
        exps = self.get_ophys_experiments(simple=False)
        names = set([exp['stimulus_name'] for exp in exps])
        return sorted(list(names))

    def get_all_stimuli(self):
        """ Return a list of all stimuli in the data set. """
        return sorted(list(stim_info.all_stimuli()))

    def get_experiment_containers(self,
                                  file_name=None,
                                  ids=None,
                                  targeted_structures=None,
                                  imaging_depths=None,
                                  cre_lines=None,
                                  transgenic_lines=None,
                                  include_failed=False,
                                  simple=True):
        """ Get a list of experiment containers matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the experiment containers.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        include_failed: boolean
            Whether or not to include failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")

        file_name = self.get_cache_path(file_name,
                                        self.EXPERIMENT_CONTAINERS_KEY)

        containers = self.api.get_experiment_containers(path=file_name,
                                                        strategy='lazy',
                                                        **Cache.cache_json())

        transgenic_lines = _merge_transgenic_lines(cre_lines, transgenic_lines)

        containers = self.api.filter_experiment_containers(
            containers,
            ids=ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            transgenic_lines=transgenic_lines,
            include_failed=include_failed)

        if simple:
            containers = [{
                'id':
                c['id'],
                'imaging_depth':
                c['imaging_depth'],
                'targeted_structure':
                c['targeted_structure']['acronym'],
                'cre_line':
                _find_specimen_cre_line(c['specimen']),
                'reporter_line':
                _find_specimen_reporter_line(c['specimen']),
                'donor_name':
                c['specimen']['donor']['external_donor_name'],
                'specimen_name':
                c['specimen']['name'],
                'tags':
                _find_container_tags(c),
                'failed':
                c['failed']
            } for c in containers]

        return containers

    def get_ophys_experiment_stimuli(self, experiment_id):
        """ For a single experiment, return the list of stimuli present in that experiment. """
        exps = self.get_ophys_experiments(ids=[experiment_id])

        if len(exps) == 0:
            return None

        return stim_info.stimuli_in_session(exps[0]['session_type'])

    def get_ophys_experiments(self,
                              file_name=None,
                              ids=None,
                              experiment_container_ids=None,
                              targeted_structures=None,
                              imaging_depths=None,
                              cre_lines=None,
                              transgenic_lines=None,
                              stimuli=None,
                              session_types=None,
                              cell_specimen_ids=None,
                              include_failed=False,
                              require_eye_tracking=False,
                              simple=True):
        """ Get a list of ophys experiments matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the ophys experiments.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of ophys experiment ids.

        experiment_container_ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        stimuli: list
            List of stimulus names.  Must be in the list returned by
            BrainObservatoryCache.get_all_stimuli().

        session_types: list
            List of stimulus session type names.  Must be in the list returned by
            BrainObservatoryCache.get_all_session_types().

        cell_specimen_ids: list
            Only include experiments that contain cells with these ids.

        include_failed: boolean
            Whether or not to include experiments from failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        require_eye_tracking: boolean
            If True, only return experiments that have eye tracking results. Default: False.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")
        _assert_not_string(stimuli, "stimuli")
        _assert_not_string(session_types, "session_types")

        file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)

        exps = self.api.get_ophys_experiments(path=file_name,
                                              strategy='lazy',
                                              **Cache.cache_json())

        transgenic_lines = _merge_transgenic_lines(cre_lines, transgenic_lines)

        if cell_specimen_ids is not None:
            cells = self.get_cell_specimens(ids=cell_specimen_ids)
            cell_container_ids = set(
                [cell['experiment_container_id'] for cell in cells])
            if experiment_container_ids is not None:
                experiment_container_ids = list(
                    set(experiment_container_ids) - cell_container_ids)
            else:
                experiment_container_ids = list(cell_container_ids)

        exps = self.api.filter_ophys_experiments(
            exps,
            ids=ids,
            experiment_container_ids=experiment_container_ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            transgenic_lines=transgenic_lines,
            stimuli=stimuli,
            session_types=session_types,
            include_failed=include_failed,
            require_eye_tracking=require_eye_tracking)

        if simple:
            exps = [{
                'id':
                e['id'],
                'imaging_depth':
                e['imaging_depth'],
                'targeted_structure':
                e['targeted_structure']['acronym'],
                'cre_line':
                _find_specimen_cre_line(e['specimen']),
                'reporter_line':
                _find_specimen_reporter_line(e['specimen']),
                'acquisition_age_days':
                _find_experiment_acquisition_age(e),
                'experiment_container_id':
                e['experiment_container_id'],
                'session_type':
                e['stimulus_name'],
                'donor_name':
                e['specimen']['donor']['external_donor_name'],
                'specimen_name':
                e['specimen']['name'],
                'fail_eye_tracking':
                e.get('fail_eye_tracking', None)
            } for e in exps]

        return exps

    def _get_stimulus_mappings(self, file_name=None):
        """ Returns a mapping of which metrics are related to which stimuli. Internal use only. """

        file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)

        mappings = self.api.get_stimulus_mappings(path=file_name,
                                                  strategy='lazy',
                                                  **Cache.cache_json())

        return mappings

    def get_cell_specimens(self,
                           file_name=None,
                           ids=None,
                           experiment_container_ids=None,
                           include_failed=False,
                           simple=True,
                           filters=None):
        """ Return cell specimens that have certain properies.

        Parameters
        ----------
        file_name: string
            File name to save/read the cell specimens.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of cell specimen ids.

        experiment_container_ids: list
            List of experiment container ids.

        include_failed: bool
            Whether to include cells from failed experiment containers

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        filters: list of dicts
            List of filter dictionaries.  The Allen Brain Observatory web site can 
            generate filters in this format to reproduce a filtered set of cells
            found there.  To see what these look like, visit 
            http://observatory.brain-map.org/visualcoding, perform a cell search
            and apply some filters (e.g. find cells in a particular area), then 
            click the "view these cells in the AllenSDK" link on the bottom-left
            of the search results page.  This will take you to a page that contains
            a code sample you can use to apply those same filters via this argument.
            For more detail on the filter syntax, see BrainObservatoryApi.dataframe_query.
            

        Returns
        -------
        list of dictionaries
        """

        file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)

        cell_specimens = self.api.get_cell_metrics(path=file_name,
                                                   strategy='lazy',
                                                   **Cache.cache_json())

        cell_specimens = self.api.filter_cell_specimens(
            cell_specimens,
            ids=ids,
            experiment_container_ids=experiment_container_ids,
            include_failed=include_failed,
            filters=filters)

        # drop the thumbnail columns
        if simple:
            mappings = self._get_stimulus_mappings()
            thumbnails = [
                m['item'] for m in mappings
                if m['item_type'] == 'T' and m['level'] == 'R'
            ]
            for cs in cell_specimens:
                for t in thumbnails:
                    del cs[t]

        return cell_specimens

    def get_ophys_experiment_data(self, ophys_experiment_id, file_name=None):
        """ Download the NWB file for an ophys_experiment (if it hasn't already been
        downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: integer
            id of the ophys_experiment to retrieve

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        file_name = self.get_cache_path(file_name, self.EXPERIMENT_DATA_KEY,
                                        ophys_experiment_id)

        self.api.save_ophys_experiment_data(ophys_experiment_id,
                                            file_name,
                                            strategy='lazy')

        return BrainObservatoryNwbDataSet(file_name)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()
        mb.set_version(self.MANIFEST_VERSION)
        mb.add_path('BASEDIR', '.')
        mb.add_path(self.EXPERIMENT_CONTAINERS_KEY,
                    'experiment_containers.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENTS_KEY,
                    'ophys_experiments.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENT_DATA_KEY,
                    'ophys_experiment_data/%d.nwb',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.CELL_SPECIMENS_KEY,
                    'cell_specimens.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.STIMULUS_MAPPINGS_KEY,
                    'stimulus_mappings.json',
                    typename='file',
                    parent_key='BASEDIR')

        mb.write_json_file(file_name)
class BrainObservatoryCache(Cache):
    """
    Cache class for storing and accessing data from the Brain Observatory.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: BrainObservatoryApi instance
        The object used for making API queries related to the Brain
        Observatory.

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ophys_experiment_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "brain_observatory_manifest.json".
    """

    EXPERIMENT_CONTAINERS_KEY = 'EXPERIMENT_CONTAINERS'
    EXPERIMENTS_KEY = 'EXPERIMENTS'
    CELL_SPECIMENS_KEY = 'CELL_SPECIMENS'
    EXPERIMENT_DATA_KEY = 'EXPERIMENT_DATA'
    ANALYSIS_DATA_KEY = 'ANALYSIS_DATA'
    EVENTS_DATA_KEY = 'EVENTS_DATA'
    STIMULUS_MAPPINGS_KEY = 'STIMULUS_MAPPINGS'
    EYE_GAZE_DATA_KEY = 'EYE_GAZE_DATA'
    MANIFEST_VERSION = '1.3'

    def __init__(self,
                 cache=True,
                 manifest_file=None,
                 base_uri=None,
                 api=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('brain_observatory')

        super(BrainObservatoryCache,
              self).__init__(manifest=manifest_file,
                             cache=cache,
                             version=self.MANIFEST_VERSION)

        if api is None:
            self.api = BrainObservatoryApi(base_uri=base_uri)
        else:
            self.api = api

    def get_all_targeted_structures(self):
        """ Return a list of all targeted structures in the data set. """
        containers = self.get_experiment_containers(simple=False)
        targeted_structures = set(
            [c['targeted_structure']['acronym'] for c in containers])
        return sorted(list(targeted_structures))

    def get_all_cre_lines(self):
        """ Return a list of all cre driver lines in the data set. """
        containers = self.get_experiment_containers(simple=True)
        cre_lines = set([c['cre_line'] for c in containers])
        return sorted(list(cre_lines))

    def get_all_reporter_lines(self):
        """ Return a list of all reporter lines in the data set. """
        containers = self.get_experiment_containers(simple=True)
        reporter_lines = set([c['reporter_line'] for c in containers])
        return sorted(list(reporter_lines))

    def get_all_imaging_depths(self):
        """ Return a list of all imaging depths in the data set. """
        containers = self.get_experiment_containers(simple=True)
        imaging_depths = set([c['imaging_depth'] for c in containers])
        return sorted(list(imaging_depths))

    def get_all_session_types(self):
        """ Return a list of all stimulus sessions in the data set. """
        exps = self.get_ophys_experiments(simple=False)
        names = set([exp['stimulus_name'] for exp in exps])
        return sorted(list(names))

    def get_all_stimuli(self):
        """ Return a list of all stimuli in the data set. """
        return sorted(list(stim_info.all_stimuli()))

    def get_experiment_containers(self,
                                  file_name=None,
                                  ids=None,
                                  targeted_structures=None,
                                  imaging_depths=None,
                                  cre_lines=None,
                                  reporter_lines=None,
                                  transgenic_lines=None,
                                  include_failed=False,
                                  simple=True):
        """ Get a list of experiment containers matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the experiment containers.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().

        reporter_lines: list
            List of reporter lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_reporter_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        include_failed: boolean
            Whether or not to include failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(reporter_lines, "reporter_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")

        file_name = self.get_cache_path(file_name,
                                        self.EXPERIMENT_CONTAINERS_KEY)

        containers = self.api.get_experiment_containers(path=file_name,
                                                        strategy='lazy',
                                                        **Cache.cache_json())

        containers = self.api.filter_experiment_containers(
            containers,
            ids=ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            cre_lines=cre_lines,
            reporter_lines=reporter_lines,
            transgenic_lines=transgenic_lines,
            include_failed=include_failed,
            simple=simple)

        return containers

    def get_ophys_experiment_stimuli(self, experiment_id):
        """ For a single experiment, return the list of stimuli present in that experiment. """
        exps = self.get_ophys_experiments(ids=[experiment_id])

        if len(exps) == 0:
            return None

        return stim_info.stimuli_in_session(exps[0]['session_type'])

    def get_ophys_experiments(self,
                              file_name=None,
                              ids=None,
                              experiment_container_ids=None,
                              targeted_structures=None,
                              imaging_depths=None,
                              cre_lines=None,
                              reporter_lines=None,
                              transgenic_lines=None,
                              stimuli=None,
                              session_types=None,
                              cell_specimen_ids=None,
                              include_failed=False,
                              require_eye_tracking=False,
                              simple=True):
        """ Get a list of ophys experiments matching certain criteria.

        Parameters
        ----------
        file_name: string
            File name to save/read the ophys experiments.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of ophys experiment ids.

        experiment_container_ids: list
            List of experiment container ids.

        targeted_structures: list
            List of structure acronyms.  Must be in the list returned by
            BrainObservatoryCache.get_all_targeted_structures().

        imaging_depths: list
            List of imaging depths.  Must be in the list returned by
            BrainObservatoryCache.get_all_imaging_depths().

        cre_lines: list
            List of cre lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines().
        
        reporter_lines: list
            List of reporter lines.  Must be in the list returned by
            BrainObservatoryCache.get_all_reporter_lines().

        transgenic_lines: list
            List of transgenic lines. Must be in the list returned by
            BrainObservatoryCache.get_all_cre_lines() or.
            BrainObservatoryCache.get_all_reporter_lines().

        stimuli: list
            List of stimulus names.  Must be in the list returned by
            BrainObservatoryCache.get_all_stimuli().

        session_types: list
            List of stimulus session type names.  Must be in the list returned by
            BrainObservatoryCache.get_all_session_types().

        cell_specimen_ids: list
            Only include experiments that contain cells with these ids.

        include_failed: boolean
            Whether or not to include experiments from failed experiment containers.

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        require_eye_tracking: boolean
            If True, only return experiments that have eye tracking results. Default: False.

        Returns
        -------
        list of dictionaries
        """
        _assert_not_string(targeted_structures, "targeted_structures")
        _assert_not_string(cre_lines, "cre_lines")
        _assert_not_string(reporter_lines, "reporter_lines")
        _assert_not_string(transgenic_lines, "transgenic_lines")
        _assert_not_string(stimuli, "stimuli")
        _assert_not_string(session_types, "session_types")

        file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)

        exps = self.api.get_ophys_experiments(path=file_name,
                                              strategy='lazy',
                                              **Cache.cache_json())

        # NOTE: Ugly hack to update the 'fail_eye_tracking' field
        # which is using True/False values for the previous eye mapping
        # implementation. This will also need to be fixed in warehouse.
        # ----- Start of ugly hack -----
        response = self.api.template_query('brain_observatory_queries',
                                           'all_eye_mapping_files')

        session_ids_with_eye_tracking: set = {
            entry['attachable_id']
            for entry in response if entry['attachable_type'] == "OphysSession"
        }

        for indx, exp in enumerate(exps):
            try:
                ophys_session_id = ophys_experiment_session_id_map[exp['id']]
                if ophys_session_id in session_ids_with_eye_tracking:
                    exps[indx]['fail_eye_tracking'] = False
                else:
                    exps[indx]['fail_eye_tracking'] = True
            except KeyError:
                exps[indx]['fail_eye_tracking'] = True
        # ----- End of ugly hack -----

        if cell_specimen_ids is not None:
            cells = self.get_cell_specimens(ids=cell_specimen_ids)
            cell_container_ids = set(
                [cell['experiment_container_id'] for cell in cells])
            if experiment_container_ids is not None:
                experiment_container_ids = list(
                    set(experiment_container_ids) - cell_container_ids)
            else:
                experiment_container_ids = list(cell_container_ids)

        exps = self.api.filter_ophys_experiments(
            exps,
            ids=ids,
            experiment_container_ids=experiment_container_ids,
            targeted_structures=targeted_structures,
            imaging_depths=imaging_depths,
            cre_lines=cre_lines,
            reporter_lines=reporter_lines,
            transgenic_lines=transgenic_lines,
            stimuli=stimuli,
            session_types=session_types,
            include_failed=include_failed,
            require_eye_tracking=require_eye_tracking,
            simple=simple)

        return exps

    def _get_stimulus_mappings(self, file_name=None):
        """ Returns a mapping of which metrics are related to which stimuli. Internal use only. """

        file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)

        mappings = self.api.get_stimulus_mappings(path=file_name,
                                                  strategy='lazy',
                                                  **Cache.cache_json())

        return mappings

    def get_cell_specimens(self,
                           file_name=None,
                           ids=None,
                           experiment_container_ids=None,
                           include_failed=False,
                           simple=True,
                           filters=None):
        """ Return cell specimens that have certain properies.

        Parameters
        ----------
        file_name: string
            File name to save/read the cell specimens.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ids: list
            List of cell specimen ids.

        experiment_container_ids: list
            List of experiment container ids.

        include_failed: bool
            Whether to include cells from failed experiment containers

        simple: boolean
            Whether or not to simplify the dictionary properties returned by this method
            to a more concise subset.

        filters: list of dicts
            List of filter dictionaries.  The Allen Brain Observatory web site can
            generate filters in this format to reproduce a filtered set of cells
            found there.  To see what these look like, visit
            http://observatory.brain-map.org/visualcoding, perform a cell search
            and apply some filters (e.g. find cells in a particular area), then
            click the "view these cells in the AllenSDK" link on the bottom-left
            of the search results page.  This will take you to a page that contains
            a code sample you can use to apply those same filters via this argument.
            For more detail on the filter syntax, see BrainObservatoryApi.dataframe_query.


        Returns
        -------
        list of dictionaries
        """

        file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)

        cell_specimens = self.api.get_cell_metrics(
            path=file_name,
            strategy='lazy',
            pre=lambda x: [y for y in x],
            **Cache.cache_json())

        cell_specimens = self.api.filter_cell_specimens(
            cell_specimens,
            ids=ids,
            experiment_container_ids=experiment_container_ids,
            include_failed=include_failed,
            filters=filters)

        # drop the thumbnail columns
        if simple:
            mappings = self._get_stimulus_mappings()
            thumbnails = [
                m['item'] for m in mappings
                if m['item_type'] == 'T' and m['level'] == 'R'
            ]
            for cs in cell_specimens:
                for t in thumbnails:
                    del cs[t]

        return cell_specimens

    def get_nwb_filepath(self, ophys_experiment_id=None):
        cache_nwb_filepath = self.get_cache_path(None,
                                                 self.EXPERIMENT_DATA_KEY,
                                                 ophys_experiment_id)
        if os.path.exists(cache_nwb_filepath):
            return cache_nwb_filepath
        else:
            return None

    def get_ophys_experiment_data(self, ophys_experiment_id, file_name=None):
        """ Download the NWB file for an ophys_experiment (if it hasn't already been
        downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: integer
            id of the ophys_experiment to retrieve

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        file_name = self.get_cache_path(file_name, self.EXPERIMENT_DATA_KEY,
                                        ophys_experiment_id)

        self.api.save_ophys_experiment_data(ophys_experiment_id,
                                            file_name,
                                            strategy='lazy')

        return BrainObservatoryNwbDataSet(file_name)

    def get_ophys_experiment_analysis(self,
                                      ophys_experiment_id,
                                      stimulus_type,
                                      file_name=None):
        """ Download the h5 analysis file for a stimulus set, for a particular ophys_experiment 
        (if it hasn't already been downloaded) and return a data accessor object.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: int
            id of the ophys_experiment to retrieve

        stimulus_name: str
            stimulus type; should be an element of self.list_stimuli()

        Returns
        -------
        BrainObservatoryNwbDataSet
        """
        data_set = self.get_ophys_experiment_data(ophys_experiment_id,
                                                  file_name=None)
        session_type = data_set.get_session_type()

        if not stimulus_type in stim_info.SESSION_STIMULUS_MAP[session_type]:
            raise RuntimeError(
                'Stimulus %s not available session type: %s' %
                (stimulus_type, stim_info.SESSION_STIMULUS_MAP[stimulus_type]))

        # Use manifest to figure out where to cache the file:
        file_name = self.get_cache_path(file_name, self.ANALYSIS_DATA_KEY,
                                        ophys_experiment_id, session_type)

        # Cache the analsis file from an RMA query:
        self.api.save_ophys_experiment_analysis_data(ophys_experiment_id,
                                                     file_name,
                                                     strategy='lazy')

        # Get the analysis class from ANALYSIS_CLASS_DICT, and build from the static method:
        if stimulus_type in stim_info.LOCALLY_SPARSE_NOISE_STIMULUS_TYPES + stim_info.NATURAL_MOVIE_STIMULUS_TYPES:
            return ANALYSIS_CLASS_DICT[stimulus_type].from_analysis_file(
                data_set, file_name, stimulus_type)
        else:
            return ANALYSIS_CLASS_DICT[stimulus_type].from_analysis_file(
                data_set, file_name)

    def get_ophys_experiment_events(self, ophys_experiment_id, file_name=None):
        """ Download the npz events file for an ophys_experiment if it hasn't
        already been downloaded and return the events array.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.
        ophys_experiment_id: int
            id of the ophys_experiment to retrieve events for
        Returns
        -------
        events: numpy.ndarray
            [N_cells,N_times] array of events.
        """
        file_name = self.get_cache_path(file_name, self.EVENTS_DATA_KEY,
                                        ophys_experiment_id)

        self.api.save_ophys_experiment_event_data(ophys_experiment_id,
                                                  file_name,
                                                  strategy='lazy')

        return np.load(file_name, allow_pickle=False)["ev"]

    def get_ophys_pupil_data(self,
                             ophys_experiment_id: int,
                             file_name: str = None,
                             suppress_pupil_data: bool = True) -> pd.DataFrame:
        """Download the h5 eye gaze mapping file for an ophys_experiment if
        it hasn't already been downloaded and return it as a pandas.DataFrame.

        Parameters
        ----------
        file_name: string
            File name to save/read the data set.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        ophys_experiment_id: int
            id of the ophys_experiment to retrieve pupil data for.

        suppress_pupil_data: bool
            Whether or not to suppress pupil data from dataset.
            Default is True.

        Returns
        -------
        pd.DataFrame
            If 'suppress_eye_gaze_data' is set to 'False':
                Contains raw/filtered columns for gaze mapping:
                    *_eye_area
                    *_pupil_area
                    *_screen_coordinates_x_cm
                    *_screen_coordinates_y_cm
                    *_screen_coordinates_spherical_x_deg
                    *_screen_coorindates_spherical_y_deg
            Otherwise:
                An empty pandas DataFrame
        """

        if suppress_pupil_data:
            print(
                "This pupil data is obtained using a new eye "
                "tracking algorithm and is in the process of being validated. "
                "If you would like to view the data anyways, "
                "please set the 'suppress_pupil_data' parameter to 'False'.")
            return pd.DataFrame()

        # NOTE: This is a really ugly hack to get around the fact that warehouse does
        # not have Ophys session ids associated with experiment ids. This should be
        # removed when warehouse session ids have associations with experiment ids.
        # ----- Start of ugly hack -----
        try:
            ophys_session_id = ophys_experiment_session_id_map[
                ophys_experiment_id]
        except KeyError:
            raise RuntimeError(
                f"Experiment id '{ophys_experiment_id}' has no associated session!"
            )
        # ----- End of ugly hack -----

        file_name = self.get_cache_path(file_name, self.EYE_GAZE_DATA_KEY,
                                        ophys_session_id)

        if not file_name:
            raise RuntimeError("Could not obtain a file_name for pupil data "
                               f"with experiment id: {ophys_experiment_id} "
                               f"(session id: {ophys_session_id})")

        # NOTE: `save_ophys_experiment_eye_gaze_data` will also need to be
        # updated to remove ophy_session_id param when ugly hack is removed.
        self.api.save_ophys_experiment_eye_gaze_data(ophys_experiment_id,
                                                     ophys_session_id,
                                                     file_name,
                                                     strategy='lazy')

        gaze_mapping_data = read_eye_gaze_mappings(Path(file_name))

        return create_eye_gaze_mapping_dataframe(gaze_mapping_data)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()
        mb.set_version(self.MANIFEST_VERSION)
        mb.add_path('BASEDIR', '.')
        mb.add_path(self.EXPERIMENT_CONTAINERS_KEY,
                    'experiment_containers.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENTS_KEY,
                    'ophys_experiments.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EXPERIMENT_DATA_KEY,
                    'ophys_experiment_data/%d.nwb',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.ANALYSIS_DATA_KEY,
                    'ophys_experiment_analysis/%d_%s_analysis.h5',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EVENTS_DATA_KEY,
                    'ophys_experiment_events/%d_events.npz',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.CELL_SPECIMENS_KEY,
                    'cell_specimens.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.STIMULUS_MAPPINGS_KEY,
                    'stimulus_mappings.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(
            self.EYE_GAZE_DATA_KEY,
            'ophys_eye_gaze_mapping/%d_eyetracking_dlc_to_screen_mapping.h5',
            typename='file',
            parent_key='BASEDIR')

        mb.write_json_file(file_name)
示例#19
0
def make_bo_api():
    from allensdk.api.queries.brain_observatory_api import BrainObservatoryApi

    endpoint = os.environ['TEST_API_ENDPOINT'] if 'TEST_API_ENDPOINT' in os.environ else 'http://twarehouse-backup'
    return BrainObservatoryApi(endpoint)