Esempio n. 1
0
 def __init__(self,
              cache=True,
              manifest_file='cell_types_manifest.json',
              base_uri=None):
     super(CellTypesCache, self).__init__(manifest=manifest_file,
                                          cache=cache)
     self.api = CellTypesApi(base_uri=base_uri)
Esempio n. 2
0
def get_observation(dataset_id, kind, cached=True, quiet=False):
    """
    Gets an observation of kind 'kind' from the dataset with id 'dataset_id', 
    optionally using the cached value retrieved previously.
    """
    
    db = shelve.open('aibs-cache') if cached else {}
    identifier = '%d_%s' % (dataset_id,kind)
    if identifier in db:
        print("Getting %s cached data value for from AIBS dataset %s" \
              % (kind.title(),dataset_id))
        value = db[identifier]
    else:
        print("Getting %s data value for from AIBS dataset %s" \
              % (kind.title(),dataset_id))    
        ct = CellTypesApi()
        cmd = ct.get_cell(dataset_id) # Cell metadata
        if kind == 'rheobase':
            sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id']
        sp = get_sweep_params(dataset_id, sweep_id)
        if kind == 'rheobase':
            value = sp['stimulus_absolute_amplitude']
            value = np.round(value,2) # Round to nearest hundredth of a pA.
            value *= pq.pA # Apply units.
        db[identifier] = value
    
    if cached:
        db.close()
    return {'value': value}
Esempio n. 3
0
    def __init__(self, cache=True, manifest_file=None, base_uri=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('cell_types')

        super(CellTypesCache, self).__init__(manifest=manifest_file,
                                             cache=cache,
                                             version=self.MANIFEST_VERSION)
        self.api = CellTypesApi(base_uri=base_uri)
Esempio n. 4
0
def get_sweep_params(dataset_id,sweep_id):
    ct = CellTypesApi()
    experiment_params = ct.get_ephys_sweeps(dataset_id)
    for sp in experiment_params:
        if sp['id']==sweep_id:
            sweep_num = sp['sweep_number']
            break
    if sweep_num is None:
        raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
    return sp
Esempio n. 5
0
def test_list_cells_mocked(mock_cells):
    ctapi = CellTypesApi()

    ctapi.model_query = MagicMock(return_value=mock_cells)

    cells = ctapi.list_cells()
    assert len(cells) == 3

    flu_cells = [
        cell for cell in cells if cell['disease_categories'] == [('influenza')]
    ]
    assert len(flu_cells) == 1

    cells = ctapi.list_cells(require_reconstruction=True)
    assert len(cells) == 1

    cells = ctapi.list_cells(require_morphology=True)
    assert len(cells) == 1

    cells = ctapi.list_cells(reporter_status=['bob'])
    assert len(cells) == 1

    cells = ctapi.list_cells(species=['H**O SAPIENS'])
    assert len(cells) == 1

    cells = ctapi.list_cells(species=['mus musculus'])
    assert len(cells) == 1
Esempio n. 6
0
def fetch_pipeline_file(request):
    specimen_id = request.param
    nwb_file_name = '{}.nwb'.format(specimen_id)

    nwb_file_full_path = os.path.join(TEST_DATA_PATH, nwb_file_name)

    if not os.path.exists(nwb_file_full_path):
        ct = CellTypesApi()
        ct.save_ephys_data(specimen_id, nwb_file_full_path)

    return nwb_file_full_path
Esempio n. 7
0
 def __init__(self, archive_dir=None):
     self.bp = BiophysicalApi('http://api.brain-map.org')
     self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file
     self.cta = CellTypesApi()
     self.rma = RmaApi()
     self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/'
     self.template_names = {}
     self.nwb_list = []
     
     if archive_dir == None:
         archive_dir = '.'
     self.archive_dir = archive_dir
Esempio n. 8
0
def get_observation(dataset_id,kind,cached=True):
    ct = CellTypesApi()
    cmd = ct.get_cell(dataset_id) # Cell metadata
    
    sweep_num = None
    if kind == 'rheobase':
        sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id']
    sp = get_sweep_params(dataset_id, sweep_id)
    if kind == 'rheobase':
        value = sp['stimulus_absolute_amplitude']
        value = np.round(value,2) # Round to nearest hundredth of a pA.
        value *= pq.pA # Apply units.  
    return {'value': value}
def extract_data(ephys_cell, sweep_number):
    """Data extraction from AllenDB

    Parameters
    ----------
    ephys_cell : int
        Cell identity from AllenDB
    sweep_number : int
        Stimulus identity for cell ephys_cell from AllenDB
    """
    t_offset = 815.
    duration = 1450.
    real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
        ephys_cell, sweep_number)
    if not os.path.isfile(real_data_path):
        from allensdk.core.cell_types_cache import CellTypesCache
        from allensdk.api.queries.cell_types_api import CellTypesApi

        manifest_file = 'cell_types/manifest.json'

        cta = CellTypesApi()
        ctc = CellTypesCache(manifest_file=manifest_file)
        data_set = ctc.get_ephys_data(ephys_cell)
        sweep_data = data_set.get_sweep(
            sweep_number)  # works with python2 and fails with python3
        sweeps = cta.get_ephys_sweeps(ephys_cell)

        sweep = sweeps[sweep_number]

        index_range = sweep_data["index_range"]
        i = sweep_data["stimulus"][0:index_range[1] + 1]  # in A
        v = sweep_data["response"][0:index_range[1] + 1]  # in V
        sampling_rate = sweep_data["sampling_rate"]  # in Hz
        dt = 1e3 / sampling_rate  # in ms
        i *= 1e6  # to muA
        v *= 1e3  # to mV
        v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
        i = i[int(t_offset / dt):int((t_offset + duration) / dt)]

        real_data_obs = np.array(v).reshape(1, -1, 1)
        I_real_data = np.array(i).reshape(-1)
        t_on = int(
            sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
        t_off = int(
            (sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
            sampling_rate) * dt - t_offset

        f = open(real_data_path, 'wb')
        pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
        f.close()
Esempio n. 10
0
def get_sweep_params(dataset_id, sweep_id):
    """
    Gets sweep parameters corresponding to the sweep with id 'sweep_id' from
    the dataset with id 'dataset_id'.
    """

    ct = CellTypesApi()
    experiment_params = ct.get_ephys_sweeps(dataset_id)
    sp = None
    for sp in experiment_params:
        if sp['id']==sweep_id:
            sweep_num = sp['sweep_number']
            if sweep_num is None:
                raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
            break
    return sp
Esempio n. 11
0
def cell_types_api():
    endpoint = None

    if 'TEST_API_ENDPOINT' in os.environ:
        endpoint = os.environ['TEST_API_ENDPOINT']
        return CellTypesApi(endpoint)
    else:
        return None
Esempio n. 12
0
    def __init__(self, cache=True, manifest_file=None, base_uri=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('cell_types')

        super(CellTypesCache, self).__init__(
            manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)
        self.api = CellTypesApi(base_uri=base_uri)
Esempio n. 13
0
class BiophysicalArchiver(object):
    def __init__(self, archive_dir=None):
        self.bp = BiophysicalApi('http://api.brain-map.org')
        self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file
        self.cta = CellTypesApi()
        self.rma = RmaApi()
        self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/'
        self.template_names = {}
        self.nwb_list = []
        
        if archive_dir == None:
            archive_dir = '.'
        self.archive_dir = archive_dir
    
    def get_template_names(self):
        template_response = self.rma.model_query('NeuronalModelTemplate')
        self.template_names = { t['id']: str(t['name']).replace(' ', '_') for t in template_response}
    
    def get_cells(self):
        return self.cta.list_cells(True, True)
    
    def get_neuronal_models(self, specimen_ids):
        return self.rma.model_query('NeuronalModel',
                                    criteria='specimen[id$in%s]' % ','.join(str(i) for i in specimen_ids),
                                    include='specimen',
                                    num_rows='all')
    
    def get_stimulus_file(self, neuronal_model_id):
        result = self.rma.model_query('NeuronalModel',
                                      criteria='[id$eq%d]' % (neuronal_model_id),
                                      include="specimen(ephys_result(well_known_files(well_known_file_type[name$il'NWB*'])))",
                                      tabular=['path'])
        
        stimulus_filename = result[0]['path']
        
        return stimulus_filename
        
        
        stimulus_filename = os.path.basename(result[0]['path'])
        
        return stimulus_filename
    
    def archive_cell(self, ephys_result_id, specimen_id, template, neuronal_model_id):
        url = self.neuronal_model_download_endpoint + "/%d" % (neuronal_model_id)
        file_name = os.path.join(self.archive_dir, 'ephys_result_%d_specimen_%d_%s_neuronal_model_%d.zip' % (ephys_result_id,
                                                                                                             specimen_id,
                                                                                                             template,
                                                                                                             neuronal_model_id))
        self.rma.retrieve_file_over_http(url, file_name)
        nwb_file = self.get_stimulus_file(neuronal_model_id)
        shutil.copy(nwb_file, self.archive_dir) 
        self.nwb_list.append("%s\t%s" % (os.path.basename(nwb_file),
                                         file_name))
Esempio n. 14
0
def get_observation(dataset_id, kind, cached=True, quiet=False):
    """Get an observation.

    Get an observation of kind 'kind' from the dataset with id 'dataset_id'.
    optionally using the cached value retrieved previously.
    """
    db = shelve.open('aibs-cache') if cached else {}
    identifier = '%d_%s' % (dataset_id, kind)
    if identifier in db:
        print("Getting %s cached data value for from AIBS dataset %s" %
              (kind.title(), dataset_id))
        value = db[identifier]
    else:
        print("Getting %s data value for from AIBS dataset %s" %
              (kind.title(), dataset_id))
        ct = CellTypesApi()
        cmd = ct.get_cell(dataset_id)  # Cell metadata

        if kind == 'rheobase':
            if 'ephys_features' in cmd:
                value = cmd['ephys_features'][0][
                    'threshold_i_long_square']  # newer API
            else:
                value = cmd['ef__threshold_i_long_square']  # older API

            value = np.round(value, 2)  # Round to nearest hundredth of a pA.
            value *= pq.pA  # Apply units.

        else:
            value = cmd[kind]

        db[identifier] = value

    if cached:
        db.close()
    return {'value': value}
Esempio n. 15
0
def test_list_cells_api_mocked(mock_cells_api):
    with patch.object(CellTypesApi, "model_query", return_value=mock_cells_api):
        ctapi = CellTypesApi()

        cells = ctapi.list_cells_api()
        assert len(cells) == 2

        fcells = ctapi.filter_cells_api(cells, require_reconstruction=True)
        assert len(fcells) == 1

        fcells = ctapi.filter_cells_api(cells, require_morphology=True)
        assert len(fcells) == 1

        fcells = ctapi.filter_cells_api(cells, species=['taco'])
        assert len(fcells) == 2

        fcells = ctapi.filter_cells_api(cells, reporter_status=['fish'])
        assert len(fcells) == 1
Esempio n. 16
0
# Based on https://github.com/stripathy/AIBS_cell_types/blob/master/Allen_ephys_playground.ipynb
'''
This script can be used to download data from the CellTypes Database
'''

from allensdk.api.queries.cell_types_api import CellTypesApi
import os
import sys

ct = CellTypesApi()

from data_helper import CURRENT_DATASETS, DATASET_TARGET_SWEEPS

test = '-test' in sys.argv

dataset_ids = CURRENT_DATASETS
if test:
    dataset_ids = [479704527]

sweep_numbers_for_data = DATASET_TARGET_SWEEPS

for dataset_id in dataset_ids:

    raw_ephys_file_name = '%d_raw_data.nwb' % dataset_id

    if not os.path.isfile(raw_ephys_file_name):
        print('Downloading data: %s' % raw_ephys_file_name)
        ct.save_ephys_data(dataset_id, raw_ephys_file_name)

        print('Saved: %s' % raw_ephys_file_name)
    else:
Esempio n. 17
0
all_experiments = mcc.get_experiments(dataframe=True)


from allensdk.api.queries.biophysical_perisomatic_api import \
    BiophysicalPerisomaticApi

from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import os
from allensdk.core.nwb_data_set import NwbDataSet
from IPython import __main__


bp = BiophysicalPerisomaticApi('http://api.brain-map.org')

ct = CellTypesApi()
cells = ct.list_cells()


#Below is one way, I can create URL, queries and get the result in a local python dictionary.
import allensdk 
import mpi4py as MPI
import numpy as np
import csv
import pickle
import sys
#import glob

def __main__():
    print 'main'
    
Esempio n. 18
0
def allen_obs_data(ephys_cell=464212183,
                   sweep_number=33,
                   A_soma=np.pi * (70. * 1e-4)**2):
    """Data for x_o. Cell from AllenDB

    Parameters
    ----------
    ephys_cell : int
        Cell identity from AllenDB
    sweep_number : int
        Stimulus identity for cell ephys_cell from AllenDB
    """
    t_offset = 815.
    duration = 1450.
    dir_cache = './support_files'
    real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
        ephys_cell, sweep_number)
    if not os.path.isfile(real_data_path):
        from allensdk.core.cell_types_cache import CellTypesCache
        from allensdk.api.queries.cell_types_api import CellTypesApi

        manifest_file = 'cell_types/manifest.json'

        cta = CellTypesApi()
        ctc = CellTypesCache(manifest_file=manifest_file)
        data_set = ctc.get_ephys_data(ephys_cell)
        sweep_data = data_set.get_sweep(
            sweep_number)  # works with python2 and fails with python3
        sweeps = cta.get_ephys_sweeps(ephys_cell)

        sweep = sweeps[sweep_number]

        index_range = sweep_data["index_range"]
        i = sweep_data["stimulus"][0:index_range[1] + 1]  # in A
        v = sweep_data["response"][0:index_range[1] + 1]  # in V
        sampling_rate = sweep_data["sampling_rate"]  # in Hz
        dt = 1e3 / sampling_rate  # in ms
        i *= 1e6  # to muA
        v *= 1e3  # to mV
        v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
        i = i[int(t_offset / dt):int((t_offset + duration) / dt)]

        real_data_obs = np.array(v).reshape(1, -1, 1)
        I_real_data = np.array(i).reshape(-1)
        t_on = int(
            sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
        t_off = int(
            (sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
            sampling_rate) * dt - t_offset

        io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
    else:

        def pickle_load(file):
            """Loads data from file."""
            f = open(file, 'rb')
            data = pickle.load(f, encoding='latin1')
            f.close()
            return data

        real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
            real_data_path)

    t = np.arange(0, duration, dt)

    # external current
    I = I_real_data / A_soma  # muA/cm2

    # return real_data_obs, I_obs
    return {
        'data': real_data_obs.reshape(-1),
        'time': t,
        'dt': dt,
        'I': I.reshape(-1),
        't_on': t_on,
        't_off': t_off
    }
Esempio n. 19
0
import numpy as np

from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk import __version__ as allensdk_ver
import time
import sys

import data_helper as DH

from pyelectro import utils
from pyelectro import __version__ as pyel_ver

import pprint
pp = pprint.PrettyPrinter(indent=4)

ct = CellTypesApi()

plot = False  # not '-nogui' in sys.argv
test = '-test' in sys.argv


def extract_info_from_nwb_file(dataset_id, raw_ephys_file_name):

    info = {}

    import h5py
    import numpy as np
    h5f = h5py.File(raw_ephys_file_name, "r")
    metas = [
        'aibs_cre_line', 'aibs_dendrite_type',
        'intracellular_ephys/Electrode 1/location'
Esempio n. 20
0
class CellTypesCache(Cache):
    """
    Cache class for storing and accessing data from the Cell Types Database.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: CellTypesApi instance
        The object used for making API queries related to the Cell Types Database

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ephys_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "cell_types_manifest.json".
    """

    # manifest keys
    CELLS_KEY = 'CELLS'
    EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
    MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
    EPHYS_DATA_KEY = 'EPHYS_DATA'
    EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
    RECONSTRUCTION_KEY = 'RECONSTRUCTION'
    MARKER_KEY = 'MARKER'
    MANIFEST_VERSION = "1.1"

    def __init__(self, cache=True, manifest_file=None, base_uri=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('cell_types')

        super(CellTypesCache, self).__init__(
            manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)
        self.api = CellTypesApi(base_uri=base_uri)

    def get_cells(self, file_name=None,
                  require_morphology=False,
                  require_reconstruction=False,
                  reporter_status=None,
                  species=None,
                  simple=True):
        """
        Download metadata for all cells in the database and optionally return a
        subset filtered by whether or not they have a morphology or reconstruction.

        Parameters
        ----------

        file_name: string
            File name to save/read the cell metadata as JSON.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        require_morphology: boolean
            Filter out cells that have no morphological images.

        require_reconstruction: boolean
            Filter out cells that have no morphological reconstructions.

        reporter_status: list
            Filter for cells that have one or more cell reporter statuses.

        species: list
            Filter for cells that belong to one or more species.  If None, return all.
            Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
        """

        file_name = self.get_cache_path(file_name, self.CELLS_KEY)

        cells = self.api.list_cells_api(path=file_name,
                                        strategy='lazy',
                                        **Cache.cache_json())

        if isinstance(reporter_status, string_types):
            reporter_status = [reporter_status]

        # filter the cells on the way out
        cells = self.api.filter_cells_api(cells,
                                          require_morphology,
                                          require_reconstruction,
                                          reporter_status,
                                          species,
                                          simple)


        return cells




    def get_ephys_sweeps(self, specimen_id, file_name=None):
        """
        Download sweep metadata for a single cell specimen.

        Parameters
        ----------

        specimen_id: int
             ID of a cell.
        """

        file_name = self.get_cache_path(
            file_name, self.EPHYS_SWEEPS_KEY, specimen_id)

        sweeps = self.api.get_ephys_sweeps(specimen_id,
                                           strategy='lazy',
                                           path=file_name,
                                           **Cache.cache_json())

        return sweeps

    def get_ephys_features(self, dataframe=False, file_name=None):
        """
        Download electrophysiology features for all cells in the database.

        Parameters
        ----------

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.
        """
        file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)

        if self.cache:
            if dataframe:
                warnings.warn("dataframe argument is deprecated.")
                args = Cache.cache_csv_dataframe()
            else:
                args = Cache.cache_csv_json()
            args['strategy'] = 'lazy'
        else:
            args = Cache.nocache_json()

        features_df = self.api.get_ephys_features(path=file_name,
                                                  **args)

        return features_df


    def get_morphology_features(self, dataframe=False, file_name=None):
        """
        Download morphology features for all cells with reconstructions in the database.

        Parameters
        ----------

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.
        """

        file_name = self.get_cache_path(
            file_name, self.MORPHOLOGY_FEATURES_KEY)

        if self.cache:
            if dataframe:
                warnings.warn("dataframe argument is deprecated.")
                args = Cache.cache_csv_dataframe()
            else:
                args = Cache.cache_csv_json()
        else:
            args = Cache.nocache_json()

        args['strategy'] = 'lazy'
        args['path'] = file_name

        return self.api.get_morphology_features(**args)


    def get_all_features(self, dataframe=False, require_reconstruction=True):
        """
        Download morphology and electrophysiology features for all cells and merge them
        into a single table.

        Parameters
        ----------

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.

        require_reconstruction: boolean
            Only return ephys and morphology features for cells that have
            reconstructions. Default True.
        """

        ephys_features = pd.DataFrame(self.get_ephys_features())
        morphology_features = pd.DataFrame(self.get_morphology_features())

        how = 'inner' if require_reconstruction else 'outer'

        all_features = ephys_features.merge(morphology_features,
                                            how=how,
                                            on='specimen_id')

        if dataframe:
            warnings.warn("dataframe argument is deprecated.")
            return all_features
        else:
            return all_features.to_dict('records')

    def get_ephys_data(self, specimen_id, file_name=None):
        """
        Download electrophysiology traces for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        NwbDataSet
            A class instance with helper methods for retrieving stimulus
            and response traces out of an NWB file.
        """

        file_name = self.get_cache_path(
            file_name, self.EPHYS_DATA_KEY, specimen_id)

        self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')

        return NwbDataSet(file_name)

    def get_reconstruction(self, specimen_id, file_name=None):
        """
        Download and open a reconstruction for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the reconstruction SWC.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        Morphology
             A class instance with methods for accessing morphology compartments.
        """

        file_name = self.get_cache_path(
            file_name, self.RECONSTRUCTION_KEY, specimen_id)

        if file_name is None:
            raise Exception(
                "Please enable caching (CellTypes.cache = True) or specify a save_file_name.")

        if not os.path.exists(file_name):
            self.api.save_reconstruction(specimen_id, file_name)

        return swc.read_swc(file_name)

    def get_reconstruction_markers(self, specimen_id, file_name=None):
        """
        Download and open a reconstruction marker file for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the reconstruction marker.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        Morphology
             A class instance with methods for accessing morphology compartments.
        """

        file_name = self.get_cache_path(
            file_name, self.MARKER_KEY, specimen_id)

        if file_name is None:
            raise Exception(
                "Please enable caching (CellTypes.cache = True) or specify a save_file_name.")

        if not os.path.exists(file_name):
            try:
                self.api.save_reconstruction_markers(specimen_id, file_name)
            except LookupError as e:
                logging.warning(e.args)
                return []

        return swc.read_marker_file(file_name)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()
        mb.set_version(self.MANIFEST_VERSION)
        mb.add_path('BASEDIR', '.')
        mb.add_path(self.CELLS_KEY, 'cells.json',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker',
                    typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json',
                    typename='file', parent_key='BASEDIR')

        mb.write_json_file(file_name)
Esempio n. 21
0
 def __init__(self, cache=True, manifest_file='cell_types_manifest.json', base_uri=None):
     super(CellTypesCache, self).__init__(
         manifest=manifest_file, cache=cache)
     self.api = CellTypesApi(base_uri=base_uri)
Esempio n. 22
0
import os
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import matplotlib.pyplot as plt
import numpy as np
import math
import pprint
import csv

#output_file = 'sholl_all.csv'
#output_file = 'sholl_apical_dendrite_only.csv'
output_file = 'sholl_basal_dendrite_only.csv'

ct = CellTypesApi()

cells = ct.list_cells(require_reconstruction=True)
print "Cells with reconstructions: ", len(cells)

download_file = False
allow_types = [3]

radii = np.array(range(0, 1000, 10))
radii_square = np.square(radii)

#
# Computation based on line-sphere intersection formula
# https://en.wikipedia.org/wiki/Line%E2%80%93sphere_intersection
#

with open(output_file, 'wb') as csvfile:
    cwriter = csv.writer(csvfile, delimiter=',', quotechar='"')
Esempio n. 23
0
from allensdk.api.queries.cell_types_api import CellTypesApi

ct = CellTypesApi()

# a list of dictionaries containing metadata for cells with reconstructions
cells = ct.list_cells(require_reconstruction=True)

# download the electrophysiology data for one cell
ct.save_ephys_data(cells[0]['id'], 'example.nwb')

# download the reconstruction for the same cell
ct.save_reconstruction(cells[0]['id'], 'example.swc')
Esempio n. 24
0
class CellTypesCache(Cache):
    '''
    Cache class for storing and accessing data from the Cell Types Database.
    By default, this class will cache any downloaded metadata or files in 
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------
    
    api: CellTypesApi instance
        The object used for making API queries related to the Cell Types Database

    Parameters
    ----------
    
    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ephys_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "manifest.json".
    '''

    CELLS_KEY = 'CELLS'
    EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
    MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
    EPHYS_DATA_KEY = 'EPHYS_DATA'
    EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
    RECONSTRUCTION_KEY = 'RECONSTRUCTION'
    
    def __init__(self, cache=True, manifest_file='manifest.json'):
        super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache)
        self.api = CellTypesApi()


    def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False):
        '''
        Download metadata for all cells in the database and optionally return a
        subset filtered by whether or not they have a morphology or reconstruction.

        Parameters
        ----------
        
        file_name: string
            File name to save/read the cell metadata as JSON.  If file_name is None, 
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        require_morphology: boolean
            Filter out cells that have no morphological images.

        require_reconstruction: boolean
            Filter out cells that have no morphological reconstructions.
        '''

        file_name = self.get_cache_path(file_name, self.CELLS_KEY)

        if os.path.exists(file_name):
            cells = json_utilities.read(file_name)
        else:
            cells = self.api.list_cells(False, False)

            if self.cache:
                json_utilities.write(file_name, cells)

        # filter the cells on the way out
        return self.api.filter_cells(cells, require_morphology, require_reconstruction)
    

    def get_ephys_sweeps(self, specimen_id, file_name=None):
        '''
        Download sweep metadata for a single cell specimen.  

        Parameters
        ----------
        
        specimen_id: int
             ID of a cell.
        '''

        file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY, specimen_id)
        
        if os.path.exists(file_name):
            sweeps = json_utilities.read(file_name)
        else:
            sweeps = self.api.get_ephys_sweeps(specimen_id)

            if self.cache:
                json_utilities.write(file_name, sweeps)

        return sweeps

    
    def get_ephys_features(self, dataframe=False, file_name=None):
        '''
        Download electrophysiology features for all cells in the database.

        Parameters
        ----------
        
        file_name: string
            File name to save/read the ephys features metadata as CSV.  
            If file_name is None, the file_name will be pulled out of the 
            manifest.  If caching is disabled, no file will be saved. 
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return 
            a list of dictionaries.
        '''

        file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)

        if os.path.exists(file_name):
            features_df = pd.DataFrame.from_csv(file_name)
        else:
            features_df = self.api.get_ephys_features(dataframe=True)

            if self.cache:
                features_df.to_csv(file_name)

        if dataframe:
            return features_df
        else:
            return features_df.to_dict('records')


    def get_morphology_features(self, dataframe=False, file_name=None):
        '''
        Download morphology features for all cells with reconstructions in the database.

        Parameters
        ----------
        
        file_name: string
            File name to save/read the ephys features metadata as CSV.  
            If file_name is None, the file_name will be pulled out of the 
            manifest.  If caching is disabled, no file will be saved. 
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return 
            a list of dictionaries.
        '''
        
        file_name = self.get_cache_path(file_name, self.MORPHOLOGY_FEATURES_KEY)
        
        if os.path.exists(file_name):
            features_df = pd.DataFrame.from_csv(file_name)
        else:
            features_df = self.api.get_morphology_features(dataframe=True)
            
            if self.cache:
                features_df.to_csv(file_name)

        if dataframe:
            return features_df
        else:
            return features_df.to_dict('records')

    
    def get_all_features(self, dataframe=False, require_reconstruction=True):
        '''
        Download morphology and electrophysiology features for all cells and merge them
        into a single table.

        Parameters
        ----------

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return 
            a list of dictionaries.

        require_reconstruction: boolean
            Only return ephys and morphology features for cells that have 
            reconstructions. Default True.
        '''

        ephys_features = self.get_ephys_features(dataframe=True)
        morphology_features = self.get_morphology_features(dataframe=True)

        how = 'inner' if require_reconstruction else 'outer'

        all_features = ephys_features.merge(morphology_features, 
                                            how=how,
                                            on='specimen_id')
        
        if dataframe:
            return all_features
        else:
            return all_features.to_dict('records')
        

    def get_ephys_data(self, specimen_id, file_name=None):
        '''
        Download electrophysiology traces for a single cell in the database.

        Parameters
        ----------
        
        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the ephys features metadata as CSV.  
            If file_name is None, the file_name will be pulled out of the 
            manifest.  If caching is disabled, no file will be saved. 
            Default is None.

        Returns
        -------
        NwbDataSet
            A class instance with helper methods for retrieving stimulus
            and response traces out of an NWB file.
        '''

        file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY, specimen_id)

        if not os.path.exists(file_name):
            self.api.save_ephys_data(specimen_id, file_name)

        return NwbDataSet(file_name)


    def get_reconstruction(self, specimen_id, file_name=None):
        '''
        Download and open a reconstruction for a single cell in the database.

        Parameters
        ----------
        
        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the ephys features metadata as CSV.  
            If file_name is None, the file_name will be pulled out of the 
            manifest.  If caching is disabled, no file will be saved. 
            Default is None.

        Returns
        -------
        Morphology
             A class instance with methods for accessing morphology compartments.
        '''

        file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY, specimen_id)

        if file_name is None:
            raise Exception("Please enable caching (CellTypes.cache = True) or specify a save_file_name.")

        if not os.path.exists(file_name):
            self.api.save_reconstruction(specimen_id, file_name)

        return swc.read_swc(file_name)


    def build_manifest(self, file_name):
        '''
        Construct a manifest for this Cache class and save it in a file.
        
        Parameters
        ----------
        
        file_name: string
            File location to save the manifest.

        '''

        mb = ManifestBuilder()

        mb.add_path('BASEDIR', '.')
        mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR')
        mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR')
        mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR')
        mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR')


        mb.write_json_file(file_name)
import os
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import matplotlib.pyplot as plt
import numpy as np
import math
import pprint 
import csv

#output_file = 'sholl_all.csv'
#output_file = 'sholl_apical_dendrite_only.csv'
output_file = 'sholl_basal_dendrite_only.csv'

ct = CellTypesApi()

cells = ct.list_cells(require_reconstruction=True)
print "Cells with reconstructions: ", len(cells)

download_file = False
allow_types = [3]

radii = np.array(range(0,1000,10))
radii_square = np.square( radii )

#
# Computation based on line-sphere intersection formula
# https://en.wikipedia.org/wiki/Line%E2%80%93sphere_intersection
#

with open( output_file, 'wb' ) as csvfile :
    cwriter = csv.writer( csvfile, delimiter=',', quotechar='"' )
Esempio n. 26
0
 def __init__(self, cache=True, manifest_file='manifest.json'):
     super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache)
     self.api = CellTypesApi()
Esempio n. 27
0
class CellTypesCache(Cache):
    """
    Cache class for storing and accessing data from the Cell Types Database.
    By default, this class will cache any downloaded metadata or files in
    well known locations defined in a manifest file.  This behavior can be
    disabled.

    Attributes
    ----------

    api: CellTypesApi instance
        The object used for making API queries related to the Cell Types Database

    Parameters
    ----------

    cache: boolean
        Whether the class should save results of API queries to locations specified
        in the manifest file.  Queries for files (as opposed to metadata) must have a
        file location.  If caching is disabled, those locations must be specified
        in the function call (e.g. get_ephys_data(file_name='file.nwb')).

    manifest_file: string
       File name of the manifest to be read.  Default is "cell_types_manifest.json".
    """

    # manifest keys
    CELLS_KEY = 'CELLS'
    EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
    MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
    EPHYS_DATA_KEY = 'EPHYS_DATA'
    EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
    RECONSTRUCTION_KEY = 'RECONSTRUCTION'
    MARKER_KEY = 'MARKER'
    MANIFEST_VERSION = "1.1"

    def __init__(self, cache=True, manifest_file=None, base_uri=None):

        if manifest_file is None:
            manifest_file = get_default_manifest_file('cell_types')

        super(CellTypesCache, self).__init__(manifest=manifest_file,
                                             cache=cache,
                                             version=self.MANIFEST_VERSION)
        self.api = CellTypesApi(base_uri=base_uri)

    def get_cells(self,
                  file_name=None,
                  require_morphology=False,
                  require_reconstruction=False,
                  reporter_status=None,
                  species=None,
                  simple=True):
        """
        Download metadata for all cells in the database and optionally return a
        subset filtered by whether or not they have a morphology or reconstruction.

        Parameters
        ----------

        file_name: string
            File name to save/read the cell metadata as JSON.  If file_name is None,
            the file_name will be pulled out of the manifest.  If caching
            is disabled, no file will be saved. Default is None.

        require_morphology: boolean
            Filter out cells that have no morphological images.

        require_reconstruction: boolean
            Filter out cells that have no morphological reconstructions.

        reporter_status: list
            Filter for cells that have one or more cell reporter statuses.

        species: list
            Filter for cells that belong to one or more species.  If None, return all.
            Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
        """

        file_name = self.get_cache_path(file_name, self.CELLS_KEY)

        cells = self.api.list_cells_api(path=file_name,
                                        strategy='lazy',
                                        **Cache.cache_json())

        if isinstance(reporter_status, string_types):
            reporter_status = [reporter_status]

        # filter the cells on the way out
        cells = self.api.filter_cells_api(cells, require_morphology,
                                          require_reconstruction,
                                          reporter_status, species, simple)

        return cells

    def get_ephys_sweeps(self, specimen_id, file_name=None):
        """
        Download sweep metadata for a single cell specimen.

        Parameters
        ----------

        specimen_id: int
             ID of a cell.
        """

        file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY,
                                        specimen_id)

        sweeps = self.api.get_ephys_sweeps(specimen_id,
                                           strategy='lazy',
                                           path=file_name,
                                           **Cache.cache_json())

        return sweeps

    def get_ephys_features(self, dataframe=False, file_name=None):
        """
        Download electrophysiology features for all cells in the database.

        Parameters
        ----------

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.
        """
        file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)

        if self.cache:
            if dataframe:
                warnings.warn("dataframe argument is deprecated.")
                args = Cache.cache_csv_dataframe()
            else:
                args = Cache.cache_csv_json()
            args['strategy'] = 'lazy'
        else:
            args = Cache.nocache_json()

        features_df = self.api.get_ephys_features(path=file_name, **args)

        return features_df

    def get_morphology_features(self, dataframe=False, file_name=None):
        """
        Download morphology features for all cells with reconstructions in the database.

        Parameters
        ----------

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.
        """

        file_name = self.get_cache_path(file_name,
                                        self.MORPHOLOGY_FEATURES_KEY)

        if self.cache:
            if dataframe:
                warnings.warn("dataframe argument is deprecated.")
                args = Cache.cache_csv_dataframe()
            else:
                args = Cache.cache_csv_json()
        else:
            args = Cache.nocache_json()

        args['strategy'] = 'lazy'
        args['path'] = file_name

        return self.api.get_morphology_features(**args)

    def get_all_features(self, dataframe=False, require_reconstruction=True):
        """
        Download morphology and electrophysiology features for all cells and merge them
        into a single table.

        Parameters
        ----------

        dataframe: boolean
            Return the output as a Pandas DataFrame.  If False, return
            a list of dictionaries.

        require_reconstruction: boolean
            Only return ephys and morphology features for cells that have
            reconstructions. Default True.
        """

        ephys_features = pd.DataFrame(self.get_ephys_features())
        morphology_features = pd.DataFrame(self.get_morphology_features())

        how = 'inner' if require_reconstruction else 'outer'

        all_features = ephys_features.merge(morphology_features,
                                            how=how,
                                            on='specimen_id')

        if dataframe:
            warnings.warn("dataframe argument is deprecated.")
            return all_features
        else:
            return all_features.to_dict('records')

    def get_ephys_data(self, specimen_id, file_name=None):
        """
        Download electrophysiology traces for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the ephys features metadata as CSV.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        NwbDataSet
            A class instance with helper methods for retrieving stimulus
            and response traces out of an NWB file.
        """

        file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY,
                                        specimen_id)

        self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')

        return NwbDataSet(file_name)

    def get_reconstruction(self, specimen_id, file_name=None):
        """
        Download and open a reconstruction for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the reconstruction SWC.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        Morphology
             A class instance with methods for accessing morphology compartments.
        """

        file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY,
                                        specimen_id)

        if file_name is None:
            raise Exception(
                "Please enable caching (CellTypes.cache = True) or specify a save_file_name."
            )

        if not os.path.exists(file_name):
            self.api.save_reconstruction(specimen_id, file_name)

        return swc.read_swc(file_name)

    def get_reconstruction_markers(self, specimen_id, file_name=None):
        """
        Download and open a reconstruction marker file for a single cell in the database.

        Parameters
        ----------

        specimen_id: int
            The ID of a cell specimen to download.

        file_name: string
            File name to save/read the reconstruction marker.
            If file_name is None, the file_name will be pulled out of the
            manifest.  If caching is disabled, no file will be saved.
            Default is None.

        Returns
        -------
        Morphology
             A class instance with methods for accessing morphology compartments.
        """

        file_name = self.get_cache_path(file_name, self.MARKER_KEY,
                                        specimen_id)

        if file_name is None:
            raise Exception(
                "Please enable caching (CellTypes.cache = True) or specify a save_file_name."
            )

        if not os.path.exists(file_name):
            try:
                self.api.save_reconstruction_markers(specimen_id, file_name)
            except LookupError as e:
                logging.warning(e.args)
                return []

        return swc.read_marker_file(file_name)

    def build_manifest(self, file_name):
        """
        Construct a manifest for this Cache class and save it in a file.

        Parameters
        ----------

        file_name: string
            File location to save the manifest.

        """

        mb = ManifestBuilder()
        mb.set_version(self.MANIFEST_VERSION)
        mb.add_path('BASEDIR', '.')
        mb.add_path(self.CELLS_KEY,
                    'cells.json',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EPHYS_DATA_KEY,
                    'specimen_%d/ephys.nwb',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EPHYS_FEATURES_KEY,
                    'ephys_features.csv',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.MORPHOLOGY_FEATURES_KEY,
                    'morphology_features.csv',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.RECONSTRUCTION_KEY,
                    'specimen_%d/reconstruction.swc',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.MARKER_KEY,
                    'specimen_%d/reconstruction.marker',
                    typename='file',
                    parent_key='BASEDIR')
        mb.add_path(self.EPHYS_SWEEPS_KEY,
                    'specimen_%d/ephys_sweeps.json',
                    typename='file',
                    parent_key='BASEDIR')

        mb.write_json_file(file_name)
import pprint
import subprocess

from neuroml import NeuroMLDocument
from neuroml import Network
from neuroml import Population
from neuroml import Location
from neuroml import Instance
from neuroml import IncludeType

import neuroml.writers as writers

pp = pprint.PrettyPrinter(indent=4)

ct = CellTypesApi()

# use the SDK API to get a list of single cells and corresponding metadata
cell_metadata_list = ct.list_cells(require_reconstruction=True)

# convert that list of cells' metadata to a python dictionary for easy access
cell_meta_dict = {}
dataset_ids = []

for i in range(len(cell_metadata_list)):
    id = cell_metadata_list[i]['id']
    dataset_ids.append(id)
    cell_meta_dict[id] = cell_metadata_list[i]
    
cells = ['467703703', '323452245']
cells = ['467703703', '471141261', '320207387', '471141261', '324493977', '464326095']
Esempio n. 29
0
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
from allensdk.core.nwb_data_set import NwbDataSet

# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'

# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)

# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
    sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])

# calculate features
cell_features = extract_cell_features(NwbDataSet(nwb_file),
                                      sweep_numbers['Ramp'],
                                      sweep_numbers['Short Square'],
                                      sweep_numbers['Long Square'])
# Based on https://github.com/stripathy/AIBS_cell_types/blob/master/Allen_ephys_playground.ipynb

from allensdk.api.queries.cell_types_api import CellTypesApi
import os

ct = CellTypesApi()

from data_helper import CURRENT_DATASETS, DATASET_TARGET_SWEEPS

dataset_ids = CURRENT_DATASETS

sweep_numbers_for_data = DATASET_TARGET_SWEEPS


for dataset_id in dataset_ids:

    raw_ephys_file_name = '%d_raw_data.nwb' % dataset_id

    if not os.path.isfile(raw_ephys_file_name):
        print('Downloading data: %s'%raw_ephys_file_name)
        ct.save_ephys_data(dataset_id, raw_ephys_file_name)

        print('Saved: %s'%raw_ephys_file_name)
    else:
        print('File: %s already present...'%raw_ephys_file_name)


    print('Loading data from: %s'%raw_ephys_file_name)

    from allensdk.core.nwb_data_set import NwbDataSet
    data_set = NwbDataSet(raw_ephys_file_name)
Esempio n. 31
0
"""
Single sweep detection
=================================

Detect spikes for a single sweep
"""

import os
import matplotlib.pyplot as plt
from allensdk.api.queries.cell_types_api import CellTypesApi
from ipfx.aibs_data_set import AibsDataSet
from ipfx.ephys_extractor import SpikeExtractor

# Download and access the experimental data
ct = CellTypesApi()

specimen_id = 595570553
nwb_file = "%d.nwb" % specimen_id
sweep_info = ct.get_ephys_sweeps(specimen_id)

if not os.path.exists(nwb_file):
    ct.save_ephys_data(specimen_id, nwb_file)

# Get the data for the sweep into a format we can use
dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file)
sweep_number = 39
sweep = dataset.sweep(sweep_number)

# Extract information about the spikes
ext = SpikeExtractor()
results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i)
Esempio n. 32
0
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict

# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'

# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)

# group the sweeps by stimulus 
sweep_numbers = defaultdict(list)
for sweep in sweeps:
    sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])

# calculate features
cell_features = extract_cell_features(nwb_file,
                                      sweep_numbers['Long Square'],
                                      sweep_numbers['Short Square'],
                                      sweep_numbers['Ramp'])

Esempio n. 33
0
import pprint
import subprocess

from neuroml import NeuroMLDocument
from neuroml import Network
from neuroml import Population
from neuroml import Location
from neuroml import Instance
from neuroml import IncludeType

import neuroml.writers as writers

pp = pprint.PrettyPrinter(indent=4)

ct = CellTypesApi()

# use the SDK API to get a list of single cells and corresponding metadata
cell_metadata_list = ct.list_cells(require_reconstruction=True)

# convert that list of cells' metadata to a python dictionary for easy access
cell_meta_dict = {}
dataset_ids = []

for i in range(len(cell_metadata_list)):
    id = cell_metadata_list[i]['id']
    dataset_ids.append(id)
    cell_meta_dict[id] = cell_metadata_list[i]

cells = ['467703703', '323452245']
cells = [
Esempio n. 34
0
def download_glif_models(cell_ids,
                         base_dir,
                         incl_ephys=True,
                         force_overwrite=False):
    """Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up
    all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/
    """
    # Determine the best url for connecting to cell-types db
    try:
        # see if we can connect to interal cell-types db
        request = requests.get('http://icelltypes/')
        if request.status_code == 200:
            base_uri = 'http://icelltypes/'
        else:
            base_uri = None
    except Exception:
        base_uri = None  # use the default url

    base_dir = base_dir if base_dir.endswith('/') else base_dir + '/'

    valid_cells = []
    ct_api = CellTypesApi(base_uri)
    for cell in ct_api.list_cells():
        if cell['id'] in cell_ids:
            # create directory for cell
            cell_home = '{}cell_{}/'.format(base_dir, cell['id'])
            if not os.path.exists(cell_home):
                os.makedirs(cell_home)

            # save metadata
            cell_metadata_file = cell_home + 'cell_metadata.json'
            if force_overwrite or not os.path.exists(cell_metadata_file):
                print('Saving metadata for cell {} in {}'.format(
                    cell['id'], cell_metadata_file))
                json_utilities.write(cell_metadata_file, cell)
            else:
                print('File {} already exists. Skipping'.format(
                    cell_metadata_file))

            # save ephys data
            if incl_ephys:
                cell_ephys_file = cell_home + 'ephys_data.nwb'
                if force_overwrite or not os.path.exists(cell_ephys_file):
                    print('Saving ephys data for cell {} in {}'.format(
                        cell['id'], cell_ephys_file))
                    ct_api.save_ephys_data(cell['id'], cell_ephys_file)
                else:
                    print('File {} already exists. Skipping'.format(
                        cell_ephys_file))

            # save sweeps file
            sweeps_file = cell_home + 'ephys_sweeps.json'
            if force_overwrite or not os.path.exists(sweeps_file):
                print('- Saving sweeps file to {}'.format(sweeps_file))
                ephys_sweeps = ct_api.get_ephys_sweeps(cell['id'])
                json_utilities.write(sweeps_file, ephys_sweeps)
            else:
                print('- File {} already exits. Skipping.'.format(sweeps_file))

            # keep track of valid ids
            valid_cells.append(cell['id'])
            cell_ids.remove(cell['id'])

    for cid in cell_ids:
        print('Warning: cell #{} was not found in cell-types database'.format(
            cid))

    # Iterate through each all available models and find ones correspoding to cell list
    glif_models = {}  # map model-id to their directory
    glif_api = GlifApi(base_uri=base_uri)
    for model in glif_api.list_neuronal_models():
        if model['specimen_id'] in valid_cells:
            # save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/
            cell_id = model['specimen_id']
            model_id = model['id']
            model_type = model[
                'neuronal_model_template_id']  #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc
            type_name = model_id2name.get(model_type, None)
            if type_name is None:
                print(
                    'Warning: Unknown model type {} ({}) for cell/model {}/{}'.
                    format(model_type,
                           model['neuronal_model_template']['name'], cell_id,
                           model_id))
                type_name = model_type
            model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id,
                                                       type_name, model_id)
            glif_models[model_id] = model_home_dir

    # go through all the found models, download necessary files
    n_models = len(glif_models)
    for i, (gid, home_dir) in enumerate(glif_models.iteritems()):
        print('Processing model {}  ({} of {})'.format(gid, (i + 1), n_models))
        model_metadata = glif_api.get_neuronal_model(gid)

        if not os.path.exists(home_dir):
            os.makedirs(home_dir)

        # save model metadata
        metadata_file = home_dir + 'metadata.json'
        if force_overwrite or not os.path.exists(metadata_file):
            print('- Saving metadata file to {}'.format(metadata_file))
            #print type(metadata_file)
            with open(metadata_file, 'wb') as fp:
                json.dump(model_metadata, fp, indent=2)
        else:
            print('- File {} already exits. Skipping.'.format(metadata_file))

        # get neuron configuration file
        config_file = home_dir + 'config.json'
        if force_overwrite or not os.path.exists(config_file):
            print('- Saving configuration file to {}'.format(config_file))
            neuron_config = glif_api.get_neuron_config()
            json_utilities.write(config_file, neuron_config)
        else:
            print('- File {} already exits. Skipping.'.format(config_file))