def get_sweep_params(dataset_id,sweep_id):
ct = CellTypesApi()
experiment_params = ct.get_ephys_sweeps(dataset_id)
for sp in experiment_params:
if sp['id']==sweep_id:
sweep_num = sp['sweep_number']
break
if sweep_num is None:
raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
return sp
def extract_data(ephys_cell, sweep_number):
"""Data extraction from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
f = open(real_data_path, 'wb')
pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
f.close()
def get_sweep_params(dataset_id, sweep_id):
"""
Gets sweep parameters corresponding to the sweep with id 'sweep_id' from
the dataset with id 'dataset_id'.
"""
ct = CellTypesApi()
experiment_params = ct.get_ephys_sweeps(dataset_id)
sp = None
for sp in experiment_params:
if sp['id']==sweep_id:
sweep_num = sp['sweep_number']
if sweep_num is None:
raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
break
return sp
class CellTypesCache(Cache):
"""
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "cell_types_manifest.json".
"""
# manifest keys
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
MARKER_KEY = 'MARKER'
MANIFEST_VERSION = "1.1"
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(
manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def get_cells(self, file_name=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have one or more cell reporter statuses.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
cells = self.api.list_cells_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
if isinstance(reporter_status, string_types):
reporter_status = [reporter_status]
# filter the cells on the way out
cells = self.api.filter_cells_api(cells,
require_morphology,
require_reconstruction,
reporter_status,
species,
simple)
return cells
def get_ephys_sweeps(self, specimen_id, file_name=None):
"""
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
"""
file_name = self.get_cache_path(
file_name, self.EPHYS_SWEEPS_KEY, specimen_id)
sweeps = self.api.get_ephys_sweeps(specimen_id,
strategy='lazy',
path=file_name,
**Cache.cache_json())
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
"""
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
args['strategy'] = 'lazy'
else:
args = Cache.nocache_json()
features_df = self.api.get_ephys_features(path=file_name,
**args)
return features_df
def get_morphology_features(self, dataframe=False, file_name=None):
"""
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(
file_name, self.MORPHOLOGY_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
else:
args = Cache.nocache_json()
args['strategy'] = 'lazy'
args['path'] = file_name
return self.api.get_morphology_features(**args)
def get_all_features(self, dataframe=False, require_reconstruction=True):
"""
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
"""
ephys_features = pd.DataFrame(self.get_ephys_features())
morphology_features = pd.DataFrame(self.get_morphology_features())
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
warnings.warn("dataframe argument is deprecated.")
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
"""
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
"""
file_name = self.get_cache_path(
file_name, self.EPHYS_DATA_KEY, specimen_id)
self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
"""
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction SWC.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(
file_name, self.RECONSTRUCTION_KEY, specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def get_reconstruction_markers(self, specimen_id, file_name=None):
"""
Download and open a reconstruction marker file for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction marker.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(
file_name, self.MARKER_KEY, specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
try:
self.api.save_reconstruction_markers(specimen_id, file_name)
except LookupError as e:
logging.warning(e.args)
return []
return swc.read_marker_file(file_name)
def build_manifest(self, file_name):
"""
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
"""
mb = ManifestBuilder()
mb.set_version(self.MANIFEST_VERSION)
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY, 'cells.json',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv',
typename='file', parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv',
typename='file', parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc',
typename='file', parent_key='BASEDIR')
mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json',
typename='file', parent_key='BASEDIR')
mb.write_json_file(file_name)
Detect spikes for a single sweep
"""
import os
import matplotlib.pyplot as plt
from allensdk.api.queries.cell_types_api import CellTypesApi
from ipfx.aibs_data_set import AibsDataSet
from ipfx.ephys_extractor import SpikeExtractor
# Download and access the experimental data
ct = CellTypesApi()
specimen_id = 595570553
nwb_file = "%d.nwb" % specimen_id
sweep_info = ct.get_ephys_sweeps(specimen_id)
if not os.path.exists(nwb_file):
ct.save_ephys_data(specimen_id, nwb_file)
# Get the data for the sweep into a format we can use
dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file)
sweep_number = 39
sweep = dataset.sweep(sweep_number)
# Extract information about the spikes
ext = SpikeExtractor()
results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i)
# Plot the results, showing two features of the detected spikes
plt.plot(sweep.t, sweep.v)
class CellTypesCache(Cache):
"""
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "cell_types_manifest.json".
"""
# manifest keys
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
MARKER_KEY = 'MARKER'
MANIFEST_VERSION = "1.1"
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache,
version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def get_cells(self,
file_name=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have one or more cell reporter statuses.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
cells = self.api.list_cells_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
if isinstance(reporter_status, string_types):
reporter_status = [reporter_status]
# filter the cells on the way out
cells = self.api.filter_cells_api(cells, require_morphology,
require_reconstruction,
reporter_status, species, simple)
return cells
def get_ephys_sweeps(self, specimen_id, file_name=None):
"""
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY,
specimen_id)
sweeps = self.api.get_ephys_sweeps(specimen_id,
strategy='lazy',
path=file_name,
**Cache.cache_json())
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
"""
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
args['strategy'] = 'lazy'
else:
args = Cache.nocache_json()
features_df = self.api.get_ephys_features(path=file_name, **args)
return features_df
def get_morphology_features(self, dataframe=False, file_name=None):
"""
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name,
self.MORPHOLOGY_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
else:
args = Cache.nocache_json()
args['strategy'] = 'lazy'
args['path'] = file_name
return self.api.get_morphology_features(**args)
def get_all_features(self, dataframe=False, require_reconstruction=True):
"""
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
"""
ephys_features = pd.DataFrame(self.get_ephys_features())
morphology_features = pd.DataFrame(self.get_morphology_features())
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
warnings.warn("dataframe argument is deprecated.")
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
"""
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY,
specimen_id)
self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
"""
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction SWC.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY,
specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name."
)
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def get_reconstruction_markers(self, specimen_id, file_name=None):
"""
Download and open a reconstruction marker file for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction marker.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(file_name, self.MARKER_KEY,
specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name."
)
if not os.path.exists(file_name):
try:
self.api.save_reconstruction_markers(specimen_id, file_name)
except LookupError as e:
logging.warning(e.args)
return []
return swc.read_marker_file(file_name)
def build_manifest(self, file_name):
"""
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
"""
mb = ManifestBuilder()
mb.set_version(self.MANIFEST_VERSION)
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY,
'cells.json',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY,
'specimen_%d/ephys.nwb',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY,
'ephys_features.csv',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY,
'morphology_features.csv',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY,
'specimen_%d/reconstruction.swc',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.MARKER_KEY,
'specimen_%d/reconstruction.marker',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY,
'specimen_%d/ephys_sweeps.json',
typename='file',
parent_key='BASEDIR')
mb.write_json_file(file_name)
def allen_obs_data(ephys_cell=464212183,
sweep_number=33,
A_soma=np.pi * (70. * 1e-4)**2):
"""Data for x_o. Cell from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
dir_cache = './support_files'
real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
else:
def pickle_load(file):
"""Loads data from file."""
f = open(file, 'rb')
data = pickle.load(f, encoding='latin1')
f.close()
return data
real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
real_data_path)
t = np.arange(0, duration, dt)
# external current
I = I_real_data / A_soma # muA/cm2
# return real_data_obs, I_obs
return {
'data': real_data_obs.reshape(-1),
'time': t,
'dt': dt,
'I': I.reshape(-1),
't_on': t_on,
't_off': t_off
}
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'
# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(nwb_file,
sweep_numbers['Long Square'],
sweep_numbers['Short Square'],
sweep_numbers['Ramp'])
def download_glif_models(cell_ids,
base_dir,
incl_ephys=True,
force_overwrite=False):
"""Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up
all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/
"""
# Determine the best url for connecting to cell-types db
try:
# see if we can connect to interal cell-types db
request = requests.get('http://icelltypes/')
if request.status_code == 200:
base_uri = 'http://icelltypes/'
else:
base_uri = None
except Exception:
base_uri = None # use the default url
base_dir = base_dir if base_dir.endswith('/') else base_dir + '/'
valid_cells = []
ct_api = CellTypesApi(base_uri)
for cell in ct_api.list_cells():
if cell['id'] in cell_ids:
# create directory for cell
cell_home = '{}cell_{}/'.format(base_dir, cell['id'])
if not os.path.exists(cell_home):
os.makedirs(cell_home)
# save metadata
cell_metadata_file = cell_home + 'cell_metadata.json'
if force_overwrite or not os.path.exists(cell_metadata_file):
print('Saving metadata for cell {} in {}'.format(
cell['id'], cell_metadata_file))
json_utilities.write(cell_metadata_file, cell)
else:
print('File {} already exists. Skipping'.format(
cell_metadata_file))
# save ephys data
if incl_ephys:
cell_ephys_file = cell_home + 'ephys_data.nwb'
if force_overwrite or not os.path.exists(cell_ephys_file):
print('Saving ephys data for cell {} in {}'.format(
cell['id'], cell_ephys_file))
ct_api.save_ephys_data(cell['id'], cell_ephys_file)
else:
print('File {} already exists. Skipping'.format(
cell_ephys_file))
# save sweeps file
sweeps_file = cell_home + 'ephys_sweeps.json'
if force_overwrite or not os.path.exists(sweeps_file):
print('- Saving sweeps file to {}'.format(sweeps_file))
ephys_sweeps = ct_api.get_ephys_sweeps(cell['id'])
json_utilities.write(sweeps_file, ephys_sweeps)
else:
print('- File {} already exits. Skipping.'.format(sweeps_file))
# keep track of valid ids
valid_cells.append(cell['id'])
cell_ids.remove(cell['id'])
for cid in cell_ids:
print('Warning: cell #{} was not found in cell-types database'.format(
cid))
# Iterate through each all available models and find ones correspoding to cell list
glif_models = {} # map model-id to their directory
glif_api = GlifApi(base_uri=base_uri)
for model in glif_api.list_neuronal_models():
if model['specimen_id'] in valid_cells:
# save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/
cell_id = model['specimen_id']
model_id = model['id']
model_type = model[
'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc
type_name = model_id2name.get(model_type, None)
if type_name is None:
print(
'Warning: Unknown model type {} ({}) for cell/model {}/{}'.
format(model_type,
model['neuronal_model_template']['name'], cell_id,
model_id))
type_name = model_type
model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id,
type_name, model_id)
glif_models[model_id] = model_home_dir
# go through all the found models, download necessary files
n_models = len(glif_models)
for i, (gid, home_dir) in enumerate(glif_models.iteritems()):
print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models))
model_metadata = glif_api.get_neuronal_model(gid)
if not os.path.exists(home_dir):
os.makedirs(home_dir)
# save model metadata
metadata_file = home_dir + 'metadata.json'
if force_overwrite or not os.path.exists(metadata_file):
print('- Saving metadata file to {}'.format(metadata_file))
#print type(metadata_file)
with open(metadata_file, 'wb') as fp:
json.dump(model_metadata, fp, indent=2)
else:
print('- File {} already exits. Skipping.'.format(metadata_file))
# get neuron configuration file
config_file = home_dir + 'config.json'
if force_overwrite or not os.path.exists(config_file):
print('- Saving configuration file to {}'.format(config_file))
neuron_config = glif_api.get_neuron_config()
json_utilities.write(config_file, neuron_config)
else:
print('- File {} already exits. Skipping.'.format(config_file))
class CellTypesCache(Cache):
'''
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "manifest.json".
'''
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
def __init__(self, cache=True, manifest_file='manifest.json'):
super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache)
self.api = CellTypesApi()
def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False):
'''
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
'''
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
if os.path.exists(file_name):
cells = json_utilities.read(file_name)
else:
cells = self.api.list_cells(False, False)
if self.cache:
json_utilities.write(file_name, cells)
# filter the cells on the way out
return self.api.filter_cells(cells, require_morphology, require_reconstruction)
def get_ephys_sweeps(self, specimen_id, file_name=None):
'''
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY, specimen_id)
if os.path.exists(file_name):
sweeps = json_utilities.read(file_name)
else:
sweeps = self.api.get_ephys_sweeps(specimen_id)
if self.cache:
json_utilities.write(file_name, sweeps)
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
'''
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if os.path.exists(file_name):
features_df = pd.DataFrame.from_csv(file_name)
else:
features_df = self.api.get_ephys_features(dataframe=True)
if self.cache:
features_df.to_csv(file_name)
if dataframe:
return features_df
else:
return features_df.to_dict('records')
def get_morphology_features(self, dataframe=False, file_name=None):
'''
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
'''
file_name = self.get_cache_path(file_name, self.MORPHOLOGY_FEATURES_KEY)
if os.path.exists(file_name):
features_df = pd.DataFrame.from_csv(file_name)
else:
features_df = self.api.get_morphology_features(dataframe=True)
if self.cache:
features_df.to_csv(file_name)
if dataframe:
return features_df
else:
return features_df.to_dict('records')
def get_all_features(self, dataframe=False, require_reconstruction=True):
'''
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
'''
ephys_features = self.get_ephys_features(dataframe=True)
morphology_features = self.get_morphology_features(dataframe=True)
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
'''
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY, specimen_id)
if not os.path.exists(file_name):
self.api.save_ephys_data(specimen_id, file_name)
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
'''
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
'''
file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY, specimen_id)
if file_name is None:
raise Exception("Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def build_manifest(self, file_name):
'''
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
'''
mb = ManifestBuilder()
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR')
mb.write_json_file(file_name)
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
from allensdk.core.nwb_data_set import NwbDataSet
# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'
# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(NwbDataSet(nwb_file),
sweep_numbers['Ramp'],
sweep_numbers['Short Square'],
sweep_numbers['Long Square'])
