def __init__(self,
cache=True,
manifest_file='cell_types_manifest.json',
base_uri=None):
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache)
self.api = CellTypesApi(base_uri=base_uri)
def get_observation(dataset_id, kind, cached=True, quiet=False):
"""
Gets an observation of kind 'kind' from the dataset with id 'dataset_id',
optionally using the cached value retrieved previously.
"""
db = shelve.open('aibs-cache') if cached else {}
identifier = '%d_%s' % (dataset_id,kind)
if identifier in db:
print("Getting %s cached data value for from AIBS dataset %s" \
% (kind.title(),dataset_id))
value = db[identifier]
else:
print("Getting %s data value for from AIBS dataset %s" \
% (kind.title(),dataset_id))
ct = CellTypesApi()
cmd = ct.get_cell(dataset_id) # Cell metadata
if kind == 'rheobase':
sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id']
sp = get_sweep_params(dataset_id, sweep_id)
if kind == 'rheobase':
value = sp['stimulus_absolute_amplitude']
value = np.round(value,2) # Round to nearest hundredth of a pA.
value *= pq.pA # Apply units.
db[identifier] = value
if cached:
db.close()
return {'value': value}
def test_list_cells_mocked(mock_cells):
with patch.object(CellTypesApi, "model_query", return_value=mock_cells):
ctapi = CellTypesApi()
cells = ctapi.list_cells()
assert len(cells) == 3
flu_cells = [
cell for cell in cells
if cell['disease_categories'] == [('influenza')]
]
assert len(flu_cells) == 1
cells = ctapi.list_cells(require_reconstruction=True)
assert len(cells) == 1
cells = ctapi.list_cells(require_morphology=True)
assert len(cells) == 1
cells = ctapi.list_cells(reporter_status=['bob'])
assert len(cells) == 1
cells = ctapi.list_cells(species=['H**O SAPIENS'])
assert len(cells) == 1
cells = ctapi.list_cells(species=['mus musculus'])
assert len(cells) == 1
def cell_types_api():
endpoint = None
if 'TEST_API_ENDPOINT' in os.environ:
endpoint = os.environ['TEST_API_ENDPOINT']
return CellTypesApi(endpoint)
else:
return None
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache,
version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def fetch_pipeline_file(request):
specimen_id = request.param
nwb_file_name = '{}.nwb'.format(specimen_id)
nwb_file_full_path = os.path.join(TEST_DATA_PATH, nwb_file_name)
if not os.path.exists(nwb_file_full_path):
ct = CellTypesApi()
ct.save_ephys_data(specimen_id, nwb_file_full_path)
return nwb_file_full_path
def __init__(self, archive_dir=None):
self.bp = BiophysicalApi('http://api.brain-map.org')
self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file
self.cta = CellTypesApi()
self.rma = RmaApi()
self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/'
self.template_names = {}
self.nwb_list = []
if archive_dir == None:
archive_dir = '.'
self.archive_dir = archive_dir
def extract_data(ephys_cell, sweep_number):
"""Data extraction from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
f = open(real_data_path, 'wb')
pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
f.close()
def get_sweep_params(dataset_id, sweep_id):
"""
Gets sweep parameters corresponding to the sweep with id 'sweep_id' from
the dataset with id 'dataset_id'.
"""
ct = CellTypesApi()
experiment_params = ct.get_ephys_sweeps(dataset_id)
sp = None
for sp in experiment_params:
if sp['id']==sweep_id:
sweep_num = sp['sweep_number']
if sweep_num is None:
raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
break
return sp
def test_list_cells_api_mocked(mock_cells_api):
with patch.object(CellTypesApi, "model_query", return_value=mock_cells_api):
ctapi = CellTypesApi()
cells = ctapi.list_cells_api()
assert len(cells) == 2
fcells = ctapi.filter_cells_api(cells, require_reconstruction=True)
assert len(fcells) == 1
fcells = ctapi.filter_cells_api(cells, require_morphology=True)
assert len(fcells) == 1
fcells = ctapi.filter_cells_api(cells, species=['taco'])
assert len(fcells) == 2
fcells = ctapi.filter_cells_api(cells, reporter_status=['fish'])
assert len(fcells) == 1
def get_observation(dataset_id, kind, cached=True, quiet=False):
"""Get an observation.
Get an observation of kind 'kind' from the dataset with id 'dataset_id'.
optionally using the cached value retrieved previously.
"""
db = shelve.open('aibs-cache') if cached else {}
identifier = '%d_%s' % (dataset_id, kind)
if identifier in db:
print("Getting %s cached data value for from AIBS dataset %s" %
(kind.title(), dataset_id))
value = db[identifier]
else:
print("Getting %s data value for from AIBS dataset %s" %
(kind.title(), dataset_id))
ct = CellTypesApi()
cmd = ct.get_cell(dataset_id) # Cell metadata
if kind == 'rheobase':
if 'ephys_features' in cmd:
value = cmd['ephys_features'][0][
'threshold_i_long_square'] # newer API
else:
value = cmd['ef__threshold_i_long_square'] # older API
value = np.round(value, 2) # Round to nearest hundredth of a pA.
value *= pq.pA # Apply units.
else:
value = cmd[kind]
db[identifier] = value
if cached:
db.close()
return {'value': value}
"""
Single sweep detection
=================================
Detect spikes for a single sweep
"""
import os
import matplotlib.pyplot as plt
from allensdk.api.queries.cell_types_api import CellTypesApi
from ipfx.aibs_data_set import AibsDataSet
from ipfx.ephys_extractor import SpikeExtractor
# Download and access the experimental data
ct = CellTypesApi()
specimen_id = 595570553
nwb_file = "%d.nwb" % specimen_id
sweep_info = ct.get_ephys_sweeps(specimen_id)
if not os.path.exists(nwb_file):
ct.save_ephys_data(specimen_id, nwb_file)
# Get the data for the sweep into a format we can use
dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file)
sweep_number = 39
sweep = dataset.sweep(sweep_number)
# Extract information about the spikes
ext = SpikeExtractor()
results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i)
def allen_obs_data(ephys_cell=464212183,
sweep_number=33,
A_soma=np.pi * (70. * 1e-4)**2):
"""Data for x_o. Cell from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
dir_cache = './support_files'
real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
else:
def pickle_load(file):
"""Loads data from file."""
f = open(file, 'rb')
data = pickle.load(f, encoding='latin1')
f.close()
return data
real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
real_data_path)
t = np.arange(0, duration, dt)
# external current
I = I_real_data / A_soma # muA/cm2
# return real_data_obs, I_obs
return {
'data': real_data_obs.reshape(-1),
'time': t,
'dt': dt,
'I': I.reshape(-1),
't_on': t_on,
't_off': t_off
}
def download_glif_models(cell_ids,
base_dir,
incl_ephys=True,
force_overwrite=False):
"""Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up
all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/
"""
# Determine the best url for connecting to cell-types db
try:
# see if we can connect to interal cell-types db
request = requests.get('http://icelltypes/')
if request.status_code == 200:
base_uri = 'http://icelltypes/'
else:
base_uri = None
except Exception:
base_uri = None # use the default url
base_dir = base_dir if base_dir.endswith('/') else base_dir + '/'
valid_cells = []
ct_api = CellTypesApi(base_uri)
for cell in ct_api.list_cells():
if cell['id'] in cell_ids:
# create directory for cell
cell_home = '{}cell_{}/'.format(base_dir, cell['id'])
if not os.path.exists(cell_home):
os.makedirs(cell_home)
# save metadata
cell_metadata_file = cell_home + 'cell_metadata.json'
if force_overwrite or not os.path.exists(cell_metadata_file):
print('Saving metadata for cell {} in {}'.format(
cell['id'], cell_metadata_file))
json_utilities.write(cell_metadata_file, cell)
else:
print('File {} already exists. Skipping'.format(
cell_metadata_file))
# save ephys data
if incl_ephys:
cell_ephys_file = cell_home + 'ephys_data.nwb'
if force_overwrite or not os.path.exists(cell_ephys_file):
print('Saving ephys data for cell {} in {}'.format(
cell['id'], cell_ephys_file))
ct_api.save_ephys_data(cell['id'], cell_ephys_file)
else:
print('File {} already exists. Skipping'.format(
cell_ephys_file))
# save sweeps file
sweeps_file = cell_home + 'ephys_sweeps.json'
if force_overwrite or not os.path.exists(sweeps_file):
print('- Saving sweeps file to {}'.format(sweeps_file))
ephys_sweeps = ct_api.get_ephys_sweeps(cell['id'])
json_utilities.write(sweeps_file, ephys_sweeps)
else:
print('- File {} already exits. Skipping.'.format(sweeps_file))
# keep track of valid ids
valid_cells.append(cell['id'])
cell_ids.remove(cell['id'])
for cid in cell_ids:
print('Warning: cell #{} was not found in cell-types database'.format(
cid))
# Iterate through each all available models and find ones correspoding to cell list
glif_models = {} # map model-id to their directory
glif_api = GlifApi(base_uri=base_uri)
for model in glif_api.list_neuronal_models():
if model['specimen_id'] in valid_cells:
# save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/
cell_id = model['specimen_id']
model_id = model['id']
model_type = model[
'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc
type_name = model_id2name.get(model_type, None)
if type_name is None:
print(
'Warning: Unknown model type {} ({}) for cell/model {}/{}'.
format(model_type,
model['neuronal_model_template']['name'], cell_id,
model_id))
type_name = model_type
model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id,
type_name, model_id)
glif_models[model_id] = model_home_dir
# go through all the found models, download necessary files
n_models = len(glif_models)
for i, (gid, home_dir) in enumerate(glif_models.iteritems()):
print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models))
model_metadata = glif_api.get_neuronal_model(gid)
if not os.path.exists(home_dir):
os.makedirs(home_dir)
# save model metadata
metadata_file = home_dir + 'metadata.json'
if force_overwrite or not os.path.exists(metadata_file):
print('- Saving metadata file to {}'.format(metadata_file))
#print type(metadata_file)
with open(metadata_file, 'wb') as fp:
json.dump(model_metadata, fp, indent=2)
else:
print('- File {} already exits. Skipping.'.format(metadata_file))
# get neuron configuration file
config_file = home_dir + 'config.json'
if force_overwrite or not os.path.exists(config_file):
print('- Saving configuration file to {}'.format(config_file))
neuron_config = glif_api.get_neuron_config()
json_utilities.write(config_file, neuron_config)
else:
print('- File {} already exits. Skipping.'.format(config_file))
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
from allensdk.core.nwb_data_set import NwbDataSet
# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'
# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(NwbDataSet(nwb_file),
sweep_numbers['Ramp'],
sweep_numbers['Short Square'],
sweep_numbers['Long Square'])
