def getExperiment(cell_id, stimtypes):
ctc = CellTypesCache()
ephys_sweeps = ctc.get_ephys_sweeps(specimen_id=cell_id)
sweeps_by_type = defaultdict(list)
sweeps = []
for sweep in ephys_sweeps:
if sweep['stimulus_name'] in stimtypes:
sweeps.append(sweep['sweep_number'])
sweeps_by_type[sweep['stimulus_name']].append(
sweep['sweep_number'])
ephys_filename = f'{cell_id}/{cell_id}_ephys.nwb'
ctc.get_ephys_data(cell_id, ephys_filename)
swc_filename = f'{cell_id}/{cell_id}.swc'
ctc.get_reconstruction(cell_id, swc_filename)
return ephys_filename, swc_filename, sweeps, sweeps_by_type
def load_realdata_ephys(cell_specimen_id=464212183):
from allensdk.core.cell_types_cache import CellTypesCache
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
cell_specimen_id =cell_specimen_id#64212183#325941643#464212183 (good)
data_set = ctc.get_ephys_data(cell_specimen_id)
sweep_number = 35
sweep_data = data_set.get_sweep(sweep_number)
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1]+1] # in A
v = sweep_data["response"][0:index_range[1]+1] # in V
i *= 1e6 # to mu A #1e12 # to pA
v *= 1e3 # to mV
sampling_rate = sweep_data["sampling_rate"] # in Hz
t = np.arange(0, len(v)) * (1.0 / sampling_rate)
D_latent= 2
signal_idx = np.logical_and(t >= 1.03, t <= 2.022)
V = v[signal_idx]
I_inj_values = i[signal_idx]
t = t[signal_idx]
t *= 1e3 #ms
downsample = 30 # downsample
V = V[::downsample]
I_inj_values = I_inj_values[::downsample]
t = t[::downsample]
return V, t, I_inj_values
def allen_id_to_sweeps(specimen_id):
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
specimen_id = int(specimen_id)
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
return sweep_numbers,data_set,sweeps
def cache_fixture():
# Instantiate the CellTypesCache instance. The manifest_file argument
# tells it where to store the manifest, which is a JSON file that tracks
# file paths. If you supply a relative path (like this), it will go
# into your current working directory
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
specimen_id = 464212183
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
data_set = ctc.get_ephys_data(specimen_id)
return ctc, data_set
def cache_fixture():
# Instantiate the CellTypesCache instance. The manifest_file argument
# tells it where to store the manifest, which is a JSON file that tracks
# file paths. If you supply a relative path (like this), it will go
# into your current working directory
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
specimen_id = 464212183
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
data_set = ctc.get_ephys_data(specimen_id)
return ctc, data_set
def get_data_sets_from_remote(upper_bound=2, lower_bound=None):
try:
with open('all_allen_cells.p', 'rb') as f:
cells = pickle.load(f)
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
except:
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
cells = ctc.get_cells()
with open('all_allen_cells.p', 'wb') as f:
pickle.dump(cells, f)
data = []
data_sets = []
path_name = 'data_nwbs'
try:
os.mkdir(path_name)
except:
print('directory already made.')
ids = [c['id'] for c in cells]
if upper_bound == None and lower_bound is None:
limited_range = ids[0:-1]
elif upper_bound is not None and lower_bound is not None:
limited_range = ids[lower_bound:upper_bound]
cnt = 0
for specimen_id in limited_range:
temp_path = str(path_name) + str('/') + str(specimen_id) + '.p'
if os.path.exists(temp_path):
cnt += 1
for specimen_id in limited_range:
temp_path = str(path_name) + str('/') + str(specimen_id) + '.p'
if os.path.exists(temp_path):
with open(temp_path, 'rb') as f:
(data_set_nwb, sweeps, specimen_id) = pickle.load(f)
data_sets.append((data_set_nwb, sweeps, specimen_id))
else:
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
file_name = 'cell_types/specimen_' + str(
specimen_id) + '/ephys.nwb'
data_set_nwb = NwbDataSet(file_name)
data_sets.append((data_set_nwb, sweeps, specimen_id))
with open(temp_path, 'wb') as f:
pickle.dump((data_set_nwb, sweeps, specimen_id), f)
return data_sets
def extract_data(ephys_cell, sweep_number):
"""Data extraction from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
f = open(real_data_path, 'wb')
pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
f.close()
def run_nwb(cell_id, model_type, neuron_config, stim_type='Ramp'):
"""Generates a injection current from nwb sweep file
Parameters
----------
cell_id : ID of cell speciment
model_type : LIF, LIF-R, LIF-R-ASC, LIF-ASC, LIF-R-ASC-A
stumilus_data : stumilus data from NWB file
"""
# get sweep/stimulus data
ctc = CellTypesCache()
ephys_sweeps = ctc.get_ephys_sweeps(cell_id)
ds = ctc.get_ephys_data(cell_id)
#ephys_sweep = next( s for s in ephys_sweeps if s['stimulus_name'] == stim_type )
ephys_sweep_stim = [
s for s in ephys_sweeps if s['stimulus_name'] == stim_type
]
ephys_sweep = ephys_sweep_stim[0]
stumilus_data = ds.get_sweep(ephys_sweep['sweep_number'])
ret = {}
n_steps = len(stumilus_data['stimulus'])
dt = 1.0 / stumilus_data['sampling_rate'] * 1.0e03
amp_times = [t * dt for t in xrange(n_steps)]
amp_values = stumilus_data['stimulus'].tolist()
total_time = n_steps * dt
output = runNestModel(model_type, neuron_config, amp_times, amp_values, dt,
total_time)
ret['nest'] = {
'times': output[0],
'voltages': output[1],
'spike_times': output[2]
}
output = runGlifNeuron(neuron_config, amp_values, dt)
ret['allen'] = {
'times': output[0],
'voltages': output[1],
'spike_times': output[2]
}
ret['I'] = amp_values
ret['dt'] = dt
return ret
class NwbDataSetTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(NwbDataSetTest, self).__init__(*args, **kwargs)
def testAllDataSets(self):
manifest_file = '/local1/projects/FHL2015/cell_types/manifest.json'
if not os.path.exists(manifest_file):
print "Cannot run this test: manifest does not exist (%s)" % manifest_file
return True
self.cache = CellTypesCache(manifest_file=manifest_file)
cells = self.cache.get_cells()
for cell in cells:
data_set = self.cache.get_ephys_data(cell['id'])
sweeps = self.cache.get_ephys_sweeps(cell['id'])
for sweep in sweeps:
metadata = data_set.get_sweep_metadata(sweep['sweep_number'])
def load_data(stim_names, reps=10, dur=3000, delay=200):
ctc = CellTypesCache(manifest_file="ctc/manifest.json")
cells = ctc.get_cells()
cell_id = cells[0]['id']
sweeps = ctc.get_ephys_sweeps(cell_id)
sweeps = [(sweep['sweep_number'], sweep['stimulus_name'])
for sweep in sweeps if sweep['stimulus_name'] in stim_names]
ds = ctc.get_ephys_data(cell_id)
vv, ii = [], []
for sn, st in sweeps:
v, i, t = load_sweep(ds, sn)
stim_start = np.argwhere(i != 0)[0][0]
for rep in range(reps):
idx0 = stim_start - delay - np.random.randint(0, dur // 2)
vr = v[idx0:]
ir = i[idx0:]
if st.startswith('Noise'):
offs = [0, 200000, 400000]
for off in offs:
vv.append(vr[off:off + dur])
ii.append(ir[off:off + dur])
else:
vv.append(vr[:dur])
ii.append(ir[:dur])
stims = np.vstack(ii)
resps = np.vstack(vv) + 74.0
print(stims.shape)
return stims, resps
# Just as we did before, we will will assign the output of `get_ephys_features()` to a variable and then convert it into a pandas dataframe.
# In[10]:
ephys_features = pd.DataFrame(
ctc.get_ephys_features()).set_index('specimen_id')
ephys_features.head()
# Again, we can combine our dataframe that contians the metadata of our cells with our electrophysiology dataframe to create one single dataframe.
# In[11]:
all_ephys_features = all_cells_df.join(ephys_features)
all_ephys_features.head()
# In[12]:
print(len(all_ephys_features))
# The `get_ephys_data()` method can download electrophysiology traces for a single cell in the database. This method returns a class instance with helper methods for retrieving stimulus and response traces out of an NWB file. In order to use this method, you must specify the id of the cell specimen whose electrophysiology you would like to download.
# In[13]:
specimen1_ephys = ctc.get_ephys_data(specimen_id=525011903)
#specimen1_ephys.get_experiment_sweep_numbers()
#specimen1_ephys.get_sweep(100)
#specimen1_ephys.get_sweep_metadata(100)
#specimen1_ephys.get_spike_times(100)
# For more information on the helper methods that can be used on this NwbDataSet instance, please click <a href="https://allensdk.readthedocs.io/en/latest/allensdk.core.nwb_data_set.html">here</a>
#-------------------------------------------------
if species =='mouse':
structured_data_directory='mouse_struc_data_dir' #Note that this is accessing data local to this folder not the global 'mouse_data' folder saved one directory up for convenience
elif species=='human':
structured_data_directory='human_struc_data_dir'
else:
raise Exception('species not recognized')
# sorting folders into an order (not necessary)
folders=np.sort([os.path.join(structured_data_directory, f) for f in os.listdir(structured_data_directory)])
def make_the_directory(dir):
if not os.path.exists(dir):
os.makedirs(dir)
for folder in folders:
specimen_id=int(os.path.basename(folder)[:9])
if species =='mouse':
dir_name=os.path.join(relative_path, 'mouse_nwb/specimen_'+ str(specimen_id))
elif species=='human':
dir_name=os.path.join(relative_path, 'human_nwb/specimen_'+ str(specimen_id))
else:
raise Exception('species not recognized')
make_the_directory(dir_name)
ctc.get_ephys_sweeps(specimen_id, os.path.join(dir_name, 'ephys_sweeps.json'))
ctc.get_ephys_data(specimen_id, os.path.join(dir_name, 'ephys.nwb'))
#=============================================================================== # example 1 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # a list of cell metadata for cells with reconstructions, download if necessary cells = ctc.get_cells(require_reconstruction=True) # open the electrophysiology data of one cell, download if necessary data_set = ctc.get_ephys_data(cells[0]['id']) # read the reconstruction, download if necessary reconstruction = ctc.get_reconstruction(cells[0]['id']) #=============================================================================== # example 2 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict # initialize the cache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # pick a cell to analyze specimen_id = 324257146
from allensdk.ephys.feature_extractor import EphysFeatureExtractor
from allensdk.core.cell_types_cache import CellTypesCache #Following jupyter notebook here
import pprint
pp = pprint.PrettyPrinter(indent=2)
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
from allensdk.api.queries.cell_types_api import CellTypesApi
#%%
ephys_features = ctc.get_ephys_features()
data_set = ctc.get_ephys_data(
464212183
) # For one particular specimen (later find one that has all models, follows trend in plots)
#ct = CellTypesApi()
#cells = ct.list_cells(require_reconstruction=False)
#ct.save_ephys_data(cells[0]['476218657'], 'example.nwb')
exp_sweeps = []
exp_spike_times = []
for sweep in data_set.get_sweep_numbers():
if data_set.get_sweep_metadata(sweep)['aibs_stimulus_name'] == 'Noise 1':
exp_sweeps.append(sweep)
exp_spike_times.append(data_set.get_spike_times(sweep))
fig, axes = plt.subplots(2, 1, sharex=True)
current2 = plt.subplot2grid((10, 2), (0, 1))
data2 = plt.subplot2grid((10, 2), (1, 1))
LIF_Ctgf = plt.subplot2grid((10, 2), (2, 1))
LIFR_Ctgf = plt.subplot2grid((10, 2), (3, 1))
LIFASC_Ctgf = plt.subplot2grid((10, 2), (4, 1), rowspan=2)
LIFRASC_Ctgf = plt.subplot2grid((10, 2), (6, 1), rowspan=2)
LIFRASCAT_Ctgf = plt.subplot2grid((10, 2), (8, 1), rowspan=2)
x_lim = [18, 18.3]
##---474637203Htr3a------
dir_name = os.path.join(relative_path, 'mouse_nwb/specimen_' + str(474637203))
the_sweeps = ctc.get_ephys_sweeps(474637203,
os.path.join(dir_name, 'ephys_sweeps.json'))
nwb = ctc.get_ephys_data(474637203, os.path.join(dir_name, 'ephys.json'))
sweeps = get_sweep_num_by_name(the_sweeps, 'Noise 2')
data = []
spike_times = []
for s in sweeps:
spike_times.append(nwb.get_spike_times(s))
data.append(nwb.get_sweep(s))
print 'loading LIF'
LIF_model = pickle.load(
open("pkl_data/474637203Htr3a-Cre_NO152_LIF_model.pkl", "rb"))
print 'loading LIFR'
LIFR_model = pickle.load(
open("pkl_data/474637203Htr3a-Cre_NO152_LIFR_model.pkl", "rb"))
print 'loading LIFASC'
LIFASC_model = pickle.load(
def CreateDB(specimenList, databaseName, resetDB, manifestFile,
host, user, password, verbose):
if verbose:
print "CreateDB importing..."
import sys
from allensdk.ephys.extract_cell_features import extract_cell_features
from allensdk.core.cell_types_cache import CellTypesCache
from collections import defaultdict
import mysql.connector
import numpy as np
from numpyconversion import NumpyMySQLConverter
from CellSurveyTableOps import dropTable, createDonorsTable
from CellSurveyTableOps import createSpecimensTable, createSpecimenFXsTable
from CellSurveyTableOps import createExperimentsTable, createExperimentFXsTable
from CellSurveyTableOps import addSpecimen, addExperiment, addDonor
from CellSurveyTableOps import addExpFX,addSpecFX
from ABISweepFX import getABIAnalysisPoints, ExtractSweepFeatures
#### Create the database from scratch if required
if verbose:
print "Connecting to the database";
try:
cnx = mysql.connector.connect(user=user, password=password,
host=host, database=databaseName,
converter_class=NumpyMySQLConverter)
if verbose:
print "Connection complete"
cursobj = cnx.cursor()
except:
cnx = mysql.connector.connect(user=user, password=password, host=host,
converter_class=NumpyMySQLConverter)
if verbose:
print cnx
cursobj = cnx.cursor()
mycmd = 'create database ' + databaseName
cursobj.execute(mycmd)
if verbose:
print "Database created"
mycmd = 'use ' + databaseName
cursobj.execute(mycmd)
if verbose:
print "Using database " + databaseName
if resetDB:
if verbose:
print "Dropping all tables"
tablenames = ['specimenFXs', 'experimentFXs', 'experiments',
'specimens', 'donors']
for tablename in tablenames:
result = dropTable(cnx, tablename)
if verbose:
if result:
print tablename + " table dropped"
else:
print " There was a problem dropping table " + tablename
# -----
if verbose:
print "Creating tables"
result = createDonorsTable(cnx)
if verbose:
if result:
print "Donors Table created"
else:
print "There was a problem creating the Donors Table"
result = createSpecimensTable(cnx)
if verbose:
if result:
print "Specimens Table created"
else:
print "There was a problem creating the Specimens Table"
result = createExperimentsTable(cnx)
if verbose:
if result:
print "Experiments Table created"
else:
print "There was a problem creating the Experiments Table"
result = createSpecimenFXsTable(cnx)
if verbose:
if result:
print "SpecimenFXs Table created"
else:
print "There was a problem creating the SpecimenFXs Table"
result = createExperimentFXsTable(cnx)
if verbose:
if result:
print "ExperimentFXs Table created"
else:
print "There was a problem creating the ExperimentFXs Table"
# ====================================================================
# Install the ABI Datasets
if verbose:
print "Installing the ABI Datasets into the database"; sys.stdout.flush()
# Instantiate the CellTypesCache instance.
ctc = CellTypesCache(manifest_file=manifestFile)
# Get metadata on all cells
cells = ctc.get_cells()
####### ALL DONORS #######
# Populate the donors table with all donors of all cells
if verbose:
print "Populating donors table"
for cell in cells:
addDonor(cnx, cell['donor_id'], cell['donor']['sex'], cell['donor']['name'])
####### ALL EPHYS FEATURES #######
try:
# for all cells
allEphysFeatures = ctc.get_ephys_features()
except:
# If no ephys features, we cannot do anything
print "No ephys features available; aborting program."
sys.exit()
####### SPECIMENS #######
# Get relevant info for each specimen in input list
if verbose:
print "Processing each specimen in turn"; sys.stdout.flush()
for specimen in specimenList:
# if verbose:
print '@@@@@ Processing specimen:', specimen
try:
specEphysData = ctc.get_ephys_data(specimen)
except:
# If no ephys data, we do not want to bother with it
print "No ephys data for specimen ", specimen, "; ignoring it."
continue
###### SPECIMEN >>> METADATA ######
# Paw through the cells to find the metadata for the current specimen
# The cell is a dictionary that has most of the "other" non-sweep stuff
# we need such as cell averages, rheobase info, transgenic line, hemisphere,
# age, sex, graph order, dendrite type, area, has_burst,...
# May be able to improve this search Pythonically
for cell in cells:
datasets = cell['data_sets']
for dataset in datasets:
dsspec = dataset['specimen_id']
if dsspec == specimen:
specCell = cell
break
# Add the specimen to the database
donorID = specCell['donor_id']
specimenTableIDX = addSpecimen(cnx, donorID, specimen)
####### SPECIMEN >>> SWEEPS/EXPERIMENTS #######
# Change these to true if show in any sweep
cellHasBursts = False
cellHasDelays = False
cellHasPauses = False
# Process each sweep in turn
sweeps = ctc.get_ephys_sweeps(specimen)
for sweep in sweeps:
sweepNum = sweep['sweep_number']
# if verbose:
msg = (" Processing sweep_number: " + str(sweepNum) +
" stimulus: " + str(sweep['stimulus_name']) +
" num_spikes = " + str(sweep['num_spikes']))
print msg
# Screen out some sweep types because they are not suitable for our
# simulations or because the stimulus type is not successful
# in use of process_spikes() (which we use for simulations)
databaseList = ['Long Square', 'Short Square', 'Noise 1', 'Noise 2',
'Square - 2s Suprathreshold', 'Square - 0.5ms Subthreshold',
'Short Square - Triple', 'Ramp', 'Ramp to Rheobase']
if sweep['stimulus_name'] not in databaseList:
print " Stimulus type", sweep['stimulus_name'], "not supported."
continue
# sweepData holds index range, response data vector, sampling_rate, and stimulus vector
sweepData = specEphysData.get_sweep(sweepNum)
# sweep_metadata holds aibs_stimulus_amplitude_pa, aibs_stimulus_name,
# gain, initial_access_resistance, and seal
sweep_metadata = specEphysData.get_sweep_metadata(sweepNum)
samplingRate = sweepData["sampling_rate"] # in Hz
# Need to check if this sweep is actually an experiment
# [not implemented]
# Add the experiment to the database
experimentIDX = (#
addExperiment(cnx, specimenTableIDX,
sweepNum, samplingRate,
sweep_metadata['aibs_stimulus_name'],
float(sweep_metadata['aibs_stimulus_amplitude_pa'])))
# Only Long Square is suitable for our simulations
fxOKList = ['Long Square']
if sweep['stimulus_name'] not in fxOKList:
print " Stimulus type", sweep['stimulus_name'], "entered into database but not supported for feature extractions."
continue
## Create the experiment feature extraction data ##
# This approach seen at
# http://alleninstitute.github.io/AllenSDK/_static/examples/nb/
# cell_types.html#Computing-Electrophysiology-Features
# index_range[0] is the "experiment" start index. 0 is the "sweep" start index
indexRange = sweepData["index_range"]
# For our purposes, we grab the data from the beginning of the sweep
# instead of the beginning of the experiment
# i = sweepData["stimulus"][indexRange[0]:indexRange[1]+1] # in A
# v = sweepData["response"][indexRange[0]:indexRange[1]+1] # in V
i = sweepData["stimulus"][0:indexRange[1]+1] # in A
v = sweepData["response"][0:indexRange[1]+1] # in V
i *= 1e12 # to pA
v *= 1e3 # to mV
t = np.arange(0, len(v)) * (1.0 / samplingRate) # in seconds
###### Do the sweep's feature extraction #######
# Determine the position and length of the analysis window with respect
# to the beginning of the sweep
stimType = sweep_metadata['aibs_stimulus_name']
analysisPoints = getABIAnalysisPoints(stimType)
analysis_start = analysisPoints['analysisStart']
stimulus_start = analysisPoints['stimulusStart']
analysis_duration = analysisPoints['analysisDuration']
if verbose:
print ('analysis_start', analysis_start, 'stimulus_start ',
stimulus_start, 'analysis_duration', analysis_duration)
# Trim the analysis to end of experiment if necessary
if (analysis_start + analysis_duration) * samplingRate >= indexRange[1]:
end_time = (indexRange[1]-1)/samplingRate
analysis_duration = end_time - analysis_start
if verbose:
print ('analysis_start', analysis_start, 'stimulus_start ',
stimulus_start, 'analysis_duration', analysis_duration)
# Now we extract the sweep features from that analysis window
swFXs = ExtractSweepFeatures(t, v, i, analysis_start,
analysis_duration, stimulus_start, verbose)
if len(swFXs) == 0:
print "Skipping experiment: ", specimen, '/', sweepNum, " and continuing..."
continue
if swFXs['hasBursts']: cellHasBursts = True
if swFXs['hasPauses']: cellHasPauses = True
if swFXs['hasDelay']: cellHasDelays = True
## Add the feature extraction to the database ##
expFXs = dict(swFXs)
# individual spike data not going into the database directly
if 'spikeData' in expFXs:
del expFXs['spikeData']
addExpFX(cnx, experimentIDX, expFXs)
# end of: for sweep in sweeps:
## Assemble the specimen feature extraction data ##
specimenEphysFeaturesList = [f for f in allEphysFeatures if f['specimen_id'] == specimen]
specimenEphysFeatures = specimenEphysFeaturesList[0]
data_set = ctc.get_ephys_data(specCell['id'])
sweeps = ctc.get_ephys_sweeps(specimen)
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
cell_features = (extract_cell_features(data_set, sweep_numbers['Ramp'],
sweep_numbers['Short Square'], sweep_numbers['Long Square']))
spFXs = {}
spFXs['hasSpikes'] = cell_features['long_squares']['spiking_sweeps'] != []
spFXs['hero_sweep_id'] = cell_features['long_squares']['hero_sweep']['id']
spFXs['hero_sweep_avg_firing_rate'] = cell_features['long_squares']['hero_sweep']['avg_rate']
spFXs['hero_sweep_adaptation'] = cell_features['long_squares']['hero_sweep']['adapt']
spFXs['hero_sweep_first_isi'] = cell_features['long_squares']['hero_sweep']['first_isi']
spFXs['hero_sweep_mean_isi'] = cell_features['long_squares']['hero_sweep']['mean_isi']
spFXs['hero_sweep_median_isi'] = cell_features['long_squares']['hero_sweep']['median_isi']
spFXs['hero_sweep_isi_cv'] = cell_features['long_squares']['hero_sweep']['isi_cv']
spFXs['hero_sweep_latency'] = cell_features['long_squares']['hero_sweep']['latency']
spFXs['hero_sweep_stim_amp'] = cell_features['long_squares']['hero_sweep']['stim_amp']
spFXs['hero_sweep_v_baseline'] = cell_features['long_squares']['hero_sweep']['v_baseline']
spFXs['dendrite_type'] = specCell['dendrite_type']
spFXs['electrode_0_pa'] = specimenEphysFeatures['electrode_0_pa']
spFXs['f_i_curve_slope'] = specimenEphysFeatures['f_i_curve_slope']
spFXs['fast_trough_t_long_square'] = specimenEphysFeatures['fast_trough_t_long_square']
spFXs['fast_trough_t_ramp'] = specimenEphysFeatures['fast_trough_t_ramp']
spFXs['fast_trough_t_short_square'] = specimenEphysFeatures['fast_trough_t_short_square']
spFXs['fast_trough_v_long_square'] = specimenEphysFeatures['fast_trough_v_long_square']
spFXs['fast_trough_v_ramp'] = specimenEphysFeatures['fast_trough_v_ramp']
spFXs['fast_trough_v_short_square'] = specimenEphysFeatures['fast_trough_v_short_square']
spFXs['has_bursts'] = cellHasBursts
spFXs['has_delays'] = cellHasDelays
spFXs['has_pauses'] = cellHasPauses
spFXs['hemisphere'] = specCell['hemisphere']
spFXs['input_resistance_mohm'] = specimenEphysFeatures['input_resistance_mohm']
spFXs['peak_t_long_square'] = specimenEphysFeatures['peak_t_long_square']
spFXs['peak_t_ramp'] = specimenEphysFeatures['peak_t_ramp']
spFXs['peak_t_short_square'] = specimenEphysFeatures['peak_t_short_square']
spFXs['peak_v_long_square'] = specimenEphysFeatures['peak_v_long_square']
spFXs['peak_v_ramp'] = specimenEphysFeatures['peak_v_ramp']
spFXs['peak_v_short_square'] = specimenEphysFeatures['peak_v_short_square']
spFXs['reporter_status'] = specCell['reporter_status']
spFXs['rheobase_current'] = cell_features['long_squares']['rheobase_i']
spFXs['ri'] = specimenEphysFeatures['ri']
spFXs['sagFraction'] = specimenEphysFeatures['sag']
spFXs['seal_gohm'] = specimenEphysFeatures['seal_gohm']
spFXs['slow_trough_t_long_square'] = specimenEphysFeatures['slow_trough_t_long_square']
spFXs['slow_trough_t_ramp'] = specimenEphysFeatures['slow_trough_t_ramp']
spFXs['slow_trough_t_short_square'] = specimenEphysFeatures['slow_trough_t_short_square']
spFXs['slow_trough_v_long_square'] = specimenEphysFeatures['slow_trough_v_long_square']
spFXs['slow_trough_v_ramp'] = specimenEphysFeatures['slow_trough_v_ramp']
spFXs['slow_trough_v_short_square'] = specimenEphysFeatures['slow_trough_v_short_square']
spFXs['structure_acronym'] = specCell['structure']['acronym']
spFXs['structure_name'] = specCell['structure']['name']
spFXs['tau'] = specimenEphysFeatures['tau']
spFXs['threshold_i_long_square'] = specimenEphysFeatures['threshold_i_long_square']
spFXs['threshold_i_ramp'] = specimenEphysFeatures['threshold_i_ramp']
spFXs['threshold_i_short_square'] = specimenEphysFeatures['threshold_i_short_square']
spFXs['threshold_t_long_square'] = specimenEphysFeatures['threshold_t_long_square']
spFXs['threshold_t_ramp'] = specimenEphysFeatures['threshold_t_ramp']
spFXs['threshold_t_short_square'] = specimenEphysFeatures['threshold_t_short_square']
spFXs['threshold_v_long_square'] = specimenEphysFeatures['threshold_v_long_square']
spFXs['threshold_v_ramp'] = specimenEphysFeatures['threshold_v_ramp']
spFXs['threshold_v_short_square'] = specimenEphysFeatures['threshold_v_short_square']
spFXs['transgenic_line'] = specCell['transgenic_line']
spFXs['trough_t_long_square'] = specimenEphysFeatures['trough_t_long_square']
spFXs['trough_t_ramp'] = specimenEphysFeatures['trough_t_ramp']
spFXs['trough_t_short_square'] = specimenEphysFeatures['trough_t_short_square']
spFXs['trough_v_long_square'] = specimenEphysFeatures['trough_v_long_square']
spFXs['trough_v_ramp'] = specimenEphysFeatures['trough_v_ramp']
spFXs['trough_v_short_square'] = specimenEphysFeatures['trough_v_short_square']
spFXs['upstroke_downstroke_ratio_long_square'] \
= specimenEphysFeatures['upstroke_downstroke_ratio_long_square']
spFXs['upstroke_downstroke_ratio_ramp'] \
= specimenEphysFeatures['upstroke_downstroke_ratio_ramp']
spFXs['upstroke_downstroke_ratio_short_square'] \
= specimenEphysFeatures['upstroke_downstroke_ratio_short_square']
spFXs['v_rest'] = specimenEphysFeatures['vrest']
spFXs['vm_for_sag'] = specimenEphysFeatures['vm_for_sag']
## Add the specimen feature extraction data to the database ##
addSpecFX(cnx, specimenTableIDX, spFXs)
# end of: for specimen in specimenList
cnx.close()
# Combine our metadata with our electrophysiology data all_ephys_features = all_cells_df.join(ephys_features) all_ephys_features.head() # The `get_ephys_data()` method can download electrophysiology traces for a single cell in the database. This method returns a class instance with helper methods for retrieving stimulus and response traces out of an NWB file. In order to use this method, you must specify the id of the cell specimen whose electrophysiology you would like to download. # The `get_experiment_sweep_numbers()` method downloads all of the sweep numbers for experiments in the file. Each sweep contains metadata and electrophysiology data. # In[6]: # Select cell id cell_id_2 = 525011903 # Get electrophysiological traces of our cell specimen_ephys_data = ctc.get_ephys_data(specimen_id=cell_id_2) # Retrieve sweep numbers for cell sweep_numbers = specimen_ephys_data.get_experiment_sweep_numbers() print(sweep_numbers[0:10]) # Now that we have sweep numbers to choose from, we can take a look at a sweep's metadata by calling `get_sweep_metadata()`. This return a dictionary contianing information such as stimulus paramaters and recording quality. # In[8]: # Select a sweep number sweep_number = 100 # Retrieve metadata for selected sweep specimen_metadata = specimen_ephys_data.get_sweep_metadata(sweep_number) print(specimen_metadata)
import allensdk.core.json_utilities as json_utilities
neuronal_model_id = 566302806
# download model metadata
glif_api = GlifApi()
nm = glif_api.get_neuronal_models_by_id([neuronal_model_id])[0]
# download the model configuration file
nc = glif_api.get_neuron_configs([neuronal_model_id])[neuronal_model_id]
neuron_config = glif_api.get_neuron_configs([neuronal_model_id])
json_utilities.write('neuron_config.json', neuron_config)
# download information about the cell
ctc = CellTypesCache()
ctc.get_ephys_data(nm['specimen_id'], file_name='stimulus.nwb')
ctc.get_ephys_sweeps(nm['specimen_id'], file_name='ephys_sweeps.json')
#===============================================================================
# example 2
#===============================================================================
import allensdk.core.json_utilities as json_utilities
from allensdk.model.glif.glif_neuron import GlifNeuron
# initialize the neuron
neuron_config = json_utilities.read('neuron_config.json')['566302806']
neuron = GlifNeuron.from_dict(neuron_config)
# make a short square pulse. stimulus units should be in Amps.
stimulus = [ 0.0 ] * 100 + [ 10e-9 ] * 100 + [ 0.0 ] * 100
from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache() # a list of cell metadata for cells with reconstructions, download if necessary cells = ctc.get_cells(require_reconstruction=True) # open the electrophysiology data of one cell, download if necessary data_set = ctc.get_ephys_data(cells[0]['id']) # read the reconstruction, download if necessary reconstruction = ctc.get_reconstruction(cells[0]['id'])
import allensdk.core.json_utilities as json_utilities
neuronal_model_id = 566302806
# download model metadata
glif_api = GlifApi()
nm = glif_api.get_neuronal_models_by_id([neuronal_model_id])[0]
# download the model configuration file
nc = glif_api.get_neuron_configs([neuronal_model_id])[neuronal_model_id]
neuron_config = glif_api.get_neuron_configs([neuronal_model_id])
json_utilities.write('neuron_config.json', neuron_config)
# download information about the cell
ctc = CellTypesCache()
ctc.get_ephys_data(nm['specimen_id'], file_name='stimulus.nwb')
ctc.get_ephys_sweeps(nm['specimen_id'], file_name='ephys_sweeps.json')
import allensdk.core.json_utilities as json_utilities
from allensdk.model.glif.glif_neuron import GlifNeuron
# initialize the neuron
neuron_config = json_utilities.read('neuron_config.json')['566302806']
neuron = GlifNeuron.from_dict(neuron_config)
# make a short square pulse. stimulus units should be in Amps.
stimulus = [0.0] * 100 + [10e-9] * 100 + [0.0] * 100
# important! set the neuron's dt value for your stimulus in seconds
neuron.dt = 5e-6
# simulate the neuron
def allen_to_model_and_features(content):
data_set, sweeps, specimen_id = content
sweep_numbers_ = defaultdict(list)
for sweep in sweeps:
sweep_numbers_[sweep['stimulus_name']].append(sweep['sweep_number'])
try:
sweep_numbers = data_set.get_sweep_numbers()
except:
print('erroneous deletion of relevant ephys.nwb file')
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
file_name = 'cell_types/specimen_' + str(specimen_id) + '/ephys.nwb'
data_set_nwb = NwbDataSet(file_name)
try:
sweep_numbers = data_set_nwb.get_sweep_numbers()
except:
return None
for sn in sweep_numbers:
spike_times = data_set.get_spike_times(sn)
sweep_data = data_set.get_sweep(sn)
##
# cell_features = extract_cell_features(data_set, sweep_numbers_['Ramp'],sweep_numbers_['Short Square'],sweep_numbers_['Long Square'])
##
cell_features = None
if cell_features is not None:
spiking_sweeps = cell_features['long_squares']['spiking_sweeps'][0]
multi_spike_features = cell_features['long_squares']['hero_sweep']
biophysics = cell_features['long_squares']
shapes = cell_features['long_squares']['spiking_sweeps'][0]['spikes'][
0]
supras = [
s for s in sweeps
if s['stimulus_name'] == str('Square - 2s Suprathreshold')
]
if len(supras) == 0:
return None
supra_numbers = [s['sweep_number'] for s in supras]
smallest_multi = 1000
all_currents = []
temp_vm = None
for sn in supra_numbers:
spike_times = data_set.get_spike_times(sn)
sweep_data = data_set.get_sweep(sn)
if len(spike_times) == 1:
inj_rheobase = np.max(sweep_data['stimulus'])
temp_vm = sweep_data['response']
break
if len(spike_times) < smallest_multi and len(spike_times) > 1:
smallest_multi = len(spike_times)
inj_multi_spike = np.max(sweep_data['stimulus'])
inj_rheobase = inj_multi_spike
temp_vm = sweep_data['response']
spike_times = data_set.get_spike_times(supras[-1]['sweep_number'])
sweep_data = data_set.get_sweep(supras[-1]['sweep_number'])
sd = sweep_data['stimulus']
# sampling rate is in Hz
sampling_rate = sweep_data['sampling_rate']
inj = AnalogSignal(sd, sampling_rate=sampling_rate * qt.Hz, units=qt.pA)
indexs = np.where(sd == np.max(sd))[0]
if temp_vm is None:
return (None, None, None, None)
vm = AnalogSignal(temp_vm, sampling_rate=sampling_rate * qt.Hz, units=qt.V)
sm = models.StaticModel(vm)
sm.allen = None
sm.allen = True
sm.protocol = {}
sm.protocol['Time_Start'] = inj.times[indexs[0]]
sm.protocol['Time_End'] = inj.times[indexs[-1]]
sm.name = specimen_id
sm.data_set = data_set
sm.sweeps = sweeps
sm.inject_square_current = MethodType(inject_square_current, sm)
sm.get_membrane_potential = MethodType(get_membrane_potential, sm)
sm.rheobase_current = inj_rheobase
current = {}
current['amplitude'] = sm.rheobase_current
sm.vm_rheobase = sm.inject_square_current(current)
try:
import asciiplotlib as apl
fig = apl.figure()
fig.plot([float(t) for t in sm.vm30.times],
[float(v) for v in sm.vm30],
label="data",
width=100,
height=80)
fig.show()
import asciiplotlib as apl
fig = apl.figure()
fig.plot([float(t) for t in sm.vm15.times],
[float(v) for v in sm.vm15],
label="data",
width=100,
height=80)
fig.show()
except:
pass
sm.get_spike_count = MethodType(get_spike_count, sm)
subs = [
s for s in sweeps
if s['stimulus_name'] == str('Square - 0.5ms Subthreshold')
]
sub_currents = [(s['stimulus_absolute_amplitude'] * qt.A).rescale(qt.pA)
for s in subs]
if len(sub_currents) == 3:
sm.druckmann2013_input_resistance_currents = [
sub_currents[0], sub_currents[1], sub_currents[2]
]
elif len(sub_currents) == 2:
sm.druckmann2013_input_resistance_currents = [
sub_currents[0], sub_currents[0], sub_currents[1]
]
elif len(sub_currents) == 1:
# unfortunately only one inhibitory current available here.
sm.druckmann2013_input_resistance_currents = [
sub_currents[0], sub_currents[0], sub_currents[0]
]
try:
sm.inject_square_current(sub_currents[0])
except:
pass
get_15_30(sm, inj_rheobase)
everything = (sm, sweep_data, cell_features, vm)
return everything
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
# initialize the cache
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
# pick a cell to analyze
specimen_id = 324257146
# download the ephys data and sweep metadata
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(data_set, sweep_numbers['Ramp'],
sweep_numbers['Short Square'],
sweep_numbers['Long Square'])
def allen_obs_data(ephys_cell=464212183,
sweep_number=33,
A_soma=np.pi * (70. * 1e-4)**2):
"""Data for x_o. Cell from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
dir_cache = './support_files'
real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
else:
def pickle_load(file):
"""Loads data from file."""
f = open(file, 'rb')
data = pickle.load(f, encoding='latin1')
f.close()
return data
real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
real_data_path)
t = np.arange(0, duration, dt)
# external current
I = I_real_data / A_soma # muA/cm2
# return real_data_obs, I_obs
return {
'data': real_data_obs.reshape(-1),
'time': t,
'dt': dt,
'I': I.reshape(-1),
't_on': t_on,
't_off': t_off
}
return True
# sort the data so that specifying start and end integers works
folders=np.sort([os.path.join(structured_data_directory, f) for f in os.listdir(structured_data_directory)])
sp_id_to_remove=[]
for ii, specimen_id_directory in enumerate(folders):
print 'looking at', ii, 'of', len(folders) #prints the sequential file number being run
specimen_id=int(os.path.basename(specimen_id_directory)[:9])
sweeps_file=os.path.join(nwb_directory,'specimen_'+ str(specimen_id), 'ephys_sweeps.json')
nwb_file=os.path.join(nwb_directory,'specimen_'+ str(specimen_id), 'ephys.nwb')
# load files
the_sweeps=ctc.get_ephys_sweeps(specimen_id, sweeps_file)
nwb=ctc.get_ephys_data(specimen_id, nwb_file)
n1_sweeps=get_sweep_num_by_name(the_sweeps, 'Noise 1')
n2_sweeps=get_sweep_num_by_name(the_sweeps, 'Noise 2')
# check to see if their are at least two noise 1 and noise 2 sweeps in the data nwb file
if not check_more_than_two_sweeps(n1_sweeps, nwb):
print specimen_id ,"has less than two noise_1 sweeps"
logging.warning(str(specimen_id) +" has less than two noise_1 sweeps")
sp_id_to_remove.append(specimen_id)
continue
if not check_more_than_two_sweeps(n2_sweeps, nwb):
print specimen_id ,"has less than two noise_2 sweeps"
logging.warning(str(specimen_id) +" has less than two noise_2 sweeps")
sp_id_to_remove.append(specimen_id)
continue
def sdk_nwb_information(specimen_id):
ctc = CellTypesCache()
nwb_data_set = ctc.get_ephys_data(specimen_id)
sweep_info = ctc.get_ephys_sweeps(specimen_id)
return nwb_data_set.file_name, sweep_info
file = get_file_path_endswith(sub_folder, '_preprocessor_values.json')
neuron_dict = ju.read(file)
R_NO_asc = neuron_dict['resistance']['R_test_list']['mean']
R_asc = neuron_dict['resistance']['R_fit_ASC_and_R']['mean']
C = neuron_dict['capacitance']['C_test_list']['mean']
El = neuron_dict['El']['El_noise']['measured']['mean']
# get the sweeps
dir_name = os.path.join(relative_path, 'mouse_nwb/specimen_' + specimen_id)
the_sweeps = ctc.get_ephys_sweeps(int(specimen_id),
os.path.join(dir_name, 'ephys_sweeps.json'))
noise1_sweeps = get_sweep_num_by_name(the_sweeps, 'Noise 1')
# put data in the format required for functions below
n1_s1_data = ctc.get_ephys_data(int(specimen_id),
os.path.join(dir_name, 'ephys.nwb')).get_sweep(
noise1_sweeps[0])
sr = n1_s1_data['sampling_rate']
dt = 1. / sr
suthresh_i = n1_s1_data['stimulus'][n1_s1_data['index_range'][0] +
int(1.2 / dt):int(3. / dt)]
suthresh_v = []
for s in noise1_sweeps:
data = ctc.get_ephys_data(int(specimen_id),
os.path.join(dir_name, 'ephys.nwb')).get_sweep(s)
suthresh_v.append(data['response'][data['index_range'][0] +
int(1.2 / dt):int(3. / dt)])
V_NO_asc = simple_model_to_eval_subthresh_params([suthresh_i], [R_NO_asc], [C],
[El], dt)[0]
#V_asc=simple_model_to_eval_subthresh_params([suthresh_i], [R_asc], [C], [El], dt)[0]
#=============================================================================== # example 1 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # a list of cell metadata for cells with reconstructions, download if necessary cells = ctc.get_cells(require_reconstruction=True) # open the electrophysiology data of one cell, download if necessary data_set = ctc.get_ephys_data(cells[0]['id']) # read the reconstruction, download if necessary reconstruction = ctc.get_reconstruction(cells[0]['id']) #=============================================================================== # example 2 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict # initialize the cache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # pick a cell to analyze specimen_id = 324257146
sp_ids=[]
cres=[]
all_neurons=[]
for folder in folders[:2]:
specimen_id=int(os.path.basename(folder)[:9])
sp_ids.append(specimen_id)
print specimen_id
cre=os.path.basename(folder)[10:]
cres.append(cre)
# get the file and open it
#---get spike times from the data
the_sweeps=ctc.get_ephys_sweeps(specimen_id)
noise_sweeps=get_sweep_num_by_name(the_sweeps, 'Noise 2')
data=ctc.get_ephys_data(specimen_id)
noise_data=[]
noise_spike_times=[]
for s in noise_sweeps:
noise_spike_times.append(data.get_spike_times(s))
noise_data.append(data.get_sweep(s))
dt=1./noise_data[0]['sampling_rate']
stim=noise_data[0]['stimulus']
stim_len=len(stim)
#convert data times to indicies
noise1_spike_ind=[]
for d in noise_spike_times:
noise1_spike_ind.append((d/dt).astype(int))
# First get the models for the neuron
