def partition_responses(rmap_bins=36,
table='cell_information',
quality='None',
smooth=True,
just_bins=False):
from scanr.cluster import PrincipalCellCriteria, get_min_quality_criterion, AND
from scanr.data import get_node
pbins = np.array(
[1.1, 0.5, 0.1, 0.05, 0.03, 0.02, 0.01, 0.005, 0.002, 0.0])
ibins = np.linspace(0, 6, 13)
if just_bins:
return pbins, ibins
N_pbins, N_ibins = len(pbins) - 1, len(ibins) - 1
R = [[[] for j in xrange(N_ibins)] for i in xrange(N_pbins)]
cell_table = get_node('/physiology', table)
cell_criteria = AND(PrincipalCellCriteria,
get_min_quality_criterion(quality))
for cell in cell_table.where('(area=="CA3")|(area=="CA1")'):
session = SessionData.get((cell['rat'], cell['day'], cell['session']),
load_clusters=False)
cluster = session.cluster_data(cell['tc'])
if not (cell['N_running'] > 30 and cell_criteria.filter(cluster)):
continue
pix = (cell['p_value'] <= pbins).nonzero()[0]
iix = (cell['I'] >= ibins).nonzero()[0]
if not len(pix) or not (0 <= pix[-1] < N_pbins):
continue
if not len(iix) or not (0 <= iix[-1] < N_ibins):
continue
pix = pix[-1]
iix = iix[-1]
R[pix][iix].append(
session.get_cluster_ratemap(cluster,
bins=rmap_bins,
smoothing=smooth,
blur_width=360. / rmap_bins,
exclude_off_track=True,
exclude=session.scan_and_pause_list))
print '...added %s to p-value bin %.4f, info bin %.2f...' % (
cell['tc'], pbins[pix], ibins[iix])
for i, row in enumerate(R):
for j, rmap in enumerate(row):
R[i][j] = r = np.asarray(rmap)
if not len(rmap):
continue
R[i][j] = r[np.argsort(np.argmax(r, axis=1))]
return R
def plot_scan_example(self, rds=(95,4,1), scan_number=22, margin=1.0):
"""Plot example scan with relevant behavior variables and scan phases
"""
data = SessionData.get(rds)
scan_table = get_node('/behavior', 'scans')
scan = get_unique_row(scan_table,
data.session_query + '&(number==%d)'%scan_number)
t_scan = { k: data.T_(scan[k]) for k in ScanPoints }
window = t_scan['downshift'] - margin, t_scan['upshift'] + margin
plt.ioff()
f = self.new_figure('scan_example',
'Scan Example: Rat %d, Day %d, M%d @ t=%.2f'%(rds + (window[0],)))
traj = data.trajectory
t = data.T_(traj.ts)
s = time_slice(t, start=window[0], end=window[1])
scan_slice = time_slice(t, start=t_scan['start'], end=t_scan['end'])
y = 0
dy = -2
norm = lambda x: x / np.max(np.abs(x))
ax = f.add_subplot(111)
ax.axhline(y, c='k', ls='-', zorder=0)
ax.plot(t[s], y + norm(traj.radius[s]), 'b-', lw=2, zorder=1); y += dy
ax.axhline(y, c='k', ls='-', zorder=0)
ax.plot(t[s], y + norm(traj.radial_velocity[s]), 'b-', lw=2, zorder=1); y += dy
ax.axhline(y, c='k', ls='-', zorder=0)
ax.plot(t[s], y + norm(traj.forward_velocity[s]), 'b-', lw=2, zorder=1); y += dy
ax.axvspan(t_scan['downshift'], t_scan['start'], lw=0, fc='g', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['start'], t_scan['max'], lw=0, fc='m', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['max'], t_scan['return'], lw=0, fc='y', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['return'], t_scan['end'], lw=0, fc='m', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['end'], t_scan['upshift'], lw=0, fc='g', alpha=0.3, zorder=-2)
for k in ScanPoints:
ax.axvline(t_scan[k], c='k', ls='-', zorder=-1)
ax.set_yticks([0, -2, -4])
ax.set_yticklabels(['r', 'rs', 'fwd'])
ax.set_ylim(-4.75, 1.25)
ax.set_xlim(*window)
ax.tick_params(top=False, right=False)
f = self.new_figure('scan_example_space', 'Scan Example', (4,4))
ax = f.add_subplot(111)
ax.plot(traj.x[s], traj.y[s], 'k-', alpha=0.6)
ax.plot(traj.x[scan_slice], traj.y[scan_slice], 'r-', lw=2)
ax.axis('equal')
ax.set_axis_off()
plot_track_underlay(ax=ax, ls='dotted')
plt.ion()
plt.show()
def plot_LFP_example(self, rds=(95,4,1), scan_number=22, margin=2.0):
"""Plot example LFP traces showing theta power during a head scan
"""
data = SessionData.get(rds)
scan_table = get_node('/behavior', 'scans')
scan = get_unique_row(scan_table,
data.session_query + '&(number==%d)'%scan_number)
t_scan = { k: data.T_(scan[k]) for k in ScanPoints }
window = t_scan['downshift'] - margin, t_scan['upshift'] + margin
plt.ioff()
f = self.new_figure('scan_LFP_example',
'Scan LFP Example: Rat %d, Day %d, M%d @ t=%.2f'%(rds + (window[0],)))
theta_tt = find_theta_tetrode(rds[:2], condn='(EEG==True)&(area=="CA1")')[0]
ts, EEG = get_eeg_timeseries(rds, theta_tt)
t_EEG = data.T_(ts)
data_file = self.get_data_file()
session = get_unique_row(data_file.root.sessions, data.session_query)
t_theta, x_theta = Theta.timeseries(t_EEG, EEG)
ZP_theta = Z(Theta.power(x_theta, filtered=True)) #data_file.getNode('/arrays', session['ZP_theta'])
f_theta = data_file.getNode('/arrays', session['f_theta'])
traj = data.trajectory
t_traj = data.T_(traj.ts)
s_traj = time_slice(t_traj, start=window[0], end=window[1])
s_EEG = time_slice(t_EEG, start=window[0], end=window[1])
s_theta = time_slice(t_theta, start=window[0], end=window[1])
y = 0
dy = -2
norm = lambda x: (x - np.mean(x)) / (1.1 * float(np.max(np.abs((x - np.mean(x))))))
ax = f.add_subplot(111)
ax.axhline(y, c='k', ls='-', zorder=0)
ax.plot(t_traj[s_traj], y + norm(traj.radius[s_traj]), 'b-', lw=2, zorder=1); y += dy
ax.plot(t_EEG[s_EEG], y + norm(EEG[s_EEG]), 'k-', lw=1, zorder=1); y += dy
ax.plot(t_theta[s_theta], y + norm(x_theta[s_theta]), 'k-', lw=1, zorder=1); y += dy
ax.plot(t_theta[s_theta], y + norm(ZP_theta[s_theta]), 'k-', lw=1, zorder=1); y += dy
ax.plot(t_theta[s_theta], y + norm(f_theta[s_theta]), 'k-', lw=1, zorder=1); y += dy
ax.axvspan(t_scan['downshift'], t_scan['start'], lw=0, fc='g', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['start'], t_scan['max'], lw=0, fc='m', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['max'], t_scan['return'], lw=0, fc='y', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['return'], t_scan['end'], lw=0, fc='m', alpha=0.3, zorder=-2)
ax.axvspan(t_scan['end'], t_scan['upshift'], lw=0, fc='g', alpha=0.3, zorder=-2)
ax.set_yticks([0, -2, -4, -6, -8])
ax.set_yticklabels(['r', 'EEG', 'theta', 'ZP_theta', 'f_theta'])
ax.set_ylim(-8.75, 1.25)
ax.set_xlim(*window)
ax.tick_params(top=False, right=False)
plt.ion()
plt.show()
def missing_quality(row):
rds = row['rat'], row['day'], row['session']
cell_id = (rds[0], rds[1], row['tc'])
if row['quality'] != 'None' or cell_id in cell_list:
return False
data = SessionData.get(rds, load_clusters=False)
cluster = data.cluster_data(row['tc'])
if PrincipalCellCriteria.filter(cluster):
cell_list.append(cell_id)
return True
return False
def run(self,
test='place',
place_field='pass',
min_quality='fair',
**kwds):
"""Compute I_pos and I_spike across all criterion place cells in CA3/CA1
Keyword arguments:
place_field -- 'pass', 'fail', or 'all' to restrict responses based on place
field criterion test results
test -- 'place', 'skaggs', or 'olypher' to use either the full place field test or
one of the component tests for the cell filtering
min_quality -- isolation quality threshold for filtering cells
Remaining keywords are passed to TetrodeSelect.
Returns (I_pos, I_spike) tuple of arrays for selected cell clusters.
"""
self.out = CPrint(prefix='ScatterInfo')
area_query = '(area=="CA3")|(area=="CA1")'
# Metadata for the plot title
self.place_field = place_field
self.test = test
self.quality = min_quality
if place_field == 'all':
self.test = 'place'
if test == 'place':
SpatialTest = SpatialInformationCriteria
elif test == 'skaggs':
SpatialTest = SkaggsCriteria
elif test == 'olypher':
SpatialTest = OlypherCriteria
else:
raise ValueError, 'bad test value: %s' % test
MinQuality = get_min_quality_criterion(min_quality)
CellCriteria = AND(PrincipalCellCriteria, SpikeCountCriteria,
MinQuality)
if place_field == 'pass':
CellCriteria = AND(CellCriteria, SpatialTest)
elif place_field == 'fail':
CellCriteria = AND(CellCriteria, NOT(SpatialTest))
elif place_field != 'all':
raise ValueError, 'bad place_field value: %s' % place_field
I = []
for dataset in TetrodeSelect.datasets(area_query):
rat, day = dataset
Criteria = AND(
CellCriteria,
TetrodeSelect.criterion(dataset, area_query, **kwds))
for maze in get_maze_list(*dataset):
data = SessionData.get((rat, day, maze))
for tc in data.get_clusters(request=Criteria):
cluster = data.cluster_data(tc)
I.append((cluster.I_pos, cluster.I_spike))
self.I = I = np.array(I).T
self.out('%d cell-sessions counted.' % I.shape[1])
return I[0], I[1]
def collect_data(self, area_query='(area=="CA3")|(area=="CA1")'):
"""Collate ripple, theta power across head-scan events
"""
scan_table = get_node('/behavior', 'scans')
pause_table = get_node('/behavior', 'pauses')
tetrode_query = '(%s)&(EEG==True)' % area_query
dataset_list = TetrodeSelect.datasets(tetrode_query,
allow_ambiguous=True)
# Tables and iterators
data_file = self.open_data_file()
scan_data_table = data_file.createTable('/',
'scan_data',
BehDescr,
title='Scan Data')
pause_data_table = data_file.createTable('/',
'pause_data',
BehDescr,
title='Pause Data')
ripple_data_table = data_file.createTable('/',
'ripple_data',
RippleDescr,
title='Ripple Data')
scan_row = scan_data_table.row
pause_row = pause_data_table.row
ripple_row = ripple_data_table.row
for dataset in dataset_list:
rat, day = dataset
# Find the tetrode based on the chosen tetrode strategy
roi_tt = find_theta_tetrode(dataset, condn=tetrode_query)
if type(roi_tt) is tuple:
roi_tt = roi_tt[0]
self.out('Rat%03d-%02d: using tetrode Sc%d' % (rat, day, roi_tt))
# Loop through sessions
for session in get_maze_list(rat, day):
rds = rat, day, session
data = SessionData.get(rds)
ts, EEG = get_eeg_timeseries(rds, roi_tt)
ripple_list = Ripple.detect(ts, EEG)
if len(ripple_list):
ripple_peaks = np.array(ripple_list)[:, 1]
else:
ripple_peaks = np.array([])
ts_theta, x_theta = Theta.timeseries(ts, EEG)
zpow = (lambda x:
(x - x.mean()) / x.std())(Theta.power(x_theta,
filtered=True))
# Loop through scans and pauses
for row, table in [(scan_row, scan_table),
(pause_row, pause_table)]:
for rec in table.where(data.session_query):
theta = zpow[select_from(ts_theta, [rec['tlim']])]
row['id'] = rec['id']
row['rat'] = rat
row['theta_avg'] = theta.mean()
row['theta_max'] = theta.max()
row['ripples'] = select_from(ripple_peaks,
[rec['tlim']]).sum()
row.append()
scan_data_table.flush()
pause_data_table.flush()
# Loop through ripples
zpow_t = interp1d(ts_theta,
zpow,
fill_value=0.0,
bounds_error=False)
for t_ripple in ripple_peaks:
ripple_row['rat'] = rat
ripple_row['theta'] = zpow_t(
t_ripple) # interpolate z-power at ripple peak
ripple_row['running'] = data.velocity_filter(t_ripple)
ripple_row['scan'] = np.any(
select_from([t_ripple], data.scan_list))
ripple_row['pause'] = np.any(
select_from([t_ripple], data.pause_list))
ripple_row.append()
ripple_data_table.flush()
self.out('All done!')
def collect_data(self):
"""Collate theta power and head-scan events across CA1 datasets
"""
tetrode_query = '(area=="CA1")&(EEG==True)'
scan_table = get_node('/behavior', 'scans')
potentiation_table = get_node('/physiology', 'potentiation')
dataset_list = TetrodeSelect.datasets(tetrode_query,
allow_ambiguous=True)
psd_kwds = dict(Fs=1001.0,
NFFT=2048,
noverlap=1024,
scale_by_freq=True)
# psd_kwds = dict(Fs=FullBand.fs, NFFT=256, noverlap=0, scale_by_freq=True)
scan_psd = {}
pause_psd = {}
running_psd = {}
for rat, day in dataset_list:
theta_tt, base_theta = find_theta_tetrode((rat, day),
condn=tetrode_query,
ambiguous=True)
self.out('Rat%03d-%02d: using tetrode Sc%d' % (rat, day, theta_tt))
lfp = np.array([])
scan_lfp = np.array([])
pause_lfp = np.array([])
running_lfp = np.array([])
for session in get_maze_list(rat, day):
self.out('Adding data from session %d...' % session)
rds = rat, day, session
data = SessionData.get(rds, load_clusters=False)
ts, EEG = get_eeg_timeseries(rds, theta_tt)
ts_full, x_full = ts, EEG #FullBand._downsample(ts), FullBand._decimate(EEG)
running_ix = data.filter_tracking_data(ts_full,
boolean_index=True,
**data.running_filter())
lfp = np.r_[lfp, x_full]
scan_lfp = np.r_[scan_lfp,
x_full[select_from(ts_full, data.scan_list)]]
pause_lfp = np.r_[
pause_lfp, x_full[select_from(ts_full, data.pause_list)]]
running_lfp = np.r_[running_lfp, x_full[running_ix]]
self.out('Computing and normalizing spectra...')
Pxx, freqs = psd(lfp, **psd_kwds)
Pxx_scan = np.squeeze(psd(scan_lfp, **psd_kwds)[0])
Pxx_pause = np.squeeze(psd(pause_lfp, **psd_kwds)[0])
Pxx_running = np.squeeze(psd(running_lfp, **psd_kwds)[0])
if 'freqs' not in self.results:
self.results['freqs'] = freqs
full_power = np.trapz(Pxx, x=freqs)
for P in Pxx_scan, Pxx_pause, Pxx_running:
P /= full_power
if rat in scan_psd:
scan_psd[rat] = np.vstack((scan_psd[rat], Pxx_scan))
pause_psd[rat] = np.vstack((pause_psd[rat], Pxx_pause))
running_psd[rat] = np.vstack((running_psd[rat], Pxx_running))
else:
scan_psd[rat] = Pxx_scan[np.newaxis]
pause_psd[rat] = Pxx_pause[np.newaxis]
running_psd[rat] = Pxx_running[np.newaxis]
rat_list = sorted(scan_psd.keys())
self.out('Averaging spectra for %d rats...' % len(rat_list))
scan_spectra = np.empty((len(rat_list), len(freqs)), 'd')
pause_spectra = np.empty_like(scan_spectra)
running_spectra = np.empty_like(scan_spectra)
for i, rat in enumerate(rat_list):
scan_spectra[i] = scan_psd[rat].mean(axis=0)
pause_spectra[i] = pause_psd[rat].mean(axis=0)
running_spectra[i] = running_psd[rat].mean(axis=0)
self.results['rat_list'] = np.array(rat_list)
self.results['scan_psd'] = scan_spectra
self.results['pause_psd'] = pause_spectra
self.results['running_psd'] = running_spectra
self.out('All done!')
def collect_data(self, area='CA1'):
SessionDescr = {
'id': tb.UInt16Col(pos=1),
'rat': tb.UInt16Col(pos=2),
'day': tb.UInt16Col(pos=3),
'session': tb.UInt16Col(pos=4),
'start': tb.UInt64Col(pos=5),
'type': tb.StringCol(itemsize=4, pos=6),
't_theta': tb.StringCol(itemsize=16, pos=7),
'P_theta': tb.StringCol(itemsize=16, pos=8),
'f_theta': tb.StringCol(itemsize=16, pos=9),
'speed': tb.StringCol(itemsize=16, pos=10),
'radial_velocity': tb.StringCol(itemsize=16, pos=11),
'hd_velocity': tb.StringCol(itemsize=16, pos=12)
}
def get_area_query(area):
if area == "CAX":
return '(area=="CA1")|(area=="CA3")'
return 'area=="%s"' % area
tetrode_query = '(%s)&(EEG==True)' % get_area_query(area)
self.out('Using tetrode query: %s' % tetrode_query)
self.results['scan_points'] = ('start', 'max', 'return', 'end')
dataset_list = TetrodeSelect.datasets(tetrode_query)
def get_dataset_sessions():
sessions = []
for dataset in dataset_list:
for maze in get_maze_list(*dataset):
sessions.append(dataset + (maze, ))
return sessions
session_list = get_dataset_sessions()
self.results['rats'] = rat_list = sorted(
list(set(map(lambda d: d[0], dataset_list))))
self.results['N_rats'] = len(rat_list)
data_file = self.open_data_file()
array_group = data_file.createGroup('/',
'arrays',
title='Scan and Signal Arrays')
session_table = data_file.createTable(
'/', 'sessions', SessionDescr,
'Sessions for Scan Cross-Correlation Analysis')
id_fmt = 'data_%06d'
array_id = 0
session_id = 0
row = session_table.row
remove = []
for rds in session_list:
rds_str = 'rat%d-%02d-m%d' % rds
data = SessionData.get(rds)
theta_tt = find_theta_tetrode(rds[:2], condn=tetrode_query)
if theta_tt is None:
remove.append(rds)
continue
theta_tt = theta_tt[0]
row['id'] = session_id
row['rat'], row['day'], row['session'] = rds
row['type'] = (data.attrs['type']
in ('STD', 'MIS')) and 'DR' or 'NOV'
row['start'] = data.start
EEG = get_eeg_timeseries(rds, theta_tt)
if EEG is None:
remove.append(rds)
continue
ts_theta, x_theta = Theta.timeseries(*EEG)
t_theta = data.T_(ts_theta)
P_theta = zscore(Theta.power(x_theta, filtered=True))
f_theta = Theta.frequency(x_theta, filtered=True)
speed = data.F_('speed')(t_theta)
radial_velocity = np.abs(data.F_('radial_velocity')(t_theta))
hd_velocity = np.abs(data.F_('hd_velocity')(t_theta))
session_signals = [('t_theta', t_theta), ('P_theta', P_theta),
('f_theta', f_theta), ('speed', speed),
('radial_velocity', radial_velocity),
('hd_velocity', hd_velocity)]
for k, d in session_signals:
data_file.createArray(array_group,
id_fmt % array_id,
d,
title='%s : %s' % (rds_str, k))
row[k] = id_fmt % array_id
array_id += 1
self.out('Saved data from %s.' % rds_str)
row.append()
if array_id % 10 == 0:
session_table.flush()
session_id += 1
for rds in remove:
session_list.remove(rds)
self.results['sessions'] = session_list
self.results['N_sessions'] = len(session_list)
self.results['signals'] = ('P_theta', 'f_theta', 'speed',
'radial_velocity', 'hd_velocity')
session_table.flush()
self.close_data_file()
self.out('All done!')
def collect_data(self):
"""Create a data structure with theta power/frequency samples with
corresponding instantaneous velocity measurements such as path
speed, head direction velocity, and radial velocity
"""
velocity_moments = ('speed', 'radial_velocity', 'hd_velocity')
self.results['velocity_moments'] = velocity_moments
tetrode_query = '(area=="CA1")&(EEG==True)'
dataset_list = TetrodeSelect.datasets(tetrode_query,
allow_ambiguous=True)
samples = AutoVivification()
def initialize_rat_samples(rat):
for v_name in velocity_moments:
samples[rat][v_name] = np.array([], float)
samples[rat]['power'] = np.array([], float)
samples[rat]['frequency'] = np.array([], float)
def add_velocity_samples(rat, session, t):
for moment in velocity_moments:
add_data_sample(rat, moment, session.F_(moment)(t))
def add_data_sample(rat, key, data):
samples[rat][key] = np.r_[samples[rat][key], data]
for rat, day in dataset_list:
theta_tt, base_theta = find_theta_tetrode((rat, day),
condn=tetrode_query,
ambiguous=True)
if rat not in samples:
initialize_rat_samples(rat)
for maze in get_maze_list(rat, day):
rds = rat, day, maze
self.out('Session rat%03d-%02d-m%d: tetrode Sc%02d' %
(rds + (theta_tt, )))
session = SessionData.get(rds, load_clusters=False)
EEG = get_eeg_timeseries(rds, theta_tt)
if EEG is None:
continue
ts, x = EEG
ts_theta, x_theta = Theta.timeseries(ts, x)
P_theta = zscore(Theta.power(x_theta, filtered=True))
f_theta = Theta.frequency(x_theta, filtered=True)
ix_scanning = select_from(ts_theta, session.scan_list)
t_theta_scanning = session.T_(ts_theta[ix_scanning])
add_velocity_samples(rat, session, t_theta_scanning)
add_data_sample(rat, 'power', P_theta[ix_scanning])
add_data_sample(rat, 'frequency', f_theta[ix_scanning])
rat_list = sorted(list(set(samples.keys())))
self.out('Finished collected data for %d rats.' % len(rat_list))
sample_description = {k: tb.FloatCol() for k in velocity_moments}
sample_description.update(rat=tb.UInt16Col(),
power=tb.FloatCol(),
frequency=tb.FloatCol())
data_file = self.open_data_file()
results_table = data_file.createTable(
'/',
'theta_velocity',
sample_description,
title='Theta and Velocity Data Across Rats')
row = results_table.row
self.out('Generating results table...')
c = 0
for rat in rat_list:
N = samples[rat]['power'].size
self.out('Adding rat %d, with %d samples.' % (rat, N))
assert len(set(samples[rat][k].size
for k in samples[rat].keys())) == 1
for i in xrange(N):
row['rat'] = rat
row['power'] = samples[rat]['power'][i]
row['frequency'] = samples[rat]['frequency'][i]
for moment in velocity_moments:
row[moment] = samples[rat][moment][i]
row.append()
if c % 100 == 0:
results_table.flush()
if c % 500 == 0:
self.out.printf('.')
c += 1
self.out.printf('\n')
self.out('Done!')
self.close_data_file()
def collect_data(self,
band='theta',
distro='frequency',
drange=None,
nbins=128):
"""Collate frequency and head-scan events across CA1 datasets
"""
self.results['distro'] = distro
tetrode_query = '(area=="CA1")&(EEG==True)'
dataset_list = TetrodeSelect.datasets(tetrode_query,
allow_ambiguous=True)
Band = get_filter(band)
was_zero_lag = Band.zero_lag
Band.zero_lag = True
self.results['band'] = band
# Rat accumulators
rat_number = []
running = {}
scan = {}
pause = {}
#if distro == 'frequency':
# smoothing = THETA_FREQ_SMOOTHING
#elif distro == 'power':
# self.out('warning: power smoothing not implemented')
# smoothing = THETA_POWER_SMOOTHING
#else:
# raise ValueError, 'distro must be frequency or power'
for dataset in dataset_list:
rat, day = dataset
roi_tt, base_theta = find_theta_tetrode(dataset,
condn=tetrode_query,
ambiguous=True)
self.out('Rat%03d-%02d: using tetrode Sc%d' % (rat, day, roi_tt))
if rat not in rat_number:
rat_number.append(rat)
running[rat] = np.array([], 'd')
scan[rat] = np.array([], 'd')
pause[rat] = np.array([], 'd')
for session in get_maze_list(rat, day):
rds = rat, day, session
data = SessionData.get(rds, load_clusters=False)
self.out('Collating %s data for rat%03d-%02d-m%d...' %
(distro, rat, day, session))
ts, x = Band.timeseries(*get_eeg_timeseries(rds, roi_tt))
if distro == 'frequency':
sig = Band.frequency(x, filtered=True)
elif distro == 'power':
sig = zscore(Band.power(x, filtered=True))
running_ix = data.filter_tracking_data(ts,
boolean_index=True,
**data.running_filter())
running[rat] = np.r_[running[rat], sig[running_ix]]
scan[rat] = np.r_[scan[rat],
sig[select_from(ts, data.scan_list)]]
pause[rat] = np.r_[pause[rat],
sig[select_from(ts, data.pause_list)]]
# Initialize data storage and accumulators
running_pdf = []
running_cdf = []
running_mu = []
scan_pdf = []
scan_cdf = []
scan_mu = []
scan_p = []
pause_pdf = []
pause_cdf = []
pause_mu = []
pause_p = []
scan_pause_p = []
# Setup distribution bins
if drange is not None:
bins = np.linspace(drange[0], drange[1], nbins + 1)
elif distro == 'frequency':
bins = np.linspace(CfgBand[band][0], CfgBand[band][1], nbins + 1)
elif distro == 'power':
bins = np.linspace(-2, 3, nbins + 1)
self.results['centers'] = (bins[1:] + bins[:-1]) / 2
def sig_distro(data, cdf=False):
distro = KT_estimate(np.histogram(data, bins=bins)[0])
if cdf:
distro = np.cumsum(distro) / np.sum(distro)
return distro
for rat in rat_number:
self.out('Computing distributions and stats for rat %d...' % rat)
running_pdf.append(sig_distro(running[rat]))
scan_pdf.append(sig_distro(scan[rat]))
pause_pdf.append(sig_distro(pause[rat]))
running_cdf.append(sig_distro(running[rat], cdf=True))
scan_cdf.append(sig_distro(scan[rat], cdf=True))
pause_cdf.append(sig_distro(pause[rat], cdf=True))
running_mu.append(running[rat].mean())
scan_mu.append(scan[rat].mean())
D, pval = st.ks_2samp(scan[rat], running[rat])
scan_p.append(pval)
pause_mu.append(pause[rat].mean())
D, pval = st.ks_2samp(pause[rat], running[rat])
pause_p.append(pval)
D, pval = st.ks_2samp(scan[rat], pause[rat])
scan_pause_p.append(pval)
# Store results data
self.results['rat_number'] = np.array(rat_number)
self.results['running_pdf'] = np.array(running_pdf)
self.results['scan_pdf'] = np.array(scan_pdf)
self.results['pause_pdf'] = np.array(pause_pdf)
self.results['running_cdf'] = np.array(running_cdf)
self.results['scan_cdf'] = np.array(scan_cdf)
self.results['pause_cdf'] = np.array(pause_cdf)
self.results['running_mu'] = np.array(running_mu)
self.results['scan_mu'] = np.array(scan_mu)
self.results['scan_p'] = np.array(scan_p)
self.results['pause_mu'] = np.array(pause_mu)
self.results['pause_p'] = np.array(pause_p)
self.results['scan_pause_p'] = np.array(scan_pause_p)
# Good-bye!
Band.zero_lag = was_zero_lag
self.out('All done!')
def collect_data(self, dataset=(57, 1), lag=0.25):
"""Detect all ripples in dataset and plot EEG, ripple-band, and power
signals along with detected event boundaries
"""
ripple_table = get_node('/physiology', 'ripples')
tetrode_query = '(area=="CA1")&(EEG==True)'
dataset_query = '(rat==%d)&(day==%d)' % dataset
pyr_tetrodes = find_pyramidale_tetrodes(dataset, verbose=False)
rat, day = dataset
# Initialize accumulators
time_slices = []
EEG_slices = []
power_slices = []
events = []
timestamps = []
Ripple = RippleFilter()
# Loop through sessions, detecting and storing ripple slices
for rds in unique_sessions(ripple_table, condn=dataset_query):
data = SessionData.get(rds)
self.out('Loading data for rat%03d-%02d-m%d...' % rds)
ts = None
EEG = None
P = None
for tt in pyr_tetrodes:
X = get_eeg_timeseries(rds, tt)
if X is None:
continue
if ts is None:
ts = X[0]
if EEG is None:
EEG = X[1]
else:
EEG = np.vstack((EEG, X[1]))
if P is None:
P = Ripple.power(X[1])
else:
P = np.vstack((P, Ripple.power(X[1])))
if P.ndim == 2:
P = np.mean(P, axis=0)
ts_ripples = [(rec['start'], rec['peak'], rec['end'])
for rec in ripple_table.where(data.session_query)]
t = data.T_(ts)
for timing in ts_ripples:
start, peak, end = data.T_(timing)
chunk = time_slice(t, peak - lag, peak + lag)
time_slices.append(t[chunk] - peak)
EEG_slices.append(EEG[..., chunk])
power_slices.append(P[chunk])
events.append((start - peak, end - peak))
timestamps.append(timing[1])
self.out('Plotting EEG traces of ripple events...')
LW = 0.4
norm = lambda x: x.astype('d') / float(CLIP)
for i, ax in self.get_plot(range(len(time_slices))):
t_chunk = time_slices[i]
traces = EEG_slices[i]
if traces.ndim == 1:
ax.plot(t_chunk,
norm(traces),
'k-',
lw=1.5 * LW,
alpha=1,
zorder=0)
else:
ax.plot(t_chunk,
norm(traces).T,
'k-',
lw=LW,
alpha=0.5,
zorder=-1)
ax.plot(t_chunk,
norm(np.mean(traces, axis=0)),
'k-',
lw=LW,
alpha=1,
zorder=0)
ax.plot(t_chunk,
power_slices[i] / power_slices[i].max(),
'b-',
lw=1.5 * LW,
alpha=1,
zorder=1)
ax.axhline(0, ls='-', c='k', lw=LW, zorder=0)
ax.axvline(events[i][0], ls='-', c='k', lw=LW, zorder=2)
ax.axvline(0, ls=':', c='k', lw=LW, zorder=2, alpha=0.5)
ax.axvline(events[i][1], ls='-', c='k', lw=LW, zorder=2)
ax.set_xlim(-lag, lag)
ax.set_ylim(-1, 1)
ax.set_axis_off()
quicktitle(ax, '%d' % timestamps[i], size='xx-small')
self.out.printf('.')
self.out.printf('\n')
def precache_sessions():
[
SessionData.get(rds, load_clusters=False)
for rds in unique_sessions(scan_cell_table)
]
def create_outcome_table(self,
event_table='potentiation',
half_windows=HALF_WINDOWS_DEFAULT):
outcome_table_description = dict(scan_cell_id=tb.UInt32Col(pos=1))
window_cols = map(lambda h: 'window_%d' % h, half_windows)
def add_halfwindow_column_descriptors(descr):
pos = 2
for col in window_cols:
descr[col] = tb.BoolCol(pos=pos)
pos += 1
add_halfwindow_column_descriptors(outcome_table_description)
data_file = self.get_data_file(mode='a')
outcome_table = create_table(data_file,
'/',
'hit_outcome',
outcome_table_description,
title='Place-Field Event (Hit) Outcomes')
row = outcome_table.row
scan_cell_table = data_file.root.scan_cell_info
scans = get_node('/behavior', 'scans')
potentiation = get_node('/physiology', event_table)
outcome_table._v_attrs['event_table'] = event_table
# Adjust hit angle for depotentiation hack, in which event is actually last active traversal
hit_angle = -360.0 # forward one lap
if event_table == 'depotentiation':
hti_angle = 0.0 # same lap
def precache_sessions():
[
SessionData.get(rds, load_clusters=False)
for rds in unique_sessions(scan_cell_table)
]
precache_sessions()
@memoize
def cell_query(session, tc):
return '(%s)&(tc=="%s")' % (session.session_query, tc)
def print_test_indicator(result):
color = result and 'green' or 'red'
self.out.printf(u'\u25a1', color=color)
def test_for_event_hit(scan_angle, h, bounds):
window = tuple(scan_angle + hit_angle + np.array([h, -h]))
start_before = bounds[0] >= window[1]
end_after = bounds[1] <= window[0]
return start_before and end_after
for pair in scan_cell_table.iterrows():
self.out.printf('|', color='cyan')
rds = pair['rat'], pair['day'], pair['session']
session = SessionData.get(rds, load_clusters=False)
event_bounds = lambda t: tuple(
session.F_('alpha_unwrapped')(session.T_(t)))
scan_angle = pair['angle']
row['scan_cell_id'] = pair['id']
for event in potentiation.where(cell_query(session, pair['tc'])):
self.out.printf('|', color='lightgray')
bounds = event_bounds(event['tlim'])
for h, col in zip(half_windows, window_cols):
row[col] = test_for_event_hit(scan_angle, h, bounds)
print_test_indicator(row[col])
row.append()
if pair['id'] % 100 == 0:
outcome_table.flush()
self.out.printf(' [flush]\n%d / %d scan-cell pairs\n' %
(outcome_table.nrows, scan_cell_table.nrows),
color='lightgray')
self.out.printf('\n')
self.close_data_file()
def generate_index_validation_set(self, N=100):
data_file = self.get_data_file(mode='r')
scan_cell_table = data_file.root.scan_cell_info
scan_table = get_node('/behavior', 'scans')
index = ScanFiringDeviationIndex()
def get_subdir(name):
subdir = os.path.join(self.datadir, name)
if not os.path.isdir(subdir):
os.makedirs(subdir)
return subdir
output_dir = get_subdir('index_validation')
figure_dir = get_subdir(os.path.join('index_validation', 'figures'))
spreadsheet = DataSpreadsheet(
os.path.join(output_dir, 'scan_index_sample.csv'),
[('sample', 'd'), ('rat', 'd'), ('day', 'd'), ('session', 'd'),
('cell', 's'), ('scan_number', 'd'), ('scan_start', 'd'),
('surprise', 'f')])
rec = spreadsheet.get_record()
sample_ix = np.random.permutation(scan_cell_table.nrows)[:N]
scan_index = scan_cell_table.col('S')[sample_ix]
sample_ix = sample_ix[np.argsort(scan_index)]
plt.ioff()
figure_files = []
sample_id = 1
self.out('Generating sample records and figures...')
for ix in sample_ix:
self.out.printf('.')
row = scan_cell_table[ix]
rds = row['rat'], row['day'], row['session']
scan = scan_table[row['scan_id']]
rec['sample'] = sample_id
rec['rat'], rec['day'], rec['session'] = rds
rec['cell'] = row['tc']
rec['scan_number'] = scan['number']
rec['scan_start'] = scan['start']
rec['surprise'] = row['S']
spreadsheet.write_record(rec)
session = SessionData.get(rds)
cluster = session.cluster_data(row['tc'])
index.compute(session, scan, cluster, plot=True)
figure_fn = os.path.join(figure_dir,
'distros_%03d.pdf' % sample_id)
figure_files.append(figure_fn)
plt.savefig(figure_fn)
plt.close()
sample_id += 1
self.out.printf('\n')
spreadsheet.close()
plt.ion()
def concatenate_distro_figures_into_report():
pdftk = '/opt/local/bin/pdftk'
if os.path.exists(pdftk):
distro_report = os.path.join(output_dir, 'distro_figures.pdf')
retcode = subprocess.call([pdftk] + figure_files +
['cat', 'output', distro_report])
if retcode == 0:
self.out('Saved distribution figures to:\n%s' %
distro_report)
else:
self.out('Error saving figure report.', error=True)
concatenate_distro_figures_into_report()
def collect_data(self,
test='place',
place_field='pass',
min_quality='fair',
allow_ambiguous=True):
"""Tally place fields across areas
Keyword arguments similar to info_scores.InfoScoreData. Remaining
keywords are passed to TetrodeSelect.
"""
# Metadata for determining valid fields
self.results['test'] = test
self.results['place_field'] = place_field
self.results['min_quality'] = min_quality
self.results['allow_ambiguous'] = allow_ambiguous
if place_field == 'all':
self.test = 'place'
# Construct place cell selection criteria based on keyword arguments
if test == 'place':
SpatialTest = SpatialInformationCriteria
elif test == 'skaggs':
SpatialTest = SkaggsCriteria
elif test == 'olypher':
SpatialTest = OlypherCriteria
else:
raise ValueError, 'bad test value: %s' % test
MinQuality = get_min_quality_criterion(min_quality)
CellCriteria = AND(PrincipalCellCriteria, SpikeCountCriteria,
MinQuality)
if place_field == 'pass':
CellCriteria = AND(CellCriteria, SpatialTest)
elif place_field == 'fail':
CellCriteria = AND(CellCriteria, NOT(SpatialTest))
elif place_field != 'all':
raise ValueError, 'bad place_field value: %s' % place_field
# Walk the tree and count place fields
N = {}
N_cells = {}
N_sessions = {}
sessions = set()
tetrodes = get_node('/metadata', 'tetrodes')
for area in AREAS.keys():
for subdiv in (['all'] + AREAS[area]):
self.out('Walking datasets for %s %s...' % (area, subdiv))
key = '%s_%s' % (area, subdiv)
N[key] = 0
N_cells[key] = 0
N_sessions[key] = 0
area_query = 'area=="%s"' % area
if subdiv != 'all':
area_query = '(%s)&(subdiv=="%s")' % (area_query, subdiv)
for dataset in TetrodeSelect.datasets(
area_query, allow_ambiguous=allow_ambiguous):
Criteria = AND(
CellCriteria,
TetrodeSelect.criterion(
dataset,
area_query,
allow_ambiguous=allow_ambiguous))
dataset_cells = set()
for maze in get_maze_list(*dataset):
rds = dataset + (maze, )
data = SessionData.get(rds)
sessions.add(rds)
place_cell_clusters = data.get_clusters(
request=Criteria)
N[key] += len(place_cell_clusters)
dataset_cells.update(place_cell_clusters)
N_sessions[key] += 1
N_cells[key] += len(dataset_cells)
self.out.timestamp = False
self.results['N'] = N
self.out('Total number of sessions = %d' % len(sessions))
for key in sorted(N.keys()):
self.out('N_cells[%s] = %d cells' % (key, N_cells[key]))
self.out('N_sessions[%s] = %d sessions' % (key, N_sessions[key]))
self.out('N_cell_sessions[%s] = %d cell-sessions' % (key, N[key]))
# Good-bye
self.out('All done!')
def collect_data(self):
"""Collate ripple and head-scan events across CA3/CA1 datasets
"""
scan_table = get_node('/behavior', 'scans')
ripple_table = get_node('/physiology', 'ripples')
# Get datasets, sessions, and rats with detected ripples
self.results['datasets'] = dataset_list = unique_datasets(ripple_table)
self.results['N_datasets'] = len(dataset_list)
session_list = []
for dataset in dataset_list:
session_list.extend([dataset + (maze,)
for maze in get_maze_list(*dataset)])
self.results['sessions'] = session_list
self.results['N_sessions'] = len(session_list)
self.results['rats'] = rat_list = unique_rats(ripple_table)
self.results['N_rats'] = len(rat_list)
# Open a new data file
data_file = self.open_data_file()
array_group = data_file.createGroup('/', 'arrays', title='Array Data')
session_table = data_file.createTable('/', 'sessions', SessionDescr,
'Sessions for Scan-Ripple Analysis')
# Loop through sessions, detecting ripples and getting head scans
id_fmt = 'data_%06d'
array_id = 0
session_id = 0
row = session_table.row
for rds in session_list:
rds_str = 'rat%d-%02d-m%d...'%rds
self.out('Loading data for %s'%rds_str)
data = SessionData.get(rds)
theta_tt = find_theta_tetrode(rds[:2], condn='(EEG==True)&(area=="CA1")')
if theta_tt is None:
continue
theta_tt = theta_tt[0]
self.out('Using theta tetrode Sc%02d.'%theta_tt)
row['id'] = session_id
row['rat'], row['day'], row['session'] = rds
if data.attrs['type'] in ('STD', 'MIS'):
row['type'] = 'DR'
else:
row['type'] = 'NOV'
# Compute smoothed theta power and frequency time-series
ts, EEG = get_eeg_timeseries(rds, theta_tt)
t = data.T_(ts) # time represented as elapsed time within session
t_theta, x_theta = Theta.timeseries(t, EEG)
ZP_theta = Z(Theta.power(x_theta, filtered=True))
f_theta = quick_boxcar(Theta.frequency(x_theta, filtered=True),
M=(lambda i: (i % 2 == 0) and (i+1) or i)(int(Theta.fs * THETA_FREQ_SMOOTHING)))
# Get scans, pauses, and ripples
scans = data.T_([tuple(map(lambda k: rec[k], ScanPoints))
for rec in scan_table.where(data.session_query)])
pauses = data.T_(data.pause_list)
ripples = data.T_([(rec['start'], rec['peak'], rec['end'])
for rec in ripple_table.where(data.session_query)])
# Save the array data as resources for analysis
data_file.createArray(array_group, id_fmt%array_id, t_theta,
title='%s t_theta'%rds_str)
row['t_theta'] = id_fmt%array_id
array_id += 1
data_file.createArray(array_group, id_fmt%array_id, ZP_theta,
title='%s ZP_theta'%rds_str)
row['ZP_theta'] = id_fmt%array_id
array_id += 1
data_file.createArray(array_group, id_fmt%array_id, f_theta,
title='%s f_theta'%rds_str)
row['f_theta'] = id_fmt%array_id
array_id += 1
data_file.createArray(array_group, id_fmt%array_id, scans,
title='%s scans'%rds_str)
row['scans'] = id_fmt%array_id
array_id += 1
data_file.createArray(array_group, id_fmt%array_id, pauses,
title='%s pauses'%rds_str)
row['pauses'] = id_fmt%array_id
array_id += 1
data_file.createArray(array_group, id_fmt%array_id, ripples,
title='%s ripples'%rds_str)
row['ripples'] = id_fmt%array_id
array_id += 1
row.append()
if array_id % 10 == 0:
session_table.flush()
session_id += 1
# Good-bye
session_table.flush()
self.out('All done!')
def collect_data(self,
area='CA1',
phase_band='theta',
amp_band='gamma',
tetrode='theta',
cycles=2,
nbins=72):
"""Collate phase-amplitude modulation data about head-scan events
"""
tetrode_query = '(area=="%s")&(EEG==True)' % area
dataset_list = TetrodeSelect.datasets(tetrode_query,
allow_ambiguous=True)
self.results['phase_band'] = phase_band
self.results['amp_band'] = amp_band
# Dataset accumulators
rat_number = self.results['rat_number'] = []
P_running = {}
P_scan = {}
P_pause = {}
for dataset in dataset_list:
rat, day = dataset
# Find the tetrode with the higheset overall relative theta power
if tetrode == 'theta':
roi_tt, _rtheta = find_theta_tetrode(dataset,
condn=tetrode_query,
ambiguous=True)
else:
roi_tt = find_pyramidale_tetrode(dataset,
condn=tetrode_query,
ambiguous=True)
self.out('Rat%03d-%02d: using tetrode Sc%d' % (rat, day, roi_tt))
# Session accumulators
phase_t = np.array([], 'i8')
phase = np.array([], 'd')
running_amp_t = np.array([], 'i8')
running_amp = np.array([], 'd')
scan_amp_t = np.array([], 'i8')
scan_amp = np.array([], 'd')
pause_amp_t = np.array([], 'i8')
pause_amp = np.array([], 'd')
self.out('Collating phase-amplitude data for rat%03d-%02d...' %
dataset)
for session in get_maze_list(rat, day):
rds = rat, day, session
data = SessionData.get(rds)
phase_data, amp_data = phase_modulation_timeseries(
*get_eeg_timeseries(rds, roi_tt),
phase=phase_band,
amp=amp_band)
t_phase, phi_x = phase_data
t_amp, A_x = amp_data
phase_t = np.r_[phase_t, t_phase]
phase = np.r_[phase, phi_x]
ix = data.velocity_filter(t_amp)
running_amp_t = np.r_[running_amp_t, t_amp[ix]]
running_amp = np.r_[running_amp, A_x[ix]]
ix = select_from(t_amp, data.scan_list)
scan_amp_t = np.r_[scan_amp_t, t_amp[ix]]
scan_amp = np.r_[scan_amp, A_x[ix]]
ix = select_from(t_amp, data.pause_list)
pause_amp_t = np.r_[pause_amp_t, t_amp[ix]]
pause_amp = np.r_[pause_amp, A_x[ix]]
# Initialize per-rat phase distributions
if rat not in rat_number:
rat_number.append(rat)
P_running[rat] = []
P_scan[rat] = []
P_pause[rat] = []
self.out('...computing phase distributions...')
phase_series = (phase_t, phase)
P_running[rat].append(
PAD(phase_series, (running_amp_t, running_amp), nbins=nbins))
P_scan[rat].append(
PAD(phase_series, (scan_amp_t, scan_amp), nbins=nbins))
P_pause[rat].append(
PAD(phase_series, (pause_amp_t, pause_amp), nbins=nbins))
self.out('Averaging dataset distributions to rat distributions...')
norm = lambda P: P / P.sum()
for rat in rat_number:
P_running[rat] = norm(np.array(P_running[rat]).mean(axis=0))
P_scan[rat] = norm(np.array(P_scan[rat]).mean(axis=0))
P_pause[rat] = norm(np.array(P_pause[rat]).mean(axis=0))
# Initialize data storage and accumulators
running_distro = []
running_index = []
scan_distro = []
scan_index = []
pause_distro = []
pause_index = []
self.out('Computing display distributions and modulation indexes...')
plottable = lambda P: plottable_phase_distribution(P, cycles=cycles)
for rat in rat_number:
phase_bins, P = plottable(P_running[rat])
if 'phase_bins' not in self.results:
self.results['phase_bins'] = phase_bins
running_distro.append(P)
running_index.append(modulation_index(P_running[rat]))
scan_distro.append(plottable(P_scan[rat])[1])
scan_index.append(modulation_index(P_scan[rat]))
pause_distro.append(plottable(P_pause[rat])[1])
pause_index.append(modulation_index(P_pause[rat]))
# Store results data
self.results['running_distro'] = np.array(running_distro)
self.results['running_index'] = np.array(running_index)
self.results['scan_distro'] = np.array(scan_distro)
self.results['scan_index'] = np.array(scan_index)
self.results['pause_distro'] = np.array(pause_distro)
self.results['pause_index'] = np.array(pause_index)
# Good-bye!
self.out('All done!')