def draw_figures(data_dir='/auto/tdrive/mschachter/data'):
preproc_file = os.path.join(data_dir, 'GreBlu9508M', 'preprocess', 'RNNPreprocess_GreBlu9508M_Site4_Call1_L.h5')
rp = RNNPreprocessTransform.load(preproc_file)
i = rp.event_df.stim_type == 'mlnoise'
print 'ml noise stims: ',rp.event_df[i].stim_id.unique()
fig = plt.figure(figsize=(23, 10), facecolor='w')
fig.subplots_adjust(hspace=0.35, wspace=0.35, right=0.95, left=0.10)
gs = plt.GridSpec(3, 2)
ax = plt.subplot(gs[0, 0])
plot_specs_with_env(ax, rp, 287)
ax = plt.subplot(gs[1, 0])
plot_specs_with_env(ax, rp, 277)
ax = plt.subplot(gs[2, 0])
plot_specs_with_env(ax, rp, 68)
ax = plt.subplot(gs[:, 1])
plot_temp_mods(ax, rp)
fname = os.path.join(get_this_dir(), 'figure.svg')
plt.savefig(fname, facecolor=fig.get_facecolor(), edgecolor='none')
plt.show()
def draw_preds(agg, data_dir='/auto/tdrive/mschachter/data'):
df_best = pd.read_csv('/auto/tdrive/mschachter/data/aggregate/rnn_best.csv')
# print 'min_err=', df_best.err.min()
bird = 'GreBlu9508M'
block = 'Site2'
segment = 'Call2'
hemi = 'L'
best_md5 = 'db5d601c8af2621f341d8e4cbd4108c1'
fname = '%s_%s_%s_%s' % (bird, block, segment, hemi)
preproc_file = os.path.join(data_dir, bird, 'preprocess', 'RNNPreprocess_%s.h5' % fname)
rnn_file = os.path.join(data_dir, bird, 'rnn', 'RNNLFPEncoder_%s_%s.h5' %(fname, best_md5))
pred_file = os.path.join(data_dir, bird, 'rnn', 'RNNLFPEncoderPred_%s_%s.h5' %(fname, best_md5))
linear_file = os.path.join(data_dir, bird, 'rnn', 'LFPEnvelope_%s.h5' % fname)
if not os.path.exists(pred_file):
rpt = RNNPreprocessTransform.load(preproc_file)
rpt.write_pred_file(rnn_file, pred_file, lfp_enc_file=linear_file)
visualize_pred_file(pred_file, 3, 0, 2.4, electrode_order=ROSTRAL_CAUDAL_ELECTRODES_LEFT[::-1], dbnoise=4.0)
leg = custom_legend(['k', 'r', 'b'], ['Real', 'Linear', 'RNN'])
plt.legend(handles=leg)
fname = os.path.join(get_this_dir(), 'rnn_preds.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_pairwise_weights_vs_dist(agg):
wdf = export_pairwise_decoder_weights(agg)
r2_thresh = 0.20
# aprops = ALL_ACOUSTIC_PROPS
aprops = ['meanspect', 'stdspect', 'sal', 'maxAmp']
aprop_clrs = {'meanspect':'#FF8000', 'stdspect':'#FFBF00', 'sal':'#088A08', 'maxAmp':'k'}
print wdf.decomp.unique()
# get scatter data for weights vs distance
scatter_data = dict()
for decomp in ['full_psds+full_cfs', 'spike_rate+spike_sync']:
i = wdf.decomp == decomp
assert i.sum() > 0
df = wdf[i]
for n,aprop in enumerate(aprops):
ii = (df.r2 > r2_thresh) & (df.aprop == aprop) & ~np.isnan(df.dist) & ~np.isinf(df.dist)
d = df.dist[ii].values
w = df.w[ii].values
scatter_data[(decomp, aprop)] = (d, w)
decomp_labels = {'full_psds+full_cfs':'LFP Pairwise Correlations', 'spike_rate+spike_sync':'Spike Synchrony'}
figsize = (14, 5)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(left=0.10, right=0.98)
for k,decomp in enumerate(['full_psds+full_cfs', 'spike_rate+spike_sync']):
ax = plt.subplot(1, 2, k+1)
for aprop in aprops:
d,w = scatter_data[(decomp, aprop)]
wz = w**2
wz /= wz.std(ddof=1)
xcenter, ymean, yerr, ymean_cs = compute_mean_from_scatter(d, wz, bins=5, num_smooth_points=300)
# plt.plot(xcenter, ymean, '-', c=aprop_clrs[aprop], linewidth=7.0, alpha=0.7)
plt.errorbar(xcenter, ymean, linestyle='-', yerr=yerr, c=aprop_clrs[aprop], linewidth=9.0, alpha=0.7,
elinewidth=8., ecolor='#d8d8d8', capsize=0.)
plt.axis('tight')
plt.ylabel('Decoder Weight Effect')
plt.xlabel('Pairwise Distance (mm)')
plt.title(decomp_labels[decomp])
aprop_lbls = [ACOUSTIC_PROP_NAMES[aprop] for aprop in aprops]
aclrs = [aprop_clrs[aprop] for aprop in aprops]
leg = custom_legend(aclrs, aprop_lbls)
plt.legend(handles=leg, loc='lower left')
fname = os.path.join(get_this_dir(), 'pairwise_decoder_effect_vs_dist.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_spike_rate_vs_power(data_dir='/auto/tdrive/mschachter/data'):
# read PairwiseCF file
pcf_file = os.path.join(data_dir, 'aggregate', 'pairwise_cf.h5')
pcf = AggregatePairwiseCF.load(pcf_file)
# concatenate the lfp and spike psds
nfreqs = len(pcf.freqs)
lfp_and_spike_psds = np.zeros([len(pcf.df), nfreqs*2 + 1])
nz = np.zeros(len(pcf.df), dtype='bool')
for k,(lfp_index,spike_index) in enumerate(zip(pcf.df['lfp_index'], pcf.df['spike_index'])):
lpsd = pcf.lfp_psds[lfp_index, :]
spsd = pcf.spike_psds[spike_index, :]
srate,sstd = pcf.spike_rates[spike_index, :]
nz[k] = np.abs(lpsd).sum() > 0 and np.abs(spsd).sum() > 0
lfp_and_spike_psds[k, :nfreqs] = lpsd
lfp_and_spike_psds[k, nfreqs:-1] = spsd
lfp_and_spike_psds[k, -1] = np.log(srate)
# throw some bad data points out
lfp_sum = lfp_and_spike_psds[:, :nfreqs].sum(axis=1)
spike_sum = lfp_and_spike_psds[:, nfreqs:-1].sum(axis=1)
nz = ~np.isinf(lfp_and_spike_psds[:, -1]) & (lfp_sum > 0) & (spike_sum > 0) & ~np.isnan(spike_sum) & ~np.isnan(lfp_sum)
print '# of good data points: %d out of %d' % (nz.sum(), lfp_and_spike_psds.shape[0])
# zscore the concatenated matrix
lfp_and_spike_psds = lfp_and_spike_psds[nz, :]
lfp_and_spike_psds -= lfp_and_spike_psds.mean(axis=0)
lfp_and_spike_psds /= lfp_and_spike_psds.std(axis=0, ddof=1)
# compute CC between spike rate and power
lfp_spike_rate_cc = np.zeros(len(pcf.freqs))
spike_spike_rate_cc = np.zeros(len(pcf.freqs))
for k,f in enumerate(pcf.freqs):
lfp_spike_rate_cc[k] = np.corrcoef(lfp_and_spike_psds[:, k], lfp_and_spike_psds[:, -1])[0, 1]
spike_spike_rate_cc[k] = np.corrcoef(lfp_and_spike_psds[:, k+len(pcf.freqs)], lfp_and_spike_psds[:, -1])[0, 1]
fig = plt.figure(figsize=(12, 7))
plt.axhline(0, c='k')
plt.plot(pcf.freqs, lfp_spike_rate_cc, '-', linewidth=7.0, alpha=0.7, c=COLOR_BLUE_LFP)
plt.plot(pcf.freqs, spike_spike_rate_cc, '-', linewidth=7.0, alpha=0.7, c=COLOR_YELLOW_SPIKE)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Correlation Coefficient')
plt.title('CC Between Log Spike Rate and Spectral Power')
plt.axis('tight')
plt.ylim(-0.1, 0.6)
leg = custom_legend([COLOR_BLUE_LFP, COLOR_YELLOW_SPIKE], ['LFP PSD', 'Spike PSD'])
plt.legend(handles=leg, fontsize='x-small')
fname = os.path.join(get_this_dir(), 'power_vs_rate.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_figures(agg, data_dir='/auto/tdrive/mschachter/data'):
stats(agg)
figsize = (23, 8)
fig = plt.figure(figsize=figsize)
ax = plt.subplot(1, 2, 1)
draw_perf_by_freq(agg)
ax = plt.subplot(1, 2, 2)
draw_rate_weight_by_dist(agg)
fname = os.path.join(get_this_dir(), 'figure.svg')
# plt.savefig(fname, facecolor='w', edgecolor='none')
draw_rate_weight_by_same(agg)
fname = os.path.join(get_this_dir(), 'figure_inset.svg')
# plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_perfs(agg):
# filter out some data points
i = (agg.df.cc > 0.05) & (agg.df.region != '?') & (agg.df.region != 'HP') \
& ~np.isnan(agg.df.dist_l2a) & ~np.isnan(agg.df.dist_midline) \
& (agg.df.dist_l2a > -1) & (agg.df.dist_l2a < 1)
df = agg.df[i]
print len(df)
print df.region.unique()
# make a scatter plot of performance by anatomical location
fig = plt.figure(figsize=(23, 10), facecolor='w')
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.25, hspace=0.25)
gs = plt.GridSpec(2, 100)
ax = plt.subplot(gs[:, :55])
plt.scatter(df.dist_midline, df.dist_l2a, c=df.cc, s=64, cmap=plt.cm.afmhot_r, alpha=0.7)
plt.ylim(-1, 1)
plt.xlim(0, 2.5)
for x, y, r in zip(df.dist_midline, df.dist_l2a, df.region):
plt.text(x, y - 0.05, r, fontsize=9)
cbar = plt.colorbar(label='Linear Encoder Performance (cc)')
plt.xlabel('Dist to Midline (mm)')
plt.ylabel('Dist to L2A (mm)')
region_stats = list()
regions_to_use = ['L1', 'L2', 'L3', 'CM', 'NCM']
for reg in regions_to_use:
i = df.region == reg
region_stats.append({'cc': df[i].cc, 'cc_mean': df[i].cc.mean(), 'region': reg})
region_stats.sort(key=operator.itemgetter('cc_mean'), reverse=True)
regions = [x['region'] for x in region_stats]
region_ccs = {x['region']: x['cc'] for x in region_stats}
ax = plt.subplot(gs[:45, 65:])
boxplot_with_colors(region_ccs, group_names=regions, ax=ax, group_colors=['k'] * len(regions), box_alpha=0.7)
plt.xlabel('Region')
plt.ylabel('Linear Encoder Performance (cc)')
fname = os.path.join(get_this_dir(), 'encoder_perf.svg')
plt.savefig(fname, facecolor=fig.get_facecolor(), edgecolor='none')
def plot_avg_psd_encoder_weights(agg, data_dir='/auto/tdrive/mschachter/data', decomp='full_psds'):
freqs,lags = get_freqs_and_lags()
bs_agg = AggregateBiosounds.load(os.path.join(data_dir, 'aggregate', 'biosound.h5'))
wdf,Wsq,Wsq_by_freq,Wsq_by_reg = get_encoder_weights_squared(agg, decomp)
print 'Wsq_by_freq.shape=',Wsq_by_freq.shape
Wsq_by_freq,aprop_order = reorder_by_row_sum(Wsq_by_freq)
figsize = (23, 13)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(top=0.95, bottom=0.02, right=0.99, left=0.05, hspace=0.25, wspace=0.25)
gs = plt.GridSpec(100, 100)
pwidth = 45
# plot the performance boxplots by frequency
freqs = list(sorted(wdf.freq.unique()))
ax = plt.subplot(gs[55:95, 62:98])
plot_lfp_and_spike_perf_boxplot(agg, ax, lfp_decomp=decomp)
"""
# plot the average encoder effect sizes
ax = plt.subplot(gs[45:100, :pwidth])
plt.imshow(Wsq_by_freq, origin='upper', interpolation='nearest', aspect='auto', vmin=0, cmap=magma)
plt.xticks(range(len(freqs)), ['%d' % int(f) for f in freqs])
aprop_lbls = [ACOUSTIC_PROP_NAMES[USED_ACOUSTIC_PROPS[k]] for k in aprop_order]
plt.yticks(range(len(aprop_lbls)), aprop_lbls)
plt.xlabel('LFP Frequency (Hz)')
plt.colorbar(label='Normalized Encoder Effect')
"""
fname = os.path.join(get_this_dir(), 'average_encoder_weights.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_curves(agg):
aprops = ['maxAmp', 'meanspect', 'entropytime', 'sal']
freqs = [0, 33, 165]
decomps = (('spike_rate', -1), ('full_psds', freqs[0]), ('full_psds', freqs[1]), ('full_psds', freqs[2]))
# get top tuning cuves for each acoustic prop
plot_perfs = False
if plot_perfs:
fig = plt.figure()
fig.subplots_adjust(wspace=0.35, hspace=0.35)
nrows = 5
ncols = 3
perf_thresh = 0.05
top_tuning_cuves = dict()
for k,aprop in enumerate(aprops):
for j,(decomp,f) in enumerate(decomps):
i = (agg.df.decomp == decomp) & (agg.df.freq == f) & (agg.df.aprop == aprop)
assert i.sum() > 0, 'aprop=%s, decomp=%s, freq=%d' % (aprop, decomp, f)
perf = agg.df.r2[i].values
assert np.sum(np.isnan(perf)) == 0
assert np.sum(np.isinf(perf)) == 0
pi = perf > perf_thresh
xindex = agg.df.xindex[i][pi]
if len(xindex) == 0:
print 'No good curves for aprop=%s, decomp=%s, freq=%d' % (aprop, decomp, f)
continue
lst = zip(xindex, perf[pi])
lst.sort(key=operator.itemgetter(1), reverse=True)
xindex = [x[0] for x in lst]
alpha = np.array([x[1] for x in lst])
alpha -= alpha.min()
alpha /= alpha.max()
cx = agg.curve_x[xindex, :]
tc = agg.tuning_curves[xindex, :]
good_indices = np.ones([cx.shape[0]], dtype='bool')
if decomp == 'spike_rate':
for k in range(cx.shape[0]):
good_indices[k] = ~np.any(tc[k, :] > 50)
top_tuning_cuves[(aprop, decomp, f)] = (cx[good_indices, :], tc[good_indices, :], alpha[good_indices])
if plot_perfs:
ax = plt.subplot(nrows, ncols, k*len(decomps) + j + 1)
plt.hist(perf[perf > 0.6], bins=20)
plt.title('%s, %s, %d' % (aprop, decomp, f))
plt.axis('tight')
figsize = (23, 13)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(top=0.95, bottom=0.02, right=0.99, left=0.05, hspace=0.25, wspace=0.25)
gs = plt.GridSpec(100, 100)
offset = 55
wpad = 5
hpad = 8
height = int(100 / len(aprops)) - hpad
width = int((100 - offset) / 3.) - wpad
topn = 30
xticks = {'meanspect':([2, 4], ['2', '4']),
'maxAmp':([0.2, 0.4], ['0.2', '0.4']),
'sal':([0.5, 0.7], ['0.5', '0.7']),
'entropytime':([0.90, 0.94, 0.98], ['0.90', '0.94', '0.98'])
}
clrs = {('spike_rate', -1):COLOR_RED_SPIKE_RATE, ('full_psds', 0):'k', ('full_psds', 33):'k', ('full_psds', 165):'k'}
for k, aprop in enumerate(aprops):
for j, (decomp, f) in enumerate(decomps):
x1 = j*(width+wpad)
x2 = x1 + width
y1 = k*(height+hpad)
y2 = y1 + height
# print 'k=%d, j=%d, x1=%d, x2=%d, y1=%d, y2=%d' % (k, j, x1, x2, y1, y2)
ax = plt.subplot(gs[y1:y2, x1:x2])
# plot the top n tuning curves
if (aprop, decomp, f) not in top_tuning_cuves:
continue
cx,tc,alpha = top_tuning_cuves[(aprop, decomp, f)]
if aprop == 'meanspect':
cx *= 1e-3
n = min(cx.shape[0], 90)
plt.axhline(0, c='k')
for x,y,a in zip(cx[:n, :], tc[:n, :], alpha[:n]):
c = clrs[(decomp, f)]
plt.plot(x, y, '-', c=c, linewidth=1.0, alpha=a)
xlbl = ACOUSTIC_PROP_NAMES[aprop]
if aprop == 'meanspect':
xlbl += ' (kHz)'
elif aprop == 'entropytime':
xlbl += '(bits)'
plt.xlabel(xlbl)
if decomp.endswith('rate'):
ylbl = 'Spike Rate\n(z-scored)'
if k == 0:
plt.title('Spike Rate', fontweight='bold')
elif decomp.endswith('psds'):
if k == 0:
plt.title('LFP Power (%d Hz)' % f, fontweight='bold')
ylbl = 'LFP Power'
plt.ylabel(ylbl)
if aprop in xticks:
plt.xticks(xticks[aprop][0], xticks[aprop][1])
plt.axis('tight')
if aprop == 'entropytime':
plt.xlim(0.89, 0.98)
else:
plt.xlim([x.min() * 1.20, x.max() * 0.9])
if decomp.endswith('rate'):
plt.ylim(-1.5, 1.5)
elif decomp.endswith('psds'):
plt.ylim(-1., 1.)
ax = plt.subplot(gs[:45, 62:])
draw_r2(agg, ax)
fname = os.path.join(get_this_dir(), 'figure.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def plot_full_data(d, syllable_index):
syllable_start = d['syllable_props'][syllable_index]['start_time'] - 0.020
syllable_end = d['syllable_props'][syllable_index]['end_time'] + 0.030
figsize = (24.0, 10)
fig = plt.figure(figsize=figsize, facecolor='w')
fig.subplots_adjust(top=0.95, bottom=0.02, right=0.97, left=0.03, hspace=0.20, wspace=0.20)
gs = plt.GridSpec(100, 100)
left_width = 55
top_height = 30
middle_height = 40
# bottom_height = 40
top_bottom_sep = 20
# plot the spectrogram
ax = plt.subplot(gs[:top_height+1, :left_width])
spec = d['spec']
spec[spec < np.percentile(spec, 15)] = 0
plot_spectrogram(d['spec_t'], d['spec_freq']*1e-3, spec, ax=ax, colormap='SpectroColorMap', colorbar=False, ticks=True)
plt.axvline(syllable_start, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
plt.axvline(syllable_end, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
plt.ylabel('Frequency (kHz)')
plt.xlabel('Time (s)')
# plot the LFPs
sr = d['lfp_sample_rate']
# lfp_mean = d['lfp'].mean(axis=0)
lfp_mean = d['lfp'][2, :, :]
lfp_t = np.arange(lfp_mean.shape[1]) / sr
nelectrodes,nt = lfp_mean.shape
gs_i = top_height + top_bottom_sep
gs_e = gs_i + middle_height + 1
ax = plt.subplot(gs[gs_i:gs_e, :left_width])
voffset = 5
for n in range(nelectrodes):
plt.plot(lfp_t, lfp_mean[nelectrodes-n-1, :] + voffset*n, 'k-', linewidth=3.0, alpha=0.75)
plt.axis('tight')
ytick_locs = np.arange(nelectrodes) * voffset
plt.yticks(ytick_locs, list(reversed(d['electrode_order'])))
plt.ylabel('Electrode')
plt.axvline(syllable_start, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
plt.axvline(syllable_end, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
plt.xlabel('Time (s)')
# plot the PSTH
"""
gs_i = gs_e + 5
gs_e = gs_i + bottom_height + 1
ax = plt.subplot(gs[gs_i:gs_e, :left_width])
ncells = d['psth'].shape[0]
plt.imshow(d['psth'], interpolation='nearest', aspect='auto', origin='upper', extent=(0, lfp_t.max(), ncells, 0),
cmap=psth_colormap(noise_level=0.1))
cell_i2e = d['cell_index2electrode']
print 'cell_i2e=',cell_i2e
last_electrode = cell_i2e[0]
for k,e in enumerate(cell_i2e):
if e != last_electrode:
plt.axhline(k, c='k', alpha=0.5)
last_electrode = e
ytick_locs = list()
for e in d['electrode_order']:
elocs = np.array([k for k,el in enumerate(cell_i2e) if el == e])
emean = elocs.mean()
ytick_locs.append(emean+0.5)
plt.yticks(ytick_locs, d['electrode_order'])
plt.ylabel('Electrode')
plt.axvline(syllable_start, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
plt.axvline(syllable_end, c='k', linestyle='--', linewidth=3.0, alpha=0.7)
"""
# plot the biosound properties
sprops = d['syllable_props'][syllable_index]
aprops = USED_ACOUSTIC_PROPS
vals = [sprops[a] for a in aprops]
ax = plt.subplot(gs[:top_height, (left_width+5):])
plt.axhline(0, c='k')
for k,(aprop,v) in enumerate(zip(aprops,vals)):
bx = k
rgb = np.array(ACOUSTIC_PROP_COLORS_BY_TYPE[aprop]).astype('int')
clr_hex = '#%s' % "".join(map(chr, rgb)).encode('hex')
plt.bar(bx, v, color=clr_hex, alpha=0.7)
# plt.bar(range(len(aprops)), vals, color='#c0c0c0')
plt.axis('tight')
plt.ylim(-1.5, 1.5)
plt.xticks(np.arange(len(aprops))+0.5, [ACOUSTIC_PROP_NAMES[aprop] for aprop in aprops], rotation=90)
plt.ylabel('Z-score')
# plot the LFP power spectra
gs_i = top_height + top_bottom_sep
gs_e = gs_i + middle_height + 1
f = d['psd_freq']
ax = plt.subplot(gs[gs_i:gs_e, (left_width+5):])
plt.imshow(sprops['lfp_psd'], interpolation='nearest', aspect='auto', origin='upper',
extent=(f.min(), f.max(), nelectrodes, 0), cmap=viridis, vmin=-2., vmax=2.)
plt.colorbar(label='Z-scored Log Power')
plt.xlabel('Frequency (Hz)')
plt.yticks(np.arange(nelectrodes)+0.5, d['electrode_order'])
plt.ylabel('Electrode')
fname = os.path.join(get_this_dir(), 'figure.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def stats(agg, data_dir='/auto/tdrive/mschachter/data'):
data = {'bird': list(), 'block': list(), 'segment': list(), 'hemi': list(), 'electrode': list(),
'linear_cc': list(), 'cc': list(), 'err': list(),
'lambda1': list(), 'lambda2': list(), 'n_unit': list(), 'region':list(), 'md5':list()}
edata = pd.read_csv(os.path.join(data_dir, 'aggregate', 'electrode_data+dist.csv'))
g = agg.df.groupby(['bird', 'block', 'segment', 'hemi'])
for (bird, block, segment, hemi), gdf in g:
perfs = list()
gg = gdf.groupby(['lambda1', 'lambda2', 'n_unit'])
for (lambda1, lambda2, n_unit), ggdf in gg:
err = ggdf.err.values[0]
perfs.append({'err': err, 'lambda1': lambda1, 'lambda2': lambda2, 'n_unit': n_unit})
perfs.sort(key=operator.itemgetter('err'))
best_lambda1 = perfs[0]['lambda1']
best_lambda2 = perfs[0]['lambda2']
best_n_unit = perfs[0]['n_unit']
best_err = perfs[0]['err']
print 'err=%0.3f, lambda1=%0.3f, lambda2=%0.3f, n_unit=%d' % (best_err, best_lambda1, best_lambda2, best_n_unit)
i = (gdf.lambda1 == best_lambda1) & (gdf.lambda2 == best_lambda2) & (gdf.n_unit == best_n_unit)
assert i.sum() == 16, 'i.sum()=%d' % i.sum()
for e in gdf[i].electrode.unique():
ii = (gdf.lambda1 == best_lambda1) & (gdf.lambda2 == best_lambda2) & (gdf.n_unit == best_n_unit) & (
gdf.electrode == e)
assert ii.sum() == 1, 'ii.sum()=%d' % ii.sum()
iii = (edata.bird == bird) & (edata.block == block) & (edata.hemisphere == hemi) & (edata.electrode == e)
assert iii.sum() == 1, 'iii.sum()=%d' % iii.sum()
reg = clean_region(edata[iii].region.values[0])
data['bird'].append(bird)
data['block'].append(block)
data['segment'].append(segment)
data['hemi'].append(hemi)
data['lambda1'].append(best_lambda1)
data['lambda2'].append(best_lambda2)
data['n_unit'].append(best_n_unit)
data['err'].append(best_err)
data['electrode'].append(e)
data['linear_cc'].append(gdf[ii].linear_cc.values[0])
data['cc'].append(gdf[ii].cc.values[0])
data['region'].append(reg)
data['md5'].append(gdf[ii].md5.values[0])
df = pd.DataFrame(data)
df.to_csv('/auto/tdrive/mschachter/data/aggregate/rnn_best.csv', header=True, index=False)
fig = plt.figure(figsize=(12, 10), facecolor='w')
x = np.linspace(0, 1, 20)
plt.plot(x, x, 'k-')
plt.plot(df.linear_cc, df.cc, 'go', alpha=0.7, markersize=12)
plt.xlabel('Linear CC')
plt.ylabel('RNN CC')
plt.xlim(0, 0.8)
plt.ylim(0, 0.8)
fname = os.path.join(get_this_dir(), 'linear_vs_rnn_cc.svg')
# plt.savefig(fname, facecolor=fig.get_facecolor(), edgecolor='none')
plt.show()
def draw_decoder_perf_barplots(data_dir='/auto/tdrive/mschachter/data', show_all=True):
aprops_to_display = list(USED_ACOUSTIC_PROPS)
aprops_to_display.remove('stdtime')
if not show_all:
decomps = ['spike_rate', 'full_psds']
sub_names = ['Spike Rate', 'LFP PSD']
sub_clrs = [COLOR_RED_SPIKE_RATE, COLOR_BLUE_LFP]
else:
decomps = ['spike_rate', 'full_psds', 'spike_rate+spike_sync']
sub_names = ['Spike Rate', 'LFP PSD', 'Spike Rate + Sync']
sub_clrs = [COLOR_RED_SPIKE_RATE, COLOR_BLUE_LFP, COLOR_CRIMSON_SPIKE_SYNC]
df_me = pd.read_csv(os.path.join(data_dir, 'aggregate', 'decoder_perfs_for_glm.csv'))
bprop_data = list()
for aprop in aprops_to_display:
bd = dict()
for decomp in decomps:
i = (df_me.decomp == decomp) & (df_me.aprop == aprop)
perfs = df_me.r2[i].values
bd[decomp] = perfs
bprop_data.append({'bd':bd, 'lfp_mean':bd['full_psds'].mean(), 'aprop':aprop})
bprop_data.sort(key=operator.itemgetter('lfp_mean'), reverse=True)
lfp_r2 = [bdict['bd']['full_psds'].mean() for bdict in bprop_data]
lfp_r2_std = [bdict['bd']['full_psds'].std(ddof=1) for bdict in bprop_data]
spike_r2 = [bdict['bd']['spike_rate'].mean() for bdict in bprop_data]
spike_r2_std = [bdict['bd']['spike_rate'].std(ddof=1) for bdict in bprop_data]
if show_all:
spike_sync_r2 = [bdict['bd']['spike_rate+spike_sync'].mean() for bdict in bprop_data]
spike_sync_r2_std = [bdict['bd']['spike_rate+spike_sync'].std(ddof=1) for bdict in bprop_data]
aprops_xticks = [ACOUSTIC_PROP_NAMES[bdict['aprop']] for bdict in bprop_data]
figsize = (23, 7.)
fig = plt.figure(figsize=figsize)
plt.subplots_adjust(top=0.95, bottom=0.15, left=0.05, right=0.99, hspace=0.20, wspace=0.20)
bar_width = 0.4
if show_all:
bar_width = 0.2
bar_data = [(spike_r2, spike_r2_std), (lfp_r2, lfp_r2_std)]
if len(decomps) == 3:
bar_data.append( (spike_sync_r2, spike_sync_r2_std) )
bar_x = np.arange(len(lfp_r2))
for k,(br2,bstd) in enumerate(bar_data):
bx = bar_x + bar_width*k
plt.bar(bx, br2, yerr=bstd, width=bar_width, color=sub_clrs[k], alpha=0.9, ecolor='k')
plt.ylabel('Decoder R2')
plt.xticks(bar_x+0.45, aprops_xticks, rotation=90, fontsize=12)
leg = custom_legend(sub_clrs, sub_names)
plt.legend(handles=leg, loc='upper right')
plt.axis('tight')
plt.xlim(-0.5, bar_x.max() + 1)
plt.ylim(0, 0.6)
fname = os.path.join(get_this_dir(), 'decoder_perf_barplots.svg')
if show_all:
fname = os.path.join(get_this_dir(), 'decoder_perf_barplots_all.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_figures(agg, data_dir='/auto/tdrive/mschachter/data'):
decomps = ['spike_rate', 'full_psds']
aprops = ['maxAmp', 'meanspect', 'sal', 'skewspect', 'skewtime']
curves_by_prop = dict()
for aprop in aprops:
for decomp in decomps:
curves_by_prop[(decomp, aprop)] = list()
g = agg.df.groupby(['bird', 'block', 'segment', 'aprop', 'decomp'])
for (bird,block,segment,aprop,decomp),gdf in g:
if aprop not in aprops:
continue
i = ~np.isnan(gdf.r2_mean) & ~np.isinf(gdf.r2_mean) & (gdf.r2_mean > 0)
lst = zip(gdf.num_units[i].values, gdf.r2_mean[i].values, gdf.r2_std[i].values)
lst.sort(key=operator.itemgetter(0))
nu = np.array([x[0] for x in lst])
r2 = np.array([x[1] for x in lst])
r2_std = np.array([x[2] for x in lst])
curves_by_prop[(decomp, aprop)].append((nu, r2, r2_std))
for decomp in decomps:
for aprop in aprops:
pcurves = curves_by_prop[(decomp, aprop)]
num_units_90 = list()
for nu,r2,r2_std in pcurves:
r2_q90 = 0.90*max(r2)
nu_90 = min(nu[r2 > r2_q90])
num_units_90.append(nu_90)
print '%s, %s: N=%d, nu90 = %0.0f +/- %0.0f' % (decomp, aprop, len(num_units_90), np.mean(num_units_90), np.std(num_units_90, ddof=1) / np.sqrt(len(num_units_90)))
return
figsize = (23, 13)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(hspace=0.40, wspace=0.40, left=0.05, right=0.95)
ncols = len(aprops)
nrows = 3
gs = plt.GridSpec(nrows, ncols)
for j,decomp in enumerate(decomps):
for k,aprop in enumerate(aprops):
ax = plt.subplot(gs[j, k])
for nu,r2,r2_std in curves_by_prop[(decomp,aprop)]:
plt.plot(nu, r2, 'k-', linewidth=3.0, alpha=0.7)
plt.title(ACOUSTIC_PROP_NAMES[aprop])
plt.axis('tight')
if decomp == 'full_psds':
plt.xlabel('# of Electrodes')
plt.ylabel('R2 (LFP PSD)')
else:
plt.xlabel('# of Neurons')
plt.ylabel('R2 (Spike Rate)')
plt.xlim(1, 60)
fname = os.path.join(get_this_dir(), 'figure.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_filters(agg):
# filter out some data points
i = (agg.df.cc > 0.20) & (agg.df.region != '?') & (agg.df.region != 'HP')
df = agg.df[i]
regions_to_use = ['L1', 'L2', 'L3', 'CM', 'NCM']
region_filters = list()
lags_ms = (agg.lags/agg.sample_rate)*1e3
for reg in regions_to_use:
i = df.region == reg
xi = df.xindex[i].values
filts = agg.filters[xi, :]
# quantify peak time
filt_peaks = compute_filt_peaks(filts, lags_ms)
#quantify center frequency
center_freqs = compute_best_freq(filts, agg.sample_rate, lags_ms)
# compute power spectra
# filt_ps_freq,filt_psds = compute_filt_power_spectra(filts, agg.sample_rate, lags_ms)
region_filters.append({'filters':filts, 'region':reg, 'peak_mean':filt_peaks.mean(), 'freqs':center_freqs})
region_filters.sort(key=operator.itemgetter('peak_mean'))
topn = 20
lag_i = (lags_ms < 100.)
fig = plt.figure(figsize=(15, 13), facecolor='w')
fig.subplots_adjust(hspace=0.35, wspace=0.35, top=0.95, bottom=0.05)
gs = plt.GridSpec(len(region_filters), 2)
nrows = len(region_filters)
for k,rdict in enumerate(region_filters):
ax = plt.subplot(gs[k, 0])
plt.axhline(0, c='k')
for f in rdict['filters'][:topn]:
plt.plot(lags_ms[lag_i], f[lag_i]*1e3, 'k-', alpha=0.7, linewidth=2.0)
plt.axis('tight')
plt.ylim(-4, 4)
plt.title(rdict['region'])
if k == len(region_filters)-1:
plt.xlabel('Filter Lag (ms)')
ax = plt.subplot(gs[k, 1])
cfreqs = rdict['freqs']
i = (cfreqs > 0)
plt.hist(cfreqs[i], bins=20, color='b', alpha=0.7, normed=False, range=(10, 30))
plt.title(rdict['region'])
if k == len(region_filters) - 1:
plt.xlabel('Filter Frequency (Hz)')
plt.ylabel('Count')
plt.axis('tight')
plt.xlim(10, 30)
fname = os.path.join(get_this_dir(), 'encoder_filters.svg')
plt.savefig(fname, facecolor=fig.get_facecolor(), edgecolor='none')
plt.show()
def plot_psds(psd_file, data_dir='/auto/tdrive/mschachter/data'):
# read PairwiseCF file
pcf_file = os.path.join(data_dir, 'aggregate', 'pairwise_cf.h5')
pcf = AggregatePairwiseCF.load(pcf_file)
# pcf.zscore_within_site()
g = pcf.df.groupby(['bird', 'block', 'segment', 'electrode'])
nsamps_electrodes = len(g)
i = pcf.df.cell_index != -1
g = pcf.df[i].groupby(['bird', 'block', 'segment', 'electrode', 'cell_index'])
nsamps_cells = len(g)
print '# of electrodes: %d' % nsamps_electrodes
print '# of cells: %d' % nsamps_cells
print '# of lfp samples: %d' % (pcf.lfp_psds.shape[0])
print '# of spike psd samples: %d' % (pcf.spike_psds.shape[0])
# compute the LFP mean and std
lfp_psds = deepcopy(pcf.lfp_psds)
print 'lfp_psds_ind: max=%f, q99=%f' % (lfp_psds.max(), np.percentile(lfp_psds.ravel(), 99))
log_transform(lfp_psds)
print 'lfp_psds_ind: max=%f, q99=%f' % (lfp_psds.max(), np.percentile(lfp_psds.ravel(), 99))
nz = lfp_psds.sum(axis=1) > 0
lfp_psds = lfp_psds[nz, :]
lfp_psd_mean = lfp_psds.mean(axis=0)
lfp_psd_std = lfp_psds.std(axis=0, ddof=1)
nsamps_lfp = lfp_psds.shape[0]
# get the spike rate
spike_rate = pcf.df.spike_rate.values
# plt.figure()
# plt.hist(spike_rate, bins=20, color='g', alpha=0.7)
# plt.title('Spike Rate Histogram, q1=%0.3f, q5=%0.3f, q10=%0.3f, q50=%0.3f, q99=%0.3f' %
# (np.percentile(spike_rate, 1), np.percentile(spike_rate, 5), np.percentile(spike_rate, 10),
# np.percentile(spike_rate, 50), np.percentile(spike_rate, 99)))
# plt.show()
# compute the covariance
lfp_psd_z = deepcopy(lfp_psds)
lfp_psd_z -= lfp_psd_mean
lfp_psd_z /= lfp_psd_std
lfp_and_spike_cov_est = LedoitWolf()
lfp_and_spike_cov_est.fit(lfp_psd_z)
lfp_and_spike_cov = lfp_and_spike_cov_est.covariance_
"""
# read CRCNS file
cell_data = dict()
hf = h5py.File(psd_file, 'r')
cnames = hf.attrs['col_names']
for c in cnames:
cell_data[c] = np.array(hf[c])
crcns_psds = np.array(hf['psds'])
freqs = hf.attrs['freqs']
hf.close()
cell_df = pd.DataFrame(cell_data)
print 'regions=',cell_df.superregion.unique()
name_map = {'brainstem':'MLd', 'thalamus':'OV', 'cortex':'Field L+CM'}
"""
# resample the lfp mean and std
freq_rs = np.linspace(pcf.freqs.min(), pcf.freqs.max(), 1000)
lfp_mean_cs = interp1d(pcf.freqs, lfp_psd_mean, kind='cubic')
lfp_mean_rs = lfp_mean_cs(freq_rs)
lfp_std_cs = interp1d(pcf.freqs, lfp_psd_std, kind='cubic')
lfp_std_rs = lfp_std_cs(freq_rs)
# concatenate the lfp psd and log spike rate
lfp_psd_and_spike_rate = list()
for k,(li,si) in enumerate(zip(pcf.df['lfp_index'], pcf.df['spike_index'])):
lpsd = pcf.lfp_psds[li, :]
srate,sstd = pcf.spike_rates[si, :]
if srate > 0:
lfp_psd_and_spike_rate.append(np.hstack([lpsd, np.log(srate)]))
lfp_psd_and_spike_rate = np.array(lfp_psd_and_spike_rate)
nfreqs = len(pcf.freqs)
lfp_rate_cc = np.zeros([nfreqs])
for k in range(nfreqs):
lfp_rate_cc[k] = np.corrcoef(lfp_psd_and_spike_rate[:, k], lfp_psd_and_spike_rate[:, -1])[0, 1]
fig = plt.figure(figsize=(24, 12))
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.30, hspace=0.30)
nrows = 2
ncols = 100
gs = plt.GridSpec(nrows, ncols)
ax = plt.subplot(gs[0, :35])
plt.errorbar(freq_rs, lfp_mean_rs, yerr=lfp_std_rs, c='k', linewidth=9.0, elinewidth=3.0,
ecolor='#D8D8D8', alpha=0.5, capthick=0.)
plt.axis('tight')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Power (dB)')
# plt.ylim(0, 1)
plt.title('Mean LFP PSD')
ax = plt.subplot(gs[1, :35])
plt.plot(pcf.freqs, lfp_rate_cc, '-', c=COLOR_BLUE_LFP, linewidth=9.0, alpha=0.7)
plt.axhline(0, c='k')
plt.axis('tight')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Correlation Coefficient')
plt.ylim(-0.05, 0.25)
plt.title('LFP Power vs log Spike Rate')
"""
fi = freqs < 200
ax = plt.subplot(gs[1, :35])
clrs = ['k', '#d60036', COLOR_YELLOW_SPIKE]
alphas = [0.8, 0.8, 0.6]
for k,reg in enumerate(['brainstem', 'thalamus', 'cortex']):
i = cell_df.superregion == reg
indices = cell_df['index'][i].values
psds = crcns_psds[indices, :]
log_psds = deepcopy(psds)
log_transform(log_psds)
# compute the mean and sd of the power spectra
psd_mean = log_psds.mean(axis=0)
psd_std = log_psds.std(axis=0, ddof=1)
psd_cv = psd_std / psd_mean
# plot the mean power spectrum on the left
plt.plot(freqs[fi], psd_mean[fi], c=clrs[k], linewidth=9.0, alpha=alphas[k])
plt.ylabel('Power (dB)')
plt.xlabel('Frequency (Hz)')
plt.axis('tight')
plt.ylim(0, 1.0)
plt.legend(['MLd', 'OV', 'Field L+CM'], fontsize='x-small', loc='upper right')
plt.title('Mean PSTH PSDs (CRCNS Data)')
"""
ax = plt.subplot(gs[:, 40:])
plt.imshow(lfp_and_spike_cov, aspect='auto', interpolation='nearest', origin='lower', cmap=magma, vmin=0, vmax=1)
plt.colorbar(label='Correlation Coefficient')
xy = np.arange(len(pcf.freqs))
lbls = ['%d' % f for f in pcf.freqs]
plt.xticks(xy, lbls, rotation=0)
plt.yticks(xy, lbls)
plt.axhline(nfreqs-0.5, c='w')
plt.axvline(nfreqs-0.5, c='w')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Frequency (Hz)')
plt.title('LFP PSD Correlation Matrix')
fname = os.path.join(get_this_dir(), 'crcns_data.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_figures(data_dir='/auto/tdrive/mschachter/data'):
bird = 'GreBlu9508M'
block = 'Site4'
segment = 'Call1'
hemi = 'L'
fname = '%s_%s_%s_%s' % (bird, block, segment, hemi)
exp_dir = os.path.join(data_dir, bird)
preproc_file = os.path.join(exp_dir, 'preprocess', 'RNNPreprocess_%s.h5' % fname)
rp = RNNPreprocessTransform.load(preproc_file)
pred_file = os.path.join(exp_dir, 'rnn', 'LFPEnvelope_%s.h5' % fname)
hf = h5py.File(pred_file, 'r')
Ypred = np.array(hf['Ypred'])
hf.close()
assert Ypred.shape[0] == rp.U.shape[0]
stim_id = 277
trial = 5
i = (rp.event_df.stim_id == stim_id) & (rp.event_df.trial == trial)
assert i.sum() == 1
start_time = rp.event_df[i].start_time.values[0]
end_time = rp.event_df[i].end_time.values[0]
si = int(start_time*rp.sample_rate)
ei = int(end_time * rp.sample_rate)
spec = rp.U[si:ei, :].T
spec_freq = rp.spec_freq
spec_env = spec.sum(axis=0)
spec_env /= spec_env.max()
lfp = rp.Yraw[si:ei, :].T
lfp_pred = Ypred[si:ei, :].T
nt = spec.shape[1]
t = np.arange(nt) / rp.sample_rate
index2electrode = list(rp.index2electrode)
electrode_order = ROSTRAL_CAUDAL_ELECTRODES_LEFT
if hemi == 'R':
electrode_order = ROSTRAL_CAUDAL_ELECTRODES_RIGHT
fig = plt.figure(figsize=(23, 13), facecolor='w')
gs = plt.GridSpec(100, 1)
ax = plt.subplot(gs[:25, :])
spec_env *= (spec_freq.max() - spec_freq.min())
spec_env += spec_freq.min()
plot_spectrogram(t, spec_freq, spec, ax=ax, ticks=True, fmax=8000., colormap='SpectroColorMap', colorbar=False)
plt.plot(t, spec_env, 'k-', linewidth=5.0, alpha=0.7)
plt.axis('tight')
ax = plt.subplot(gs[30:, :])
nelectrodes = len(index2electrode)
lfp_spacing = 5.
for k in range(nelectrodes):
e = electrode_order[nelectrodes-k-1]
n = index2electrode.index(e)
offset = k*lfp_spacing
plt.plot(t, lfp[n, :] + offset, 'k-', alpha=0.7, linewidth=5.0)
plt.plot(t, lfp_pred[n, :] + offset, 'r-', alpha=0.7, linewidth=5.0)
plt.yticks([])
plt.axis('tight')
plt.xlabel('Time (s)')
plt.ylabel('LFP')
fname = os.path.join(get_this_dir(), 'encoder_pred.svg')
plt.savefig(fname, facecolor=fig.get_facecolor(), edgecolor='none')
plt.show()
def plot_maps(agg, data_dir='/auto/tdrive/mschachter/data'):
edata = pd.read_csv(os.path.join(data_dir, 'aggregate', 'electrode_data+dist.csv'))
data = {'bird':list(), 'block':list(), 'segment':list(), 'hemi':list(),
'electrode':list(), 'reg':list(), 'dm':list(), 'dl':list(),
'aprop':list(), 'r2':list()}
df = agg.df
# encoder performance maps
aprops_to_show = APROPS_TO_SHOW
# build a dataset that makes it easy to plot single decoder performance
g = df.groupby(['bird', 'block', 'segment', 'hemi', 'electrode', 'aprop'])
for (bird,block,segment,hemi,electrode,aprop),gdf in g:
assert len(gdf) == 1
ei = (edata.bird == bird) & (edata.block == block) & (edata.hemisphere == hemi) & (edata.electrode == electrode)
assert ei.sum() == 1
reg = clean_region(edata.region[ei].values[0])
dist_l2a = edata.dist_l2a[ei].values[0]
dist_midline = edata.dist_midline[ei].values[0]
if bird == 'GreBlu9508M':
dist_l2a *= 4
data['bird'].append(bird)
data['block'].append(block)
data['segment'].append(segment)
data['hemi'].append(hemi)
data['dm'].append(dist_midline)
data['dl'].append(dist_l2a)
data['r2'].append(gdf.r2.values[0])
data['reg'].append(reg)
data['electrode'].append(electrode)
data['aprop'].append(aprop)
df = pd.DataFrame(data)
i = ~np.isnan(df.dm) & ~np.isnan(df.dl) & ~np.isnan(df.r2) & (df.r2 > 0)
df = df[i]
print df.describe()
def _plot_map(_pdata, _ax, _cmap, _maxval, _bgcolor=None, _perf_alpha=False, _plot_region=False, _msize=60, _region_only=False):
if _bgcolor is not None:
_ax.set_axis_bgcolor(_bgcolor)
_pval = _pdata['df'].r2.values
_x = _pdata['df'].dm.values
_y = _pdata['df'].dl.values
_regs = _pdata['df'].reg.values
plt.sca(_ax)
_alpha = np.ones([len(_pval)])
if _perf_alpha:
_alpha = _pdata['df'].r2.values
_alpha /= _alpha.max()
_alpha[_alpha > 0.9] = 1.
_clrs = _cmap(_pval / _maxval)
else:
_clrs = _cmap(_pval / _maxval)
if not _region_only:
plt.scatter(_x, _y, c=_pval, marker='o', cmap=_cmap, vmin=0, s=_msize, alpha=0.6)
_cbar = plt.colorbar(label='Decoder R2')
_new_ytks = ['%0.2f' % float(_yt.get_text()) for _yt in _cbar.ax.get_yticklabels()]
# print '_new_ytks=', _new_ytks
_cbar.ax.set_yticklabels(_new_ytks)
plt.xlabel('Dist to Midline (mm)')
plt.ylabel('Dist to L2A (mm)')
# print 'ytks=',_ytks
plt.xlim(0, 2.5)
plt.ylim(-1, 1)
if _plot_region:
for _k,(_xx,_yy) in enumerate(zip(_x, _y)):
if _regs[_k] not in ['HP', '?'] and '-' not in _regs[k]:
plt.text(_xx, _yy, _regs[_k], fontsize=10, color='k', alpha=0.7)
def rb_cmap(x):
assert np.abs(x).max() <= 1
_rgb = np.zeros([len(x), 3])
_pos = x >= 0
_neg = x < 0
_rgb[_pos, 0] = x[_pos]
_rgb[_neg, 2] = np.abs(x[_neg])
return _rgb
figsize = (23, 13)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(left=0.05, right=0.98, hspace=0.25, wspace=0.25)
nrows = 2
ncols = 3
for k, aprop in enumerate(aprops_to_show):
ax = plt.subplot(nrows, ncols, k+1)
i = df.aprop == aprop
max_r2 = df[i].r2.max()
print 'k=%d, %s: max_r2=%0.2f' % (k, aprop, max_r2)
# _plot_map({'df':df[i]}, ax, magma, max_r2, _bgcolor='k', _perf_alpha=False, _plot_region=False)
_plot_map({'df': df[i]}, ax, plt.cm.afmhot_r, max_r2,_bgcolor='w',)
plt.title(ACOUSTIC_PROP_NAMES[aprop])
ax = plt.subplot(nrows, ncols, 5)
# _plot_map({'df': df[df.aprop == 'maxAmp']}, ax, plt.cm.afmhot_r, 1., _bgcolor='w', _plot_region=True, _region_only=True)
plot_r2_region_prop(ax)
fname = os.path.join(get_this_dir(), 'single_electrode_decoder_r2.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def draw_encoder_perfs(agg):
freqs,lags = get_freqs_and_lags()
electrodes = [ROSTRAL_CAUDAL_ELECTRODES_LEFT,
ROSTRAL_CAUDAL_ELECTRODES_RIGHT,
ROSTRAL_CAUDAL_ELECTRODES_LEFT]
sites = [('GreBlu9508M', 'Site4', 'Call1', 'L'),
('WhiWhi4522M', 'Site2', 'Call2', 'R'),
('YelBlu6903F', 'Site3', 'Call3', 'L')
]
weights = list()
for bird,block,segment,hemi in sites:
i = (agg.df.bird == bird) & (agg.df.block == block) & (agg.df.segment == segment) & (agg.df.hemi == hemi)
ii = i & (agg.df.decomp == 'self_locked')
assert ii.sum() == 1
wkey = agg.df[ii]['wkey'].values[0]
lfp_eperf = agg.encoder_perfs[wkey]
lfp_decoder_weights = agg.decoder_weights[wkey]
print 'lfp_decoder_weights.shape=',lfp_decoder_weights.shape
lfp_sal_weights = lfp_decoder_weights[:, :, REDUCED_ACOUSTIC_PROPS.index('sal')]
lfp_q2_weights = lfp_decoder_weights[:, :, REDUCED_ACOUSTIC_PROPS.index('q2')]
weights.append( (lfp_eperf, lfp_sal_weights, lfp_q2_weights) )
figsize = (24, 13)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(top=0.95, bottom=0.02, right=0.97, left=0.03, hspace=0.25, wspace=0.25)
nrows = 3
ncols = 3
for k,(lfp_eperf,lfp_sal_weights,lfp_q2_weights) in enumerate(weights):
index2electrode = electrodes[k]
ax = plt.subplot(nrows, ncols, k*ncols + 1)
plt.imshow(lfp_eperf, interpolation='nearest', aspect='auto', vmin=0, cmap=magma)
plt.yticks(range(len(index2electrode)), ['%d' % e for e in index2electrode])
plt.xticks(range(len(freqs)), ['%d' % f for f in freqs], rotation=45)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Electrode')
plt.colorbar(label='Encoder R2')
ax = plt.subplot(nrows, ncols, k*ncols + 2)
absmax = np.abs(lfp_sal_weights).max()
plt.imshow(lfp_sal_weights, interpolation='nearest', aspect='auto', vmin=-absmax, vmax=absmax, cmap=plt.cm.seismic)
plt.yticks(range(len(index2electrode)), ['%d' % e for e in index2electrode])
plt.xticks(range(len(freqs)), ['%d' % f for f in freqs], rotation=45)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Electrode')
plt.colorbar(label='Decoder Weight')
ax = plt.subplot(nrows, ncols, k*ncols + 3)
absmax = np.abs(lfp_sal_weights).max()
plt.imshow(lfp_q2_weights, interpolation='nearest', aspect='auto', vmin=-absmax, vmax=absmax, cmap=plt.cm.seismic)
plt.yticks(range(len(index2electrode)), ['%d' % e for e in index2electrode])
plt.xticks(range(len(freqs)), ['%d' % f for f in freqs], rotation=45)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Electrode')
plt.colorbar(label='Decoder Weight')
fname = os.path.join(get_this_dir(), 'encoder_weights.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
def plot_syllable_comps(agg, stim_id=43, syllable_order=1, data_dir='/auto/tdrive/mschachter/data'):
sprops = get_syllable_props(agg, stim_id, syllable_order, data_dir)
wave = sprops['wave']
wave_t = sprops['wave_t']
wave_si = sprops['wave_si']
wave_ei = sprops['wave_ei']
ps_freq = sprops['ps_freq']
ps = sprops['ps']
amp_env = sprops['amp_env']
aprops = sprops['aprops']
start_time = aprops['start_time']
spec = sprops['spec']
spec_t = sprops['spec_t']
spec_freq = sprops['spec_freq']
spec_si = sprops['spec_si']
spec_ei = sprops['spec_ei']
aprop_specs = dict()
aprop_spec_props = ['maxAmp', 'meanspect', 'sal']
for aprop in aprop_spec_props:
aprop_specs[aprop] = get_syllable_examples(agg, aprop, data_dir, bird='GreBlu9508M')
figsize = (23, 13)
fig = plt.figure(figsize=figsize, facecolor='w')
fig.subplots_adjust(top=0.95, bottom=0.08, right=0.97, left=0.06, hspace=0.30, wspace=0.30)
gs = plt.GridSpec(3, 100)
sp_width = 20
ax = plt.subplot(gs[0, :sp_width])
plt.plot((wave_t[wave_si:wave_ei] - start_time)*1e3, wave[wave_si:wave_ei], 'k-', linewidth=2.)
plt.plot((wave_t[wave_si:wave_ei] - start_time)*1e3, amp_env[wave_si:wave_ei], 'r-', linewidth=4., alpha=0.7)
plt.xlabel('Time (ms)')
plt.ylabel('Waveform')
plt.axis('tight')
aprops_to_get = ['skewtime', 'kurtosistime', 'entropytime', 'maxAmp']
units = ['s', '', '', 'bits', '']
ax = plt.subplot(gs[0, sp_width:(sp_width+18)])
txt_space = 0.1
for k,aprop in enumerate(aprops_to_get):
aval = aprops[aprop]
if aval > 10:
txt = '%s: %d' % (ACOUSTIC_PROP_NAMES[aprop], aval)
else:
txt = '%s: %0.2f' % (ACOUSTIC_PROP_NAMES[aprop], aval)
txt += ' %s' % units[k]
plt.text(0.1, 1-((k+1)*txt_space), txt, fontsize=18)
ax.set_axis_off()
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xticks([])
plt.yticks([])
ax = plt.subplot(gs[1, :sp_width])
fi = (ps_freq > 0) & (ps_freq <= 8000.)
plt.plot(ps_freq[fi]*1e-3, ps[fi], 'k-', linewidth=3., alpha=1.)
for aprop in ['q1', 'q2', 'q3']:
plt.axvline(aprops[aprop]*1e-3, color='#606060', linestyle='--', linewidth=3.0, alpha=0.9)
# plt.text(aprops[aprop]*1e-3 - 0.6, 3000., aprop, fontsize=14)
plt.ylabel("Power")
plt.yticks([])
plt.xlabel('Frequency (kHz)')
plt.axis('tight')
aprops_to_get = ['meanspect', 'stdspect', 'skewspect', 'kurtosisspect', 'entropyspect', 'q1', 'q2', 'q3']
units = ['Hz', 'Hz', '', '', 'bits', 'Hz', 'Hz', 'Hz']
ax = plt.subplot(gs[1, sp_width:(sp_width+18)])
for k,aprop in enumerate(aprops_to_get):
aval = aprops[aprop]
if aval > 10:
txt = '%s: %d' % (ACOUSTIC_PROP_NAMES[aprop], aval)
else:
txt = '%s: %0.2f' % (ACOUSTIC_PROP_NAMES[aprop], aval)
txt += ' %s' % units[k]
plt.text(0.1, 1-((k+1)*txt_space), txt, fontsize=18)
ax.set_axis_off()
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xticks([])
plt.yticks([])
spec_colormap()
ax = plt.subplot(gs[2, :sp_width])
plot_spectrogram((spec_t[spec_si:spec_ei]-start_time)*1e3, spec_freq*1e-3, spec[:, spec_si:spec_ei], ax, colormap='SpectroColorMap', colorbar=False)
plt.axhline(aprops['fund']*1e-3, color='k', linestyle='--', linewidth=3.0)
plt.ylabel("Frequency (kHz)")
plt.xlabel('Time (ms)')
for k,aprop in enumerate(aprop_spec_props):
full_spec_freq, full_spec, centers, propvals, stypes = aprop_specs[aprop]
ax = plt.subplot(gs[k, (sp_width+20):])
full_spec_t = np.arange(full_spec.shape[1]) / sprops['spec_sample_rate']
plot_spectrogram(full_spec_t, full_spec_freq*1e-3, full_spec, ax, colormap='SpectroColorMap', colorbar=False)
for c,st in zip(centers,stypes):
plt.text(c/sprops['spec_sample_rate'], 7., CALL_TYPE_NAMES[st], fontsize=20)
pstrs = list()
for p in propvals:
if p > 10:
if aprop == 'meanspect':
pstrs.append('%d Hz' % p)
else:
pstrs.append('%d' % p)
else:
pstrs.append('%0.3f' % p)
for c,p in zip(centers,pstrs):
plt.text(c, 6, p, fontsize=20)
# plt.xticks(centers, pstrs)
plt.xticks([])
plt.ylabel('Frequency (kHz)')
scale_t = (full_spec_t.max()*0.95) - np.linspace(0, 0.050, 20)
scale_y = np.zeros_like(scale_t) + 3
plt.plot(scale_t, scale_y, 'k-', linewidth=4.0)
plt.text(scale_t.min(), 2.0, '50ms', fontsize=20, fontweight='bold')
upper_right_x = np.percentile(full_spec_t, 80)
upper_right_y = 7
plt.title(ACOUSTIC_PROP_FULLNAMES[aprop], fontsize=22)
aprops_to_get = ['fund', 'fund2', 'sal', 'voice2percent', 'maxfund', 'minfund', 'cvfund']
units = ['Hz', 'Hz', '', '', 'Hz', 'Hz', 'Hz']
ax = plt.subplot(gs[2, sp_width:(sp_width + 18)])
for k, aprop in enumerate(aprops_to_get):
aval = aprops[aprop]
if aval > 10:
txt = '%s: %d' % (ACOUSTIC_PROP_NAMES[aprop], aval)
else:
txt = '%s: %0.2f' % (ACOUSTIC_PROP_NAMES[aprop], aval)
txt += ' %s' % units[k]
plt.text(0.1, 1 - ((k + 1) * txt_space), txt, fontsize=18)
ax.set_axis_off()
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xticks([])
plt.yticks([])
fname = os.path.join(get_this_dir(), 'figure.svg')
plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()
def plot_acoustic_stats(agg, data_dir='/auto/tdrive/mschachter/data'):
aprops = ['fund', 'maxfund', 'minfund', 'cvfund', 'fund2', 'voice2percent',
'stdspect', 'kurtosisspect', 'entropyspect', 'sal',
'meanspect', 'q1', 'q2', 'q3', 'skewspect',
'skewtime', 'kurtosistime', 'entropytime', 'maxAmp']
Xz,good_indices = agg.remove_duplicates(aprops=aprops)
i = np.zeros(len(agg.df), dtype='bool')
i[good_indices] = True
dur = (agg.df.end_time - agg.df.start_time).values * 1e3
dur_q1 = np.percentile(dur, 1)
dur_q5 = np.percentile(dur, 5)
dur_q25 = np.percentile(dur, 25)
dur_q50 = np.percentile(dur, 50)
# get rid of syllables outside of a desired duration range
dur_thresh = 40
i &= (dur > dur_thresh) & (dur < 400)
# get rid of syllables where no fundamental could be estimated (they should already be removed...)
fund = agg.Xraw[:, agg.acoustic_props.index('fund')]
i &= fund > 0
print '# of valid samples: %d' % i.sum()
# compute the correlation matrix
C = np.corrcoef(Xz.T)
# build an undirected graph from the correlation matrix
g = nx.Graph()
for aprop in aprops:
rgb = ACOUSTIC_PROP_COLORS_BY_TYPE[aprop]
viz = {'color':{'r':rgb[0], 'g':rgb[1], 'b':rgb[2], 'a':0}}
g.add_node(aprop, name=str(aprop), viz=viz)
# connect nodes if their cc is above a given threshold
cc_thresh = 0.25
for k,aprop1 in enumerate(aprops):
for j in range(k):
if np.abs(C[k, j]) > cc_thresh:
aprop2 = aprops[j]
viz = {'color': {'r': 0, 'g': .5, 'b': 0, 'a': 0.7}}
g.add_edge(aprop1, aprop2, weight=abs(float(C[k, j])))
# pos = nx.spring_layout(g)
# nx.draw(g, pos, labels={aprop:aprop for aprop in aprops})
# plt.show()
nx.write_gexf(g, '/tmp/acoustic_feature_graph.gexf')
print 'Acoustic Feature Clusters:'
for grp in sorted(nx.connected_components(g), key=len, reverse=True):
print grp
figsize = (23, 12)
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(left=0.07, right=0.99, top=0.90, bottom=0.20, hspace=0.01, wspace=0.01)
gs = plt.GridSpec(1, 100)
aprop_lbls = [ACOUSTIC_PROP_NAMES[aprop] for aprop in aprops]
# plot data matrix
absmax = 4
ax = plt.subplot(gs[0, :40])
plt.imshow(Xz, interpolation='nearest', aspect='auto', vmin=-absmax, vmax=absmax, cmap=plt.cm.seismic)
xt = range(len(aprops))
plt.xticks(xt, aprop_lbls, rotation=90)
plt.colorbar()
plt.title('Z-scored Acoustic Feature Matrix')
ax = plt.subplot(gs[0, 50:])
plt.imshow(C, interpolation='nearest', aspect='auto', vmin=-1, vmax=1, cmap=plt.cm.seismic, origin='lower')
xt = range(len(aprops))
plt.xticks(xt, aprop_lbls, rotation=90)
plt.yticks(xt, aprop_lbls)
plt.colorbar(label='Correlation Coefficient')
plt.title('Acoustic Feature Correlation Matrix')
fname = os.path.join(get_this_dir(), 'acoustic_data_and_corr_matrix.svg')
# plt.savefig(fname, facecolor='w', edgecolor='none')
plt.show()