np.save('%s%s_%s' % (target, exp, src), crr_w)
src_res = []
trials = np.arange(dt.shape[2])
ys = []
preds = []
for t in trials:
X = dt[cell_idx != cell]
X = X[:, :, trials != t].reshape(X.shape[0], -1).T
y = dt[cell, :, trials != t].ravel()
XX = dt[cell_idx != cell, :, t].reshape(X.shape[1], -1).T
yy = dt[cell, :, t].ravel()
c = clf(**clf_args)
c.fit(X, y)
ys.append(yy)
preds.append(c.predict(XX))
crr = do_thresh_corr(np.array(ys).ravel(), np.array(preds).ravel())
cell_res.append(crr)
res.append(cell_res)
res = np.array(res)
print res.shape
plt.figure(figsize=(16, 8))
plt.subplot(131, aspect='equal')
plt.scatter(res[:, 0], res[:, 1])
plt.plot([0, 1], [0, 1], '--')
plt.xlim([-0.01, res[:, 0].max() * 1.1])
plt.ylim([-0.01, res[:, 1].max() * 1.1])
plt.xlabel('Pred Corr Centre')
plt.ylabel('Pred Corr Whole')
plt.subplot(132)
plt.scatter(res[:, 0], crr_c[d])
plt.plot([0, 1], [0, 1], '--')
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
folds=5, filt=0.2,
alpha=0.001, randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb, targ_type,
src_type,
e, cellid, exp_type, four_downsample,
shift=shift, randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X, y, folds=folds, clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]: -edges[1]]
mn = y[edges[0]: -edges[1]]
std = np.std(y[edges[0]: -edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
if np.diff(y).sum() == 0:
print 'no activity, skipping'
continue
#y = np.tile(y, [X.shape[0], 1])
pred_trial = np.zeros([X.shape[0], y.shape[1]])
trls = np.arange(X.shape[0])
pred_time, coefs = CV_time(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edges=[0, 0])
pred_time = pred_time.ravel()
y = y.ravel()
crr = do_thresh_corr(pred_time,
y) #, corr_type='pearsonr')
crrs.append(crr)
res = 'Predict: %s, Using: %s, Dimension: %d, Crr: %.2f' % (
d, m, dim, crr)
pred_time = (pred_time -
pred_time.mean()) / np.std(pred_time)
y = (y - y.mean()) / np.std(y)
# print res
if crr > 0.6:
plt.figure(figsize=(14, 8))
plt.hold(True)
plt.plot(pred_time, label='pred')
plt.plot(y, label='movie')
plt.legend()
plt.title(res)
plt.show()
def do_lag_classification(exp_type='SOM',
combs=[
'Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'
],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
folds=5,
filt=0.2,
alpha=0.001,
randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb,
targ_type,
src_type,
e,
cellid,
exp_type,
four_downsample,
shift=shift,
randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]:-edges[1]]
mn = y[edges[0]:-edges[1]]
std = np.std(y[edges[0]:-edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn,
pred,
do_abs=True,
corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
np.save('%s%s_%s' % (target, exp, src), crr_w)
src_res = []
trials = np.arange(dt.shape[2])
ys = []
preds = []
for t in trials:
X = dt[cell_idx != cell]
X = X[:, :, trials != t].reshape(X.shape[0], -1).T
y = dt[cell, :, trials != t].ravel()
XX = dt[cell_idx != cell, :, t].reshape(X.shape[1], -1).T
yy = dt[cell, :, t].ravel()
c = clf(**clf_args)
c.fit(X, y)
ys.append(yy)
preds.append(c.predict(XX))
crr = do_thresh_corr(np.array(ys).ravel(), np.array(preds).ravel())
cell_res.append(crr)
res.append(cell_res)
res = np.array(res)
print res.shape
plt.figure(figsize=(16, 8))
plt.subplot(131, aspect='equal')
plt.scatter(res[:, 0], res[:, 1])
plt.plot([0, 1], [0, 1], '--')
plt.xlim([-0.01, res[:, 0].max() * 1.1])
plt.ylim([-0.01, res[:, 1].max() * 1.1])
plt.xlabel('Pred Corr Centre')
plt.ylabel('Pred Corr Whole')
plt.subplot(132)
plt.scatter(res[:, 0], crr_c[d])
plt.plot([0, 1], [0, 1], '--')
if psth_s is not None:
ax = plt.subplot(536)
cnts, _, _ = plt.hist(mn_s.ravel(), bins)
y = plt.ylim()[1]
plt.text(0.7, y * 0.9, 'Avg Act: %.2f' % avg_s)
hist_mx = np.maximum(cnts.max(), hist_mx)
plt.xticks(tks)
adjust_spines(ax, ['bottom', 'left'])
ax.set_ylim(0, hist_mx)
hst_ax1.set_ylim(0, hist_mx)
hst_ax2.set_ylim(0, hist_mx)
ax = plt.subplot(537)
plt.hold(True)
cr = do_thresh_corr(mn_w, mn_c)
vals.append(cr)
plt.plot(mn_w, label='Whole', color=clr_w, linewidth=1.5)
#plt.fill_between(range(psth_w.shape[1]), mn_c, mn_w,
# facecolor=clr2)
plt.plot(mn_c, color=clr_c, label='Center', linewidth=2)
plt.legend(bbox_to_anchor=(0.55, 0.95, 0.4, 0.1),
frameon=False,
ncol=2)
plt.text(10, 0.9, 'corr: %.2f' % (cr))
adjust_spines(ax, ['bottom', 'left'])
ylim = plt.ylim(0, 1)
plt.ylim([-0.01, ylim[1]])
ax = plt.subplot(538)
plt.title('Mean Comparison')
y = ys[:, :, dim]
samples = np.minimum(y.shape[1], X.shape[1])
y = y[:, :samples]
X = X[:, :samples]
if np.diff(y).sum() == 0:
print 'no activity, skipping'
continue
#y = np.tile(y, [X.shape[0], 1])
pred_trial = np.zeros([X.shape[0], y.shape[1]])
trls = np.arange(X.shape[0])
pred_time, coefs = CV_time(clf, X, y, folds=folds, clf_args=clf_args,
edges=[0, 0])
pred_time = pred_time.ravel()
y = y.ravel()
crr = do_thresh_corr(pred_time, y)#, corr_type='pearsonr')
crrs.append(crr)
res= 'Predict: %s, Using: %s, Dimension: %d, Crr: %.2f' %(
d, m, dim, crr)
pred_time = (pred_time - pred_time.mean()) / np.std(pred_time)
y = (y - y.mean()) / np.std(y)
# print res
if crr > 0.6:
plt.figure(figsize=(14, 8))
plt.hold(True)
plt.plot(pred_time, label='pred')
plt.plot(y, label='movie')
plt.legend()
plt.title(res)
plt.show()
crrs = np.array(crrs)
def do_classification(
exp_type='SOM',
combs=['Luminance', 'Contrast', 'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center',
'Whole'], ['Whole', 'Center'],
['Whole', 'Whole'], ['Surround', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
randomise=None,
folds=5,
filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]: #, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb
or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][
targ_type]:
cell_results[cellid][full_comb][targ_type][
src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type,
four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path, randomise,
str(filt), cellid)
else:
fname = '%s%s/%s/' % (fig_path, str(filt), cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge:-edge]
clf_vals.append([
targ_type,
])
mn = y.mean(0)[edge:-edge]
std = np.std(y, 0)[edge:-edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][
src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge:-edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [
pred, mn, std, crr_pred, crr_y, coefs, plot_params
]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def plot_corrs(c_vals, w_vals, mn_c_crr, mn_w_crr, n_cells, header, fname):
fig = plt.figure(figsize=(14, 8))
fig.set_facecolor('white')
c_vals = average_corrs(c_vals)
w_vals = average_corrs(w_vals)
c_vals_mn = average_corrs(c_vals)
w_vals_mn = average_corrs(w_vals)
mn_c_vals_mn = average_corrs(mn_c_crr)
mn_w_vals_mn = average_corrs(mn_w_crr)
print mn_c_crr.min(), mn_c_crr.max()
# ax = plt.subplot(411)
# plt.hold(True)
# plt.plot(c_vals.T, '0.8')
# plt.plot(c_vals_mn, '0.4', linewidth=2)
# plt.xlim(0, c_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.title('Masked')
# plt.ylabel('Mean R')
#
# ax = plt.subplot(412)
# plt.hold(True)
# plt.plot(w_vals.T, '0.8')
# plt.plot(w_vals_mn, '0.4', linewidth=2)
# plt.xlim(0, w_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.title('Whole Field')
# plt.ylabel('Mean R')
#
# ax = plt.subplot(413)
# plt.hold(True)
# plt.plot(w_vals_mn, 'g', linewidth=2, label='Whole')
# plt.plot(c_vals_mn, 'k', linewidth=2, label='Centre')
# crr = do_thresh_corr(w_vals_mn, c_vals_mn)
# leg = plt.legend(ncol=2)
# leg.draw_frame(False)
# plt.xlim(0, c_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.ylabel('Mean R')
# plt.xlabel('Sample')
# plt.title('Whole vs Masked: Crr: {0:.2f}'.format(crr))
ax = plt.subplot(211)
plt.hold(True)
plt.plot(mn_w_vals_mn, 'g', linewidth=2, label='Whole')
plt.plot(mn_c_vals_mn, 'k', linewidth=2, label='Centre')
crr = do_thresh_corr(mn_w_vals_mn, mn_c_vals_mn)
leg = plt.legend(ncol=2)
leg.draw_frame(False)
plt.xlim(0, c_vals.shape[1])
adjust_spines(ax, ['bottom', 'left'])
plt.title('Mean Whole vs Masked: R^2: {0:.2f}'.format(crr))
plt.ylabel('Mean R')
plt.xlabel('Sample')
ax = plt.subplot(212)
plt.hold(True)
plt.plot(mn_c_vals_mn - mn_w_vals_mn, '0.3', linewidth=2)
leg.draw_frame(False)
plt.xlim(0, c_vals.shape[1])
adjust_spines(ax, ['bottom', 'left'])
plt.title('Difference - Mean Whole vs Masked')
plt.ylabel('Mean R')
plt.xlabel('Sample')
plt.suptitle('%s - Intercell R^2 over Trials - #Cells: %d' %
(header, n_cells))
plt.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.9,
wspace=0.2,
hspace=0.25)
fig.savefig(fname + '.eps')
fig.savefig(fname + '.png')
#plt.show()
plt.close(fig)
print fname
print n_cells
print '\tMean\tMax'
print 'Centre:\t%.3f\t%.3f' % (c_vals_mn.mean(), c_vals.max())
print 'Whole:\t%.3f\t%.3f' % (w_vals.mean(), w_vals.max())
print
if psth_s is not None:
ax = plt.subplot(536)
cnts, _, _ = plt.hist(mn_s.ravel(), bins)
y = plt.ylim()[1]
plt.text(0.7, y * 0.9, 'Avg Act: %.2f' % avg_s)
hist_mx = np.maximum(cnts.max(), hist_mx)
plt.xticks(tks)
adjust_spines(ax, ['bottom', 'left'])
ax.set_ylim(0, hist_mx)
hst_ax1.set_ylim(0, hist_mx)
hst_ax2.set_ylim(0, hist_mx)
ax = plt.subplot(537)
plt.hold(True)
cr = do_thresh_corr(mn_w, mn_c)
vals.append(cr)
plt.plot(mn_w, label='Whole', color=clr_w, linewidth=1.5)
#plt.fill_between(range(psth_w.shape[1]), mn_c, mn_w,
# facecolor=clr2)
plt.plot(mn_c, color=clr_c,
label='Center', linewidth=2)
plt.legend(bbox_to_anchor=(0.55, 0.95, 0.4, 0.1), frameon=False, ncol=2)
plt.text(10, 0.9, 'corr: %.2f' % (cr))
adjust_spines(ax, ['bottom', 'left'])
ylim = plt.ylim(0, 1)
plt.ylim([-0.01, ylim[1]])
ax = plt.subplot(538)
plt.title('Mean Comparison')
if psth_s is not None:
changed = False
for shift in e[cell_id]['shifts']:
print shift
if shift not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type]:
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift] = {}
dt = dat[shift]
strt = dt.shape[1] - pred.shape[0]
dt = dt[:, strt:]
for cell, cell_dat in enumerate(pred.T):
if cell not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift]:
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell] = {}
if 'cell_crr' in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]:
print 'skipping ', cell
continue
crr = corr_trial_to_mean(dt, cell_dat)
crr_mn = do_thresh_corr(dt.mean(0), cell_dat)
crr_mn_r2 = do_thresh_corr(dt.mean(0), cell_dat, corr_type=None)
cell_crr = corr_trial_to_mean(dt)
changed = True
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr'] = crr
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn'] = crr_mn
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['cell_crr'] = cell_crr
if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3:
print '%s %s %s %s %s: cell: %d, corr: %.3f, pred corr: %.3f, crr mn: %.3f' % (
exp_type, cell_id, rbm_type,
act_type, shift, cell, cell_crr, crr, crr_mn)
if changed:
print 'saving'
np.save('mask_data_complete_%s' % stim_type, patches)
# [rbm_type]
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast',
'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center', 'Whole'],
['Whole', 'Center'], ['Whole', 'Whole'],
['Surround', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
randomise=None, folds=5, filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]:#, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][targ_type]:
cell_results[cellid][full_comb][targ_type][src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type, four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path,
randomise, str(filt),
cellid)
else:
fname = '%s%s/%s/' % (fig_path,
str(filt),
cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X, y, folds=folds, clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge: -edge]
clf_vals.append([targ_type, ])
mn = y.mean(0)[edge: -edge]
std = np.std(y, 0)[edge: -edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge: -edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [pred, mn, std, crr_pred, crr_y,
coefs, plot_params]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def plot_corrs(c_vals, w_vals, mn_c_crr, mn_w_crr, n_cells, header, fname):
fig = plt.figure(figsize=(14, 8))
fig.set_facecolor('white')
c_vals = average_corrs(c_vals)
w_vals = average_corrs(w_vals)
c_vals_mn = average_corrs(c_vals)
w_vals_mn = average_corrs(w_vals)
mn_c_vals_mn = average_corrs(mn_c_crr)
mn_w_vals_mn = average_corrs(mn_w_crr)
print mn_c_crr.min(), mn_c_crr.max()
# ax = plt.subplot(411)
# plt.hold(True)
# plt.plot(c_vals.T, '0.8')
# plt.plot(c_vals_mn, '0.4', linewidth=2)
# plt.xlim(0, c_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.title('Masked')
# plt.ylabel('Mean R')
#
# ax = plt.subplot(412)
# plt.hold(True)
# plt.plot(w_vals.T, '0.8')
# plt.plot(w_vals_mn, '0.4', linewidth=2)
# plt.xlim(0, w_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.title('Whole Field')
# plt.ylabel('Mean R')
#
# ax = plt.subplot(413)
# plt.hold(True)
# plt.plot(w_vals_mn, 'g', linewidth=2, label='Whole')
# plt.plot(c_vals_mn, 'k', linewidth=2, label='Centre')
# crr = do_thresh_corr(w_vals_mn, c_vals_mn)
# leg = plt.legend(ncol=2)
# leg.draw_frame(False)
# plt.xlim(0, c_vals.shape[1])
# adjust_spines(ax, ['bottom', 'left'])
# plt.ylabel('Mean R')
# plt.xlabel('Sample')
# plt.title('Whole vs Masked: Crr: {0:.2f}'.format(crr))
ax = plt.subplot(211)
plt.hold(True)
plt.plot(mn_w_vals_mn, 'g', linewidth=2, label='Whole')
plt.plot(mn_c_vals_mn, 'k', linewidth=2, label='Centre')
crr = do_thresh_corr(mn_w_vals_mn, mn_c_vals_mn)
leg = plt.legend(ncol=2)
leg.draw_frame(False)
plt.xlim(0, c_vals.shape[1])
adjust_spines(ax, ['bottom', 'left'])
plt.title('Mean Whole vs Masked: R^2: {0:.2f}'.format(crr))
plt.ylabel('Mean R')
plt.xlabel('Sample')
ax = plt.subplot(212)
plt.hold(True)
plt.plot(mn_c_vals_mn - mn_w_vals_mn, '0.3', linewidth=2)
leg.draw_frame(False)
plt.xlim(0, c_vals.shape[1])
adjust_spines(ax, ['bottom', 'left'])
plt.title('Difference - Mean Whole vs Masked')
plt.ylabel('Mean R')
plt.xlabel('Sample')
plt.suptitle('%s - Intercell R^2 over Trials - #Cells: %d' %
(header, n_cells))
plt.subplots_adjust(left=0.05, bottom=0.05, right=0.95, top=0.9,
wspace=0.2, hspace=0.25)
fig.savefig(fname + '.eps')
fig.savefig(fname + '.png')
#plt.show()
plt.close(fig)
print fname
print n_cells
print '\tMean\tMax'
print 'Centre:\t%.3f\t%.3f' % (c_vals_mn.mean(), c_vals.max())
print 'Whole:\t%.3f\t%.3f' % (w_vals.mean(), w_vals.max())
print
psth_w = d['psth_w']
mn_c = psth_c.mean(0)
mn_w = psth_w.mean(0)
mx = np.maximum(mn_c.max(), mn_w.max())
if psth_s is not None:
mn_s = psth_s.mean(0)
mx = np.maximum(mx, mn_s.max())
cr_c_w = None
cr_c_s = None
cr_w_s = None
mn_c = mn_c / mx
mn_w = mn_w / mx
cr_c_w = do_thresh_corr(mn_w, mn_c, corr_type=corr_type)
if psth_s is not None:
mn_s = mn_s / mx
cr_c_s = do_thresh_corr(mn_s, mn_c, corr_type=corr_type)
cr_w_s = do_thresh_corr(mn_s, mn_w, corr_type=corr_type)
corrs.append([cr_c_w, cr_c_s, cr_w_s])
cells.append([exp_type, cellid])
corrs = np.array(corrs)
print corrs
cells = np.array(cells)
fig = plt.figure(figsize=(10, 12))
fig.set_facecolor('white')
x = np.arange(len(corrs))
cnts, _, _ = plt.hist(mn_w.ravel(), bins)
y = plt.ylim()[1]
plt.text(0.7, y * 0.9, 'Avg Act: %.2f' % avg_w)
hist_mx = np.maximum(cnts.max(), hist_mx)
plt.xticks(tks)
plt.title('Mean Activation Histogram')
adjust_spines(hst_ax2, ['bottom', 'left'])
hst_ax1.set_ylim(0, hist_mx)
hst_ax2.set_ylim(0, hist_mx)
ax = plt.subplot(425)
plt.hold(True)
crrs = []
for c, w in zip(dat_c.mean(2), dat_w.mean(2)):
crrs.append(do_thresh_corr(c, w))
cr = average_corrs(crrs)
vals.append(cr)
csv_vals[4] += crrs
plt.plot(mn_w, label='Whole', color=clr_w, linewidth=1.5)
#plt.fill_between(range(psth_w.shape[1]), mn_c, mn_w,
# facecolor=clr2)
plt.plot(mn_c, color=clr_c,
label='Center', linewidth=2)
plt.legend(bbox_to_anchor=(0.55, 0.95, 0.4, 0.1), frameon=False, ncol=2)
plt.text(10, 0.9, 'corr: %.2f' % (cr))
adjust_spines(ax, ['bottom', 'left'])
ylim = plt.ylim(0, 1)
plt.ylim([-0.01, ylim[1]])
plt.title('Mean Comparison')
cnts, _, _ = plt.hist(mn_w.ravel(), bins)
y = plt.ylim()[1]
plt.text(0.7, y * 0.9, 'Avg Act: %.2f' % avg_w)
hist_mx = np.maximum(cnts.max(), hist_mx)
plt.xticks(tks)
plt.title('Mean Activation Histogram')
adjust_spines(hst_ax2, ['bottom', 'left'])
hst_ax1.set_ylim(0, hist_mx)
hst_ax2.set_ylim(0, hist_mx)
ax = plt.subplot(425)
plt.hold(True)
crrs = []
for c, w in zip(dat_c.mean(2), dat_w.mean(2)):
crrs.append(do_thresh_corr(c, w))
cr = average_corrs(crrs)
vals.append(cr)
csv_vals[4] += crrs
plt.plot(mn_w, label='Whole', color=clr_w, linewidth=1.5)
#plt.fill_between(range(psth_w.shape[1]), mn_c, mn_w,
# facecolor=clr2)
plt.plot(mn_c, color=clr_c, label='Center', linewidth=2)
plt.legend(bbox_to_anchor=(0.55, 0.95, 0.4, 0.1),
frameon=False,
ncol=2)
plt.text(10, 0.9, 'corr: %.2f' % (cr))
adjust_spines(ax, ['bottom', 'left'])
ylim = plt.ylim(0, 1)
plt.ylim([-0.01, ylim[1]])
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift] = {}
dt = dat[shift]
strt = dt.shape[1] - pred.shape[0]
dt = dt[:, strt:]
for cell, cell_dat in enumerate(pred.T):
if cell not in patches[exp_type][cell_id]['corrs'][
rbm_type][act_type][shift]:
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell] = {}
if 'cell_crr' in patches[exp_type][cell_id]['corrs'][
rbm_type][act_type][shift][cell]:
print 'skipping ', cell
continue
crr = corr_trial_to_mean(dt, cell_dat)
crr_mn = do_thresh_corr(dt.mean(0), cell_dat)
crr_mn_r2 = do_thresh_corr(dt.mean(0),
cell_dat,
corr_type=None)
cell_crr = corr_trial_to_mean(dt)
changed = True
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr'] = crr
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr_mn'] = crr_mn
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['cell_crr'] = cell_crr
if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3:
print '%s %s %s %s %s: cell: %d, corr: %.3f, pred corr: %.3f, crr mn: %.3f' % (
psth_w = d['psth_w']
mn_c = psth_c.mean(0)
mn_w = psth_w.mean(0)
mx = np.maximum(mn_c.max(), mn_w.max())
if psth_s is not None:
mn_s = psth_s.mean(0)
mx = np.maximum(mx, mn_s.max())
cr_c_w = None
cr_c_s = None
cr_w_s = None
mn_c = mn_c / mx
mn_w = mn_w / mx
cr_c_w = do_thresh_corr(mn_w, mn_c, corr_type=corr_type)
if psth_s is not None:
mn_s = mn_s / mx
cr_c_s = do_thresh_corr(mn_s, mn_c, corr_type=corr_type)
cr_w_s = do_thresh_corr(mn_s, mn_w, corr_type=corr_type)
corrs.append([cr_c_w, cr_c_s, cr_w_s])
cells.append([exp_type, cellid])
corrs = np.array(corrs)
print corrs
cells = np.array(cells)
fig = plt.figure(figsize=(10, 12))
fig.set_facecolor('white')
x = np.arange(len(corrs))
