rbm_type][act_type]:
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift] = {}
dt = dat[shift]
strt = dt.shape[1] - pred.shape[0]
dt = dt[:, strt:]
for cell, cell_dat in enumerate(pred.T):
if cell not in patches[exp_type][cell_id]['corrs'][
rbm_type][act_type][shift]:
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell] = {}
if 'cell_crr' in patches[exp_type][cell_id]['corrs'][
rbm_type][act_type][shift][cell]:
print 'skipping ', cell
continue
crr = corr_trial_to_mean(dt, cell_dat)
crr_mn = do_thresh_corr(dt.mean(0), cell_dat)
crr_mn_r2 = do_thresh_corr(dt.mean(0),
cell_dat,
corr_type=None)
cell_crr = corr_trial_to_mean(dt)
changed = True
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr'] = crr
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr_mn'] = crr_mn
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2
patches[exp_type][cell_id]['corrs'][rbm_type][
act_type][shift][cell]['cell_crr'] = cell_crr
if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3:
pred = patches[exp_type][cell_id]['responses'][rbm_type][act_type][1]
changed = False
for shift in e[cell_id]['shifts']:
print shift
if shift not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type]:
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift] = {}
dt = dat[shift]
strt = dt.shape[1] - pred.shape[0]
dt = dt[:, strt:]
for cell, cell_dat in enumerate(pred.T):
if cell not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift]:
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell] = {}
if 'cell_crr' in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]:
print 'skipping ', cell
continue
crr = corr_trial_to_mean(dt, cell_dat)
crr_mn = do_thresh_corr(dt.mean(0), cell_dat)
crr_mn_r2 = do_thresh_corr(dt.mean(0), cell_dat, corr_type=None)
cell_crr = corr_trial_to_mean(dt)
changed = True
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr'] = crr
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn'] = crr_mn
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2
patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['cell_crr'] = cell_crr
if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3:
print '%s %s %s %s %s: cell: %d, corr: %.3f, pred corr: %.3f, crr mn: %.3f' % (
exp_type, cell_id, rbm_type,
act_type, shift, cell, cell_crr, crr, crr_mn)
if changed:
print 'saving'
np.save('mask_data_complete_%s' % stim_type, patches)
elif randomise == 'random':
psth_c = d['psth_c_rand']
psth_s = d['psth_s_rand']
psth_w = d['psth_w_rand']
elif randomise == 'generated':
psth_c = d['psth_c_gen']
psth_s = d['psth_s_gen']
psth_w = d['psth_w_gen']
if not os.path.exists(f_path):
os.makedirs(f_path)
edge = d['edge']
mn_c = psth_c.mean(0)
std_c = np.std(psth_c, 0)
xcorr_c = corr_trial_to_mean(d['psth_c_raw'])
avg_c = d['psth_c_raw'].ravel().mean()
vals.append(xcorr_c)
mn_w = psth_w.mean(0)
std_w = np.std(psth_w, 0)
xcorr_w = corr_trial_to_mean(d['psth_w_raw'])
avg_w = d['psth_w_raw'].ravel().mean()
vals.append(xcorr_w)
mx = np.maximum(mn_c.max(), mn_w.max())
if psth_s is not None:
mn_s = psth_s.mean(0)
std_s = np.std(psth_s, 0)
xcorr_s = corr_trial_to_mean(d['psth_s_raw'])
avg_s = d['psth_s_raw'].ravel().mean()
vals.append(xcorr_s)
mx = np.maximum(mx, mn_s.max())
def do_lag_classification(exp_type='SOM',
combs=[
'Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'
],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
folds=5,
filt=0.2,
alpha=0.001,
randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_' + str(filt)
else:
dat_path = dat_path + str(filt)
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb,
targ_type,
src_type,
e,
cellid,
exp_type,
four_downsample,
shift=shift,
randomise=randomise)
# ignore edge effects
pred, coefs = CV(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]:-edges[1]]
mn = y[:, edges[0]:-edges[1]].mean(0)
std = np.std(y[:, edges[0]:-edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn,
pred,
do_abs=True,
corr_type='pearsonr')
crr_exp = corr_trial_to_mean(y[:, edges[0]:-edges[1]],
mn)
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = crr_exp
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
folds=5, filt=0.2,
alpha=0.001, randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_' + str(filt)
else:
dat_path = dat_path + str(filt)
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb, targ_type,
src_type,
e, cellid, exp_type, four_downsample,
shift=shift, randomise=randomise)
# ignore edge effects
pred, coefs = CV(clf,
X, y, folds=folds, clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]: -edges[1]]
mn = y[:, edges[0]: -edges[1]].mean(0)
std = np.std(y[:, edges[0]: -edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr')
crr_exp = corr_trial_to_mean(y[:, edges[0]: -edges[1]],
mn)
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = crr_exp
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
elif randomise == 'random':
psth_c = d['psth_c_rand']
psth_s = d['psth_s_rand']
psth_w = d['psth_w_rand']
elif randomise == 'generated':
psth_c = d['psth_c_gen']
psth_s = d['psth_s_gen']
psth_w = d['psth_w_gen']
if not os.path.exists(f_path):
os.makedirs(f_path)
edge = d['edge']
mn_c = psth_c.mean(0)
std_c = np.std(psth_c, 0)
xcorr_c = corr_trial_to_mean(d['psth_c_raw'])
avg_c = d['psth_c_raw'].ravel().mean()
vals.append(xcorr_c)
mn_w = psth_w.mean(0)
std_w = np.std(psth_w, 0)
xcorr_w = corr_trial_to_mean(d['psth_w_raw'])
avg_w = d['psth_w_raw'].ravel().mean()
vals.append(xcorr_w)
mx = np.maximum(mn_c.max(), mn_w.max())
if psth_s is not None:
mn_s = psth_s.mean(0)
std_s = np.std(psth_s, 0)
xcorr_s = corr_trial_to_mean(d['psth_s_raw'])
avg_s = d['psth_s_raw'].ravel().mean()
vals.append(xcorr_s)
mx = np.maximum(mx, mn_s.max())
