rbm_type][act_type]: patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift] = {} dt = dat[shift] strt = dt.shape[1] - pred.shape[0] dt = dt[:, strt:] for cell, cell_dat in enumerate(pred.T): if cell not in patches[exp_type][cell_id]['corrs'][ rbm_type][act_type][shift]: patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift][cell] = {} if 'cell_crr' in patches[exp_type][cell_id]['corrs'][ rbm_type][act_type][shift][cell]: print 'skipping ', cell continue crr = corr_trial_to_mean(dt, cell_dat) crr_mn = do_thresh_corr(dt.mean(0), cell_dat) crr_mn_r2 = do_thresh_corr(dt.mean(0), cell_dat, corr_type=None) cell_crr = corr_trial_to_mean(dt) changed = True patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift][cell]['crr'] = crr patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift][cell]['crr_mn'] = crr_mn patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2 patches[exp_type][cell_id]['corrs'][rbm_type][ act_type][shift][cell]['cell_crr'] = cell_crr if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3:
pred = patches[exp_type][cell_id]['responses'][rbm_type][act_type][1] changed = False for shift in e[cell_id]['shifts']: print shift if shift not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type]: patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift] = {} dt = dat[shift] strt = dt.shape[1] - pred.shape[0] dt = dt[:, strt:] for cell, cell_dat in enumerate(pred.T): if cell not in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift]: patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell] = {} if 'cell_crr' in patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]: print 'skipping ', cell continue crr = corr_trial_to_mean(dt, cell_dat) crr_mn = do_thresh_corr(dt.mean(0), cell_dat) crr_mn_r2 = do_thresh_corr(dt.mean(0), cell_dat, corr_type=None) cell_crr = corr_trial_to_mean(dt) changed = True patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr'] = crr patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn'] = crr_mn patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['crr_mn_r2'] = crr_mn_r2 patches[exp_type][cell_id]['corrs'][rbm_type][act_type][shift][cell]['cell_crr'] = cell_crr if (crr > 0.2 and crr >= cell_crr) or crr_mn > 0.3: print '%s %s %s %s %s: cell: %d, corr: %.3f, pred corr: %.3f, crr mn: %.3f' % ( exp_type, cell_id, rbm_type, act_type, shift, cell, cell_crr, crr, crr_mn) if changed: print 'saving' np.save('mask_data_complete_%s' % stim_type, patches)
elif randomise == 'random': psth_c = d['psth_c_rand'] psth_s = d['psth_s_rand'] psth_w = d['psth_w_rand'] elif randomise == 'generated': psth_c = d['psth_c_gen'] psth_s = d['psth_s_gen'] psth_w = d['psth_w_gen'] if not os.path.exists(f_path): os.makedirs(f_path) edge = d['edge'] mn_c = psth_c.mean(0) std_c = np.std(psth_c, 0) xcorr_c = corr_trial_to_mean(d['psth_c_raw']) avg_c = d['psth_c_raw'].ravel().mean() vals.append(xcorr_c) mn_w = psth_w.mean(0) std_w = np.std(psth_w, 0) xcorr_w = corr_trial_to_mean(d['psth_w_raw']) avg_w = d['psth_w_raw'].ravel().mean() vals.append(xcorr_w) mx = np.maximum(mn_c.max(), mn_w.max()) if psth_s is not None: mn_s = psth_s.mean(0) std_s = np.std(psth_s, 0) xcorr_s = corr_trial_to_mean(d['psth_s_raw']) avg_s = d['psth_s_raw'].ravel().mean() vals.append(xcorr_s) mx = np.maximum(mx, mn_s.max())
def do_lag_classification(exp_type='SOM', combs=[ 'Fourier', 'Frequency', 'Luminance', 'Contrast', 'Orientation', 'Flow' ], targets=[['Center', 'Center'], ['Whole', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., folds=5, filt=0.2, alpha=0.001, randomise=None): # Sub directory of the figure path to put the plots in dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: dat_path = dat_path + randomise + '_' + str(filt) else: dat_path = dat_path + str(filt) dat_file = dat_path + '/preds.pkl' cell_results = {} if os.path.exists(dat_file): print ' already exists. loading %s' % dat_file with open(dat_file, 'rb') as infile: cell_results = pickle.load(infile) else: print '%s doesnt exist, starting New' % dat_file if not os.path.exists(dat_path): os.makedirs(dat_path) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 for num_combs in [1]: for comb in itertools.combinations(combs, num_combs): full_comb = str(num_combs) + '_' + "_".join(comb) for e in dat.values(): cellid = str(e['cellid']) if cellid not in cell_results: cell_results[cellid] = {} if randomise is None: shifts = e['shifts'] else: shifts = [0] edge = e['edge'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] changed = False for shift in shifts: for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if k not in cell_results[cellid]: cell_results[cellid][k] = {} if full_comb not in cell_results[cellid][k]: cell_results[cellid][k][full_comb] = {} if shift not in cell_results[cellid][k][full_comb]: cell_results[cellid][k][full_comb][shift] = None else: print 'done - continue' continue changed = True edges = [edge, np.abs(np.minimum(-edge, shift))] X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, shift=shift, randomise=randomise) # ignore edge effects pred, coefs = CV(clf, X, y, folds=folds, clf_args=clf_args, edges=edges) pred, _ = filter(pred, e['bin_freq']) pred = pred[edges[0]:-edges[1]] mn = y[:, edges[0]:-edges[1]].mean(0) std = np.std(y[:, edges[0]:-edges[1]]) crr_pred_r2 = np.maximum(r2_score(mn, pred), 0) crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr') crr_exp = corr_trial_to_mean(y[:, edges[0]:-edges[1]], mn) print exp_type, comb, k, shift, crr_pred, crr_pred_r2 res = {} res['pred'] = pred res['mn'] = mn res['std'] = std res['crr_pred_r2'] = crr_pred_r2 res['crr_pred'] = crr_pred res['crr_exp'] = crr_exp res['coefs'] = coefs res['plot_params'] = plot_params cell_results[cellid][k][full_comb][shift] = res if changed: with open(dat_file, 'wb') as outfile: pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast', 'Orientation', 'Flow'], targets=[['Center', 'Center'], ['Whole', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., folds=5, filt=0.2, alpha=0.001, randomise=None): # Sub directory of the figure path to put the plots in dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: dat_path = dat_path + randomise + '_' + str(filt) else: dat_path = dat_path + str(filt) dat_file = dat_path + '/preds.pkl' cell_results = {} if os.path.exists(dat_file): print ' already exists. loading %s' % dat_file with open(dat_file, 'rb') as infile: cell_results = pickle.load(infile) else: print '%s doesnt exist, starting New' % dat_file if not os.path.exists(dat_path): os.makedirs(dat_path) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 for num_combs in [1]: for comb in itertools.combinations(combs, num_combs): full_comb = str(num_combs) + '_' + "_".join(comb) for e in dat.values(): cellid = str(e['cellid']) if cellid not in cell_results: cell_results[cellid] = {} if randomise is None: shifts = e['shifts'] else: shifts = [0] edge = e['edge'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] changed = False for shift in shifts: for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if k not in cell_results[cellid]: cell_results[cellid][k] = {} if full_comb not in cell_results[cellid][k]: cell_results[cellid][k][full_comb] = {} if shift not in cell_results[cellid][k][full_comb]: cell_results[cellid][k][full_comb][shift] = None else: print 'done - continue' continue changed = True edges = [edge, np.abs(np.minimum(-edge, shift))] X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, shift=shift, randomise=randomise) # ignore edge effects pred, coefs = CV(clf, X, y, folds=folds, clf_args=clf_args, edges=edges) pred, _ = filter(pred, e['bin_freq']) pred = pred[edges[0]: -edges[1]] mn = y[:, edges[0]: -edges[1]].mean(0) std = np.std(y[:, edges[0]: -edges[1]]) crr_pred_r2 = np.maximum(r2_score(mn, pred), 0) crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr') crr_exp = corr_trial_to_mean(y[:, edges[0]: -edges[1]], mn) print exp_type, comb, k, shift, crr_pred, crr_pred_r2 res = {} res['pred'] = pred res['mn'] = mn res['std'] = std res['crr_pred_r2'] = crr_pred_r2 res['crr_pred'] = crr_pred res['crr_exp'] = crr_exp res['coefs'] = coefs res['plot_params'] = plot_params cell_results[cellid][k][full_comb][shift] = res if changed: with open(dat_file, 'wb') as outfile: pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)