import pylab as plt sys.path.append('..') from data_utils.movie import load_movie_data from data_utils.load_ephys import load_EphysData from data_utils.utils import corr_trial_to_mean, corr_trial_to_trial, do_thresh_corr import os stim_type = 'psth_c_shift' if os.path.exists('mask_data_complete_%s.npy' % stim_type): patches = np.load('mask_data_complete_%s.npy' % stim_type) else: patches = np.load('mask_data_complete.npy') patches = patches.item() for exp_type in patches.keys(): e = load_EphysData(exp_type) for cell_id in patches[exp_type].keys(): dat = e[cell_id][stim_type] if 'corrs' not in patches[exp_type][cell_id]: patches[exp_type][cell_id]['corrs'] = {} for rbm_type in patches[exp_type][cell_id]['responses'].keys(): if rbm_type == 'movie' or rbm_type == 'responses': continue if rbm_type not in patches[exp_type][cell_id]['corrs']: patches[exp_type][cell_id]['corrs'][rbm_type] = {} for act_type in patches[exp_type][cell_id]['responses'][ rbm_type].keys(): print act_type if act_type not in patches[exp_type][cell_id]['corrs'][ rbm_type]: patches[exp_type][cell_id]['corrs'][rbm_type][
mov.close() return lum_mask, con_mask, flow_mask, four_mask, four_mask_shape,\ freq_mask, orient_mask,\ lum_surr, con_surr, flow_surr, four_surr, four_surr_shape,\ freq_surr, orient_surr,\ lum_whole, con_whole, flow_whole, four_whole, four_whole_shape,\ freq_whole, orient_whole if __name__ == "__main__": for exp_type in ['SOM', 'FS', 'PYR']: from data_utils.load_ephys import load_EphysData import Image import pylab as plt size = 15 ephys = load_EphysData(exp_type) for e in ephys.values(): cellid = e['cellid'] mov = load_movie_data(cellid, exp_type) mov = mov['four_mask'][0] print mov assert False # for i in range(5): # plt.subplot(2, 5, i + 1) # plt.imshow(mov[i*10], cmap=plt.cm.gray) # plt.subplot(2, 5, i + 6) # xx = mov[i*10] # xx = np.array(Image.fromarray(xx).resize([size, size]).getdata()).reshape([size, size]) # plt.imshow(xx, cmap=plt.cm.gray) # plt.show()
def process_movie_ephys(exp_type='SOM'): failed_count = 0 success_count = 0 ephys = load_EphysData(exp_type) dat_path = startup.data_path + 'Sparseness/%s/' % exp_type fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type if not os.path.exists(fig_path): os.makedirs(fig_path) for e in ephys.values(): print 'recording ', e['cellid'] target_fname = e['cellid'] + '_processed.npz' flow_fname = e['expdate'] + '_flow.mat' if os.path.exists(dat_path + target_fname): print 'already exists, skipping ', target_fname success_count += 1 # continue if not os.path.exists(dat_path + flow_fname): print 'flow data missing, skipping ', flow_fname continue movie = scipy.io.loadmat(dat_path + e['expdate'] + '.mat') movie = movie['dat'] flow = scipy.io.loadmat(dat_path + flow_fname) flow = flow['uv'] flow = np.swapaxes(flow, 1, 2) actual_len = e['psth_c'].shape[1] flow = flow[:actual_len] movie = movie[:actual_len] if (np.abs(e['adjustedMovResolution'] - e['movResolution'])).sum() != 0: movie = movie[:, :e['adjustedMovResolution'][0], : e['adjustedMovResolution'][1]] flow = flow[:, :e['adjustedMovResolution'][0], : e['adjustedMovResolution'][1]] #try: masked, surround = get_masked_data(movie, e['maskSizePixel'], e['maskLocationPixel']) # except: # print 'FAILED: ', e['cellid'] # failed_count += 1 # continue success_count += 1 lum_mask = get_luminance(masked)[np.newaxis, :] con_mask = get_contrast(masked)[np.newaxis, :] four_mask, freq_mask, orient_mask = get_fourier2D(masked) four_mask = four_mask[np.newaxis, :, :, :] freq_mask = freq_mask[np.newaxis, :, :] orient_mask = orient_mask[np.newaxis, :, :] lum_surr = [] con_surr = [] four_surr = [] freq_surr = [] orient_surr = [] for s in range(surround.shape[0]): lum_surr.append(get_luminance(surround[s])) con_surr.append(get_contrast(surround[s])) f, fr, o = get_fourier2D(surround[s]) four_surr.append(f) freq_surr.append(fr) orient_surr.append(o) lum_surr = np.array(lum_surr) con_surr = np.array(con_surr) four_surr = np.array(four_surr) freq_surr = np.array(freq_surr) orient_surr = np.array(orient_surr) lum_whole = get_luminance(movie)[np.newaxis, :] con_whole = get_contrast(movie)[np.newaxis, :] four_whole, freq_whole, orient_whole = get_fourier2D(movie) four_whole = four_whole[np.newaxis, :, :, :] freq_whole = freq_whole[np.newaxis, :, :] orient_whole = orient_whole[np.newaxis, :, :] flow_whole, flow_mask, flow_surr = get_flow(flow, e['maskSizePixel'], e['maskLocationPixel']) movie = movie[np.newaxis, :, :, :] masked = masked[np.newaxis, :, :, :] np.savez(dat_path + target_fname, lum_mask=lum_mask, con_mask=con_mask, flow_mask=flow_mask, four_mask=four_mask, freq_mask=freq_mask, orient_mask=orient_mask, masked=masked, lum_whole=lum_whole, con_whole=con_whole, flow_whole=flow_whole, four_whole=four_whole, freq_whole=freq_whole, orient_whole=orient_whole, movie=movie, lum_surr=lum_surr, con_surr=con_surr, flow_surr=flow_surr, four_surr=four_surr, freq_surr=freq_surr, orient_surr=orient_surr, surround=surround) scipy.io.savemat( dat_path + target_fname[:-3] + '.mat', { 'lum_mask': lum_mask, 'con_mask': con_mask, 'flow_mask': flow_mask, 'four_mask': four_mask, 'freq_mask': freq_mask, 'orient_mask': orient_mask, 'masked': masked, 'lum_whole': lum_whole, 'con_whole': con_whole, 'flow_whole': flow_whole, 'four_whole': four_whole, 'freq_whole': freq_whole, 'orient_whole': orient_whole, 'movie': movie, 'lum_surr': lum_surr, 'con_surr': con_surr, 'flow_surr': flow_surr, 'four_surr': four_surr, 'freq_surr': freq_surr, 'orient_surr': orient_surr, 'surround': surround }) plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked, fig_path + e['cellid'] + '_masked') plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie, fig_path + e['cellid'] + '_whole') plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround, fig_path + e['cellid'] + '_surround') print 'Failed: ', failed_count print 'Successful: ', success_count
def process_movie_ephys(exp_type='SOM'): failed_count = 0 success_count = 0 ephys = load_EphysData(exp_type) dat_path = startup.data_path + 'Sparseness/%s/' % exp_type fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type if not os.path.exists(fig_path): os.makedirs(fig_path) for e in ephys.values(): print 'recording ', e['cellid'] target_fname = e['cellid'] + '_processed.npz' flow_fname = e['expdate'] + '_flow.mat' if os.path.exists(dat_path + target_fname): print 'already exists, skipping ', target_fname success_count += 1 # continue if not os.path.exists(dat_path + flow_fname): print 'flow data missing, skipping ', flow_fname continue movie = scipy.io.loadmat(dat_path + e['expdate'] + '.mat') movie = movie['dat'] flow = scipy.io.loadmat(dat_path + flow_fname) flow = flow['uv'] flow = np.swapaxes(flow, 1, 2) actual_len = e['psth_c'].shape[1] flow = flow[:actual_len] movie = movie[:actual_len] if (np.abs(e['adjustedMovResolution'] - e['movResolution'])).sum() != 0: movie = movie[:, :e['adjustedMovResolution'][0], :e['adjustedMovResolution'][1]] flow = flow[:, :e['adjustedMovResolution'][0], :e['adjustedMovResolution'][1]] #try: masked, surround = get_masked_data(movie, e['maskSizePixel'], e['maskLocationPixel']) # except: # print 'FAILED: ', e['cellid'] # failed_count += 1 # continue success_count += 1 lum_mask = get_luminance(masked)[np.newaxis, :] con_mask = get_contrast(masked)[np.newaxis, :] four_mask, freq_mask, orient_mask = get_fourier2D(masked) four_mask = four_mask[np.newaxis, :, :, :] freq_mask = freq_mask[np.newaxis, :, :] orient_mask = orient_mask[np.newaxis, :, :] lum_surr = [] con_surr = [] four_surr = [] freq_surr = [] orient_surr = [] for s in range(surround.shape[0]): lum_surr.append(get_luminance(surround[s])) con_surr.append(get_contrast(surround[s])) f, fr, o = get_fourier2D(surround[s]) four_surr.append(f) freq_surr.append(fr) orient_surr.append(o) lum_surr = np.array(lum_surr) con_surr = np.array(con_surr) four_surr = np.array(four_surr) freq_surr = np.array(freq_surr) orient_surr = np.array(orient_surr) lum_whole = get_luminance(movie)[np.newaxis, :] con_whole = get_contrast(movie)[np.newaxis, :] four_whole, freq_whole, orient_whole = get_fourier2D(movie) four_whole = four_whole[np.newaxis, :, :, :] freq_whole = freq_whole[np.newaxis, :, :] orient_whole = orient_whole[np.newaxis, :, :] flow_whole, flow_mask, flow_surr = get_flow(flow, e['maskSizePixel'], e['maskLocationPixel']) movie = movie[np.newaxis, :, :, :] masked = masked[np.newaxis, :, :, :] np.savez(dat_path + target_fname, lum_mask=lum_mask, con_mask=con_mask, flow_mask=flow_mask, four_mask=four_mask, freq_mask=freq_mask, orient_mask=orient_mask, masked=masked, lum_whole=lum_whole, con_whole=con_whole, flow_whole=flow_whole, four_whole=four_whole, freq_whole=freq_whole, orient_whole=orient_whole, movie=movie, lum_surr=lum_surr, con_surr=con_surr, flow_surr=flow_surr, four_surr=four_surr, freq_surr=freq_surr, orient_surr=orient_surr, surround=surround) scipy.io.savemat(dat_path + target_fname[:-3] + '.mat', {'lum_mask': lum_mask, 'con_mask': con_mask, 'flow_mask': flow_mask, 'four_mask': four_mask, 'freq_mask': freq_mask, 'orient_mask': orient_mask, 'masked': masked, 'lum_whole': lum_whole, 'con_whole': con_whole, 'flow_whole': flow_whole, 'four_whole': four_whole, 'freq_whole': freq_whole, 'orient_whole': orient_whole, 'movie': movie, 'lum_surr': lum_surr, 'con_surr': con_surr, 'flow_surr': flow_surr, 'four_surr': four_surr, 'freq_surr': freq_surr, 'orient_surr': orient_surr, 'surround': surround}) plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked, fig_path + e['cellid'] + '_masked') plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie, fig_path + e['cellid'] + '_whole') plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround, fig_path + e['cellid'] + '_surround') print 'Failed: ', failed_count print 'Successful: ', success_count
def do_lag_classification(exp_type='SOM', combs=[ 'Fourier', 'Frequency', 'Luminance', 'Contrast', 'Orientation', 'Flow' ], targets=[['Center', 'Center'], ['Whole', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., folds=5, filt=0.2, alpha=0.001, randomise=None): # Sub directory of the figure path to put the plots in dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: dat_path = dat_path + randomise + '_mn' else: dat_path = dat_path + 'mn' dat_file = dat_path + '/preds.pkl' cell_results = {} if os.path.exists(dat_file): print ' already exists. loading %s' % dat_file with open(dat_file, 'rb') as infile: cell_results = pickle.load(infile) else: print '%s doesnt exist, starting New' % dat_file if not os.path.exists(dat_path): os.makedirs(dat_path) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 for num_combs in [1]: for comb in itertools.combinations(combs, num_combs): full_comb = str(num_combs) + '_' + "_".join(comb) for e in dat.values(): cellid = str(e['cellid']) if cellid not in cell_results: cell_results[cellid] = {} if randomise is None: shifts = e['shifts'] else: shifts = [0] edge = e['edge'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] changed = False for shift in shifts: for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if k not in cell_results[cellid]: cell_results[cellid][k] = {} if full_comb not in cell_results[cellid][k]: cell_results[cellid][k][full_comb] = {} if shift not in cell_results[cellid][k][full_comb]: cell_results[cellid][k][full_comb][shift] = None else: print 'done - continue' continue changed = True edges = [edge, np.abs(np.minimum(-edge, shift))] X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, shift=shift, randomise=randomise) # ignore edge effects pred, coefs = CV_time(clf, X, y, folds=folds, clf_args=clf_args, edges=edges) pred, _ = filter(pred, e['bin_freq']) pred = pred[edges[0]:-edges[1]] mn = y[edges[0]:-edges[1]] std = np.std(y[edges[0]:-edges[1]]) crr_pred_r2 = np.maximum(r2_score(mn, pred), 0) crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr') print exp_type, comb, k, shift, crr_pred, crr_pred_r2 res = {} res['pred'] = pred res['mn'] = mn res['std'] = std res['crr_pred_r2'] = crr_pred_r2 res['crr_pred'] = crr_pred res['crr_exp'] = None res['coefs'] = coefs res['plot_params'] = plot_params cell_results[cellid][k][full_comb][shift] = res if changed: with open(dat_file, 'wb') as outfile: pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast', 'Orientation', 'Flow'], targets=[['Center', 'Center'], ['Whole', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., folds=5, filt=0.2, alpha=0.001, randomise=None): # Sub directory of the figure path to put the plots in dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: dat_path = dat_path + randomise + '_mn' else: dat_path = dat_path + 'mn' dat_file = dat_path + '/preds.pkl' cell_results = {} if os.path.exists(dat_file): print ' already exists. loading %s' % dat_file with open(dat_file, 'rb') as infile: cell_results = pickle.load(infile) else: print '%s doesnt exist, starting New' % dat_file if not os.path.exists(dat_path): os.makedirs(dat_path) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 for num_combs in [1]: for comb in itertools.combinations(combs, num_combs): full_comb = str(num_combs) + '_' + "_".join(comb) for e in dat.values(): cellid = str(e['cellid']) if cellid not in cell_results: cell_results[cellid] = {} if randomise is None: shifts = e['shifts'] else: shifts = [0] edge = e['edge'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] changed = False for shift in shifts: for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if k not in cell_results[cellid]: cell_results[cellid][k] = {} if full_comb not in cell_results[cellid][k]: cell_results[cellid][k][full_comb] = {} if shift not in cell_results[cellid][k][full_comb]: cell_results[cellid][k][full_comb][shift] = None else: print 'done - continue' continue changed = True edges = [edge, np.abs(np.minimum(-edge, shift))] X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, shift=shift, randomise=randomise) # ignore edge effects pred, coefs = CV_time(clf, X, y, folds=folds, clf_args=clf_args, edges=edges) pred, _ = filter(pred, e['bin_freq']) pred = pred[edges[0]: -edges[1]] mn = y[edges[0]: -edges[1]] std = np.std(y[edges[0]: -edges[1]]) crr_pred_r2 = np.maximum(r2_score(mn, pred), 0) crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr') print exp_type, comb, k, shift, crr_pred, crr_pred_r2 res = {} res['pred'] = pred res['mn'] = mn res['std'] = std res['crr_pred_r2'] = crr_pred_r2 res['crr_pred'] = crr_pred res['crr_exp'] = None res['coefs'] = coefs res['plot_params'] = plot_params cell_results[cellid][k][full_comb][shift] = res if changed: with open(dat_file, 'wb') as outfile: pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
for randomise in [None]: # , 'generated', 'random']: for exp_type in exp_types: for filt in filters: if randomise is None: f_path = fig_path + 'Sparseness/%s/initial/%.2f/' % (exp_type, filt) elif randomise == 'random': f_path = fig_path + 'Sparseness/%s/initial/%.2f_random/' % ( exp_type, filt) elif randomise == 'generated': f_path = fig_path + 'Sparseness/%s/initial/%.2f_generated/' % ( exp_type, filt) fname = '%s%s' % (f_path, 'initial_summary.png') if os.path.exists(fname): print fname + ' exist, SKIPPING' # continue dat = load_EphysData(exp_type, filt=filt) csv_vals = [] cellids = [] for k in sorted(dat.keys()): vals = [] d = dat[k] print 'doing ', d['cellid'] if randomise is None: psth_s = d['psth_s'] psth_c = d['psth_c'] psth_w = d['psth_w'] elif randomise == 'random': psth_c = d['psth_c_rand'] psth_s = d['psth_s_rand'] psth_w = d['psth_w_rand'] elif randomise == 'generated':
def do_classification( exp_type='SOM', combs=['Luminance', 'Contrast', 'Orientation', 'Frequency', 'Fourier'], targets=[['Center', 'Center'], ['Center', 'Whole'], ['Whole', 'Center'], ['Whole', 'Whole'], ['Surround', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., randomise=None, folds=5, filt=0.2, alpha=0.001): # Sub directory of the figure path to put the plots in fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: pth = fig_path + randomise + '_' + str(filt) else: pth = fig_path + str(filt) if os.path.exists(pth): if os.path.exists(pth + '/summary_all.csv'): print pth, ' already exists. Skipping' return ['skipped'] else: os.makedirs(pth) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) comb_corrs = [] if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 cell_results = {} num_combs = [1] if len(combs) > 1: num_combs.append(len(combs)) for num_combs in [1, 2]: #, len(combs)]: for comb in itertools.combinations(combs, num_combs): print comb full_comb = str(num_combs) + '_' + "_".join(comb) if num_combs == 2 and ('Frequency' not in comb or 'Orientation' not in comb): continue comb_vals = {'Overall': []} for i, e in enumerate(dat.values()): if max_exp is not None and i >= max_exp: break expdate = e['expdate'] cellid = str(e['cellid']) edge = e['edge'] if cellid not in cell_results: cell_results[cellid] = {} if full_comb not in cell_results[cellid]: cell_results[cellid][full_comb] = {} if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] clf_vals = [] sig_found = False results = {} for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if targ_type not in cell_results[cellid][full_comb]: cell_results[cellid][full_comb][targ_type] = {} if src_type not in cell_results[cellid][full_comb][ targ_type]: cell_results[cellid][full_comb][targ_type][ src_type] = {} X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, randomise) if randomise is not None: fname = '%s%s_%s/%s/' % (fig_path, randomise, str(filt), cellid) else: fname = '%s%s/%s/' % (fig_path, str(filt), cellid) if not os.path.exists(fname): os.makedirs(fname) print fname # ignore edge effects pred, coefs = CV(clf, X, y, folds=folds, clf_args=clf_args, edge=edge) coefs = np.array(coefs).mean(0) pred, _ = filter(pred, e['bin_freq']) pred = pred[edge:-edge] clf_vals.append([ targ_type, ]) mn = y.mean(0)[edge:-edge] std = np.std(y, 0)[edge:-edge] #crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr') #crr_pred = mean_squared_error(mn, pred) crr_pred = np.maximum(r2_score(mn, pred), 0) #crr_pred = explained_variance_score(mn, pred) if crr_pred > sig_thresh: sig_found = True cell_results[cellid][full_comb][targ_type][ src_type] = crr_pred comb_vals['Overall'] += [crr_pred] if k in comb_vals: comb_vals[k] += [crr_pred] else: comb_vals[k] = [crr_pred] xcorr = [] for i in range(y.shape[0]): xcorr.append(do_thresh_corr(y[i][edge:-edge], mn)) crr_y = np.array(xcorr).mean() print crr_pred results[k] = [ pred, mn, std, crr_pred, crr_y, coefs, plot_params ] # only do plots for the fourier trained classifier #if sig_found: if True: cmb = "_".join(comb) if 'Fourier' in cmb: plot_preds_fourier(cellid, results, full_targets, fname + cmb) else: plot_preds(cellid, results, full_targets, cmb, fname + cmb) comb_corrs.append([comb, comb_vals]) # make this a boxplot if randomise is not None: fp = fig_path + randomise + '_' else: fp = fig_path plot_summary(comb_corrs, full_targets, fp, str(filt)) plot_cell_summary(cell_results, fp + str(filt) + '/') return comb_corrs
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast', 'Orientation', 'Frequency', 'Fourier'], targets=[['Center', 'Center'], ['Center', 'Whole'], ['Whole', 'Center'], ['Whole', 'Whole'], ['Surround', 'Whole']], max_comb=None, min_comb=None, four_downsample=None, max_exp=None, sig_thresh=0., randomise=None, folds=5, filt=0.2, alpha=0.001): # Sub directory of the figure path to put the plots in fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if randomise is not None: pth = fig_path + randomise + '_' + str(filt) else: pth = fig_path + str(filt) if os.path.exists(pth): if os.path.exists(pth + '/summary_all.csv'): print pth, ' already exists. Skipping' return ['skipped'] else: os.makedirs(pth) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type, filt=filt) comb_corrs = [] if max_comb is None: max_comb = len(combs) if min_comb is None: min_comb = 0 cell_results = {} num_combs = [1] if len(combs) > 1: num_combs.append(len(combs)) for num_combs in [1, 2]:#, len(combs)]: for comb in itertools.combinations(combs, num_combs): print comb full_comb = str(num_combs) + '_' + "_".join(comb) if num_combs == 2 and ('Frequency' not in comb or 'Orientation' not in comb): continue comb_vals = {'Overall': []} for i, e in enumerate(dat.values()): if max_exp is not None and i >= max_exp: break expdate = e['expdate'] cellid = str(e['cellid']) edge = e['edge'] if cellid not in cell_results: cell_results[cellid] = {} if full_comb not in cell_results[cellid]: cell_results[cellid][full_comb] = {} if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] clf_vals = [] sig_found = False results = {} for [targ_type, src_type] in targets: k = '%s_%s' % (targ_type, src_type) if targ_type not in cell_results[cellid][full_comb]: cell_results[cellid][full_comb][targ_type] = {} if src_type not in cell_results[cellid][full_comb][targ_type]: cell_results[cellid][full_comb][targ_type][src_type] = {} X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample, randomise) if randomise is not None: fname = '%s%s_%s/%s/' % (fig_path, randomise, str(filt), cellid) else: fname = '%s%s/%s/' % (fig_path, str(filt), cellid) if not os.path.exists(fname): os.makedirs(fname) print fname # ignore edge effects pred, coefs = CV(clf, X, y, folds=folds, clf_args=clf_args, edge=edge) coefs = np.array(coefs).mean(0) pred, _ = filter(pred, e['bin_freq']) pred = pred[edge: -edge] clf_vals.append([targ_type, ]) mn = y.mean(0)[edge: -edge] std = np.std(y, 0)[edge: -edge] #crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr') #crr_pred = mean_squared_error(mn, pred) crr_pred = np.maximum(r2_score(mn, pred), 0) #crr_pred = explained_variance_score(mn, pred) if crr_pred > sig_thresh: sig_found = True cell_results[cellid][full_comb][targ_type][src_type] = crr_pred comb_vals['Overall'] += [crr_pred] if k in comb_vals: comb_vals[k] += [crr_pred] else: comb_vals[k] = [crr_pred] xcorr = [] for i in range(y.shape[0]): xcorr.append(do_thresh_corr(y[i][edge: -edge], mn)) crr_y = np.array(xcorr).mean() print crr_pred results[k] = [pred, mn, std, crr_pred, crr_y, coefs, plot_params] # only do plots for the fourier trained classifier #if sig_found: if True: cmb = "_".join(comb) if 'Fourier' in cmb: plot_preds_fourier(cellid, results, full_targets, fname + cmb) else: plot_preds(cellid, results, full_targets, cmb, fname + cmb) comb_corrs.append([comb, comb_vals]) # make this a boxplot if randomise is not None: fp = fig_path + randomise + '_' else: fp = fig_path plot_summary(comb_corrs, full_targets, fp, str(filt)) plot_cell_summary(cell_results, fp + str(filt) + '/') return comb_corrs
def do_classification(exp_type='SOM', targets=['Center', 'Surround', 'Whole'], four_downsample=None, max_exp=None): # Sub directory of the figure path to put the plots in fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if not os.path.exists(fig_path): os.makedirs(fig_path) if not os.path.exists(fig_path + str(downsample)): os.makedirs(fig_path + str(downsample)) dat = load_EphysData(exp_type) all_corrs = [] surr_corrs = [] mask_corrs = [] whole_corrs = [] for i, e in enumerate(dat.values()): if max_exp is not None and i >= max_exp: break expdate = e['expdate'] cellid = e['cellid'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] clf_vals = [] sig_found = False for targ_type in targets: fname = '%s%s/%s_%s_pred' % (fig_path, str(four_downsample), cellid, targ_type) print fname X, y, plot_params = get_mov_data(targ_type, e, cellid, exp_type, four_downsample) for xi in range(X.shape[2]): cv = LeaveOneOut(X.shape[1], indices=True) cv = KFold(X.shape[1], 20, indices=True, shuffle=True) pred = np.zeros(y[0].shape) X_dims = X.shape[2] coefs = [] for train, test in cv: regr = clf(alpha=0.01) regr.fit(X[:, train, xi].reshape([-1, 1]), y[:, train].ravel()) coefs.append(regr.coef_) pred[test] = regr.predict(X[0, test, xi].reshape([-1, 1])) coefs = np.array(coefs).mean(0) clf_vals.append([targ_type, coefs, plot_params]) mn = y.mean(0) std = np.std(y, 0) [crr_pred, p_pred] = corr(mn, pred) if p_pred < 0.05: crr_pred = np.nan_to_num(crr_pred) else: crr_pred = 0. p_pred = 0. if crr_pred > 0: sig_found = True if crr_pred > 0.3: plt.figure() plt.hold(True) plt.plot(pred, 'r') plt.plot(mn, 'k') plt.title('Corr: %.2f' % crr_pred) plt.show() all_corrs += [crr_pred] if targ_type == 'Center': mask_corrs += [crr_pred] elif targ_type == 'Surround': surr_corrs += [crr_pred] elif targ_type == 'Whole': whole_corrs += [crr_pred] plt.subplot(211) plt.scatter(range(len(mask_corrs)), mask_corrs) plt.subplot(212) plt.scatter(range(len(whole_corrs)), whole_corrs) plt.show() # xcorr = [] # for i in range(y.shape[0]): # xcorr.append(corr(y[i], mn)) # xcorr = np.nan_to_num(np.array(xcorr)) # xcorr[xcorr[:, 1] > 0.05, 0] = 0 # [crr_y, p_y] = xcorr.mean(0) # print crr_pred, p_pred # only do plots for the fourier trained classifier comb_vals = [] comb_vals.append(['mask', mask_corrs]) comb_vals.append(['surround', surr_corrs]) comb_vals.append(['whole', whole_corrs]) comb_vals.append(['overall', all_corrs]) comb_corrs.append([comb, comb_vals]) # make this a boxplot plot_summary(comb_corrs, fig_path, str(downsample)) return comb_corrs
import os # Sub directory of the figure path to put the plots in corr_type = 'spearmanr' corr_type = 'pearsonr' colors = ['r', 'g', 'b'] #fig_path = fig_path + 'Sparseness/%s/initial/' % (exp_type) if not os.path.exists(fig_path): os.makedirs(fig_path) exp_types = ['SOM', 'FS', 'PYR'] corrs = [] cells = [] for exp_type in exp_types: dat = load_EphysData(exp_type) for d in dat.values(): cellid = d['cellid'] print 'doing ', cellid psth_s = d['psth_s'] psth_c = d['psth_c'] psth_w = d['psth_w'] mn_c = psth_c.mean(0) mn_w = psth_w.mean(0) mx = np.maximum(mn_c.max(), mn_w.max()) if psth_s is not None: mn_s = psth_s.mean(0) mx = np.maximum(mx, mn_s.max()) cr_c_w = None
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast', 'Flow', 'Fourier'], targets=[['Center', 'Center'], ['Center', 'Whole'], ['Whole', 'Center'], ['Whole', 'Whole'], ['Surround', 'Whole']], four_downsample=None, random=False): # Sub directory of the figure path to put the plots in fig_path = startup.fig_path + 'Sparseness/%s/xcorr/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if random is not None: if not os.path.exists(fig_path + str(four_downsample) + '_%s' % random): os.makedirs(fig_path + str(four_downsample) + '_%s' % random) else: if not os.path.exists(fig_path + str(four_downsample)): os.makedirs(fig_path + str(four_downsample)) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type) comb_corrs = [] for comb in combs: print comb comb_vals = {'Overall': []} for i, e in enumerate(dat.values()): expdate = e['expdate'] cellid = e['cellid'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] results = {} for [targ_type, src_type] in targets: X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample) # y[y == 0] = 0.001 # y = np.log(y) if random is not None: fname = '%s%s_%s' % (fig_path, str(four_downsample), random) if random == 'shift': idx = deque(np.arange(y.shape[1])) idx.rotate(-23) elif random == 'random': idx = np.arange(y.shape[1]) np.random.shuffle(idx) y = y[:, idx] else: fname = '%s%s/%s_pred' % (fig_path, str(four_downsample), cellid) print fname X_dims = X.shape[2] coefs = [] # lassoCV(X.reshape(-1, X_dims), # y.ravel()) pred = gauss_filter(pred, e['bin_freq']) coefs = np.array(coefs).mean(0) clf_vals.append([targ_type, ]) mn = y.mean(0) std = np.std(y, 0) [crr_pred, p_pred] = corr(mn, pred) if p_pred < 0.05: crr_pred = np.nan_to_num(crr_pred) else: crr_pred = 0. p_pred = 0. if crr_pred > sig_thresh: sig_found = True comb_vals['Overall'] += [crr_pred] k = '%s_%s' % (targ_type, src_type) if comb_vals.has_key(k): comb_vals[k] += [crr_pred] else: comb_vals[k] = [crr_pred] xcorr = [] for i in range(y.shape[0]): xcorr.append(corr(y[i][5:-5], mn[5:-5])) xcorr = np.nan_to_num(np.array(xcorr)) xcorr[xcorr[:, 1] > 0.05, 0] = 0 [crr_y, _] = xcorr.mean(0) print crr_pred, p_pred results[k] = [pred, mn, std, crr_pred, crr_y, coefs, plot_params] # only do plots for the fourier trained classifier if sig_found and len(comb) == 1: if comb[0] == 'Fourier': plot_preds_fourier(cellid, results, full_targets, fname + '_%s' % comb[0]) else: plot_preds(cellid, results, full_targets, comb[0], fname + '_%s' % comb[0]) comb_corrs.append([comb, comb_vals]) # make this a boxplot if random: plot_summary(comb_corrs, full_targets, fig_path + 'Rand_', str(downsample)) else: plot_summary(comb_corrs, full_targets, fig_path, str(downsample)) return comb_corrs
for exp_type in exp_types: for filt in filters: if randomise is None: f_path = fig_path + 'Sparseness/%s/initial/%.2f/' % (exp_type, filt) elif randomise == 'random': f_path = fig_path + 'Sparseness/%s/initial/%.2f_random/' % ( exp_type, filt) elif randomise == 'generated': f_path = fig_path + 'Sparseness/%s/initial/%.2f_generated/' % ( exp_type, filt) fname = '%s%s' % (f_path, 'initial_summary.png') if os.path.exists(fname): print fname + ' exist, SKIPPING' # continue dat = load_EphysData(exp_type, filt=filt) csv_vals = [] cellids = [] for k in sorted(dat.keys()): vals = [] d = dat[k] print 'doing ', d['cellid'] if randomise is None: psth_s = d['psth_s'] psth_c = d['psth_c'] psth_w = d['psth_w'] elif randomise == 'random': psth_c = d['psth_c_rand'] psth_s = d['psth_s_rand'] psth_w = d['psth_w_rand'] elif randomise == 'generated':
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast', 'Flow', 'Fourier'], targets=[['Center', 'Center'], ['Center', 'Whole'], ['Whole', 'Center'], ['Whole', 'Whole'], ['Surround', 'Whole']], four_downsample=None, random=False): # Sub directory of the figure path to put the plots in fig_path = startup.fig_path + 'Sparseness/%s/xcorr/' % (exp_type) mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type) if random is not None: if not os.path.exists(fig_path + str(four_downsample) + '_%s' % random): os.makedirs(fig_path + str(four_downsample) + '_%s' % random) else: if not os.path.exists(fig_path + str(four_downsample)): os.makedirs(fig_path + str(four_downsample)) full_targets = [] for [targ_type, src_type] in targets: full_targets.append('%s_%s' % (targ_type, src_type)) dat = load_EphysData(exp_type) comb_corrs = [] for comb in combs: print comb comb_vals = {'Overall': []} for i, e in enumerate(dat.values()): expdate = e['expdate'] cellid = e['cellid'] if not os.path.exists(mov_path + cellid + '_processed.npz'): print '\nNo movie found ', cellid continue else: print '\ndoing ', e['cellid'] results = {} for [targ_type, src_type] in targets: X, y, plot_params = get_mov_data(comb, targ_type, src_type, e, cellid, exp_type, four_downsample) # y[y == 0] = 0.001 # y = np.log(y) if random is not None: fname = '%s%s_%s' % (fig_path, str(four_downsample), random) if random == 'shift': idx = deque(np.arange(y.shape[1])) idx.rotate(-23) elif random == 'random': idx = np.arange(y.shape[1]) np.random.shuffle(idx) y = y[:, idx] else: fname = '%s%s/%s_pred' % (fig_path, str(four_downsample), cellid) print fname X_dims = X.shape[2] coefs = [] # lassoCV(X.reshape(-1, X_dims), # y.ravel()) pred = gauss_filter(pred, e['bin_freq']) coefs = np.array(coefs).mean(0) clf_vals.append([ targ_type, ]) mn = y.mean(0) std = np.std(y, 0) [crr_pred, p_pred] = corr(mn, pred) if p_pred < 0.05: crr_pred = np.nan_to_num(crr_pred) else: crr_pred = 0. p_pred = 0. if crr_pred > sig_thresh: sig_found = True comb_vals['Overall'] += [crr_pred] k = '%s_%s' % (targ_type, src_type) if comb_vals.has_key(k): comb_vals[k] += [crr_pred] else: comb_vals[k] = [crr_pred] xcorr = [] for i in range(y.shape[0]): xcorr.append(corr(y[i][5:-5], mn[5:-5])) xcorr = np.nan_to_num(np.array(xcorr)) xcorr[xcorr[:, 1] > 0.05, 0] = 0 [crr_y, _] = xcorr.mean(0) print crr_pred, p_pred results[k] = [ pred, mn, std, crr_pred, crr_y, coefs, plot_params ] # only do plots for the fourier trained classifier if sig_found and len(comb) == 1: if comb[0] == 'Fourier': plot_preds_fourier(cellid, results, full_targets, fname + '_%s' % comb[0]) else: plot_preds(cellid, results, full_targets, comb[0], fname + '_%s' % comb[0]) comb_corrs.append([comb, comb_vals]) # make this a boxplot if random: plot_summary(comb_corrs, full_targets, fig_path + 'Rand_', str(downsample)) else: plot_summary(comb_corrs, full_targets, fig_path, str(downsample)) return comb_corrs