import pylab as plt
sys.path.append('..')
from data_utils.movie import load_movie_data
from data_utils.load_ephys import load_EphysData
from data_utils.utils import corr_trial_to_mean, corr_trial_to_trial, do_thresh_corr
import os
stim_type = 'psth_c_shift'
if os.path.exists('mask_data_complete_%s.npy' % stim_type):
patches = np.load('mask_data_complete_%s.npy' % stim_type)
else:
patches = np.load('mask_data_complete.npy')
patches = patches.item()
for exp_type in patches.keys():
e = load_EphysData(exp_type)
for cell_id in patches[exp_type].keys():
dat = e[cell_id][stim_type]
if 'corrs' not in patches[exp_type][cell_id]:
patches[exp_type][cell_id]['corrs'] = {}
for rbm_type in patches[exp_type][cell_id]['responses'].keys():
if rbm_type == 'movie' or rbm_type == 'responses':
continue
if rbm_type not in patches[exp_type][cell_id]['corrs']:
patches[exp_type][cell_id]['corrs'][rbm_type] = {}
for act_type in patches[exp_type][cell_id]['responses'][
rbm_type].keys():
print act_type
if act_type not in patches[exp_type][cell_id]['corrs'][
rbm_type]:
patches[exp_type][cell_id]['corrs'][rbm_type][
mov.close()
return lum_mask, con_mask, flow_mask, four_mask, four_mask_shape,\
freq_mask, orient_mask,\
lum_surr, con_surr, flow_surr, four_surr, four_surr_shape,\
freq_surr, orient_surr,\
lum_whole, con_whole, flow_whole, four_whole, four_whole_shape,\
freq_whole, orient_whole
if __name__ == "__main__":
for exp_type in ['SOM', 'FS', 'PYR']:
from data_utils.load_ephys import load_EphysData
import Image
import pylab as plt
size = 15
ephys = load_EphysData(exp_type)
for e in ephys.values():
cellid = e['cellid']
mov = load_movie_data(cellid, exp_type)
mov = mov['four_mask'][0]
print mov
assert False
# for i in range(5):
# plt.subplot(2, 5, i + 1)
# plt.imshow(mov[i*10], cmap=plt.cm.gray)
# plt.subplot(2, 5, i + 6)
# xx = mov[i*10]
# xx = np.array(Image.fromarray(xx).resize([size, size]).getdata()).reshape([size, size])
# plt.imshow(xx, cmap=plt.cm.gray)
# plt.show()
def process_movie_ephys(exp_type='SOM'):
failed_count = 0
success_count = 0
ephys = load_EphysData(exp_type)
dat_path = startup.data_path + 'Sparseness/%s/' % exp_type
fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type
if not os.path.exists(fig_path):
os.makedirs(fig_path)
for e in ephys.values():
print 'recording ', e['cellid']
target_fname = e['cellid'] + '_processed.npz'
flow_fname = e['expdate'] + '_flow.mat'
if os.path.exists(dat_path + target_fname):
print 'already exists, skipping ', target_fname
success_count += 1
# continue
if not os.path.exists(dat_path + flow_fname):
print 'flow data missing, skipping ', flow_fname
continue
movie = scipy.io.loadmat(dat_path + e['expdate'] + '.mat')
movie = movie['dat']
flow = scipy.io.loadmat(dat_path + flow_fname)
flow = flow['uv']
flow = np.swapaxes(flow, 1, 2)
actual_len = e['psth_c'].shape[1]
flow = flow[:actual_len]
movie = movie[:actual_len]
if (np.abs(e['adjustedMovResolution'] -
e['movResolution'])).sum() != 0:
movie = movie[:, :e['adjustedMovResolution'][0], :
e['adjustedMovResolution'][1]]
flow = flow[:, :e['adjustedMovResolution'][0], :
e['adjustedMovResolution'][1]]
#try:
masked, surround = get_masked_data(movie, e['maskSizePixel'],
e['maskLocationPixel'])
# except:
# print 'FAILED: ', e['cellid']
# failed_count += 1
# continue
success_count += 1
lum_mask = get_luminance(masked)[np.newaxis, :]
con_mask = get_contrast(masked)[np.newaxis, :]
four_mask, freq_mask, orient_mask = get_fourier2D(masked)
four_mask = four_mask[np.newaxis, :, :, :]
freq_mask = freq_mask[np.newaxis, :, :]
orient_mask = orient_mask[np.newaxis, :, :]
lum_surr = []
con_surr = []
four_surr = []
freq_surr = []
orient_surr = []
for s in range(surround.shape[0]):
lum_surr.append(get_luminance(surround[s]))
con_surr.append(get_contrast(surround[s]))
f, fr, o = get_fourier2D(surround[s])
four_surr.append(f)
freq_surr.append(fr)
orient_surr.append(o)
lum_surr = np.array(lum_surr)
con_surr = np.array(con_surr)
four_surr = np.array(four_surr)
freq_surr = np.array(freq_surr)
orient_surr = np.array(orient_surr)
lum_whole = get_luminance(movie)[np.newaxis, :]
con_whole = get_contrast(movie)[np.newaxis, :]
four_whole, freq_whole, orient_whole = get_fourier2D(movie)
four_whole = four_whole[np.newaxis, :, :, :]
freq_whole = freq_whole[np.newaxis, :, :]
orient_whole = orient_whole[np.newaxis, :, :]
flow_whole, flow_mask, flow_surr = get_flow(flow, e['maskSizePixel'],
e['maskLocationPixel'])
movie = movie[np.newaxis, :, :, :]
masked = masked[np.newaxis, :, :, :]
np.savez(dat_path + target_fname,
lum_mask=lum_mask,
con_mask=con_mask,
flow_mask=flow_mask,
four_mask=four_mask,
freq_mask=freq_mask,
orient_mask=orient_mask,
masked=masked,
lum_whole=lum_whole,
con_whole=con_whole,
flow_whole=flow_whole,
four_whole=four_whole,
freq_whole=freq_whole,
orient_whole=orient_whole,
movie=movie,
lum_surr=lum_surr,
con_surr=con_surr,
flow_surr=flow_surr,
four_surr=four_surr,
freq_surr=freq_surr,
orient_surr=orient_surr,
surround=surround)
scipy.io.savemat(
dat_path + target_fname[:-3] + '.mat', {
'lum_mask': lum_mask,
'con_mask': con_mask,
'flow_mask': flow_mask,
'four_mask': four_mask,
'freq_mask': freq_mask,
'orient_mask': orient_mask,
'masked': masked,
'lum_whole': lum_whole,
'con_whole': con_whole,
'flow_whole': flow_whole,
'four_whole': four_whole,
'freq_whole': freq_whole,
'orient_whole': orient_whole,
'movie': movie,
'lum_surr': lum_surr,
'con_surr': con_surr,
'flow_surr': flow_surr,
'four_surr': four_surr,
'freq_surr': freq_surr,
'orient_surr': orient_surr,
'surround': surround
})
plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked,
fig_path + e['cellid'] + '_masked')
plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie,
fig_path + e['cellid'] + '_whole')
plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround,
fig_path + e['cellid'] + '_surround')
print 'Failed: ', failed_count
print 'Successful: ', success_count
def process_movie_ephys(exp_type='SOM'):
failed_count = 0
success_count = 0
ephys = load_EphysData(exp_type)
dat_path = startup.data_path + 'Sparseness/%s/' % exp_type
fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type
if not os.path.exists(fig_path):
os.makedirs(fig_path)
for e in ephys.values():
print 'recording ', e['cellid']
target_fname = e['cellid'] + '_processed.npz'
flow_fname = e['expdate'] + '_flow.mat'
if os.path.exists(dat_path + target_fname):
print 'already exists, skipping ', target_fname
success_count += 1
# continue
if not os.path.exists(dat_path + flow_fname):
print 'flow data missing, skipping ', flow_fname
continue
movie = scipy.io.loadmat(dat_path + e['expdate'] + '.mat')
movie = movie['dat']
flow = scipy.io.loadmat(dat_path + flow_fname)
flow = flow['uv']
flow = np.swapaxes(flow, 1, 2)
actual_len = e['psth_c'].shape[1]
flow = flow[:actual_len]
movie = movie[:actual_len]
if (np.abs(e['adjustedMovResolution']
- e['movResolution'])).sum() != 0:
movie = movie[:, :e['adjustedMovResolution'][0],
:e['adjustedMovResolution'][1]]
flow = flow[:, :e['adjustedMovResolution'][0],
:e['adjustedMovResolution'][1]]
#try:
masked, surround = get_masked_data(movie,
e['maskSizePixel'], e['maskLocationPixel'])
# except:
# print 'FAILED: ', e['cellid']
# failed_count += 1
# continue
success_count += 1
lum_mask = get_luminance(masked)[np.newaxis, :]
con_mask = get_contrast(masked)[np.newaxis, :]
four_mask, freq_mask, orient_mask = get_fourier2D(masked)
four_mask = four_mask[np.newaxis, :, :, :]
freq_mask = freq_mask[np.newaxis, :, :]
orient_mask = orient_mask[np.newaxis, :, :]
lum_surr = []
con_surr = []
four_surr = []
freq_surr = []
orient_surr = []
for s in range(surround.shape[0]):
lum_surr.append(get_luminance(surround[s]))
con_surr.append(get_contrast(surround[s]))
f, fr, o = get_fourier2D(surround[s])
four_surr.append(f)
freq_surr.append(fr)
orient_surr.append(o)
lum_surr = np.array(lum_surr)
con_surr = np.array(con_surr)
four_surr = np.array(four_surr)
freq_surr = np.array(freq_surr)
orient_surr = np.array(orient_surr)
lum_whole = get_luminance(movie)[np.newaxis, :]
con_whole = get_contrast(movie)[np.newaxis, :]
four_whole, freq_whole, orient_whole = get_fourier2D(movie)
four_whole = four_whole[np.newaxis, :, :, :]
freq_whole = freq_whole[np.newaxis, :, :]
orient_whole = orient_whole[np.newaxis, :, :]
flow_whole, flow_mask, flow_surr = get_flow(flow,
e['maskSizePixel'], e['maskLocationPixel'])
movie = movie[np.newaxis, :, :, :]
masked = masked[np.newaxis, :, :, :]
np.savez(dat_path + target_fname,
lum_mask=lum_mask, con_mask=con_mask, flow_mask=flow_mask,
four_mask=four_mask,
freq_mask=freq_mask, orient_mask=orient_mask,
masked=masked,
lum_whole=lum_whole, con_whole=con_whole,
flow_whole=flow_whole,
four_whole=four_whole, freq_whole=freq_whole,
orient_whole=orient_whole, movie=movie,
lum_surr=lum_surr, con_surr=con_surr, flow_surr=flow_surr,
four_surr=four_surr, freq_surr=freq_surr,
orient_surr=orient_surr, surround=surround)
scipy.io.savemat(dat_path + target_fname[:-3] + '.mat',
{'lum_mask': lum_mask, 'con_mask': con_mask,
'flow_mask': flow_mask, 'four_mask': four_mask,
'freq_mask': freq_mask, 'orient_mask': orient_mask,
'masked': masked,
'lum_whole': lum_whole, 'con_whole': con_whole,
'flow_whole': flow_whole, 'four_whole': four_whole,
'freq_whole': freq_whole,
'orient_whole': orient_whole, 'movie': movie,
'lum_surr': lum_surr, 'con_surr': con_surr, 'flow_surr': flow_surr,
'four_surr': four_surr, 'freq_surr': freq_surr,
'orient_surr': orient_surr, 'surround': surround})
plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked,
fig_path + e['cellid'] + '_masked')
plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie,
fig_path + e['cellid'] + '_whole')
plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround,
fig_path + e['cellid'] + '_surround')
print 'Failed: ', failed_count
print 'Successful: ', success_count
def do_lag_classification(exp_type='SOM',
combs=[
'Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'
],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
folds=5,
filt=0.2,
alpha=0.001,
randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb,
targ_type,
src_type,
e,
cellid,
exp_type,
four_downsample,
shift=shift,
randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]:-edges[1]]
mn = y[edges[0]:-edges[1]]
std = np.std(y[edges[0]:-edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn,
pred,
do_abs=True,
corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
folds=5, filt=0.2,
alpha=0.001, randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb, targ_type,
src_type,
e, cellid, exp_type, four_downsample,
shift=shift, randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X, y, folds=folds, clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]: -edges[1]]
mn = y[edges[0]: -edges[1]]
std = np.std(y[edges[0]: -edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
for randomise in [None]: # , 'generated', 'random']:
for exp_type in exp_types:
for filt in filters:
if randomise is None:
f_path = fig_path + 'Sparseness/%s/initial/%.2f/' % (exp_type, filt)
elif randomise == 'random':
f_path = fig_path + 'Sparseness/%s/initial/%.2f_random/' % (
exp_type, filt)
elif randomise == 'generated':
f_path = fig_path + 'Sparseness/%s/initial/%.2f_generated/' % (
exp_type, filt)
fname = '%s%s' % (f_path, 'initial_summary.png')
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
# continue
dat = load_EphysData(exp_type, filt=filt)
csv_vals = []
cellids = []
for k in sorted(dat.keys()):
vals = []
d = dat[k]
print 'doing ', d['cellid']
if randomise is None:
psth_s = d['psth_s']
psth_c = d['psth_c']
psth_w = d['psth_w']
elif randomise == 'random':
psth_c = d['psth_c_rand']
psth_s = d['psth_s_rand']
psth_w = d['psth_w_rand']
elif randomise == 'generated':
def do_classification(
exp_type='SOM',
combs=['Luminance', 'Contrast', 'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center',
'Whole'], ['Whole', 'Center'],
['Whole', 'Whole'], ['Surround', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
randomise=None,
folds=5,
filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]: #, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb
or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][
targ_type]:
cell_results[cellid][full_comb][targ_type][
src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type,
four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path, randomise,
str(filt), cellid)
else:
fname = '%s%s/%s/' % (fig_path, str(filt), cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge:-edge]
clf_vals.append([
targ_type,
])
mn = y.mean(0)[edge:-edge]
std = np.std(y, 0)[edge:-edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][
src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge:-edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [
pred, mn, std, crr_pred, crr_y, coefs, plot_params
]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast',
'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center', 'Whole'],
['Whole', 'Center'], ['Whole', 'Whole'],
['Surround', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
randomise=None, folds=5, filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]:#, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][targ_type]:
cell_results[cellid][full_comb][targ_type][src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type, four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path,
randomise, str(filt),
cellid)
else:
fname = '%s%s/%s/' % (fig_path,
str(filt),
cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X, y, folds=folds, clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge: -edge]
clf_vals.append([targ_type, ])
mn = y.mean(0)[edge: -edge]
std = np.std(y, 0)[edge: -edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge: -edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [pred, mn, std, crr_pred, crr_y,
coefs, plot_params]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def do_classification(exp_type='SOM',
targets=['Center', 'Surround', 'Whole'],
four_downsample=None,
max_exp=None):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if not os.path.exists(fig_path):
os.makedirs(fig_path)
if not os.path.exists(fig_path + str(downsample)):
os.makedirs(fig_path + str(downsample))
dat = load_EphysData(exp_type)
all_corrs = []
surr_corrs = []
mask_corrs = []
whole_corrs = []
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = e['cellid']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
for targ_type in targets:
fname = '%s%s/%s_%s_pred' % (fig_path, str(four_downsample),
cellid, targ_type)
print fname
X, y, plot_params = get_mov_data(targ_type, e, cellid, exp_type,
four_downsample)
for xi in range(X.shape[2]):
cv = LeaveOneOut(X.shape[1], indices=True)
cv = KFold(X.shape[1], 20, indices=True, shuffle=True)
pred = np.zeros(y[0].shape)
X_dims = X.shape[2]
coefs = []
for train, test in cv:
regr = clf(alpha=0.01)
regr.fit(X[:, train, xi].reshape([-1, 1]),
y[:, train].ravel())
coefs.append(regr.coef_)
pred[test] = regr.predict(X[0, test, xi].reshape([-1, 1]))
coefs = np.array(coefs).mean(0)
clf_vals.append([targ_type, coefs, plot_params])
mn = y.mean(0)
std = np.std(y, 0)
[crr_pred, p_pred] = corr(mn, pred)
if p_pred < 0.05:
crr_pred = np.nan_to_num(crr_pred)
else:
crr_pred = 0.
p_pred = 0.
if crr_pred > 0:
sig_found = True
if crr_pred > 0.3:
plt.figure()
plt.hold(True)
plt.plot(pred, 'r')
plt.plot(mn, 'k')
plt.title('Corr: %.2f' % crr_pred)
plt.show()
all_corrs += [crr_pred]
if targ_type == 'Center':
mask_corrs += [crr_pred]
elif targ_type == 'Surround':
surr_corrs += [crr_pred]
elif targ_type == 'Whole':
whole_corrs += [crr_pred]
plt.subplot(211)
plt.scatter(range(len(mask_corrs)), mask_corrs)
plt.subplot(212)
plt.scatter(range(len(whole_corrs)), whole_corrs)
plt.show()
# xcorr = []
# for i in range(y.shape[0]):
# xcorr.append(corr(y[i], mn))
# xcorr = np.nan_to_num(np.array(xcorr))
# xcorr[xcorr[:, 1] > 0.05, 0] = 0
# [crr_y, p_y] = xcorr.mean(0)
# print crr_pred, p_pred
# only do plots for the fourier trained classifier
comb_vals = []
comb_vals.append(['mask', mask_corrs])
comb_vals.append(['surround', surr_corrs])
comb_vals.append(['whole', whole_corrs])
comb_vals.append(['overall', all_corrs])
comb_corrs.append([comb, comb_vals])
# make this a boxplot
plot_summary(comb_corrs, fig_path, str(downsample))
return comb_corrs
def do_classification(exp_type='SOM',
targets=['Center', 'Surround', 'Whole'],
four_downsample=None, max_exp=None):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if not os.path.exists(fig_path):
os.makedirs(fig_path)
if not os.path.exists(fig_path + str(downsample)):
os.makedirs(fig_path + str(downsample))
dat = load_EphysData(exp_type)
all_corrs = []
surr_corrs = []
mask_corrs = []
whole_corrs = []
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = e['cellid']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
for targ_type in targets:
fname = '%s%s/%s_%s_pred' % (fig_path,
str(four_downsample),
cellid, targ_type)
print fname
X, y, plot_params = get_mov_data(targ_type, e,
cellid, exp_type, four_downsample)
for xi in range(X.shape[2]):
cv = LeaveOneOut(X.shape[1], indices=True)
cv = KFold(X.shape[1], 20, indices=True, shuffle=True)
pred = np.zeros(y[0].shape)
X_dims = X.shape[2]
coefs = []
for train, test in cv:
regr = clf(alpha=0.01)
regr.fit(X[:, train, xi].reshape([-1, 1]),
y[:, train].ravel())
coefs.append(regr.coef_)
pred[test] = regr.predict(X[0, test, xi].reshape([-1, 1]))
coefs = np.array(coefs).mean(0)
clf_vals.append([targ_type, coefs, plot_params])
mn = y.mean(0)
std = np.std(y, 0)
[crr_pred, p_pred] = corr(mn, pred)
if p_pred < 0.05:
crr_pred = np.nan_to_num(crr_pred)
else:
crr_pred = 0.
p_pred = 0.
if crr_pred > 0:
sig_found = True
if crr_pred > 0.3:
plt.figure()
plt.hold(True)
plt.plot(pred, 'r')
plt.plot(mn, 'k')
plt.title('Corr: %.2f' % crr_pred)
plt.show()
all_corrs += [crr_pred]
if targ_type == 'Center':
mask_corrs += [crr_pred]
elif targ_type == 'Surround':
surr_corrs += [crr_pred]
elif targ_type == 'Whole':
whole_corrs += [crr_pred]
plt.subplot(211)
plt.scatter(range(len(mask_corrs)), mask_corrs)
plt.subplot(212)
plt.scatter(range(len(whole_corrs)), whole_corrs)
plt.show()
# xcorr = []
# for i in range(y.shape[0]):
# xcorr.append(corr(y[i], mn))
# xcorr = np.nan_to_num(np.array(xcorr))
# xcorr[xcorr[:, 1] > 0.05, 0] = 0
# [crr_y, p_y] = xcorr.mean(0)
# print crr_pred, p_pred
# only do plots for the fourier trained classifier
comb_vals = []
comb_vals.append(['mask', mask_corrs])
comb_vals.append(['surround', surr_corrs])
comb_vals.append(['whole', whole_corrs])
comb_vals.append(['overall', all_corrs])
comb_corrs.append([comb, comb_vals])
# make this a boxplot
plot_summary(comb_corrs, fig_path, str(downsample))
return comb_corrs
import os
# Sub directory of the figure path to put the plots in
corr_type = 'spearmanr'
corr_type = 'pearsonr'
colors = ['r', 'g', 'b']
#fig_path = fig_path + 'Sparseness/%s/initial/' % (exp_type)
if not os.path.exists(fig_path):
os.makedirs(fig_path)
exp_types = ['SOM', 'FS', 'PYR']
corrs = []
cells = []
for exp_type in exp_types:
dat = load_EphysData(exp_type)
for d in dat.values():
cellid = d['cellid']
print 'doing ', cellid
psth_s = d['psth_s']
psth_c = d['psth_c']
psth_w = d['psth_w']
mn_c = psth_c.mean(0)
mn_w = psth_w.mean(0)
mx = np.maximum(mn_c.max(), mn_w.max())
if psth_s is not None:
mn_s = psth_s.mean(0)
mx = np.maximum(mx, mn_s.max())
cr_c_w = None
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast',
'Flow', 'Fourier'],
targets=[['Center', 'Center'], ['Center', 'Whole'],
['Whole', 'Center'], ['Whole', 'Whole'],
['Surround', 'Whole']],
four_downsample=None, random=False):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/xcorr/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if random is not None:
if not os.path.exists(fig_path + str(four_downsample) + '_%s' % random):
os.makedirs(fig_path + str(four_downsample) + '_%s' % random)
else:
if not os.path.exists(fig_path + str(four_downsample)):
os.makedirs(fig_path + str(four_downsample))
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type)
comb_corrs = []
for comb in combs:
print comb
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
expdate = e['expdate']
cellid = e['cellid']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
results = {}
for [targ_type, src_type] in targets:
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type, four_downsample)
# y[y == 0] = 0.001
# y = np.log(y)
if random is not None:
fname = '%s%s_%s' % (fig_path,
str(four_downsample),
random)
if random == 'shift':
idx = deque(np.arange(y.shape[1]))
idx.rotate(-23)
elif random == 'random':
idx = np.arange(y.shape[1])
np.random.shuffle(idx)
y = y[:, idx]
else:
fname = '%s%s/%s_pred' % (fig_path,
str(four_downsample),
cellid)
print fname
X_dims = X.shape[2]
coefs = []
# lassoCV(X.reshape(-1, X_dims),
# y.ravel())
pred = gauss_filter(pred, e['bin_freq'])
coefs = np.array(coefs).mean(0)
clf_vals.append([targ_type, ])
mn = y.mean(0)
std = np.std(y, 0)
[crr_pred, p_pred] = corr(mn, pred)
if p_pred < 0.05:
crr_pred = np.nan_to_num(crr_pred)
else:
crr_pred = 0.
p_pred = 0.
if crr_pred > sig_thresh:
sig_found = True
comb_vals['Overall'] += [crr_pred]
k = '%s_%s' % (targ_type, src_type)
if comb_vals.has_key(k):
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(corr(y[i][5:-5], mn[5:-5]))
xcorr = np.nan_to_num(np.array(xcorr))
xcorr[xcorr[:, 1] > 0.05, 0] = 0
[crr_y, _] = xcorr.mean(0)
print crr_pred, p_pred
results[k] = [pred, mn, std, crr_pred, crr_y,
coefs, plot_params]
# only do plots for the fourier trained classifier
if sig_found and len(comb) == 1:
if comb[0] == 'Fourier':
plot_preds_fourier(cellid, results, full_targets,
fname + '_%s' % comb[0])
else:
plot_preds(cellid, results, full_targets, comb[0],
fname + '_%s' % comb[0])
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if random:
plot_summary(comb_corrs, full_targets, fig_path + 'Rand_', str(downsample))
else:
plot_summary(comb_corrs, full_targets, fig_path, str(downsample))
return comb_corrs
for exp_type in exp_types:
for filt in filters:
if randomise is None:
f_path = fig_path + 'Sparseness/%s/initial/%.2f/' % (exp_type,
filt)
elif randomise == 'random':
f_path = fig_path + 'Sparseness/%s/initial/%.2f_random/' % (
exp_type, filt)
elif randomise == 'generated':
f_path = fig_path + 'Sparseness/%s/initial/%.2f_generated/' % (
exp_type, filt)
fname = '%s%s' % (f_path, 'initial_summary.png')
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
# continue
dat = load_EphysData(exp_type, filt=filt)
csv_vals = []
cellids = []
for k in sorted(dat.keys()):
vals = []
d = dat[k]
print 'doing ', d['cellid']
if randomise is None:
psth_s = d['psth_s']
psth_c = d['psth_c']
psth_w = d['psth_w']
elif randomise == 'random':
psth_c = d['psth_c_rand']
psth_s = d['psth_s_rand']
psth_w = d['psth_w_rand']
elif randomise == 'generated':
def do_classification(exp_type='SOM',
combs=['Luminance', 'Contrast', 'Flow', 'Fourier'],
targets=[['Center', 'Center'], ['Center', 'Whole'],
['Whole', 'Center'], ['Whole', 'Whole'],
['Surround', 'Whole']],
four_downsample=None,
random=False):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/xcorr/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if random is not None:
if not os.path.exists(fig_path + str(four_downsample) +
'_%s' % random):
os.makedirs(fig_path + str(four_downsample) + '_%s' % random)
else:
if not os.path.exists(fig_path + str(four_downsample)):
os.makedirs(fig_path + str(four_downsample))
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type)
comb_corrs = []
for comb in combs:
print comb
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
expdate = e['expdate']
cellid = e['cellid']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
results = {}
for [targ_type, src_type] in targets:
X, y, plot_params = get_mov_data(comb, targ_type, src_type, e,
cellid, exp_type,
four_downsample)
# y[y == 0] = 0.001
# y = np.log(y)
if random is not None:
fname = '%s%s_%s' % (fig_path, str(four_downsample),
random)
if random == 'shift':
idx = deque(np.arange(y.shape[1]))
idx.rotate(-23)
elif random == 'random':
idx = np.arange(y.shape[1])
np.random.shuffle(idx)
y = y[:, idx]
else:
fname = '%s%s/%s_pred' % (fig_path, str(four_downsample),
cellid)
print fname
X_dims = X.shape[2]
coefs = []
# lassoCV(X.reshape(-1, X_dims),
# y.ravel())
pred = gauss_filter(pred, e['bin_freq'])
coefs = np.array(coefs).mean(0)
clf_vals.append([
targ_type,
])
mn = y.mean(0)
std = np.std(y, 0)
[crr_pred, p_pred] = corr(mn, pred)
if p_pred < 0.05:
crr_pred = np.nan_to_num(crr_pred)
else:
crr_pred = 0.
p_pred = 0.
if crr_pred > sig_thresh:
sig_found = True
comb_vals['Overall'] += [crr_pred]
k = '%s_%s' % (targ_type, src_type)
if comb_vals.has_key(k):
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(corr(y[i][5:-5], mn[5:-5]))
xcorr = np.nan_to_num(np.array(xcorr))
xcorr[xcorr[:, 1] > 0.05, 0] = 0
[crr_y, _] = xcorr.mean(0)
print crr_pred, p_pred
results[k] = [
pred, mn, std, crr_pred, crr_y, coefs, plot_params
]
# only do plots for the fourier trained classifier
if sig_found and len(comb) == 1:
if comb[0] == 'Fourier':
plot_preds_fourier(cellid, results, full_targets,
fname + '_%s' % comb[0])
else:
plot_preds(cellid, results, full_targets, comb[0],
fname + '_%s' % comb[0])
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if random:
plot_summary(comb_corrs, full_targets, fig_path + 'Rand_',
str(downsample))
else:
plot_summary(comb_corrs, full_targets, fig_path, str(downsample))
return comb_corrs
