def baseline_filt(dat, baseline_samples, flatten=False):
all_bs = []
all_cell = []
for cell in range(len(dat)):
cell_dat = filter(dat[cell].T, scan_freq, window=filt_window,
prm=decay)[0].T
all_cell.append(cell_dat[baseline_samples:])
all_bs.append(np.array([cell_dat[:baseline_samples].mean(0),
np.std(cell_dat[:baseline_samples], 0)]).T)
return np.array(all_cell), np.array(all_bs)
def baseline_filt(dat, baseline_samples, flatten=False):
all_bs = []
all_cell = []
for cell in range(len(dat)):
cell_dat = filter(dat[cell].T,
scan_freq,
window=filt_window,
prm=decay)[0].T
all_cell.append(cell_dat[baseline_samples:])
all_bs.append(
np.array([
cell_dat[:baseline_samples].mean(0),
np.std(cell_dat[:baseline_samples], 0)
]).T)
return np.array(all_cell), np.array(all_bs)
def baseline_filt_old(dat, baselines, flatten=False):
all_bs = []
for cell in range(len(dat)):
cell_bs = []
for trial in range(dat.shape[2]):
bs_mn = baselines[cell, :, trial].mean()
bs_std = np.std(baselines[cell, :, trial])
if flatten:
bs = bs_mn + 2 * bs_std
dat[cell, dat[cell, :, trial] < bs, trial] = 0.
cell_bs.append([bs_mn, bs_std])
all_bs.append(cell_bs)
dat[cell] = filter(dat[cell].T, scan_freq, window=filt_window,
prm=decay)[0].T
return np.array(all_bs)
def baseline_filt_old(dat, baselines, flatten=False):
all_bs = []
for cell in range(len(dat)):
cell_bs = []
for trial in range(dat.shape[2]):
bs_mn = baselines[cell, :, trial].mean()
bs_std = np.std(baselines[cell, :, trial])
if flatten:
bs = bs_mn + 2 * bs_std
dat[cell, dat[cell, :, trial] < bs, trial] = 0.
cell_bs.append([bs_mn, bs_std])
all_bs.append(cell_bs)
dat[cell] = filter(dat[cell].T,
scan_freq,
window=filt_window,
prm=decay)[0].T
return np.array(all_bs)
def do_lag_classification(exp_type='SOM', combs=['Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
folds=5, filt=0.2,
alpha=0.001, randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb, targ_type,
src_type,
e, cellid, exp_type, four_downsample,
shift=shift, randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X, y, folds=folds, clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]: -edges[1]]
mn = y[edges[0]: -edges[1]]
std = np.std(y[edges[0]: -edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn, pred, do_abs=True, corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results, outfile, protocol=pickle.HIGHEST_PROTOCOL)
def do_lag_classification(exp_type='SOM',
combs=[
'Fourier', 'Frequency', 'Luminance', 'Contrast',
'Orientation', 'Flow'
],
targets=[['Center', 'Center'], ['Whole', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
folds=5,
filt=0.2,
alpha=0.001,
randomise=None):
# Sub directory of the figure path to put the plots in
dat_path = startup.data_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
dat_path = dat_path + randomise + '_mn'
else:
dat_path = dat_path + 'mn'
dat_file = dat_path + '/preds.pkl'
cell_results = {}
if os.path.exists(dat_file):
print ' already exists. loading %s' % dat_file
with open(dat_file, 'rb') as infile:
cell_results = pickle.load(infile)
else:
print '%s doesnt exist, starting New' % dat_file
if not os.path.exists(dat_path):
os.makedirs(dat_path)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
for num_combs in [1]:
for comb in itertools.combinations(combs, num_combs):
full_comb = str(num_combs) + '_' + "_".join(comb)
for e in dat.values():
cellid = str(e['cellid'])
if cellid not in cell_results:
cell_results[cellid] = {}
if randomise is None:
shifts = e['shifts']
else:
shifts = [0]
edge = e['edge']
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
changed = False
for shift in shifts:
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if k not in cell_results[cellid]:
cell_results[cellid][k] = {}
if full_comb not in cell_results[cellid][k]:
cell_results[cellid][k][full_comb] = {}
if shift not in cell_results[cellid][k][full_comb]:
cell_results[cellid][k][full_comb][shift] = None
else:
print 'done - continue'
continue
changed = True
edges = [edge, np.abs(np.minimum(-edge, shift))]
X, y, plot_params = get_mov_data(comb,
targ_type,
src_type,
e,
cellid,
exp_type,
four_downsample,
shift=shift,
randomise=randomise)
# ignore edge effects
pred, coefs = CV_time(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edges=edges)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edges[0]:-edges[1]]
mn = y[edges[0]:-edges[1]]
std = np.std(y[edges[0]:-edges[1]])
crr_pred_r2 = np.maximum(r2_score(mn, pred), 0)
crr_pred = do_thresh_corr(mn,
pred,
do_abs=True,
corr_type='pearsonr')
print exp_type, comb, k, shift, crr_pred, crr_pred_r2
res = {}
res['pred'] = pred
res['mn'] = mn
res['std'] = std
res['crr_pred_r2'] = crr_pred_r2
res['crr_pred'] = crr_pred
res['crr_exp'] = None
res['coefs'] = coefs
res['plot_params'] = plot_params
cell_results[cellid][k][full_comb][shift] = res
if changed:
with open(dat_file, 'wb') as outfile:
pickle.dump(cell_results,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
def load_EphysData(exp_type='SOM', filt=0.1):
#ignore 120201
mat = scipy.io.loadmat(extern_data_path +
'Sparseness/EphysData/analysis/EphysData_%s.mat'
% exp_type)
dat_dir = extern_data_path + 'Sparseness/EphysData/%s/' % exp_type
mov_path = data_path + 'Sparseness/%s/' % exp_type
dat = mat['EphysData_%s' % exp_type]
print len(dat[0])
#do_firing_rate([1], 10)
all_dat = {}
for dt in dat[0]:
expdate = str(dt[0][0])
if exp_type == 'SOM':
cellid = dt[0][0] + '_' + dt[1][0]
elif exp_type == 'PYR':
cellid = dt[0][0] + '_' + str(dt[1][0][0])
else:
cellid = dt[1][0]
print cellid
if expdate == '120201':
print 'skipping'
continue
if not cell_data.has_key(cellid):
print 'no mask size found'
continue
#files = os.listdir(dat_dir + '/' + expdate + '/Spreadsheets@100Hz/')
fname = cell_data[cellid][0]
# for ff in files:
# if 'RF.mat' in ff:
# fname = ff
# break
rec_dat = scipy.io.loadmat(dat_dir + expdate +
'/Spreadsheets@100Hz/' + fname)
mov = scipy.io.loadmat(mov_path + expdate + '.mat')
mov = mov['dat']
movResolution = np.array(mov.shape[1:])
vnResolution = np.array(rec_dat['vnResolution'][0])
#vfSizeDegrees = np.array(rec_dat['vfSizeDegrees'][0])
maskLocationDeg = np.array(rec_dat['maskLocationDeg'][0])
vfPixelsPerDegree = np.array(rec_dat['vfPixelsPerDegree'][0]).mean()
maskSizeDeg = cell_data[cellid][1]
# in screen pixels
maskSizePixel = maskSizeDeg * np.array([vfPixelsPerDegree,
vfPixelsPerDegree])
#convert to movie pixels
scale_width = vnResolution[0] / (movResolution[0] * 1.)
scale_height = vnResolution[1] / (movResolution[1] * 1.)
## check this
if cell_data[cellid][2] == 1:
# scale to screen, no clipping
if scale_height < scale_width:
scale = scale_height
else:
scale = scale_width
maskSizePixel = (maskSizePixel / scale).astype(np.int)
maskLocationPixel = maskLocationDeg * vfPixelsPerDegree / scale
maskLocationPixel += (movResolution / 2.)
maskLocationPixel = maskLocationPixel.astype(np.int)
adjustedMovResolution = movResolution
else:
#scale to screen with clipping
if scale_height < scale_width:
scale = scale_width
overshoot = scale_height / scale_width
# not all of the actual movie is shown anymore
adjustedMovResolution = movResolution * np.array([1, overshoot])
adjustedMovResolution = adjustedMovResolution.astype(np.int)
else:
scale = scale_height
overshoot = scale_width / scale_height
# not all of the actual movie is shown anymore
adjustedMovResolution = movResolution * np.array([overshoot, 1])
adjustedMovResolution = adjustedMovResolution.astype(np.int)
maskSizePixel = (maskSizePixel / scale).astype(np.int)
if (maskSizePixel[0] % 2) == 0:
maskSizePixel += 1
maskLocationPixel = maskLocationDeg * vfPixelsPerDegree / scale
maskLocationPixel += (adjustedMovResolution / 2.)
maskLocationPixel = maskLocationPixel.astype(np.int)
bin_freq = dt[5][0][0]
movie_duration = dt[6][0][0]
if exp_type == 'PYR':
psth_c = np.array(dt[6], dtype=np.int)
psth_w = np.array(dt[7], dtype=np.int)
psth_s = None
else:
psth_c = np.array(dt[7], dtype=np.int)
psth_w = np.array(dt[8], dtype=np.int)
psth_s = np.array(dt[9], dtype=np.int)
# do shift
psth_c_shift = {}
psth_w_shift = {}
psth_s_shift = {}
#shifts = np.arange(-15, 16, 5).tolist()
shifts = range(-12, 5)
# shifts = np.array(list(set(shifts)))
# shifts.sort()
for shift in shifts:
idx = deque(np.arange(psth_c.shape[1]))
idx.rotate(shift)
c, _ = filter(psth_c[:, idx], bin_freq, prm=filt)
psth_c_shift[shift] = c
w, _ = filter(psth_w[:, idx], bin_freq, prm=filt)
psth_w_shift[shift] = w
if psth_s is not None:
s, _ = filter(psth_s[:, idx], bin_freq, prm=filt)
psth_s_shift[shift] = s
# psth_c_shift = np.array(psth_c_shift)
# psth_w_shift = np.array(psth_w_shift)
# psth_s_shift = np.array(psth_s_shift)
idx = deque(np.arange(psth_c.shape[1]))
idx.rotate(-4)
psth_c = psth_c[:, idx]
psth_w = psth_w[:, idx]
if psth_s is not None:
psth_s = psth_s[:, idx]
# do generate
psth_c_gen = generate_psth(psth_c)
psth_w_gen = generate_psth(psth_w)
psth_s_gen = generate_psth(psth_s)
psth_c_gen, _ = filter(psth_c_gen, bin_freq, prm=filt)
psth_w_gen, _ = filter(psth_w_gen, bin_freq, prm=filt)
psth_s_gen, _ = filter(psth_s_gen, bin_freq, prm=filt)
psth_c_raw = psth_c
psth_w_raw = psth_w
psth_s_raw = psth_s
psth_c, edge = filter(psth_c, bin_freq, prm=filt)
psth_w, _ = filter(psth_w, bin_freq, prm=filt)
psth_s, _ = filter(psth_s, bin_freq, prm=filt)
# random version
idx = np.arange(psth_c.shape[1])
np.random.shuffle(idx)
psth_c_rand = psth_c[:, idx]
psth_w_rand = psth_w[:, idx]
if psth_s is not None:
psth_s_rand = psth_s[:, idx]
else:
psth_s_rand = None
# idx = np.arange(y.shape[1])
# if randomise == 'shift':
# idx = deque(idx)
# idx.rotate(-23)
# elif randomise == 'random':
# np.random.shuffle(idx)
# y = y[:, idx]
all_dat[cellid] = {'expdate': expdate, 'cellid': cellid,
#'matfile': matfile,
#'conditions': conditions,
#'conditions_used': conditions_used,
'bin_freq': bin_freq,
'movie_duration': movie_duration,
'psth_c': psth_c, 'psth_w': psth_w,
'psth_s': psth_s,
'psth_c_raw': psth_c_raw,
'psth_w_raw': psth_w_raw,
'psth_s_raw': psth_s_raw,
'psth_c_rand': psth_c_rand,
'psth_w_rand': psth_w_rand,
'psth_s_rand': psth_s_rand,
'psth_c_gen': psth_c_gen,
'psth_w_gen': psth_w_gen,
'psth_s_gen': psth_s_gen,
'psth_c_shift': psth_c_shift,
'psth_w_shift': psth_w_shift,
'psth_s_shift': psth_s_shift,
'shifts': shifts,
'maskLocationDeg': maskLocationDeg,
'maskSizeDeg': maskSizeDeg,
#'vfSizeDegrees': vfSizeDegrees,
'vfPixelsPerDegree': vfPixelsPerDegree,
'vnResolution': vnResolution,
'movResolution': movResolution,
'adjustedMovResolution': adjustedMovResolution,
'maskSizePixel': maskSizePixel,
'maskLocationPixel': maskLocationPixel,
'scale': scale,
'edge': edge
}
return all_dat
def do_classification(
exp_type='SOM',
combs=['Luminance', 'Contrast', 'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center',
'Whole'], ['Whole', 'Center'],
['Whole', 'Whole'], ['Surround', 'Whole']],
max_comb=None,
min_comb=None,
four_downsample=None,
max_exp=None,
sig_thresh=0.,
randomise=None,
folds=5,
filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]: #, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb
or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][
targ_type]:
cell_results[cellid][full_comb][targ_type][
src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type,
four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path, randomise,
str(filt), cellid)
else:
fname = '%s%s/%s/' % (fig_path, str(filt), cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X,
y,
folds=folds,
clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge:-edge]
clf_vals.append([
targ_type,
])
mn = y.mean(0)[edge:-edge]
std = np.std(y, 0)[edge:-edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][
src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge:-edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [
pred, mn, std, crr_pred, crr_y, coefs, plot_params
]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def gen_data(dat, freq):
idx = np.arange(dat.shape[0])
np.random.shuffle(idx)
for d in range(dat.shape[1]):
dat[:, d], _ = filter(dat[idx, d], freq)
return dat
def do_classification(exp_type='SOM', combs=['Luminance', 'Contrast',
'Orientation', 'Frequency', 'Fourier'],
targets=[['Center', 'Center'], ['Center', 'Whole'],
['Whole', 'Center'], ['Whole', 'Whole'],
['Surround', 'Whole']],
max_comb=None, min_comb=None,
four_downsample=None, max_exp=None, sig_thresh=0.,
randomise=None, folds=5, filt=0.2,
alpha=0.001):
# Sub directory of the figure path to put the plots in
fig_path = startup.fig_path + 'Sparseness/%s/pred/' % (exp_type)
mov_path = startup.data_path + 'Sparseness/%s/' % (exp_type)
if randomise is not None:
pth = fig_path + randomise + '_' + str(filt)
else:
pth = fig_path + str(filt)
if os.path.exists(pth):
if os.path.exists(pth + '/summary_all.csv'):
print pth, ' already exists. Skipping'
return ['skipped']
else:
os.makedirs(pth)
full_targets = []
for [targ_type, src_type] in targets:
full_targets.append('%s_%s' % (targ_type, src_type))
dat = load_EphysData(exp_type, filt=filt)
comb_corrs = []
if max_comb is None:
max_comb = len(combs)
if min_comb is None:
min_comb = 0
cell_results = {}
num_combs = [1]
if len(combs) > 1:
num_combs.append(len(combs))
for num_combs in [1, 2]:#, len(combs)]:
for comb in itertools.combinations(combs, num_combs):
print comb
full_comb = str(num_combs) + '_' + "_".join(comb)
if num_combs == 2 and ('Frequency' not in comb or 'Orientation' not in comb):
continue
comb_vals = {'Overall': []}
for i, e in enumerate(dat.values()):
if max_exp is not None and i >= max_exp:
break
expdate = e['expdate']
cellid = str(e['cellid'])
edge = e['edge']
if cellid not in cell_results:
cell_results[cellid] = {}
if full_comb not in cell_results[cellid]:
cell_results[cellid][full_comb] = {}
if not os.path.exists(mov_path + cellid + '_processed.npz'):
print '\nNo movie found ', cellid
continue
else:
print '\ndoing ', e['cellid']
clf_vals = []
sig_found = False
results = {}
for [targ_type, src_type] in targets:
k = '%s_%s' % (targ_type, src_type)
if targ_type not in cell_results[cellid][full_comb]:
cell_results[cellid][full_comb][targ_type] = {}
if src_type not in cell_results[cellid][full_comb][targ_type]:
cell_results[cellid][full_comb][targ_type][src_type] = {}
X, y, plot_params = get_mov_data(comb, targ_type, src_type,
e, cellid, exp_type, four_downsample,
randomise)
if randomise is not None:
fname = '%s%s_%s/%s/' % (fig_path,
randomise, str(filt),
cellid)
else:
fname = '%s%s/%s/' % (fig_path,
str(filt),
cellid)
if not os.path.exists(fname):
os.makedirs(fname)
print fname
# ignore edge effects
pred, coefs = CV(clf,
X, y, folds=folds, clf_args=clf_args,
edge=edge)
coefs = np.array(coefs).mean(0)
pred, _ = filter(pred, e['bin_freq'])
pred = pred[edge: -edge]
clf_vals.append([targ_type, ])
mn = y.mean(0)[edge: -edge]
std = np.std(y, 0)[edge: -edge]
#crr_pred = do_thresh_corr(mn, pred)#, 'pearsonr')
#crr_pred = mean_squared_error(mn, pred)
crr_pred = np.maximum(r2_score(mn, pred), 0)
#crr_pred = explained_variance_score(mn, pred)
if crr_pred > sig_thresh:
sig_found = True
cell_results[cellid][full_comb][targ_type][src_type] = crr_pred
comb_vals['Overall'] += [crr_pred]
if k in comb_vals:
comb_vals[k] += [crr_pred]
else:
comb_vals[k] = [crr_pred]
xcorr = []
for i in range(y.shape[0]):
xcorr.append(do_thresh_corr(y[i][edge: -edge], mn))
crr_y = np.array(xcorr).mean()
print crr_pred
results[k] = [pred, mn, std, crr_pred, crr_y,
coefs, plot_params]
# only do plots for the fourier trained classifier
#if sig_found:
if True:
cmb = "_".join(comb)
if 'Fourier' in cmb:
plot_preds_fourier(cellid, results, full_targets,
fname + cmb)
else:
plot_preds(cellid, results, full_targets, cmb,
fname + cmb)
comb_corrs.append([comb, comb_vals])
# make this a boxplot
if randomise is not None:
fp = fig_path + randomise + '_'
else:
fp = fig_path
plot_summary(comb_corrs, full_targets, fp, str(filt))
plot_cell_summary(cell_results, fp + str(filt) + '/')
return comb_corrs
def gen_data(dat, freq):
idx = np.arange(dat.shape[0])
np.random.shuffle(idx)
for d in range(dat.shape[1]):
dat[:, d], _ = filter(dat[idx, d], freq)
return dat
def load_EphysData(exp_type="SOM", filt=0.1):
# ignore 120201
mat = scipy.io.loadmat(extern_data_path + "Sparseness/EphysData/analysis/EphysData_%s.mat" % exp_type)
dat_dir = extern_data_path + "Sparseness/EphysData/%s/" % exp_type
mov_path = data_path + "Sparseness/%s/" % exp_type
dat = mat["EphysData_%s" % exp_type]
print len(dat[0])
# do_firing_rate([1], 10)
all_dat = {}
for dt in dat[0]:
expdate = str(dt[0][0])
if exp_type == "SOM":
cellid = dt[0][0] + "_" + dt[1][0]
elif exp_type == "PYR":
cellid = dt[0][0] + "_" + str(dt[1][0][0])
else:
cellid = dt[1][0]
print cellid
if expdate == "120201":
print "skipping"
continue
if not cell_data.has_key(cellid):
print "no mask size found"
continue
# files = os.listdir(dat_dir + '/' + expdate + '/Spreadsheets@100Hz/')
fname = cell_data[cellid][0]
# for ff in files:
# if 'RF.mat' in ff:
# fname = ff
# break
rec_dat = scipy.io.loadmat(dat_dir + expdate + "/Spreadsheets@100Hz/" + fname)
mov = scipy.io.loadmat(mov_path + expdate + ".mat")
mov = mov["dat"]
movResolution = np.array(mov.shape[1:])
vnResolution = np.array(rec_dat["vnResolution"][0])
# vfSizeDegrees = np.array(rec_dat['vfSizeDegrees'][0])
maskLocationDeg = np.array(rec_dat["maskLocationDeg"][0])
vfPixelsPerDegree = np.array(rec_dat["vfPixelsPerDegree"][0]).mean()
maskSizeDeg = cell_data[cellid][1]
# in screen pixels
maskSizePixel = maskSizeDeg * np.array([vfPixelsPerDegree, vfPixelsPerDegree])
# convert to movie pixels
scale_width = vnResolution[0] / (movResolution[0] * 1.0)
scale_height = vnResolution[1] / (movResolution[1] * 1.0)
## check this
if cell_data[cellid][2] == 1:
# scale to screen, no clipping
if scale_height < scale_width:
scale = scale_height
else:
scale = scale_width
maskSizePixel = (maskSizePixel / scale).astype(np.int)
maskLocationPixel = maskLocationDeg * vfPixelsPerDegree / scale
maskLocationPixel += movResolution / 2.0
maskLocationPixel = maskLocationPixel.astype(np.int)
adjustedMovResolution = movResolution
else:
# scale to screen with clipping
if scale_height < scale_width:
scale = scale_width
overshoot = scale_height / scale_width
# not all of the actual movie is shown anymore
adjustedMovResolution = movResolution * np.array([1, overshoot])
adjustedMovResolution = adjustedMovResolution.astype(np.int)
else:
scale = scale_height
overshoot = scale_width / scale_height
# not all of the actual movie is shown anymore
adjustedMovResolution = movResolution * np.array([overshoot, 1])
adjustedMovResolution = adjustedMovResolution.astype(np.int)
maskSizePixel = (maskSizePixel / scale).astype(np.int)
if (maskSizePixel[0] % 2) == 0:
maskSizePixel += 1
maskLocationPixel = maskLocationDeg * vfPixelsPerDegree / scale
maskLocationPixel += adjustedMovResolution / 2.0
maskLocationPixel = maskLocationPixel.astype(np.int)
bin_freq = dt[5][0][0]
movie_duration = dt[6][0][0]
if exp_type == "PYR":
psth_c = np.array(dt[6], dtype=np.int)
psth_w = np.array(dt[7], dtype=np.int)
psth_s = None
else:
psth_c = np.array(dt[7], dtype=np.int)
psth_w = np.array(dt[8], dtype=np.int)
psth_s = np.array(dt[9], dtype=np.int)
# do shift
psth_c_shift = {}
psth_w_shift = {}
psth_s_shift = {}
# shifts = np.arange(-15, 16, 5).tolist()
shifts = range(-12, 5)
# shifts = np.array(list(set(shifts)))
# shifts.sort()
for shift in shifts:
idx = deque(np.arange(psth_c.shape[1]))
idx.rotate(shift)
c, _ = filter(psth_c[:, idx], bin_freq, prm=filt)
psth_c_shift[shift] = c
w, _ = filter(psth_w[:, idx], bin_freq, prm=filt)
psth_w_shift[shift] = w
if psth_s is not None:
s, _ = filter(psth_s[:, idx], bin_freq, prm=filt)
psth_s_shift[shift] = s
# psth_c_shift = np.array(psth_c_shift)
# psth_w_shift = np.array(psth_w_shift)
# psth_s_shift = np.array(psth_s_shift)
idx = deque(np.arange(psth_c.shape[1]))
idx.rotate(-4)
psth_c = psth_c[:, idx]
psth_w = psth_w[:, idx]
if psth_s is not None:
psth_s = psth_s[:, idx]
# do generate
psth_c_gen = generate_psth(psth_c)
psth_w_gen = generate_psth(psth_w)
psth_s_gen = generate_psth(psth_s)
psth_c_gen, _ = filter(psth_c_gen, bin_freq, prm=filt)
psth_w_gen, _ = filter(psth_w_gen, bin_freq, prm=filt)
psth_s_gen, _ = filter(psth_s_gen, bin_freq, prm=filt)
psth_c_raw = psth_c
psth_w_raw = psth_w
psth_s_raw = psth_s
psth_c, edge = filter(psth_c, bin_freq, prm=filt)
psth_w, _ = filter(psth_w, bin_freq, prm=filt)
psth_s, _ = filter(psth_s, bin_freq, prm=filt)
# random version
idx = np.arange(psth_c.shape[1])
np.random.shuffle(idx)
psth_c_rand = psth_c[:, idx]
psth_w_rand = psth_w[:, idx]
if psth_s is not None:
psth_s_rand = psth_s[:, idx]
else:
psth_s_rand = None
# idx = np.arange(y.shape[1])
# if randomise == 'shift':
# idx = deque(idx)
# idx.rotate(-23)
# elif randomise == 'random':
# np.random.shuffle(idx)
# y = y[:, idx]
all_dat[cellid] = {
"expdate": expdate,
"cellid": cellid,
#'matfile': matfile,
#'conditions': conditions,
#'conditions_used': conditions_used,
"bin_freq": bin_freq,
"movie_duration": movie_duration,
"psth_c": psth_c,
"psth_w": psth_w,
"psth_s": psth_s,
"psth_c_raw": psth_c_raw,
"psth_w_raw": psth_w_raw,
"psth_s_raw": psth_s_raw,
"psth_c_rand": psth_c_rand,
"psth_w_rand": psth_w_rand,
"psth_s_rand": psth_s_rand,
"psth_c_gen": psth_c_gen,
"psth_w_gen": psth_w_gen,
"psth_s_gen": psth_s_gen,
"psth_c_shift": psth_c_shift,
"psth_w_shift": psth_w_shift,
"psth_s_shift": psth_s_shift,
"shifts": shifts,
"maskLocationDeg": maskLocationDeg,
"maskSizeDeg": maskSizeDeg,
#'vfSizeDegrees': vfSizeDegrees,
"vfPixelsPerDegree": vfPixelsPerDegree,
"vnResolution": vnResolution,
"movResolution": movResolution,
"adjustedMovResolution": adjustedMovResolution,
"maskSizePixel": maskSizePixel,
"maskLocationPixel": maskLocationPixel,
"scale": scale,
"edge": edge,
}
return all_dat
