def process_movie_pop(exp_type='POP'):
failed_count = 0
success_count = 0
exps = list_PopExps()
dat_path = startup.data_path + 'Sparseness/%s/' % exp_type
fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type
if not os.path.exists(fig_path):
os.makedirs(fig_path)
for exp_id in exps:
# print 'recording ', exp_id
target_fname = exp_id + '_processed.npz'
flow_fname = exp_id + '_flow.mat'
e = load_PopData(exp_id)
if os.path.exists(dat_path + target_fname):
print 'already exists, skipping ', target_fname
success_count += 1
continue
flow_found = True
if not os.path.exists(dat_path + flow_fname):
flow_found = False
print 'flow data missing ', flow_fname
# continue
movie = scipy.io.loadmat(dat_path + e['movie'])
movie = movie['dat']
exp_len = e['dat_c'].shape[1] / e['scan_freq']
movie_len = movie.shape[0] / frame_rate
if exp_len > movie_len:
mov_frames = movie.shape[0]
exp_samples = np.ceil(e['scan_freq'] * movie_len)
else:
mov_frames = np.ceil(frame_rate * exp_len)
exp_samples = e['dat_c'].shape[1]
print
print 'Exp: %s,\tScan Rate: %.2f \tFrame Rate: %.2f' % (
exp_id, e['scan_freq'], frame_rate)
print '\t\t#Samples: %d\t\t#Frames: %d' % (e['dat_c'].shape[1],
movie.shape[0])
print '\t\tExp Length: %.2fsec\tMovie Length: %.2fsec' % (exp_len,
movie_len)
print 'Selected\t#Samples: %d\t\t#Frames: %d' % (exp_samples,
mov_frames)
movie = movie[:mov_frames]
if flow_found:
flow = scipy.io.loadmat(dat_path + flow_fname)
flow = flow['uv']
flow = np.swapaxes(flow, 1, 2)
flow = flow[:mov_frames]
if (np.abs(e['adjustedMovResolution']
- e['movResolution'])).sum() != 0:
movie = movie[:, :e['adjustedMovResolution'][0],
:e['adjustedMovResolution'][1]]
if flow_found:
flow = flow[:, :e['adjustedMovResolution'][0],
:e['adjustedMovResolution'][1]]
#try:
masked, surround = get_masked_data(movie,
e['maskSizePixel'], e['maskLocationPixel'])
# except:
# print 'FAILED: ', e['cellid']
# failed_count += 1
# continue
success_count += 1
exp_time = np.arange(exp_samples) / scan_freq
mov_time = np.arange(mov_frames) / frame_rate
lum_mask = downsample(get_luminance(masked),
mov_time, exp_time)[np.newaxis, :]
con_mask = downsample(get_contrast(masked),
mov_time, exp_time)[np.newaxis, :]
four_mask, freq_mask, orient_mask = get_fourier2D(masked)
four_mask = downsample_multi_dim(four_mask, mov_time,
exp_time, is_complex=True)[np.newaxis, :, :, :]
freq_mask = downsample_multi_dim(freq_mask, mov_time,
exp_time)[np.newaxis, :, :]
orient_mask = downsample_multi_dim(orient_mask, mov_time,
exp_time)[np.newaxis, :, :]
# lum_surr = []
# con_surr = []
# four_surr = []
# freq_surr = []
# orient_surr = []
# for s in range(surround.shape[0]):
# lum_surr.append(get_luminance(surround[s]))
# con_surr.append(get_contrast(surround[s]))
# f, fr, o = get_fourier2D(surround[s])
# four_surr.append(f)
# freq_surr.append(fr)
# orient_surr.append(o)
# lum_surr = np.array(lum_surr)
# con_surr = np.array(con_surr)
# four_surr = np.array(four_surr)
# freq_surr = np.array(freq_surr)
# orient_surr = np.array(orient_surr)
# lum_whole = downsample(get_luminance(movie), mov_time,
# exp_time)[np.newaxis, :]
# con_whole = downsample(get_contrast(movie), mov_time,
# exp_time)[np.newaxis, :]
# four_whole, freq_whole, orient_whole = get_fourier2D(movie)
# four_whole = downsample_multi_dim(four_whole, mov_time,
# exp_time)[np.newaxis, :, :, :]
# freq_whole = downsample_multi_dim(freq_whole, mov_time,
# exp_time)[np.newaxis, :, :]
# orient_whole = downsample_multi_dim(orient_whole, mov_time,
# exp_time)[np.newaxis, :, :]
if flow_found:
flow_whole, flow_mask, flow_surr = get_flow(flow,
e['maskSizePixel'], e['maskLocationPixel'])
else:
flow_whole, flow_mask, flow_surr = np.array([]), np.array([]), np.array([])
movie = movie[np.newaxis, :, :, :]
masked = masked[np.newaxis, :, :, :]
np.savez(dat_path + target_fname,
lum_mask=lum_mask, con_mask=con_mask, flow_mask=flow_mask,
four_mask=four_mask,
freq_mask=freq_mask, orient_mask=orient_mask,
masked=masked,
#lum_whole=lum_whole, con_whole=con_whole,
flow_whole=flow_whole,
#four_whole=four_whole, freq_whole=freq_whole,
#orient_whole=orient_whole,
movie=movie,
mov_frames=mov_frames, exp_samples=exp_samples
# lum_surr=lum_surr, con_surr=con_surr, flow_surr=flow_surr,
# four_surr=four_surr, freq_surr=freq_surr,
# orient_surr=orient_surr, surround=surround
)
scipy.io.savemat(dat_path + target_fname[:-3] + '.mat',
{'lum_mask': lum_mask, 'con_mask': con_mask,
'flow_mask': flow_mask, 'four_mask': four_mask,
'freq_mask': freq_mask, 'orient_mask': orient_mask,
'masked': masked,
# 'lum_whole': lum_whole, 'con_whole': con_whole,
# 'flow_whole': flow_whole, 'four_whole': four_whole,
# 'freq_whole': freq_whole,
# 'orient_whole': orient_whole,
'movie': movie,
'mov_frames': mov_frames, 'exp_samples': exp_samples
# 'lum_surr': lum_surr, 'con_surr': con_surr, 'flow_surr': flow_surr,
# 'four_surr': four_surr, 'freq_surr': freq_surr,
# 'orient_surr': orient_surr, 'surround': surround
})
plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked,
fig_path + exp_id + '_masked')
# plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie,
# fig_path + exp_id + '_whole')
# plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround,
# fig_path + exp_id + '_surround')
print 'Failed: ', failed_count
print 'Successful: ', success_count
import cPickle
from data_utils.tsne import calc_tsne
from sklearn.linear_model import LinearRegression as clf
clf_args = {}
target = data_path + 'Sparseness/POP/predict/'
if not os.path.exists(target):
os.makedirs(target)
pca_dims = 30
out_dims = 2
exps = list_PopExps()
for exp in exps:
out_fname = exp + '.pkl'
dat = load_PopData(exp)
active = dat['active']
d = np.where(active[:, 1])[0]
print d
res = []
cell_idx = np.arange(dat['dat_c'].shape[0])
crr_c = None
crr_w = None
for cell in cell_idx:
if cell in d:
print 'active'
else:
continue
print cell
cell_res = []
for i, src in enumerate(['dat_raw_c', 'dat_raw_w']):
if randomise is None:
f_path = fig_path + 'Sparseness/%s/initial/norm/' % (exp_type)
elif randomise == 'random':
f_path = fig_path + 'Sparseness/%s/initial/random/' % (exp_type)
fname = '%s%s' % (f_path, 'initial_summary.png')
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
# continue
exps = list_PopExps()
csv_vals = [[], [], [], [], []]
sum_vals = []
cellids = []
#exps = ['121127']
for exp_id in exps:
dat = load_PopData(exp_id, True)
vals = []
print 'doing ', exp_id
if randomise is None:
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
elif randomise == 'random':
assert(False)
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
if not os.path.exists(f_path):
os.makedirs(f_path)
mn_c = dat_c.mean(2).mean(0)
std_c = np.std(dat_c, 2).mean(0)
def process_movie_pop(exp_type='POP'):
failed_count = 0
success_count = 0
exps = list_PopExps()
dat_path = startup.data_path + 'Sparseness/%s/' % exp_type
fig_path = startup.fig_path + 'Sparseness/%s/movies/' % exp_type
if not os.path.exists(fig_path):
os.makedirs(fig_path)
for exp_id in exps:
# print 'recording ', exp_id
target_fname = exp_id + '_processed.npz'
flow_fname = exp_id + '_flow.mat'
e = load_PopData(exp_id)
if os.path.exists(dat_path + target_fname):
print 'already exists, skipping ', target_fname
success_count += 1
continue
flow_found = True
if not os.path.exists(dat_path + flow_fname):
flow_found = False
print 'flow data missing ', flow_fname
# continue
movie = scipy.io.loadmat(dat_path + e['movie'])
movie = movie['dat']
exp_len = e['dat_c'].shape[1] / e['scan_freq']
movie_len = movie.shape[0] / frame_rate
if exp_len > movie_len:
mov_frames = movie.shape[0]
exp_samples = np.ceil(e['scan_freq'] * movie_len)
else:
mov_frames = np.ceil(frame_rate * exp_len)
exp_samples = e['dat_c'].shape[1]
print
print 'Exp: %s,\tScan Rate: %.2f \tFrame Rate: %.2f' % (
exp_id, e['scan_freq'], frame_rate)
print '\t\t#Samples: %d\t\t#Frames: %d' % (e['dat_c'].shape[1],
movie.shape[0])
print '\t\tExp Length: %.2fsec\tMovie Length: %.2fsec' % (exp_len,
movie_len)
print 'Selected\t#Samples: %d\t\t#Frames: %d' % (exp_samples,
mov_frames)
movie = movie[:mov_frames]
if flow_found:
flow = scipy.io.loadmat(dat_path + flow_fname)
flow = flow['uv']
flow = np.swapaxes(flow, 1, 2)
flow = flow[:mov_frames]
if (np.abs(e['adjustedMovResolution'] -
e['movResolution'])).sum() != 0:
movie = movie[:, :e['adjustedMovResolution'][0], :
e['adjustedMovResolution'][1]]
if flow_found:
flow = flow[:, :e['adjustedMovResolution'][0], :
e['adjustedMovResolution'][1]]
#try:
masked, surround = get_masked_data(movie, e['maskSizePixel'],
e['maskLocationPixel'])
# except:
# print 'FAILED: ', e['cellid']
# failed_count += 1
# continue
success_count += 1
exp_time = np.arange(exp_samples) / scan_freq
mov_time = np.arange(mov_frames) / frame_rate
lum_mask = downsample(get_luminance(masked), mov_time,
exp_time)[np.newaxis, :]
con_mask = downsample(get_contrast(masked), mov_time,
exp_time)[np.newaxis, :]
four_mask, freq_mask, orient_mask = get_fourier2D(masked)
four_mask = downsample_multi_dim(four_mask,
mov_time,
exp_time,
is_complex=True)[np.newaxis, :, :, :]
freq_mask = downsample_multi_dim(freq_mask, mov_time,
exp_time)[np.newaxis, :, :]
orient_mask = downsample_multi_dim(orient_mask, mov_time,
exp_time)[np.newaxis, :, :]
# lum_surr = []
# con_surr = []
# four_surr = []
# freq_surr = []
# orient_surr = []
# for s in range(surround.shape[0]):
# lum_surr.append(get_luminance(surround[s]))
# con_surr.append(get_contrast(surround[s]))
# f, fr, o = get_fourier2D(surround[s])
# four_surr.append(f)
# freq_surr.append(fr)
# orient_surr.append(o)
# lum_surr = np.array(lum_surr)
# con_surr = np.array(con_surr)
# four_surr = np.array(four_surr)
# freq_surr = np.array(freq_surr)
# orient_surr = np.array(orient_surr)
# lum_whole = downsample(get_luminance(movie), mov_time,
# exp_time)[np.newaxis, :]
# con_whole = downsample(get_contrast(movie), mov_time,
# exp_time)[np.newaxis, :]
# four_whole, freq_whole, orient_whole = get_fourier2D(movie)
# four_whole = downsample_multi_dim(four_whole, mov_time,
# exp_time)[np.newaxis, :, :, :]
# freq_whole = downsample_multi_dim(freq_whole, mov_time,
# exp_time)[np.newaxis, :, :]
# orient_whole = downsample_multi_dim(orient_whole, mov_time,
# exp_time)[np.newaxis, :, :]
if flow_found:
flow_whole, flow_mask, flow_surr = get_flow(
flow, e['maskSizePixel'], e['maskLocationPixel'])
else:
flow_whole, flow_mask, flow_surr = np.array([]), np.array(
[]), np.array([])
movie = movie[np.newaxis, :, :, :]
masked = masked[np.newaxis, :, :, :]
np.savez(
dat_path + target_fname,
lum_mask=lum_mask,
con_mask=con_mask,
flow_mask=flow_mask,
four_mask=four_mask,
freq_mask=freq_mask,
orient_mask=orient_mask,
masked=masked,
#lum_whole=lum_whole, con_whole=con_whole,
flow_whole=flow_whole,
#four_whole=four_whole, freq_whole=freq_whole,
#orient_whole=orient_whole,
movie=movie,
mov_frames=mov_frames,
exp_samples=exp_samples
# lum_surr=lum_surr, con_surr=con_surr, flow_surr=flow_surr,
# four_surr=four_surr, freq_surr=freq_surr,
# orient_surr=orient_surr, surround=surround
)
scipy.io.savemat(
dat_path + target_fname[:-3] + '.mat',
{
'lum_mask': lum_mask,
'con_mask': con_mask,
'flow_mask': flow_mask,
'four_mask': four_mask,
'freq_mask': freq_mask,
'orient_mask': orient_mask,
'masked': masked,
# 'lum_whole': lum_whole, 'con_whole': con_whole,
# 'flow_whole': flow_whole, 'four_whole': four_whole,
# 'freq_whole': freq_whole,
# 'orient_whole': orient_whole,
'movie': movie,
'mov_frames': mov_frames,
'exp_samples': exp_samples
# 'lum_surr': lum_surr, 'con_surr': con_surr, 'flow_surr': flow_surr,
# 'four_surr': four_surr, 'freq_surr': freq_surr,
# 'orient_surr': orient_surr, 'surround': surround
})
plot_movie(lum_mask, con_mask, flow_mask, four_mask, masked,
fig_path + exp_id + '_masked')
# plot_movie(lum_whole, con_whole, flow_whole, four_whole, movie,
# fig_path + exp_id + '_whole')
# plot_surround(lum_surr, con_surr, flow_surr, four_surr, surround,
# fig_path + exp_id + '_surround')
print 'Failed: ', failed_count
print 'Successful: ', success_count
filter = [] #'120425']
folds = 5
exps = list_PopExps()
for alpha in [0.001, 0.002, 0.005, 0.01, 0.15, 0.5, 1.][::-1]:
clf_args = {'alpha': alpha}
clf_args = {}
crrs = []
for exp in exps:
if len(filter) > 0 and exp not in filter:
print exp, ' not in filter, skipping'
continue
# print 'Doing ', exp
dat = load_PopData(exp)
dat_c = dat['dat_c'].mean(2)
dat_c = normalize(dat_c, axis=0)
dat_w = dat['dat_w'].mean(2)
dat_w = normalize(dat_w, axis=0)
active = dat['active']
d = np.where(active[:, 1])[0]
lum_mask, con_mask, flow_mask, four_mask, four_mask_shape,\
freq_mask, orient_mask,\
lum_surr, con_surr, flow_surr, four_surr, four_surr_shape,\
freq_surr, orient_surr,\
lum_whole, con_whole, flow_whole, four_whole, four_whole_shape,\
freq_whole, orient_whole = load_parsed_movie_dat(exp, 'POP', None)
all_dat = {'Data': {'Centre': dat_c, 'Whole': dat_w}, 'Movie': {}}
all_dat['Movie']['Contrast'] = con_mask
#print vals
return vals
# Sub directory of the figure path to put the plots in
d_path = data_path + 'Sparseness/POP/'
exps = list_PopExps()
all_res = []
for exp_id in exps:
fname = '%s%s%s' % (d_path, 'xcorr_active_', exp_id)
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
#continue
try:
dat = load_PopData(exp_id)
print fname
except:
continue
rf_cells = dat['rf_cells']
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
bs_c = dat['bs_raw_c']
bs_w = dat['bs_raw_w']
res = []
for cell in xrange(dat_c.shape[0]):
res.append(responsive_shuffle_xcorr(cell, dat_c[cell], dat_w[cell]))
# res.append(responsive_baseline_thresh(cell, dat_c[cell], dat_w[cell],
# bs_c[cell], bs_w[cell]))
res = np.array(res)
return vals
# Sub directory of the figure path to put the plots in
d_path = data_path + 'Sparseness/POP/'
exps = list_PopExps()
all_res = []
for exp_id in exps:
fname = '%s%s%s' % (d_path, 'xcorr_active_', exp_id)
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
#continue
try:
dat = load_PopData(exp_id)
print fname
except:
continue
rf_cells = dat['rf_cells']
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
bs_c = dat['bs_raw_c']
bs_w = dat['bs_raw_w']
res = []
for cell in xrange(dat_c.shape[0]):
res.append(responsive_shuffle_xcorr(cell, dat_c[cell], dat_w[cell]))
# res.append(responsive_baseline_thresh(cell, dat_c[cell], dat_w[cell],
# bs_c[cell], bs_w[cell]))
if randomise is None:
f_path = fig_path + 'Sparseness/%s/initial/norm/' % (exp_type)
elif randomise == 'random':
f_path = fig_path + 'Sparseness/%s/initial/random/' % (exp_type)
fname = '%s%s' % (f_path, 'initial_summary.png')
if os.path.exists(fname):
print fname + ' exist, SKIPPING'
# continue
exps = list_PopExps()
csv_vals = [[], [], [], [], []]
sum_vals = []
cellids = []
#exps = ['121127']
for exp_id in exps:
dat = load_PopData(exp_id, True)
vals = []
print 'doing ', exp_id
if randomise is None:
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
elif randomise == 'random':
assert (False)
dat_c = dat['dat_raw_c']
dat_w = dat['dat_raw_w']
if not os.path.exists(f_path):
os.makedirs(f_path)
mn_c = dat_c.mean(2).mean(0)
std_c = np.std(dat_c, 2).mean(0)
