def extract_fit_displacement(dsfile, expttype='size_contrast_opto_0'):
displacement = {}
with ut.hdf5read(dsfile) as f:
keylist = [key for key in f.keys()]
for ikey in range(len(keylist)):
session = f[keylist[ikey]]
if expttype in session:
sc0 = session[expttype]
if 'rf_displacement_deg' in sc0:
pval = sc0['rf_mapping_pval'][:]
sigma = session['retinotopy_0']['rf_sigma'][:]
X = session['cell_center'][:]
y = sc0['rf_displacement_deg'][:].T
if sigma.max() > 0:
lkat = ut.k_and(pval < 0.05, ~np.isnan(X[:, 0]),
~np.isnan(y[:, 0]), sigma > 5)
else:
lkat = ut.k_and(pval < 0.05, ~np.isnan(X[:, 0]),
~np.isnan(y[:, 0]))
linreg = sklearn.linear_model.LinearRegression().fit(
X[lkat], y[lkat])
displacement[keylist[ikey]] = np.zeros_like(y)
displacement[
keylist[ikey]][~np.isnan(X[:, 0])] = linreg.predict(
X[~np.isnan(X[:, 0])])
return displacement
def compute_encoding_axis_auroc(reg, proc):
auroc = [None for k in range(len(proc))]
uangle, usize, ucontrast, ulight = [[None for k in range(len(proc))]
for iparam in range(4)]
icutoff = 0
for iexpt in range(len(proc)):
if not reg[iexpt] is None and not reg[iexpt][icutoff] is None:
cutoff = proc[iexpt]['cutoffs'][icutoff]
desired_outputs = [
'angle', 'size', 'contrast', 'light', 'running', 'sigma', 'v',
'uangle', 'usize', 'ucontrast', 'ulight', 'train'
]
angle, size, contrast, light, running, sigma, v, uangle[
iexpt], usize[iexpt], ucontrast[iexpt], ulight[
iexpt], train = [
proc[iexpt][output].copy()
for output in desired_outputs
]
zero_contrast = ut.k_and(
contrast == 0,
running) #,eye_dist < np.nanpercentile(eye_dist,50))
nsize = len(usize[iexpt])
ncontrast = len(ucontrast[iexpt])
nangle = len(uangle[iexpt])
nlight = len(ulight[iexpt])
auroc[iexpt] = np.zeros((nlight, nsize, ncontrast, nangle))
for ilight in range(nlight):
for isize in range(nsize):
for icontrast in range(ncontrast):
for iangle in range(nangle):
this_contrast = ut.k_and(
light == ilight, angle == iangle,
size == isize, contrast == icontrast, running,
~train
) #,eye_dist < np.nanpercentile(eye_dist,50))
if this_contrast.sum():
X0 = (np.diag(sigma[:cutoff])
@ v[:cutoff, :]).T[zero_contrast]
X1 = (np.diag(sigma[:cutoff])
@ v[:cutoff, :]).T[this_contrast]
# this_model = reg[iexpt][icutoff][isize][iangle].copy()
y0 = reg[iexpt][icutoff][ilight][isize][
iangle].predict(X0)
y1 = reg[iexpt][icutoff][ilight][isize][
iangle].predict(X1)
auroc[iexpt][ilight, isize, icontrast,
iangle] = ut.compute_auroc(
y0, y1)
else:
auroc[iexpt][ilight, isize, icontrast,
iangle] = np.nan
return auroc
def keep_rois_ret_aligned(subdict, dcutoff=20):
r0 = subdict['retinotopy_0']
to_keep = [None for rtype in r0]
for itype, rtype in enumerate(r0):
sig = rtype[:, paramname_to_ind('rf_mapping_pval')] < 0.05
well_fit = rtype[:, paramname_to_ind('rf_sq_error')] < 1
near = rtype[:, paramname_to_ind('rf_distance_deg')] < dcutoff
to_keep[itype] = ut.k_and(sig, well_fit, near)
return to_keep
def compute_tuning(dsfile,
running=True,
center=True,
fieldname='decon',
keylist=None,
pval_cutoff=np.inf,
dcutoff=np.inf,
nbefore=8,
nafter=8,
run_cutoff=10):
with ut.hdf5read(dsfile) as f:
if keylist is None:
keylist = [key for key in f.keys()]
tuning = [None for ikey in range(len(keylist))]
cell_criteria = [None for ikey in range(len(keylist))]
uparam = [None for ikey in range(len(keylist))]
for ikey in range(len(keylist)):
session = f[keylist[ikey]]
if 'size_contrast_opto_0' in session:
sc0 = session['size_contrast_opto_0']
# print([key for key in session.keys()])
data = sc0[fieldname][:]
stim_id = sc0['stimulus_id'][:]
paramnames = params = sc0['stimulus_parameters']
uparam[ikey] = [sc0[paramname][:] for paramname in paramnames]
trialrun = sc0['running_speed_cm_s'][:, nbefore:-nafter].mean(
-1) > run_cutoff
if not running:
trialrun = ~trialrun
if 'rf_displacement_deg' in sc0:
pval = sc0['rf_mapping_pval'][:]
X = session['cell_center'][:]
y = sc0['rf_displacement_deg'][:].T
lkat = ut.k_and(pval < 0.05, ~np.isnan(X[:, 0]),
~np.isnan(y[:, 0]))
linreg = sklearn.linear_model.LinearRegression().fit(
X[lkat], y[lkat])
displacement = np.zeros_like(y)
displacement[~np.isnan(X[:, 0])] = linreg.predict(
X[~np.isnan(X[:, 0])])
if center:
cell_criteria[ikey] = (np.sqrt((displacement**2).sum(1)) <
dcutoff) & (pval < pval_cutoff)
else:
cell_criteria[ikey] = (np.sqrt((displacement**2).sum(1)) >
dcutoff) & (pval < pval_cutoff)
tuning[ikey] = ut.compute_tuning(
data,
stim_id,
cell_criteria=cell_criteria[ikey],
trial_criteria=trialrun) #cell_criteria
print('%s: %.1f' % (keylist[ikey], trialrun.mean()))
else:
print('could not do ' + keylist[ikey])
return tuning, cell_criteria, uparam
def run_mi_plotting(network_resps,
target_bin,
plot_mi_fn=plot_smi_errorbars,
plot_mi_lbl='pc_smi_vs_contrast_l4',
ext='eps',
iconns=[0, 2, 3, 4],
itype=0,
opt={}):
# run a plotting function e.g. plot_mi_errorbars, or plot_mimi_bars_with_lines
# on each of many opto stim directions, and many perturbations of the modeled network
# with a given desired metric, plotting the outcome of that metric on PCs
# network_resps a list of lists, opto stim direction x perturbation x (fit x size x contrast x cell type)
# save the figure, indicating the type of plot, opto stim direction, perturbation, filename and extension
ibaseline = 15
idir = 2
lkat = ut.k_and(*[
network_resps[idir][iconn].max(1).max(1).max(1) < 10
for iconn in [0, 2, 3, 4]
])
light_lbls = ['silencing', 'activation']
conn_lbls = [
'baseline', 'pcpc_deleted', 'pcpv_deleted', 'pcvip_deleted',
'pcsst_deleted', ''
]
cs = [np.array((1, 0.8, 0)), np.array((1, 0, 0))]
for iconn in iconns: #[0,2,3,4]:
ihigh = int(
np.round(np.mean(target_bin[lkat]))
) #np.minimum(np.maximum(target_bin[lkat],0),20)#np.maximum(target_bin[lkat]-2,15)#ilight_drmax[2][iconn][lkat]
ilow = ihigh #30-ihigh#4#ilight_drmax[1][iconn][lkat]
idir = 0
xdata = network_resps[idir][iconn][lkat][:, ibaseline, :,
itype].reshape((-1, 6, 6))
xdata_norm = xdata / xdata.mean(1).mean(1)[:, np.newaxis, np.newaxis]
ylims = [None, None]
for idir, ilight, light_lbl, c, ylim in zip([1, 2], [ilow, ihigh],
light_lbls, cs, ylims):
#ydata = network_resps[idir][iconn][lkat][:,:,:,itype][np.arange(lkat.sum()),ilight].reshape((-1,6,6))
ydata = network_resps[idir][iconn][lkat][:, ilight, :,
itype].reshape((-1, 6, 6))
ydata_norm = ydata / xdata.mean(1).mean(1)[:, np.newaxis,
np.newaxis]
save_string = 'figures/%s_%s_%s.%s' % (plot_mi_lbl, light_lbl,
conn_lbls[iconn], ext)
opt['c'] = cs[idir - 1]
opt['save_string'] = save_string
plot_mi_fn(xdata_norm, ydata_norm, **opt)
def get_ret_info(dsfile, expttype='size_contrast_0'):
# take in an HDF5 data struct, and convert to an n-dimensional matrix
# describing the tuning curve of each neuron. For size-contrast stimuli,
# the dimensions of this matrix are ROI index x size x contrast x direction x time.
# This outputs a list of such matrices, where each element is one imaging session
with h5py.File(dsfile, mode='r') as f:
keylist = [key for key in f.keys()]
ret_info = [None for i in range(len(keylist))]
for ikey in range(len(keylist)):
session = f[keylist[ikey]]
print(session)
#print([key for key in session.keys()])
if expttype in session:
sc0 = session[expttype]
if 'rf_displacement_deg' in sc0:
pval = sc0['rf_mapping_pval'][:]
sqerror = session['retinotopy_0']['rf_sq_error'][:]
sigma = session['retinotopy_0']['rf_sigma'][:]
try:
amplitude = session['retinotopy_0']['rf_amplitude'][:]
except:
amplitude = np.nan * np.ones_like(sigma)
cell_center = session['cell_center'][:]
rf_center = sc0['rf_displacement_deg'][:].T
X = cell_center
y = rf_center
rf_conditions = [
ut.k_and(~np.isnan(X[:, 0]), ~np.isnan(y[:, 0])),
sqerror < 0.75, sigma > 3.3, pval < 0.1
]
lkat = np.ones((X.shape[0], ), dtype='bool')
for cond in rf_conditions:
lkat_new = (lkat & cond)
if lkat_new.sum() >= 5:
lkat = lkat_new.copy()
linreg = sklearn.linear_model.LinearRegression().fit(
X[lkat], y[lkat])
ret_map_loc = np.zeros_like(y)
ret_map_loc[~np.isnan(X[:, 0])] = linreg.predict(
X[~np.isnan(X[:, 0])])
ret_info[ikey] = {'pval':pval,'sqerror':sqerror,'sigma':sigma,\
'cell_center':cell_center,'rf_center':rf_center,\
'ret_map_loc':ret_map_loc,'amplitude':amplitude}
return ret_info
def compute_tuning(dsfile,
datafield='decon',
running=True,
expttype='size_contrast_0',
running_pct_cutoff=default_running_pct_cutoff,
fill_nans_under_cutoff=False,
run_speed_cutoff=default_run_speed_cutoff):
# take in an HDF5 data struct, and convert to an n-dimensional matrix
# describing the tuning curve of each neuron. For size-contrast stimuli,
# the dimensions of this matrix are ROI index x size x contrast x direction x time.
# This outputs a list of such matrices, where each element is one imaging session
with h5py.File(dsfile, mode='r') as f:
keylist = [key for key in f.keys()]
tuning = [None for i in range(len(keylist))]
uparam = [None for i in range(len(keylist))]
displacement = [None for i in range(len(keylist))]
pval = [None for i in range(len(keylist))]
for ikey in range(len(keylist)):
session = f[keylist[ikey]]
print(session)
#print([key for key in session.keys()])
if expttype in session and datafield in session[expttype]:
sc0 = session[expttype]
print(datafield)
data = sc0[datafield][:]
stim_id = sc0['stimulus_id'][:]
nbefore = sc0['nbefore'][()]
nafter = sc0['nafter'][()]
if running:
trialrun = sc0[
'running_speed_cm_s'][:, nbefore:-nafter].mean(
-1) > run_speed_cutoff #
else:
trialrun = sc0['running_speed_cm_s'][:,
nbefore:-nafter].mean(
-1
) < run_speed_cutoff
#print(sc0['running_speed_cm_s'].shape)
print(np.nanmean(trialrun))
if np.nanmean(trialrun) > running_pct_cutoff:
tuning[ikey] = ut.compute_tuning(
data, stim_id, trial_criteria=trialrun)[:]
elif fill_nans_under_cutoff:
tuning[ikey] = ut.compute_tuning(
data, stim_id, trial_criteria=trialrun)[:]
tuning[ikey] = np.nan * np.ones_like(tuning[ikey])
uparam[ikey] = []
for param in sc0['stimulus_parameters']:
uparam[ikey] = uparam[ikey] + [sc0[param][:]]
if 'rf_displacement_deg' in sc0:
pval[ikey] = sc0['rf_mapping_pval'][:]
sqerror = session['retinotopy_0']['rf_sq_error'][:]
sigma = session['retinotopy_0']['rf_sigma'][:]
X = session['cell_center'][:]
y = sc0['rf_displacement_deg'][:].T
rf_conditions = [
ut.k_and(~np.isnan(X[:, 0]), ~np.isnan(y[:, 0])),
sqerror < 0.75, sigma > 3.3, pval[ikey] < 0.1
]
lkat = np.ones((X.shape[0], ), dtype='bool')
for cond in rf_conditions:
lkat_new = (lkat & cond)
if lkat_new.sum() >= 5:
lkat = lkat_new.copy()
linreg = sklearn.linear_model.LinearRegression().fit(
X[lkat], y[lkat])
displacement[ikey] = np.zeros_like(y)
displacement[ikey][~np.isnan(X[:, 0])] = linreg.predict(
X[~np.isnan(X[:, 0])])
return tuning, uparam, displacement, pval
def compute_encoding_axes(dsname,
expttype='size_contrast_0',
cutoffs=(20, ),
alphas=np.logspace(-2, 2, 50),
running_trials=False,
training_set=None):
na = len(alphas)
with ut.hdf5read(dsname) as ds:
keylist = list(ds.keys())
nkey = len(keylist)
R = [None for k in range(nkey)]
reg = [None for k in range(nkey)]
# pred = [None for k in range(nkey)]
top_score = [None for k in range(nkey)]
proc = [{} for k in range(nkey)]
auroc = [None for k in range(nkey)]
for k in range(len(keylist)):
if expttype in ds[keylist[k]]:
R[k] = [None for icutoff in range(len(cutoffs))]
reg[k] = [None for icutoff in range(len(cutoffs))]
# pred[k] = [None for icutoff in range(len(cutoffs))]
sc0 = ds[keylist[k]][expttype]
nbefore = sc0['nbefore'][()]
nafter = sc0['nafter'][()]
decon = np.nanmean(sc0['decon'][()][:, :, nbefore:-nafter], -1)
data = sst.zscore(decon, axis=1)
data[np.isnan(data)] = 0
if training_set is None:
train = np.ones((decon.shape[1], ), dtype='bool')
else:
if isinstance(training_set[keylist[k]], list):
train = training_set[keylist[k]][0]
else:
train = training_set[keylist[k]]
pval_ret = sc0['rf_mapping_pval'][()]
dist_ret = sc0['rf_distance_deg'][()]
ontarget_ret_lax = np.logical_and(dist_ret < 40,
pval_ret < 0.05)
#ntokeep = 20
#if ontarget_ret_lax.sum() > ntokeep:
# ot = np.where(ontarget_ret_lax)[0]
# throw_out = ot[np.random.choice(len(ot),len(ot)-ntokeep,replace=False)]
# ontarget_ret_lax[throw_out] = False
ntokeep = -np.inf
data = data[ontarget_ret_lax]
print(data.shape[0])
u, sigma, v = np.linalg.svd(data)
running_speed_cm_s = sc0['running_speed_cm_s'][()]
running = np.nanmean(running_speed_cm_s[:, nbefore:-nafter],
1) > 7
if not running_trials:
running = ~running
size = sc0['stimulus_id'][()][0]
contrast = sc0['stimulus_id'][()][1]
angle = sc0['stimulus_id'][()][2]
light = sc0['stimulus_id'][()][3]
proc[k]['u'] = u
proc[k]['sigma'] = sigma
proc[k]['v'] = v
proc[k]['pval_ret'] = pval_ret
proc[k]['dist_ret'] = dist_ret
proc[k]['ontarget_ret_lax'] = ontarget_ret_lax
proc[k]['running_speed_cm_s'] = running_speed_cm_s
proc[k]['running'] = running
proc[k]['size'] = size
proc[k]['contrast'] = contrast
proc[k]['angle'] = angle
proc[k]['light'] = light
proc[k]['cutoffs'] = cutoffs
proc[k]['train'] = train
uangle, usize, ucontrast, ulight = [
sc0[key][()] for key in [
'stimulus_direction_deg', 'stimulus_size_deg',
'stimulus_contrast', 'stimulus_light'
]
]
proc[k]['uangle'], proc[k]['usize'], proc[k][
'ucontrast'], proc[k][
'ulight'] = uangle, usize, ucontrast, ulight
if np.logical_and(ontarget_ret_lax.sum() >= ntokeep,
running.mean() > 0.5):
#uangle,usize,ucontrast = [np.unique(arr) for arr in [angle,size,contrast]]
nlight = len(ulight)
nsize = len(usize)
ncontrast = len(ucontrast)
nangle = len(uangle)
top_score[k] = np.zeros(
(len(cutoffs), nlight, nsize, nangle))
auroc[k] = np.zeros((nlight, nsize, ncontrast - 1, nangle))
for icutoff, cutoff in enumerate(cutoffs):
R[k][icutoff] = np.zeros(
(nlight, nsize, nangle, cutoff))
reg[k][icutoff] = [None for ilight in range(nlight)]
# pred[k][icutoff] = [None for ilight in range(nlight)]
# actual[k][icutoff][ilight] = [None for ilight in range(nlight)]
for ilight in range(nlight):
reg[k][icutoff][ilight] = [
None for s in range(nsize)
]
# pred[k][icutoff][ilight] = [None for s in range(nsize)]
# actual[k][icutoff][ilight] = [None for s in range(nsize)]
for s in range(nsize):
reg[k][icutoff][ilight][s] = [
None for i in range(nangle)
]
# pred[k][icutoff][ilight][s] = [None for i in range(nangle)]
# actual[k][icutoff][ilight][s] = [None for i in range(nangle)]
for i in range(nangle):
stim_of_interest_all_contrast = ut.k_and(
np.logical_or(
np.logical_and(
angle == i, size == s),
contrast == 0), running,
light == ilight, train
) #,eye_dist < np.nanpercentile(eye_dist,50))
X = (
np.diag(sigma[:cutoff]) @ v[:cutoff, :]
).T[stim_of_interest_all_contrast]
y = contrast[
stim_of_interest_all_contrast] #>0
sc = np.zeros((na, ))
for ia, alpha in enumerate(alphas):
linreg = sklearn.linear_model.Ridge(
alpha=alpha, normalize=True)
reg1 = linreg.fit(X, y)
scores = sklearn.model_selection.cross_validate(
linreg, X, y, cv=5)
sc[ia] = scores['test_score'].mean()
best_alpha = np.argmax(sc)
top_score[k][icutoff, ilight, s,
i] = sc.max()
linreg = sklearn.linear_model.Ridge(
alpha=alphas[best_alpha],
normalize=True)
pred = sklearn.model_selection.cross_val_predict(
linreg, X, y, cv=5)
actual = y # [k][icutoff][ilight][s][i]
auroc[k][ilight, s, :,
i] = compute_detection_auroc(
pred,
actual,
nvals=contrast.max())
reg[k][icutoff][ilight][s][i] = linreg.fit(
X, y)
return reg, proc, top_score, auroc