def scat_states_crh(df,
x_model,
y_model,
area=None,
colors=None,
highlight_cellids={},
pup_state=False,
hue=False,
save=False,
xlabel=None,
ylabel=None,
title=None,
xlim=None,
ylim=None,
marker='o',
marker_size=15,
ax=None,
bootstats=False,
nboots=1000):
"""
This function makes a scatter plots of identified arguments.
sig_list = ~sig_state, sig_state, sig_ubeh, sig_upup, sig_both]
color_list = ['#D3D3D3', '#595959', '#82418B', '#2E7E3E', '#000000']
crh copy of scat_states 04/16/2020. Tweak some fn arguments to make a little
more user friendly, I think.
params:
df - pandas dataframe with results
x_model - string name of x column
y_model - string name of y column
area - string (A1, ICC, or ICX)
colors - list of length 5 (color of not sig cells, sig state, sig beh, sig pup, sig both).
If none, set to defaults.
highlight_cellids - Dict of cellid / color pairs. If specified, will highlight these cellids
xlabel - string, label of X axis
ylabel - string, label of Y axis
title - string, title of axis
xlim - tuple, limits of x axis
ylim - tuple, limits of y axis
ax - axis object on which to make the scatter plot
marker - matplotlib marker
save - bool, if True, save pdf of figure
hue - string column name, if specified, use this column to determine groups
(using seaborn grouping of dataframe on this column)
"""
# generate sig_list (a list of boolean masks for each of the following conditions:
# not sig cells, sig state, sig task, sig pup, sig both)
sig_list = [(~df['sig_state'] & ~df['sig_upupil'] & ~df['sig_utask']),
(df['sig_state'] & ~df['sig_upupil'] & ~df['sig_utask']),
(df['sig_utask'] & ~df['sig_upupil']),
(~df['sig_utask'] & df['sig_upupil']),
(df['sig_utask'] & df['sig_upupil'])]
# generate area mask
if area is not None:
s_area = area
area = df.area.str.contains(area, regex=True)
else:
s_area = 'All'
area = np.ones(df.shape[0]).astype(bool)
if ax is None:
plt.figure(figsize=(6, 6))
ax = plt.gca()
else:
plt.sca(ax)
if colors is None:
colors = color_list
# need a slope and c to fix the position of line
if xlim is not None:
xlim = xlim
ylim = ylim
slope = 1
c = xlim[0]
x_min = xlim[0]
x_max = xlim[1]
y_min, y_max = c, c + slope * (x_max - x_min)
plt.plot([x_min, x_max], [y_min, y_max],
linewidth=0.5,
linestyle='--',
color='k',
dashes=(4, 2))
plt.ylabel(ylabel)
plt.xlabel(xlabel)
plt.title(title)
plt.axvline(0, linestyle='--', linewidth=0.5, color='k', dashes=(4, 2))
plt.axhline(0, linestyle='--', linewidth=0.5, color='k', dashes=(4, 2))
if hue:
sns.scatterplot(x=df.loc[x_beh_state & area, x_column].tolist(),
y=df.loc[y_beh_state & area, y_column].tolist(),
s=marker_size,
hue=df.loc[x_model & x_beh_state & area, hue],
marker=marker,
edgecolors='white',
linewidth=0.5)
elif pup_state:
# plot not significant units
plt.scatter(x=df.loc[x_model & pup_state & area & sig_list[0],
x_column].tolist(),
y=df.loc[y_model & x_beh_state & area & sig_list[0],
y_column].tolist(),
s=marker_size,
color=colors[0],
marker=marker,
edgecolors='white',
linewidth=0.5)
# plot significant state units
plt.scatter(x=df.loc[x_model & pup_state & area & sig_list[1],
x_column].tolist(),
y=df.loc[y_model & x_beh_state & area & sig_list[1],
y_column].tolist(),
s=marker_size,
color=colors[1],
marker=marker,
edgecolors='white',
linewidth=0.5)
# plot significant unique behavior
plt.scatter(x=df.loc[x_model & pup_state & area & sig_list[2],
x_column].tolist(),
y=df.loc[y_model & x_beh_state & area & sig_list[2],
y_column].tolist(),
s=marker_size,
color=colors[2],
marker=marker,
edgecolors='white',
linewidth=0.5)
# plot significant unique pupil
plt.scatter(x=df.loc[x_model & pup_state & area & sig_list[3],
x_column].tolist(),
y=df.loc[y_model & x_beh_state & area & sig_list[3],
y_column].tolist(),
s=marker_size,
color=colors[3],
marker=marker,
edgecolors='white',
linewidth=0.5)
# plot significant unique both
plt.scatter(x=df.loc[x_model & pup_state & area & sig_list[4],
x_column].tolist(),
y=df.loc[y_model & x_beh_state & area & sig_list[4],
y_column].tolist(),
s=marker_size,
color=colors[4],
marker=marker,
edgecolors='white',
linewidth=0.5)
else:
# iterate: not significant units, sig state, sig u beh, sig u pup, sig u both
for i, sig in enumerate(sig_list):
x = df.loc[area & sig, x_model].values
y = df.loc[area & sig, y_model].values
out_ix = (x <= xlim[0]) | (x >= xlim[1]) | (y <= xlim[0]) | (
y >= xlim[1])
x0, y0 = x, y
x = np.clip(x, xlim[0], xlim[1])
y = np.clip(y, ylim[0], ylim[1])
if i == 0:
s = 75
else:
s = 100
# plot current group
plt.scatter(x=x,
y=y,
s=marker_size,
color=colors[i],
marker=marker,
edgecolors='white',
linewidth=0.25)
for _x, _y, _x0, _y0 in zip(x[out_ix], y[out_ix], x0[out_ix],
y0[out_ix]):
plt.text(_x,
_y,
f'({_x0:.2f},{_y0:.2f})',
fontsize=5,
color=colors[i])
# plot a cellid (e.g. TAR010c-27-2 (A1 behavior cell) or TAR010c-06-1 (A1 pupil cell)) with special color
if type(highlight_cellids) is not dict:
raise ValueError('highlight_cellids has got to be a dict!')
else:
for cellid, color in highlight_cellids.items():
plt.scatter(x=df.loc[x_model & x_beh_state & area &
(df['cellid'] == cellid), x_column].tolist(),
y=df.loc[y_model & y_beh_state & area &
(df['cellid'] == cellid), y_column].tolist(),
s=200,
color=color,
marker=marker,
edgecolors='white',
linewidth=0.5)
ax.set_aspect('equal', 'box')
nplt.ax_remove_box(ax)
# print some statistics:
print(
f'Area={s_area} X={x_model}={df.loc[area,x_model].median():.3f}, Y={y_model}={df.loc[area,y_model].median():.3f}'
)
stat, p = sci.wilcoxon(df.loc[area, x_model], df.loc[area, y_model])
statr, pr = sci.pearsonr(df.loc[area, x_model], df.loc[area, y_model])
print(
f' Wilcoxon sign test: stat={stat:.1f}, p={p:.3e} R: {statr:.3f}, p={pr:.3e}'
)
# print some more statistics using hierarchical bootstrap for recording site
if bootstats:
# add siteid to df
df['siteid'] = [c[:7] for c in df.index.get_level_values(0)]
np.random.seed(123)
s1 = {
s: df.loc[(df.siteid == s) & area, x_model].values -
df.loc[(df.siteid == s) & area, y_model].values
for s in df[area].siteid.unique()
}
ds1 = get_bootstrapped_sample(s1, nboot=nboots)
p = 1 - get_direct_prob(ds1, np.zeros(ds1.shape[0]))[0]
print(f" Hierarcichal bootstrap probability: {p}")
if save:
plt.savefig(title + ylabel + xlabel + '.pdf')
s=10,
edgecolor='white',
color='tab:blue')
mi = np.min(pd.concat([resga[dp_metric], resgp[dp_metric]]))
m = np.max(pd.concat([resga[dp_metric], resgp[dp_metric]]))
ax[0].plot([mi, m], [mi, m], linestyle='--', color='k')
ax[0].set_xlabel('Passive')
ax[0].set_ylabel('Active')
d = {
s: resga.loc[pd.IndexSlice[:, str(s)], dp_metric].values -
resgp.loc[pd.IndexSlice[:, str(s)], dp_metric].values
for s in resga.index.get_level_values(1).unique()
}
bootsamp = get_bootstrapped_sample(d,
metric='mean',
even_sample=False,
nboot=1000)
p = get_direct_prob(bootsamp, np.zeros(len(bootsamp)))[0]
print(
r"Target vs. Reference Discriminability ($d'$)" +
f"\n active: {round(resga[dp_metric].mean().astype(float), 3)}, passive: {round(resgp[dp_metric].mean().astype(float), 3)}, pval: {round(p, 3)}"
)
# correlation with overall behavior
if diff_norm:
diff = (resga[dp_metric] - resgp[dp_metric]) / (resga[dp_metric] +
resgp[dp_metric])
else:
diff = resga[dp_metric] - resgp[dp_metric]
#ax[1].scatter(resga[di_metric], diff, s=50, edgecolor='white', color='tab:blue')
sns.regplot(x=resga[di_metric],
# pvals
easy_v_medium = round(
ss.ranksums(A1[A1.difficulty.isin(easy) & A1[sig_col]][yaxis],
A1[A1.difficulty.isin(medium) & A1[sig_col]][yaxis]).pvalue, 3)
medium_v_hard = round(
ss.ranksums(A1[A1.difficulty.isin(medium) & A1[sig_col]][yaxis],
A1[A1.difficulty.isin(hard) & A1[sig_col]][yaxis]).pvalue, 3)
easy_v_hard = round(
ss.ranksums(A1[A1.difficulty.isin(easy) & A1[sig_col]][yaxis],
A1[A1.difficulty.isin(hard) & A1[sig_col]][yaxis]).pvalue, 3)
A1['site'] = [c[:7] for c in A1.index]
e = get_bootstrapped_sample(
{
s: A1[A1.difficulty.isin(easy) & (A1.site == s)][yaxis]
for s in A1.site.unique()
},
nboot=1000)
m = get_bootstrapped_sample(
{
s: A1[A1.difficulty.isin(medium) & (A1.site == s)][yaxis]
for s in A1.site.unique()
},
nboot=1000)
h = get_bootstrapped_sample(
{
s: A1[A1.difficulty.isin(hard) & (A1.site == s)][yaxis]
for s in A1.site.unique()
},
nboot=1000)
# add bootstrapped test of correlation
np.random.seed(123)
# unique site / active combos
d['siteid'] = [
c[:7] + d['state_chan_alt'].iloc[i] for i, c in enumerate(d.index)
]
x = {
s: d.loc[(d.siteid == s) & (d.area == 'A1'), perf_stat].values
for s in d[(d.area == 'A1')].siteid.unique()
}
y = {
s: d.loc[(d.siteid == s) & (d.area == 'A1'), yaxis].values
for s in d[(d.area == 'A1')].siteid.unique()
}
cc = get_bootstrapped_sample(x, y, metric='corrcoef', nboot=100)
pboot = get_direct_prob(cc, np.zeros(cc.shape[0]))[0]
ax[1].set_title(f'A1 r={r:.3f}, p={p:.4f}, pboot={pboot:.4f}')
ax[1].set_xlabel(perf_stat)
ax[1].set_ylabel(yaxis)
if perf_stat == 'DI':
x0 = np.array([58, 102])
else:
x0 = np.array([0.5, 3.5])
ax[1].plot(x0, x0 * beta[0] + beta[1], 'k--')
ax[1].set_xlim(x0)
ax[1].legend(A1.animal.unique(), frameon=False)
nplt.ax_remove_box(ax[1])
for a in IC.animal.unique():
_k = (d.area == 'IC') & (d.animal == a)
print(f' signed delta task unique: stat={stat:.3f}, p={p:.4e}')
print(f' Ratio: {ratio_IC:.3f}')
# run the above two comparisons with bootstrapped test
np.random.seed(123)
signed_diff_A1_wSite = pd.DataFrame(signed_diff_A1, columns=['signed_diff'])
signed_diff_A1_wSite['siteid'] = [c[:7] for c in signed_diff_A1_wSite.index]
signed_diff_IC_wSite = pd.DataFrame(signed_diff_IC, columns=['signed_diff'])
signed_diff_IC_wSite['siteid'] = [c[:7] for c in signed_diff_IC_wSite.index]
a1 = {
s: signed_diff_A1_wSite.loc[(signed_diff_A1_wSite.siteid == s),
'signed_diff'].values
for s in signed_diff_A1_wSite.siteid.unique()
}
a1 = get_bootstrapped_sample(a1, nboot=1000)
p = get_direct_prob(a1, np.zeros(a1.shape[0]))[0]
print(f"\n A1 task only vs. task unique bootstrapped prob: {p}\n")
ic = {
s: signed_diff_IC_wSite.loc[(signed_diff_IC_wSite.siteid == s),
'signed_diff'].values
for s in signed_diff_IC_wSite.siteid.unique()
}
ic = get_bootstrapped_sample(ic, nboot=1000)
p = get_direct_prob(ic, np.zeros(ic.shape[0]))[0]
print(f"\n IC task only vs. task unique bootstrapped prob: {p}\n")
# ICC vs. ICX comparison with bootstrap
signed_diff_ICC_wSite = pd.DataFrame(signed_diff_ICC, columns=['signed_diff'])
signed_diff_ICC_wSite['siteid'] = [c[:7] for c in signed_diff_ICC_wSite.index]
signed_diff_ICX_wSite = pd.DataFrame(signed_diff_ICX, columns=['signed_diff'])
# add statistical test to directly test if r_pupil/task is different between areas
np.random.seed(123)
df['siteid'] = [c[:7] for c in df.index]
# pupil test
da1 = {
s: df.loc[(df.siteid == s) & (df.area == 'A1'), 'r_pupil_unique'].values
for s in df[(df.area == 'A1')].siteid.unique()
}
dic = {
s: df.loc[(df.siteid == s) & df.area.isin(['ICC', 'ICX']),
'r_pupil_unique'].values
for s in df[df.area.isin(['ICC', 'ICX'])].siteid.unique()
}
a1 = get_bootstrapped_sample(da1, nboot=1000)
ic = get_bootstrapped_sample(dic, nboot=1000)
p = 1 - get_direct_prob(a1, ic)[0]
print("\n")
print(f" Median r_pupil_unique IC: {df[df.area.isin(['ICC', 'ICX'])]['r_pupil_unique'].median()}\n"\
f" Median r_pupil_unique A1: {df[df.area.isin(['A1'])]['r_pupil_unique'].median()}\n"\
f" Bootstrapped probability A1 > IC: {p}\n")
# task test
da1 = {
s: df.loc[(df.siteid == s) & (df.area == 'A1'), 'r_task_unique'].values
for s in df[(df.area == 'A1')].siteid.unique()
}
dic = {
s: df.loc[(df.siteid == s) & df.area.isin(['ICC', 'ICX']),
'r_task_unique'].values
ax=axs[1],
bootstats=True)
# CRH adding scipy test for correlation significance -- it's in the ms, but code doesn't seem to exist?
a1cc, p = sci.pearsonr(df[df.area=='A1']['MI_task_unique'], df[df.area=='A1']['MI_pupil_unique'])
print(f"A1 \n correlation MI_task_unique vs. MI_pupil_unique: {round(a1cc, 3)}, {round(p, 3)}")
iccc, p = sci.pearsonr(df[df.area.isin(['ICX', 'ICC'])]['MI_task_unique'], df[df.area.isin(['ICX', 'ICC'])]['MI_pupil_unique'])
print(f"IC \n correlation MI_task_unique vs. MI_pupil_unique: {round(iccc, 3)}, {round(p, 3)}")
# test correlation using hierarchical bootstrap
np.random.seed(123)
print("Using hierarchical bootstrap:")
da1_task = {s: df.loc[(df.siteid==s) & (df.area=='A1'), 'MI_pupil_unique'].values for s in df[(df.area=='A1')].siteid.unique()}
da1_pupil = {s: df.loc[(df.siteid==s) & (df.area=='A1'), 'MI_task_unique'].values for s in df[(df.area=='A1')].siteid.unique()}
a1_boot_cc = get_bootstrapped_sample(da1_task, da1_pupil, metric='corrcoef', nboot=100)
p = 1 - get_direct_prob(a1_boot_cc, np.zeros(a1_boot_cc.shape[0]))[0]
print(f"A1 \n correlation MI_task_unique vs. MI_pupil_unique: {round(a1cc, 3)}, {round(p, 5)}")
dic_task = {s: df.loc[(df.siteid==s) & (df.area.isin(['ICC', 'ICX'])), 'MI_pupil_unique'].values for s in df[(df.area.isin(['ICC', 'ICX']))].siteid.unique()}
dic_pupil = {s: df.loc[(df.siteid==s) & (df.area.isin(['ICC', 'ICX'])), 'MI_task_unique'].values for s in df[(df.area.isin(['ICC', 'ICX']))].siteid.unique()}
ic_boot_cc = get_bootstrapped_sample(dic_task, dic_pupil, metric='corrcoef', nboot=100)
p = 1 - get_direct_prob(ic_boot_cc, np.zeros(ic_boot_cc.shape[0]))[0]
print(f"IC \n correlation MI_task_unique vs. MI_pupil_unique: {round(iccc, 3)}, {round(p, 5)}")
for s_area in ['A1', 'ICC|ICX', 'ICC', 'ICX']:
for varname in ['MI_task_unique','MI_pupil_unique']:
area = df.area.str.contains(s_area, regex=True) & df['sig_state']
m=df.loc[area, varname].mean()
stat,p = sci.wilcoxon(df.loc[area, varname].values)
def hlf_analysis(df, state_list, pas_df=None, norm_sign=True,
sig_cells_only=False, states=None, scatter_sig_cells=None, bootstats=False):
"""
Copied/modified version of mod_per_state.hlf_analysis. Rewritten by crh 04/17/2020
"""
# figure out what cells show significant state effect. Can just use
# pupil for this, so that there's one entry per cell (rtest is the same for all states)
if states is None:
states = ['ACTIVE_1','PASSIVE_1', 'ACTIVE_2', 'PASSIVE_2']
da = df[df['state_chan']=='pupil']
dp = pd.pivot_table(da, index='cellid',columns='state_sig',values=['r','r_se'])
sig = (dp.loc[:, pd.IndexSlice['r', state_list[3]]] - dp.loc[:, pd.IndexSlice['r', state_list[0]]]) > \
(dp.loc[:, pd.IndexSlice['r_se', state_list[3]]] + dp.loc[:, pd.IndexSlice['r_se', state_list[0]]])
sig_cells = sig[sig].index
dfull = df[df['state_sig']==state_list[3]]
dpup = df[df['state_sig']==state_list[2]]
dbeh = df[df['state_sig']==state_list[1]]
dp = pd.pivot_table(dfull, index='cellid',columns='state_chan',values=['MI'])
dp_beh = pd.pivot_table(dbeh, index='cellid',columns='state_chan',values=['MI'])
dp0 = pd.pivot_table(dpup, index='cellid',columns='state_chan',values=['MI'])
dMI = dp.loc[:, pd.IndexSlice['MI', states]]
dMIbeh = dp_beh.loc[:, pd.IndexSlice['MI', states]]
dMI0 = dp0.loc[:, pd.IndexSlice['MI', states]]
dMIu = dMI - dMI0
if pas_df is not None:
dfull_pas = pas_df[pas_df['state_sig']=='st.pup.pas']
dbeh_pas = pas_df[pas_df['state_sig']=='st.pup0.pas']
dpup_pas = pas_df[pas_df['state_sig']=='st.pup.pas0']
dp_pas = pd.pivot_table(dfull_pas, index='cellid',columns='state_chan_alt',values=['MI'])
dp_beh_pas = pd.pivot_table(dbeh_pas, index='cellid',columns='state_chan_alt',values=['MI'])
dp0_pas = pd.pivot_table(dpup_pas, index='cellid',columns='state_chan_alt',values=['MI'])
dMI_pas = dp_pas.loc[:, pd.IndexSlice['MI', states]]
dMI0_pas = dp0_pas.loc[:, pd.IndexSlice['MI', states]]
dMIu_pas = dMI_pas - dMI0_pas
dMI_pas = dp_beh_pas.loc[:, pd.IndexSlice['MI', states]]
# add zeros for "PASSIVE_0" col
dMI.loc[:, pd.IndexSlice['MI', 'PASSIVE_0']] = 0
dMI0.loc[:, pd.IndexSlice['MI', 'PASSIVE_0']] = 0
dMIu.loc[:, pd.IndexSlice['MI', 'PASSIVE_0']] = 0
active_idx = [c for c in dMI.columns.get_level_values('state_chan') if 'ACTIVE' in c]
passive_idx = [c for c in dMI.columns.get_level_values('state_chan') if ('PASSIVE' in c)]
# force reorder the columns of all dataframes for the plot
new_col_order = sorted(dMI.columns.get_level_values('state_chan'), key=lambda x: x[-1])
new_cols = pd.MultiIndex.from_product([['MI'], new_col_order], names=[None, 'state_chan'])
dMI = dMI.reindex(columns=new_cols, fill_value=0)
dMI0 = dMI0.reindex(columns=new_cols, fill_value=0)
dMIu = dMIu.reindex(columns=new_cols, fill_value=0)
# define data to use for scatter plot
if pas_df is not None:
dMI_all = dMI_pas.copy()
dMIu_all = dMIu_pas.copy()
else:
# dMI_all = dMI.copy()
dMI_all = dMIbeh.copy()
dMIu_all = dMIu.copy()
if norm_sign:
b = dMI.loc[:, pd.IndexSlice['MI', passive_idx]].mean(axis=1).fillna(0)
#dMI = dMI.subtract(b, axis=0)
#dMIu = dMIu.subtract(b, axis=0)
#dMI0 = dMI0.subtract(b, axis=0)
# make it so that active MI > 0
sg = dMI.loc[:, pd.IndexSlice['MI', active_idx]].mean(axis=1) - \
dMI.loc[:, pd.IndexSlice['MI', passive_idx]].mean(axis=1)
#sg = dMI.loc[:, pd.IndexSlice['MI', active_idx]].mean(axis=1)
#import pdb;pdb.set_trace()
sg = sg.apply(np.sign)
dMI = dMI.multiply(sg, axis=0)
dMIu = dMIu.multiply(sg, axis=0)
dMI0 = dMI0.multiply(sg, axis=0)
# plot only significant state cells, with data for all state_chan conditions
state_mask = (dMI.isna().sum(axis=1) == 0)
cell_mask = dMI.index.isin(sig_cells)
if sig_cells_only:
dMI = dMI.loc[cell_mask & state_mask, :]
dMI0 = dMI0.loc[cell_mask & state_mask, :]
dMIu = dMIu.loc[cell_mask & state_mask, :]
else:
dMI = dMI.loc[state_mask, :]
dMI0 = dMI0.loc[state_mask, :]
dMIu = dMIu.loc[state_mask, :]
total_cells = len(df.cellid.unique())
sig_state_cells = len(sig_cells)
stable_cells = state_mask.sum()
f = plt.figure(figsize=(6,6))
ax = [plt.subplot(2,2,1), plt.subplot(2,1,2)]
# scatter plot of raw post passive MI vs. unique post passive MI
# e.g. does pupil account for some persistent effects?
ax[0].scatter(dMI_all.loc[:, pd.IndexSlice['MI', 'PASSIVE_1']],
dMIu_all.loc[:, pd.IndexSlice['MI', 'PASSIVE_1']], color='lightgrey',
linewidth=0.5, edgecolor='white', s=15, label='all cells')
if scatter_sig_cells is None:
pass
else:
# if sig cells, overlay colors on sig cells
for category in scatter_sig_cells:
sig_cells = scatter_sig_cells[category]
if category == 'task_only':
color = common.color_b
elif category == 'pupil_only':
color = common.color_p
elif category == 'both':
color = common.color_both
elif category == 'task_or_pupil':
color = common.color_either
else:
color = 'k'
ax[0].scatter(dMI_all.loc[sig_cells, pd.IndexSlice['MI', 'PASSIVE_1']],
dMIu_all.loc[sig_cells, pd.IndexSlice['MI', 'PASSIVE_1']], color=color,
linewidth=0.5, edgecolor='white', s=15, label=category)
nncells = np.isfinite(dMI_all.loc[:, pd.IndexSlice['MI', 'PASSIVE_1']])
#import pdb;pdb.set_trace()
_dmi=dMI_all.loc[nncells, pd.IndexSlice['MI', 'PASSIVE_1']]
_dmiu=dMIu_all.loc[nncells, pd.IndexSlice['MI', 'PASSIVE_1']]
sn=np.sign(_dmi+_dmiu)
stat, p = st.wilcoxon(_dmi*sn,_dmiu*sn)
print(f' postall vs postu: Wilcoxon stat={stat} p={p:.3e}')
print(f' mean: {_dmi.mean():.3f} mean0: {_dmiu.mean():.3f}')
if bootstats:
# also do hierarchical bootstrap test
_dmi_df = pd.DataFrame((_dmi*sn).values, index=_dmi.index, columns=['mi'])
_dmiu_df = pd.DataFrame((_dmiu*sn).values, index=_dmiu.index, columns=['miu'])
_dmi_df['siteid'] = [c[:7] for c in _dmi_df.index]
_dmiu_df['siteid'] = [c[:7] for c in _dmiu_df.index]
diff = {s: _dmi_df.loc[(_dmi_df.siteid==s), 'mi'].values - _dmiu_df.loc[(_dmiu_df.siteid==s), 'miu'].values for s in _dmi_df.siteid.unique()}
bootsamp = get_bootstrapped_sample(diff, nboot=500)
p = get_direct_prob(bootsamp, np.zeros(bootsamp.shape[0]))[0]
print(f"Hierarchical bootstrap probability unique > block only: {p}")
ax[0].legend(frameon=False, fontsize=5)
ax[0].set_xlabel('Pre vs. post MI, task only')
ax[0].set_ylabel('Pre vs. post MI, task unique')
ax[0].plot([-0.7, 0.7], [-0.7, 0.7], 'k--', linewidth=0.5, dashes=(4,2))
ax[0].axhline(0, linestyle='--', color='k', linewidth=0.5, dashes=(4,2))
ax[0].axvline(0, linestyle='--', color='k', linewidth=0.5, dashes=(4,2))
ax[0].axis('square')
nplt.ax_remove_box(ax[0])
# plot mean MI over cells for pupil, task unique, and overall state
ax[1].set_title('total cells: {0}, \n state cells: {1}, \n stable across all blocks: {2}'.format(total_cells, sig_state_cells, stable_cells),
fontsize=8)
ax[1].set_title('Total cells going into average: {0}'.format(dMI.shape[0]))
ax[1].plot(dMIu.mean(axis=0).values, '-', lw=2, color=common.color_b, marker='o', label='unique task')
ax[1].plot(dMI.mean(axis=0).values, '--', lw=2, color=common.color_b, marker='o', label='overall')
ax[1].plot(dMI0.mean(axis=0).values, '--', color=common.color_p, lw=2, marker='o', label='pupil')
ax[1].legend()
ax[1].axhline(0, linestyle='--', color='grey', lw=2)
ax[1].set_ylabel('mean MI')
ax[1].set_xticks(np.arange(dMI.shape[1]))
ax[1].set_xticklabels(dMI.columns.get_level_values('state_chan'))
ax[1].set_xlabel('behavioral block')
nplt.ax_remove_box(ax[1])
f.tight_layout()
print("raw: ", np.nanmean((dMI), axis=0))
print("u: ", np.nanmean((dMIu), axis=0))
#return f, dMI, dMI0
return f, dMIu_all, dMI_all
def aud_vs_state(df, nb=5, title=None, state_list=None,
colors=['r','g','b','k'], norm_by_null=False):
"""
d = dataframe output by get_model_results_per_state_model()
nb = number of bins
"""
if state_list is None:
state_list = ['st.pup0.beh0','st.pup0.beh','st.pup.beh0','st.pup.beh']
f = plt.figure(figsize=(5.0,5.0))
dr = df.copy()
if len(state_list)==4:
dr['bp_common'] = dr['r_full'] - df['r_task_unique'] - df['r_pupil_unique'] - dr['r_shuff']
dr = dr.sort_values('r_shuff')
mfull = dr[['r_shuff', 'r_full', 'bp_common', 'r_task_unique', 'r_pupil_unique', 'sig_state']].values
elif len(state_list)==2:
dr['bp_common'] = dr['r_full'] - dr['r_shuff']
dr = dr.sort_values('r_shuff')
dr['b_unique'] = dr['bp_common']*0
dr['p_unique'] = dr['bp_common']*0
mfull=dr[['r_shuff', 'r_full', 'bp_common', 'b_unique', 'p_unique', 'sig_state']].values
mfull=mfull.astype(float)
if nb > 0:
mm=np.zeros((nb,mfull.shape[1]))
for i in range(nb):
x01=(mfull[:,0]>i/nb) & (mfull[:,0]<=(i+1)/nb)
if np.sum(x01):
mm[i,:]=np.nanmean(mfull[x01,:],axis=0)
print(np.round(mm,3))
m = mm.copy()
else:
# alt to look at each cell individually:
m = mfull.copy()
mall = np.nanmean(mfull, axis=0, keepdims=True)
# remove sensory component, which swamps everything else
mall = mall[:, 2:]
mb=m[:,2:]
ax1 = plt.subplot(2,2,1)
stateplots.beta_comp(mfull[:,0],mfull[:,1],n1='State independent',n2='Full state-dep',
ax=ax1, highlight=mfull[:, -1], hist_range=[-0.1, 1])
plt.subplot(2,2,3)
width=0.8
mplots=np.concatenate((mall, mb), axis=0)
ind = np.arange(mplots.shape[0])
plt.bar(ind, mplots[:,0], width=width, color=colors[1])
plt.bar(ind, mplots[:,1], width=width, bottom=mplots[:,0], color=colors[2])
plt.bar(ind, mplots[:,2], width=width, bottom=mplots[:,0]+mplots[:,1], color=colors[3])
plt.legend(('common','b-unique','p_unique'))
if title is not None:
plt.title(title)
plt.xlabel('behavior-independent quintile')
plt.ylabel('mean r2')
ax3 = plt.subplot(2,2,2)
if norm_by_null:
d=(mfull[:,1]-mfull[:,0]) / (1-np.abs(mfull[:,0]))
ylabel = "dep-indep normed"
else:
d=(mfull[:,1]-mfull[:,0])
ylabel = "dep-indep"
stateplots.beta_comp(mfull[:,0], d, n1='State independent',n2=ylabel,
ax=ax3, highlight=mfull[:,-1], hist_range=[-0.1, 1], markersize=4)
if not norm_by_null:
ax3.plot([1,0], [0,1], 'k--', linewidth=0.5)
slope, intercept, r, p, std_err = st.linregress(mfull[:,0],d)
dr['site'] = [c[:7] for c in dr.index.get_level_values(0)]
x = get_bootstrapped_sample({s: mfull[(dr.site==s).values, 0] for s in dr.site.unique()},
{s: d[(dr.site==s).values] for s in dr.site.unique()}, metric='corrcoef', nboot=10000)
pboot, _ = get_direct_prob(x, np.zeros(x.shape[0]))
mm = np.array([np.min(mfull[:,0]), np.max(mfull[:,0])])
ax3.plot(mm,intercept+slope*mm,'k--', linewidth=0.5)
plt.title('n={} cc={:.3} p={:.4}, pboot={:.5f}'.format(len(d),r,p,1-pboot),fontsize=7)
ax4 = plt.subplot(2,2,4)
if norm_by_null:
d=(mfull[:,1]-mfull[:,0]) / (1-np.abs(mfull[:,0]))
ylabel = "dep-indep normed"
else:
d=(mfull[:,1]-mfull[:,0])
ylabel = "dep-indep"
snr = np.log(dr['SNR'].values)
_ok = np.isfinite(d) & np.isfinite(snr)
ax4.plot(snr[_ok], d[_ok], 'k.', markersize=4)
#stateplots.beta_comp(snr[_ok], d[_ok], n1='SNR',n2='dep - indep',
# ax=ax4, highlight=mfull[_ok,-1], hist_range=[-0.1, 1], markersize=4)
slope, intercept, r, p, std_err = st.linregress(snr[_ok], d[_ok])
x = get_bootstrapped_sample({s: snr[(dr.site==s).values & _ok] for s in dr.site.unique()},
{s: d[(dr.site==s).values & _ok] for s in dr.site.unique()}, metric='corrcoef', nboot=10000)
pboot, _ = get_direct_prob(x, np.zeros(x.shape[0]))
mm = np.array([np.min(snr[_ok]), np.max(snr[_ok])])
ax4.plot(mm,intercept+slope*mm,'k--', linewidth=0.5)
ax4.set_xlabel('log(SNR)')
ax4.set_ylabel(ylabel)
ax4.set_title('n={} cc={:.3} p={:.4}, pboot={:.5f}'.format(len(d),r,p, 1-pboot),fontsize=7)
nplt.ax_remove_box(ax4)
f.tight_layout()
return f
path=dump_path)
IC = IC[IC.sig_psth]
f, ax = plt.subplots(1, 2, figsize=(5,3), sharey='row')
sns.stripplot(x='sig_utask', y=yaxis_task, data=A1, hue='ON_BF', dodge=True, edgecolor='white', linewidth=0.5,
marker='o', size=5, ax=ax[0])
ax[0].axhline(0, linestyle='--', lw=2, color='grey')
pval = round(ss.ranksums(A1[A1.ON_BF & A1.sig_utask][yaxis_task], A1[~A1.ON_BF & A1.sig_utask][yaxis_task]).pvalue, 3)
off_med = round(A1[~A1.ON_BF & A1.sig_utask][yaxis_task].median(), 3)
on_med = round(A1[A1.ON_BF & A1.sig_utask][yaxis_task].median(), 3)
# get bootstrapped pval
A1['site'] = [c[:7] for c in A1.index]
sig_a1_on = get_bootstrapped_sample({s: A1[A1.ON_BF & A1.sig_utask & (A1.site==s)][yaxis_task] for s in A1.site.unique()}, nboot=1000)
sig_a1_off = get_bootstrapped_sample({s: A1[~A1.ON_BF & A1.sig_utask & (A1.site==s)][yaxis_task] for s in A1.site.unique()}, nboot=1000)
pboot, jm = get_direct_prob(sig_a1_on, sig_a1_off)
pval_ns = round(ss.ranksums(A1[A1.ON_BF & ~A1.sig_utask][yaxis_task], A1[~A1.ON_BF & ~A1.sig_utask][yaxis_task]).pvalue, 3)
off_med_ns = round(A1[~A1.ON_BF & ~A1.sig_utask][yaxis_task].median(), 3)
on_med_ns = round(A1[A1.ON_BF & ~A1.sig_utask][yaxis_task].median(), 3)
ns_a1_on = get_bootstrapped_sample({s: A1[A1.ON_BF & ~A1.sig_utask & (A1.site==s)][yaxis_task] for s in A1.site.unique()}, nboot=1000)
ns_a1_off = get_bootstrapped_sample({s: A1[~A1.ON_BF & ~A1.sig_utask & (A1.site==s)][yaxis_task] for s in A1.site.unique()}, nboot=1000)
pboot_ns, jm = get_direct_prob(ns_a1_on, ns_a1_off)
ax[0].set_title('A1 \n sig_cells: ON: {0}, OFF: {1}, pval: {2}, pboot: {6:.4f} \n'
'ns cells: ON: {3}, OFF: {4}, pval: {5}, pboot: {7:.4f}'.format(on_med, off_med, pval, on_med_ns, off_med_ns, pval_ns, pboot, pboot_ns))
nplt.ax_remove_box(ax[0])
np.random.seed(123)
df['siteid'] = [c[:7] for c in df.index.get_level_values(0)]
for area in [['A1'], ['ICC', 'ICX']]:
am = df.area.isin(area)
r, p = sci.pearsonr(df[am]['r_full'] - df[am]['r_shuff'],
df[am]['r_shuff'])
diff = {
s: df[(df.siteid == s) & am]['r_full'].values -
df[(df.siteid == s) & am]['r_shuff'].values
for s in df[am].siteid.unique()
}
null = {
s: df[(df.siteid == s) & am]['r_shuff'].values
for s in df[am].siteid.unique()
}
cc = get_bootstrapped_sample(diff, null, metric='corrcoef', nboot=100)
pboot = 1 - get_direct_prob(cc, np.zeros(cc.shape[0]))[0]
print(f"{area}\n r={r:.3f}, p={p:.3f}, pboot={pboot:.3f}")
# ==================================================== behavior only data ===========================================================
# Figures 6C-D - beh only effects, bigger set of cells
# later figure -- beh only (ignore pupil, can use larger stim set)
# SPECIFY models
USE_AFL = True
if USE_AFL:
dump_results = 'd_afl_sdexp.csv'
model_string = 'st.afl'
p0_model = None
b0_model = 'st.afl0'
shuf_model = 'st.afl0'
else: