def dis_group_diff(self, plot=True):
group = self.dis_group(plot=False)
agg = group.pivot_table(index=['layer','n'], columns='kind',
values='similarity').reset_index()
agg['diff'] = agg['shape'] - agg['px']
if self.bootstrap:
dfs = []
for layer in agg.layer.unique():
sel = agg[agg.layer==layer]['diff']
pct = stats.bootstrap_resample(sel, ci=None, func=np.mean)
d = OrderedDict([('kind', ['diff'] * len(pct)),
('layer', [layer]*len(pct)),
('preference', sel.mean()),
('iter', range(len(pct))),
('bootstrap', pct)])
dfs.append(pandas.DataFrame.from_dict(d))
df = pandas.concat(dfs)
else:
df = agg.groupby('layer').mean().reset_index()
df = df.rename(columns={'diff':'preference'})
df['kind'] = 'diff'
df['iter'] = 0
df['bootstrap'] = np.nan
del df['px']
del df['shape']
return df
def _pred_corr(self, human, value='accuracy', method='corr'):
nname = models.NICE_NAMES[self.model_name].lower()
acc = self.acc_single()
acc[value] = acc[value].astype(np.float)
# import pdb; pdb.set_trace()
# sel = acc.kind!='unknown'
sel = acc.sel.copy()
acc = acc[sel][value]
sns.set_palette(sns.color_palette('Set2')[1:])
df = []
human = human[sel.values]
if method == 'corr':
f = lambda machine, human: (1 + stats.corr(machine, human)) / 2.
elif method == 'diff':
f = lambda machine, human: 1 - np.mean(np.abs(machine - human))
elif method == 'euclidean':
f = lambda machine, human: 1 - scipy.spatial.distance.sqeuclidean(
machine, human) / len(machine)
else:
raise Exception('Method {} not recognized'.format(method))
corr = f(acc, human)
if self.bootstrap:
print('bootstrapping stats...')
bf = stats.bootstrap_resample(acc, human, func=f, ci=None, seed=0)
c = np.vstack([np.repeat(corr, len(bf)), np.arange(len(bf)), bf])
df.extend(c.T.tolist())
else:
df.append([corr, 0, np.nan])
df = pandas.DataFrame(df, columns=['consistency', 'iter', 'bootstrap'])
# self.save(df, pref='pred_corr')
return df
def dis_group_diff(self, plot=True):
group = self.dis_group(plot=False)
agg = group.pivot_table(index=['layer', 'n'],
columns='kind',
values='similarity').reset_index()
agg['diff'] = agg['shape'] - agg['px']
if self.bootstrap:
dfs = []
for layer in agg.layer.unique():
sel = agg[agg.layer == layer]['diff']
pct = stats.bootstrap_resample(sel, ci=None, func=np.mean)
d = OrderedDict([('kind', ['diff'] * len(pct)),
('layer', [layer] * len(pct)),
('preference', sel.mean()),
('iter', range(len(pct))),
('bootstrap', pct)])
dfs.append(pandas.DataFrame.from_dict(d))
df = pandas.concat(dfs)
else:
df = agg.groupby('layer').mean().reset_index()
df = df.rename(columns={'diff': 'preference'})
df['kind'] = 'diff'
df['iter'] = 0
df['bootstrap'] = np.nan
del df['px']
del df['shape']
return df
def _pred_corr(self, human, value='accuracy', method='corr'):
nname = models.NICE_NAMES[self.model_name].lower()
acc = self.acc_single()
acc[value] = acc[value].astype(np.float)
# import pdb; pdb.set_trace()
# sel = acc.kind!='unknown'
sel = acc.sel.copy()
acc = acc[sel][value]
sns.set_palette(sns.color_palette('Set2')[1:])
df = []
human = human[sel.values]
if method == 'corr':
f = lambda machine, human: (1 + stats.corr(machine, human)) / 2.
elif method == 'diff':
f = lambda machine, human: 1 - np.mean(np.abs(machine-human))
elif method == 'euclidean':
f = lambda machine, human: 1 - scipy.spatial.distance.sqeuclidean(machine, human) / len(machine)
else:
raise Exception('Method {} not recognized'.format(method))
corr = f(acc, human)
if self.bootstrap:
print('bootstrapping stats...')
bf = stats.bootstrap_resample(acc, human, func=f, ci=None, seed=0)
c = np.vstack([np.repeat(corr, len(bf)), np.arange(len(bf)), bf])
df.extend(c.T.tolist())
else:
df.append([corr, 0, np.nan])
df = pandas.DataFrame(df, columns=['consistency', 'iter', 'bootstrap'])
# self.save(df, pref='pred_corr')
return df
def _plot_behav(self):
human = self.myexp.behav()
hacc = human.groupby(['kind', 'subjid']).acc.mean()
for sno, subset in enumerate(ORDER):
floor, ceiling = stats.bootstrap_resample(hacc[subset])
# floor = np.percentile(hacc[subset], 2.5)
# ceiling = np.percentile(hacc[subset], 97.5)
sns.plt.axhspan(floor, ceiling, xmin=(2*sno+1)/6.-1/7., xmax=(2*sno+1)/6.+1/7., facecolor='0.9', edgecolor='0.9', zorder=0)
sns.plt.axhline(hacc[subset].mean(), xmin=(2*sno+1)/6.-1/7., xmax=(2*sno+1)/6.+1/7., color='0.65')
def _plot_behav(self):
human = self.myexp.behav()
hacc = human.groupby(['kind', 'subjid']).acc.mean()
for sno, subset in enumerate(ORDER):
floor, ceiling = stats.bootstrap_resample(hacc[subset])
# floor = np.percentile(hacc[subset], 2.5)
# ceiling = np.percentile(hacc[subset], 97.5)
sns.plt.axhspan(floor,
ceiling,
xmin=(2 * sno + 1) / 6. - 1 / 7.,
xmax=(2 * sno + 1) / 6. + 1 / 7.,
facecolor='0.9',
edgecolor='0.9',
zorder=0)
sns.plt.axhline(hacc[subset].mean(),
xmin=(2 * sno + 1) / 6. - 1 / 7.,
xmax=(2 * sno + 1) / 6. + 1 / 7.,
color='0.65')
