def plot_ROC(label_list, pred_list, names=None, **args):
"""
複数の ROC 曲線をプロットする
:param: label_list: 正解ラベルリストの配列. [(y1, y2, ...), (y1, y2, ...)] のようにして与える, pred_list に対応させる
:param: pred_list: 予測確率リストの配列. label_list と同じ長さにすること
:param: names=None: モデルの名称. None または同じ長さにすること. 指定しない場合,
ラベルの組が 2~3 ならば ['train', 'valid', 'test'] を与える. 3より多い場合は通し番号にする.
:param args: sklearn.metrics.roc_curve に与えるパラメータ
:return: plotnine オブジェクト
"""
if names is None:
if len(label_list) == 2:
names = ('train', 'test')
elif len(label_list) == 3:
names = ('train', 'valid', 'test')
else:
names = list(range(len(label_list)))
else:
pass
roc = [roc_curve(y, p, **args) for y, p in zip(label_list, pred_list)]
fpr, tpr = tuple([list(chain.from_iterable(x)) for x in zip(*roc)][0:2])
models = chain.from_iterable([[name] * l for name, l in zip(names, [len(x) for x, y, _ in roc])])
d_roc = pd.DataFrame({'fpr': fpr, 'tpr': tpr, 'model': models})
return ggplot(
d_roc,
aes(x='fpr', y='tpr', group='model', color='model')
) + geom_segment(x=0, y=0, xend=1, yend=1, linetype=':', color='grey'
) + geom_line(
) + scale_color_discrete(breaks=names
) + labs(x='false positive rate', y='true positive rate'
) + coord_equal(ratio=1, xlim=[0, 1], ylim=[0, 1]
) + theme_classic() + theme(figure_size=(4, 4))
def ggimg(image, mapping=None, data=None, dpi=80):
w, h = image.size
return (
ggplot(mapping=mapping, data=data)
+ scale_y_reverse(limits=(0, h))
+ xlim(0, w)
+ scale_color_discrete(guide=False) # removes legend for line color
+ theme_image(w, h, dpi=dpi)
+ annotate(
"rect", xmin=0, xmax=w, ymin=0, ymax=h, color="black", fill=None
) # box around image
)
def plot_runtimes(ags):
threshold = -50 / (400 / np.log(10))
best = (
ags.query('test_nodes == 64').loc[lambda df: df.elo > threshold].
sort_values('train_time').groupby('boardsize').first().reset_index())
return (pn.ggplot(
best, pn.aes(x='boardsize', y='train_time', color='factor(boardsize)'))
+ pn.geom_point(size=2, show_legend=False)
# + pn.geom_line(size=.5, show_legend=False)
+ pn.scale_y_continuous(trans='log10') +
pn.scale_color_discrete(l=.4) +
pn.labs(x='Board size', y='Training time (seconds)') + plot.IEEE())
def plot_training_curves(ags):
df = ags[ags.test_nodes == 64].copy()
df['g'] = df.run + df.test_nodes.astype(str)
return (pn.ggplot(
df,
pn.aes(x='train_flops',
y='400/np.log(10)*elo',
group='g',
color='factor(boardsize)')) + pn.geom_line() +
pn.geom_point(size=.5) + pn.scale_x_continuous(trans='log10') +
pn.scale_color_discrete(name='Boardsize') +
pn.labs(x='Training FLOPS',
y='Elo v. perfect play',
title='All agents\' training curves') + plot.mpl_theme() +
plot.poster_sizes())
def plot_resid_var_trends(ags):
resid_var = data.residual_vars(ags)
return (
pn.ggplot(
resid_var,
pn.aes(x='ratio',
y='rv',
color='factor(predicted)',
group='predicted')) + pn.geom_line(size=2) +
pn.geom_text(pn.aes(label='seen'), nudge_y=-.1, size=14) +
pn.geom_point(size=4) + pn.scale_x_continuous(trans='log10') +
pn.scale_y_continuous(trans='log10') +
pn.scale_color_discrete(name='Predicted frontier') + pn.labs(
x='(cost of observed frontier)/(cost of predicted frontier)',
y='residual variance in performance',
title=
'Frontiers of small problems are good, cheap proxies for frontiers of expensive problems'
) + plot.mpl_theme() + plot.poster_sizes())
def plot_flops_frontier(ags):
df = data.modelled_elos(ags)
return (
pn.ggplot(
df,
pn.aes(
x='train_flops', color='factor(boardsize)', group='boardsize'))
+ pn.geom_line(pn.aes(y='400/np.log(10)*elo'), size=2) + pn.geom_line(
pn.aes(y='400/np.log(10)*elohat'), size=1, linetype='dashed') +
pn.labs(
x='Training FLOPS',
y='Elo v. perfect play',
title=
'Performance is a sigmoid of compute, linearly scaled by board size'
) + pn.scale_x_continuous(trans='log10') +
pn.scale_color_discrete(name='Boardsize') +
pn.coord_cartesian(None,
(None, 0)) + plot.mpl_theme() + plot.poster_sizes())
def plot_resid_var(ags):
resid_var = data.residual_vars(ags)
resid_var['diff'] = resid_var.predicted - resid_var.seen
labels = resid_var.sort_values('seen').groupby(
'predicted').last().reset_index()
return (pn.ggplot(
resid_var,
pn.aes(x='seen', y='rv', color='factor(predicted)',
group='predicted')) + pn.geom_line(size=.5, show_legend=False) +
pn.geom_text(pn.aes(label='predicted'),
labels,
nudge_x=+.15,
size=6,
show_legend=False) +
pn.geom_point(size=.5, show_legend=False) +
pn.scale_y_continuous(trans='log10') +
pn.scale_color_discrete(l=.4, limits=list(range(3, 10))) +
pn.labs(x='Max board size observed', y='Residual Variance') +
plot.IEEE())
def plot_calibration(label_list, pred_list, names=None, **args):
"""
カリブレーションカーブを複数描く.
:param: label_list: 正解ラベルリストの配列. [(y1, y2, ...), (y1, y2, ...)] のようにして与える, pred_list に対応させる
:param: pred_list: 予測確率リストの配列. label_list と同じ長さにすること
:param: names=None: モデルの名称. None または同じ長さにすること. 指定しない場合, ラベルの組が 2~3 ならば ['train', 'valid', 'test'] を与える. 3より多い場合は通し番号にする.
:param: args: sklearn.metrics.roc_curve に与えるパラメータ.
:param: strategy='quantile': 分割方法. 'quantile' または 'uniform'
:param: n_bins=10: ビン数.
:param: normalize=False: 予測確率の0-1正規化が必要かどうか
:return: plotnine オブジェクト
TODO: 入力データがすごい偏ってるときの表示範囲
"""
if names is None:
if len(label_list) == 2:
names = ('train', 'test')
elif len(label_list) == 3:
names = ('train', 'valid', 'test')
elif len(label_list) == 1:
names = 'model',
else:
names = list(range(len(label_list)))
else:
pass
if args is None:
args = {'strategy': 'quantile', 'n_bins': 5}
else:
args['strategy'] = args['strategy'] if 'strategy' in args.keys() else 'quantile'
args['n_bins'] = args['n_bins'] if 'n_bins' in args.keys() else 10
calib = [calibration_curve(y, p, **args) for y, p in zip(label_list, pred_list)]
frac, pred = tuple([list(chain.from_iterable(x)) for x in zip(*calib)][0:2])
models = chain.from_iterable([[name] * l for name, l in zip(names, [len(x) for x, y in calib])])
d_calib = pd.DataFrame({'pred': pred, 'frac': frac, 'model': models})
return ggplot(
d_calib,
aes(x='pred', y='frac', group='model', color='model')
) + geom_segment(x=0, y=0, xend=1, yend=1, linetype=':', color='grey'
) + geom_line(
) + geom_point(
) + scale_color_discrete(breaks=names
) + labs(x='mean estimated probability', y='fraction of positives'
) + coord_equal(ratio=1) + theme_classic() + theme(figure_size=(4, 4))
def plot_frontiers(ags):
df, model = data.modelled_elos(ags)
labels = df.sort_values('train_flops').groupby(
'boardsize').first().reset_index()
return (pn.ggplot(
df,
pn.aes(x='train_flops', color='factor(boardsize)', group='boardsize'))
+ pn.geom_line(pn.aes(y='ELO*elo'), size=.5, show_legend=False) +
pn.geom_line(pn.aes(y='ELO*elohat'),
size=.25,
linetype='dashed',
show_legend=False) +
pn.geom_text(pn.aes(y='ELO*elohat', label='boardsize'),
data=labels,
show_legend=False,
size=6,
nudge_x=-.25,
nudge_y=-15) +
pn.labs(x='Training compute (FLOPS-seconds)',
y='Elo v. perfect play') + pn.scale_color_discrete(l=.4) +
pn.scale_x_continuous(trans='log10') +
pn.coord_cartesian(None, (None, 0)) + plot.IEEE())
pr_perform_df = (pd.DataFrame({
'precision': gen_precision,
'recall': gen_recall
}).assign(model='gen_model').append(
pd.DataFrame({
'precision': disc_precision,
'recall': disc_recall
}).assign(model='disc_model')))
# In[15]:
(p9.ggplot(pr_perform_df,
p9.aes(x="recall", y="precision", color="factor(model)")) +
p9.geom_point() + p9.geom_line() +
p9.labs(title="Validation PR Curve", color="Model") +
p9.scale_color_discrete(l=.4) + p9.theme_seaborn())
# In[16]:
precision_recall_fscore_support(candidate_df.query(
f"candidate_id in {disc_model_dict[value].candidate_id.values.tolist()}").
curated_gig.values,
gen_predicton.apply(lambda x: 1
if x > 0.5 else 0),
average='binary')
# In[17]:
precision_recall_fscore_support(candidate_df.query(
f"candidate_id in {disc_model_dict[value].candidate_id.values.tolist()}").
curated_gig.values,
def run_cluster_sim(n_sims = 1000, param = (.1, .5), n = 1000,
n_cluster = 50, rho = .5, cluster_robust = False):
res = [cluster_sim(param = param, n = n, rho = rho,
n_cluster = n_cluster,
cluster_robust = cluster_robust) for x in range(n_sims)]
df = pd.DataFrame(res)
df.columns = ('b1', 'se_b1', 'ci95_lower', 'ci95_upper')
df['param_caught'] = (df['ci95_lower'] <= param[1]) & (param[1] <= df['ci95_upper'])
df['id'] = df.index
return df
# Simulation clustered SE
sim_params = [.4, 0] # beta1 = 0: no effect of x on y
sim_nocluster = run_cluster_sim(n_sims=1000, param = sim_params, cluster_robust = False)
p.ggplot(sim_nocluster.sample(100).sort_values('b1'),
p.aes(x = 'factor(id)', y = 'b1',
ymin = 'ci95_lower', ymax = 'ci95_upper',
color = 'param_caught')) +\
p.geom_hline(yintercept = sim_params[1], linetype = 'dashed') +\
p.geom_pointrange() +\
p.labs(x = 'sim ID', y = 'b1', title = 'Randomly Chosen 100 95% CIs') +\
p.scale_color_discrete(name = 'True param value', labels = ('missed', 'hit')) +\
p.coord_flip()
# In[22]:
# Visualize UMAP results
clone_ae_umap_gg = (
gg.ggplot(embedding_cloneAE_df)
+ gg.geom_point(
gg.aes('x', 'y',
shape="Metadata_Plate",
size='Metadata_treatment',
color="Metadata_clone_number"),
alpha=0.8
)
+ gg.theme_bw()
+ gg.scale_shape_manual(name="Plate", values=[".", "+"])
+ gg.scale_color_discrete(name="Clone")
+ gg.scale_size_manual(name="Treatment", values=[1, 3, 5, 7])
+ gg.xlab("UMAP X")
+ gg.ylab("UMAP Y")
+ gg.ggtitle("Clone A and E - Merged")
+ gg.theme(
legend_key=gg.element_rect(color="black", fill = "white"),
strip_background=gg.element_rect(colour="black", fill="#fdfff4")
)
)
file = os.path.join("figures", "umap", "cloneAE_umap")
for extension in save_file_extensions:
clone_ae_umap_gg.save(
filename='{}{}'.format(file, extension), height=3, width=3.5, dpi=400
)
def syntactic_diversity_plots():
with open('data/external/syntactic_diversity_table.json') as f:
rows = json.load(f)
parse_df = pd.DataFrame(rows)
parse_df['parse_ratio'] = parse_df['unique_parses'] / parse_df['parses']
melt_df = pd.melt(
parse_df,
id_vars=['dataset', 'depth', 'overlap', 'parses'],
value_vars=['parse_ratio', 'unique_parses'],
var_name='metric',
value_name='y'
)
def label_facet(name):
if name == 'parse_ratio':
return 'Average Unique Parses per Instance'
elif name == 'unique_parses':
return 'Count of Unique Parses'
def label_y(ys):
formatted_ys = []
for y in ys:
y = str(y)
if y.endswith('000.0'):
formatted_ys.append(y[:-5] + 'K')
else:
formatted_ys.append(y)
return formatted_ys
p = (
ggplot(melt_df)
+ aes(x='depth', y='y', color='dataset')
+ facet_wrap('metric', scales='free_y', nrow=2, labeller=label_facet)
+ geom_line() + geom_point()
+ xlab('Parse Truncation Depth') + ylab('')
+ scale_color_discrete(name='Dataset')
+ scale_y_continuous(labels=label_y)
+ scale_x_continuous(
breaks=list(range(1, 11)),
minor_breaks=list(range(1, 11)),
limits=[1, 10])
+ theme_fs()
)
p.save(path.join(output_path, 'syn_div_plot.pdf'))
p = (
ggplot(parse_df)
+ aes(x='depth', y='unique_parses', color='dataset')
+ geom_line() + geom_point()
+ xlab('Parse Truncation Depth')
+ ylab('Count of Unique Parses')
+ scale_color_discrete(name='Dataset')
+ scale_x_continuous(
breaks=list(range(1, 11)),
minor_breaks=list(range(1, 11)),
limits=[1, 10])
+ theme_fs()
)
p.save(path.join(output_path, 'n_unique_parses.pdf'))
p = (
ggplot(parse_df)
+ aes(x='depth', y='parse_ratio', color='dataset')
+ geom_line() + geom_point()
+ xlab('Parse Truncation Depth')
+ ylab('Average Unique Parses per Instance')
+ scale_color_discrete(name='Dataset')
+ scale_x_continuous(breaks=list(range(1, 11)), minor_breaks=list(range(1, 11)), limits=[1, 10])
+ scale_y_continuous(limits=[0, 1])
+ theme_fs()
)
p.save(path.join(output_path, 'parse_ratio.pdf'))
import numpy as np
import pandas as pd
import statsmodels.api as sm
import statsmodels.formula.api as smf
from itertools import combinations
import plotnine as p
# read data
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
def read_data(file):
return pd.read_stata(
"https://raw.github.com/scunning1975/mixtape/master/" + file)
tb = pd.DataFrame({
'd':
np.concatenate((np.repeat(0, 20), np.repeat(1, 20))),
'y': (0.22, -0.87, -2.39, -1.79, 0.37, -1.54, 1.28, -0.31, -0.74, 1.72,
0.38, -0.17, -0.62, -1.10, 0.30, 0.15, 2.30, 0.19, -0.50, -0.9,
-5.13, -2.19, 2.43, -3.83, 0.5, -3.25, 4.32, 1.63, 5.18, -0.43, 7.11,
4.87, -3.10, -5.81, 3.76, 6.31, 2.58, 0.07, 5.76, 3.50)
})
p.ggplot() + p.geom_density(tb, p.aes(x='y', color='factor(d)')) + p.xlim(
-7, 8) + p.labs(title="Kolmogorov-Smirnov Test") + p.scale_color_discrete(
labels=("Control", "Treatment"))
#######################################
# ----------- (5) FIGURES ----------- #
di_tt = {'bbd': 'BBD', 'phn': 'HN'}
tmp = res_fisher.merge(df_inf[cn_gg + ['pval_rep']])
tmp = tmp.query('pval_rep <= 0.05')
tmp.tt = tmp.tt.map(di_tt)
# (i) Reported versus actual p-value
gg_rep_act = (pn.ggplot(tmp, pn.aes(x='pval_rep', y='pval_bl', color='tt')) +
pn.theme_bw() + pn.geom_point() +
pn.labs(y="Calculated (Fisher's Exact)", x='Reported') +
pn.ggtitle('P-values') +
pn.geom_hline(yintercept=alpha, linetype='--') +
pn.scale_color_discrete(name='Literature'))
gg_rep_act.save(os.path.join(dir_output, 'gg_rep_act.png'), width=8, height=4)
di_notes = {
'chi2': 'χ2-correction',
'insig': 'Erroneous',
'specification': 'Specification',
'non-replicable': 'Inconsistent'
}
# (ii) Breakdown of counts
tmp = acc_tt.merge(
res_fisher.tt.value_counts().reset_index().rename(columns={
'index': 'tt',
'tt': 'n_lit'
}))
tmp = tmp.assign(tt=lambda x: x.tt.map(di_tt),
def mixed_linear_plots(df, x_axis, x_label):
plotnine.options.figure_size = (8, 10)
md = smf.mixedlm('log_score ~ percent_broken + percent_fail_runs',
df,
groups=df.index.values)
mdf_rul = md.fit()
print('#' * 18 + 'Log RUL' + '#' * 18)
print(mdf_rul.summary())
md = smf.mixedlm('mse ~ percent_broken + percent_fail_runs',
df,
groups=df.index.values)
mdf_mse = md.fit()
print('#' * 18 + 'RMSE' + '#' * 18)
print(mdf_mse.summary())
df['percent_broken'] = df['percent_broken'].round().astype(np.int)
df['percent_fail_runs'] = df['percent_fail_runs'].round().astype(np.int)
gg = (plotnine.ggplot(
df, plotnine.aes(x=x_axis, y='log_score', color='method')) +
plotnine.geom_jitter(width=2.5, show_legend=False) +
plotnine.geom_abline(
plotnine.aes(intercept=mdf_rul.params['Intercept'],
slope=mdf_rul.params[x_axis])) +
plotnine.stat_smooth(method='gls', show_legend=False) +
plotnine.xlab(x_label) + plotnine.ylab('Logarithmic RUL-Score') +
plotnine.scale_color_discrete(name='Method', labels=['DAAN', 'JAN'])
+ plotnine.theme_classic(base_size=20))
gg.save('%s_log_rul_by_method.pdf' % x_axis)
gg = (plotnine.ggplot(
df, plotnine.aes(x=x_axis, y='log_score', color='task')) +
plotnine.geom_jitter(width=2.5, show_legend=False) +
plotnine.geom_abline(
plotnine.aes(intercept=mdf_rul.params['Intercept'],
slope=mdf_rul.params[x_axis])) +
plotnine.stat_smooth(method='gls', show_legend=False) +
plotnine.xlab(x_label) + plotnine.ylab('Logarithmic RUL-Score') +
plotnine.scale_color_discrete(
name='Task',
labels=['4→3', '4→2', '1→3', '1→2', '3→4', '3→1', '2→4', '2→1'
]) + plotnine.theme_classic(base_size=20))
gg.save('%s_log_rul_by_task.pdf' % x_axis)
gg = (
plotnine.ggplot(df, plotnine.aes(x=x_axis, y='mse', color='method')) +
plotnine.geom_jitter(width=2.5) + plotnine.geom_abline(
plotnine.aes(intercept=mdf_mse.params['Intercept'],
slope=mdf_mse.params[x_axis])) +
plotnine.stat_smooth(method='gls') + plotnine.ylab('RMSE') +
plotnine.xlab(x_label) +
plotnine.scale_color_discrete(name='Method', labels=['DAAN', 'JAN']) +
plotnine.theme_classic(base_size=20))
gg.save('%s_mse_by_method.pdf' % x_axis)
gg = (plotnine.ggplot(df, plotnine.aes(x=x_axis, y='mse', color='task')) +
plotnine.geom_jitter(width=2.5) + plotnine.geom_abline(
plotnine.aes(intercept=mdf_mse.params['Intercept'],
slope=mdf_mse.params[x_axis])) +
plotnine.stat_smooth(method='gls') + plotnine.ylab('RMSE') +
plotnine.scale_color_discrete(
name='Task',
labels=['4→3', '4→2', '1→3', '1→2', '3→4', '3→1', '2→4', '2→1'
]) + plotnine.theme_classic(base_size=20))
gg.save('%s_mse_by_task.pdf' % x_axis)