def plot_training_curves(ags):
df = ags[ags.test_nodes == 64].copy()
df['g'] = df.run + df.test_nodes.astype(str)
return (pn.ggplot(
df,
pn.aes(x='train_flops',
y='400/np.log(10)*elo',
group='g',
color='factor(boardsize)')) + pn.geom_line() +
pn.geom_point(size=.5) + pn.scale_x_continuous(trans='log10') +
pn.scale_color_discrete(name='Boardsize') +
pn.labs(x='Training FLOPS',
y='Elo v. perfect play',
title='All agents\' training curves') + plot.mpl_theme() +
plot.poster_sizes())
def plot_params(ags):
df = ags.query('boardsize == 9 & test_nodes == 64').copy()
df['params'] = df.train_flops / df.samples
return (
pn.ggplot(
df,
pn.aes(x='train_flops',
y='400/np.log(10)*elo',
color='params',
group='run')) + pn.geom_line() + pn.geom_point() +
pn.scale_x_continuous(trans='log10') +
pn.scale_color_continuous(trans='log10', name='Params') + pn.labs(
title=
'Smaller networks are more compute efficient for lower performances, but plateau earlier',
y='Elo v. perfect play',
x='Train FLOPS') + plot.mpl_theme() + plot.poster_sizes() +
plot.no_colorbar_ticks())
def plot_sample_efficiency(ags):
df = ags.query('boardsize == 9 & test_nodes == 64').copy()
df['params'] = df.train_flops / df.samples
return (
pn.ggplot(
df,
pn.aes(x='samples',
y='400/np.log(10)*elo',
color='params',
group='run')) + pn.geom_line() + pn.geom_point() +
pn.scale_x_continuous(trans='log10') +
pn.scale_color_continuous(trans='log10', name='Params') + pn.labs(
title=
'Bigger networks might not be comute efficient, but they are sample efficient',
y='Elo v. perfect play',
x='Train FLOPS') + plot.mpl_theme() + plot.poster_sizes() +
plot.no_colorbar_ticks())
def plot_resid_var_trends(ags):
resid_var = data.residual_vars(ags)
return (
pn.ggplot(
resid_var,
pn.aes(x='ratio',
y='rv',
color='factor(predicted)',
group='predicted')) + pn.geom_line(size=2) +
pn.geom_text(pn.aes(label='seen'), nudge_y=-.1, size=14) +
pn.geom_point(size=4) + pn.scale_x_continuous(trans='log10') +
pn.scale_y_continuous(trans='log10') +
pn.scale_color_discrete(name='Predicted frontier') + pn.labs(
x='(cost of observed frontier)/(cost of predicted frontier)',
y='residual variance in performance',
title=
'Frontiers of small problems are good, cheap proxies for frontiers of expensive problems'
) + plot.mpl_theme() + plot.poster_sizes())
def plot_flops_frontier(ags):
df = data.modelled_elos(ags)
return (
pn.ggplot(
df,
pn.aes(
x='train_flops', color='factor(boardsize)', group='boardsize'))
+ pn.geom_line(pn.aes(y='400/np.log(10)*elo'), size=2) + pn.geom_line(
pn.aes(y='400/np.log(10)*elohat'), size=1, linetype='dashed') +
pn.labs(
x='Training FLOPS',
y='Elo v. perfect play',
title=
'Performance is a sigmoid of compute, linearly scaled by board size'
) + pn.scale_x_continuous(trans='log10') +
pn.scale_color_discrete(name='Boardsize') +
pn.coord_cartesian(None,
(None, 0)) + plot.mpl_theme() + plot.poster_sizes())