def plot_ROC(label_list, pred_list, names=None, **args): """ 複数の ROC 曲線をプロットする :param: label_list: 正解ラベルリストの配列. [(y1, y2, ...), (y1, y2, ...)] のようにして与える, pred_list に対応させる :param: pred_list: 予測確率リストの配列. label_list と同じ長さにすること :param: names=None: モデルの名称. None または同じ長さにすること. 指定しない場合, ラベルの組が 2~3 ならば ['train', 'valid', 'test'] を与える. 3より多い場合は通し番号にする. :param args: sklearn.metrics.roc_curve に与えるパラメータ :return: plotnine オブジェクト """ if names is None: if len(label_list) == 2: names = ('train', 'test') elif len(label_list) == 3: names = ('train', 'valid', 'test') else: names = list(range(len(label_list))) else: pass roc = [roc_curve(y, p, **args) for y, p in zip(label_list, pred_list)] fpr, tpr = tuple([list(chain.from_iterable(x)) for x in zip(*roc)][0:2]) models = chain.from_iterable([[name] * l for name, l in zip(names, [len(x) for x, y, _ in roc])]) d_roc = pd.DataFrame({'fpr': fpr, 'tpr': tpr, 'model': models}) return ggplot( d_roc, aes(x='fpr', y='tpr', group='model', color='model') ) + geom_segment(x=0, y=0, xend=1, yend=1, linetype=':', color='grey' ) + geom_line( ) + scale_color_discrete(breaks=names ) + labs(x='false positive rate', y='true positive rate' ) + coord_equal(ratio=1, xlim=[0, 1], ylim=[0, 1] ) + theme_classic() + theme(figure_size=(4, 4))
def ggimg(image, mapping=None, data=None, dpi=80): w, h = image.size return ( ggplot(mapping=mapping, data=data) + scale_y_reverse(limits=(0, h)) + xlim(0, w) + scale_color_discrete(guide=False) # removes legend for line color + theme_image(w, h, dpi=dpi) + annotate( "rect", xmin=0, xmax=w, ymin=0, ymax=h, color="black", fill=None ) # box around image )
def plot_runtimes(ags): threshold = -50 / (400 / np.log(10)) best = ( ags.query('test_nodes == 64').loc[lambda df: df.elo > threshold]. sort_values('train_time').groupby('boardsize').first().reset_index()) return (pn.ggplot( best, pn.aes(x='boardsize', y='train_time', color='factor(boardsize)')) + pn.geom_point(size=2, show_legend=False) # + pn.geom_line(size=.5, show_legend=False) + pn.scale_y_continuous(trans='log10') + pn.scale_color_discrete(l=.4) + pn.labs(x='Board size', y='Training time (seconds)') + plot.IEEE())
def plot_training_curves(ags): df = ags[ags.test_nodes == 64].copy() df['g'] = df.run + df.test_nodes.astype(str) return (pn.ggplot( df, pn.aes(x='train_flops', y='400/np.log(10)*elo', group='g', color='factor(boardsize)')) + pn.geom_line() + pn.geom_point(size=.5) + pn.scale_x_continuous(trans='log10') + pn.scale_color_discrete(name='Boardsize') + pn.labs(x='Training FLOPS', y='Elo v. perfect play', title='All agents\' training curves') + plot.mpl_theme() + plot.poster_sizes())
def plot_resid_var_trends(ags): resid_var = data.residual_vars(ags) return ( pn.ggplot( resid_var, pn.aes(x='ratio', y='rv', color='factor(predicted)', group='predicted')) + pn.geom_line(size=2) + pn.geom_text(pn.aes(label='seen'), nudge_y=-.1, size=14) + pn.geom_point(size=4) + pn.scale_x_continuous(trans='log10') + pn.scale_y_continuous(trans='log10') + pn.scale_color_discrete(name='Predicted frontier') + pn.labs( x='(cost of observed frontier)/(cost of predicted frontier)', y='residual variance in performance', title= 'Frontiers of small problems are good, cheap proxies for frontiers of expensive problems' ) + plot.mpl_theme() + plot.poster_sizes())
def plot_flops_frontier(ags): df = data.modelled_elos(ags) return ( pn.ggplot( df, pn.aes( x='train_flops', color='factor(boardsize)', group='boardsize')) + pn.geom_line(pn.aes(y='400/np.log(10)*elo'), size=2) + pn.geom_line( pn.aes(y='400/np.log(10)*elohat'), size=1, linetype='dashed') + pn.labs( x='Training FLOPS', y='Elo v. perfect play', title= 'Performance is a sigmoid of compute, linearly scaled by board size' ) + pn.scale_x_continuous(trans='log10') + pn.scale_color_discrete(name='Boardsize') + pn.coord_cartesian(None, (None, 0)) + plot.mpl_theme() + plot.poster_sizes())
def plot_resid_var(ags): resid_var = data.residual_vars(ags) resid_var['diff'] = resid_var.predicted - resid_var.seen labels = resid_var.sort_values('seen').groupby( 'predicted').last().reset_index() return (pn.ggplot( resid_var, pn.aes(x='seen', y='rv', color='factor(predicted)', group='predicted')) + pn.geom_line(size=.5, show_legend=False) + pn.geom_text(pn.aes(label='predicted'), labels, nudge_x=+.15, size=6, show_legend=False) + pn.geom_point(size=.5, show_legend=False) + pn.scale_y_continuous(trans='log10') + pn.scale_color_discrete(l=.4, limits=list(range(3, 10))) + pn.labs(x='Max board size observed', y='Residual Variance') + plot.IEEE())
def plot_calibration(label_list, pred_list, names=None, **args): """ カリブレーションカーブを複数描く. :param: label_list: 正解ラベルリストの配列. [(y1, y2, ...), (y1, y2, ...)] のようにして与える, pred_list に対応させる :param: pred_list: 予測確率リストの配列. label_list と同じ長さにすること :param: names=None: モデルの名称. None または同じ長さにすること. 指定しない場合, ラベルの組が 2~3 ならば ['train', 'valid', 'test'] を与える. 3より多い場合は通し番号にする. :param: args: sklearn.metrics.roc_curve に与えるパラメータ. :param: strategy='quantile': 分割方法. 'quantile' または 'uniform' :param: n_bins=10: ビン数. :param: normalize=False: 予測確率の0-1正規化が必要かどうか :return: plotnine オブジェクト TODO: 入力データがすごい偏ってるときの表示範囲 """ if names is None: if len(label_list) == 2: names = ('train', 'test') elif len(label_list) == 3: names = ('train', 'valid', 'test') elif len(label_list) == 1: names = 'model', else: names = list(range(len(label_list))) else: pass if args is None: args = {'strategy': 'quantile', 'n_bins': 5} else: args['strategy'] = args['strategy'] if 'strategy' in args.keys() else 'quantile' args['n_bins'] = args['n_bins'] if 'n_bins' in args.keys() else 10 calib = [calibration_curve(y, p, **args) for y, p in zip(label_list, pred_list)] frac, pred = tuple([list(chain.from_iterable(x)) for x in zip(*calib)][0:2]) models = chain.from_iterable([[name] * l for name, l in zip(names, [len(x) for x, y in calib])]) d_calib = pd.DataFrame({'pred': pred, 'frac': frac, 'model': models}) return ggplot( d_calib, aes(x='pred', y='frac', group='model', color='model') ) + geom_segment(x=0, y=0, xend=1, yend=1, linetype=':', color='grey' ) + geom_line( ) + geom_point( ) + scale_color_discrete(breaks=names ) + labs(x='mean estimated probability', y='fraction of positives' ) + coord_equal(ratio=1) + theme_classic() + theme(figure_size=(4, 4))
def plot_frontiers(ags): df, model = data.modelled_elos(ags) labels = df.sort_values('train_flops').groupby( 'boardsize').first().reset_index() return (pn.ggplot( df, pn.aes(x='train_flops', color='factor(boardsize)', group='boardsize')) + pn.geom_line(pn.aes(y='ELO*elo'), size=.5, show_legend=False) + pn.geom_line(pn.aes(y='ELO*elohat'), size=.25, linetype='dashed', show_legend=False) + pn.geom_text(pn.aes(y='ELO*elohat', label='boardsize'), data=labels, show_legend=False, size=6, nudge_x=-.25, nudge_y=-15) + pn.labs(x='Training compute (FLOPS-seconds)', y='Elo v. perfect play') + pn.scale_color_discrete(l=.4) + pn.scale_x_continuous(trans='log10') + pn.coord_cartesian(None, (None, 0)) + plot.IEEE())
pr_perform_df = (pd.DataFrame({ 'precision': gen_precision, 'recall': gen_recall }).assign(model='gen_model').append( pd.DataFrame({ 'precision': disc_precision, 'recall': disc_recall }).assign(model='disc_model'))) # In[15]: (p9.ggplot(pr_perform_df, p9.aes(x="recall", y="precision", color="factor(model)")) + p9.geom_point() + p9.geom_line() + p9.labs(title="Validation PR Curve", color="Model") + p9.scale_color_discrete(l=.4) + p9.theme_seaborn()) # In[16]: precision_recall_fscore_support(candidate_df.query( f"candidate_id in {disc_model_dict[value].candidate_id.values.tolist()}"). curated_gig.values, gen_predicton.apply(lambda x: 1 if x > 0.5 else 0), average='binary') # In[17]: precision_recall_fscore_support(candidate_df.query( f"candidate_id in {disc_model_dict[value].candidate_id.values.tolist()}"). curated_gig.values,
def run_cluster_sim(n_sims = 1000, param = (.1, .5), n = 1000, n_cluster = 50, rho = .5, cluster_robust = False): res = [cluster_sim(param = param, n = n, rho = rho, n_cluster = n_cluster, cluster_robust = cluster_robust) for x in range(n_sims)] df = pd.DataFrame(res) df.columns = ('b1', 'se_b1', 'ci95_lower', 'ci95_upper') df['param_caught'] = (df['ci95_lower'] <= param[1]) & (param[1] <= df['ci95_upper']) df['id'] = df.index return df # Simulation clustered SE sim_params = [.4, 0] # beta1 = 0: no effect of x on y sim_nocluster = run_cluster_sim(n_sims=1000, param = sim_params, cluster_robust = False) p.ggplot(sim_nocluster.sample(100).sort_values('b1'), p.aes(x = 'factor(id)', y = 'b1', ymin = 'ci95_lower', ymax = 'ci95_upper', color = 'param_caught')) +\ p.geom_hline(yintercept = sim_params[1], linetype = 'dashed') +\ p.geom_pointrange() +\ p.labs(x = 'sim ID', y = 'b1', title = 'Randomly Chosen 100 95% CIs') +\ p.scale_color_discrete(name = 'True param value', labels = ('missed', 'hit')) +\ p.coord_flip()
# In[22]: # Visualize UMAP results clone_ae_umap_gg = ( gg.ggplot(embedding_cloneAE_df) + gg.geom_point( gg.aes('x', 'y', shape="Metadata_Plate", size='Metadata_treatment', color="Metadata_clone_number"), alpha=0.8 ) + gg.theme_bw() + gg.scale_shape_manual(name="Plate", values=[".", "+"]) + gg.scale_color_discrete(name="Clone") + gg.scale_size_manual(name="Treatment", values=[1, 3, 5, 7]) + gg.xlab("UMAP X") + gg.ylab("UMAP Y") + gg.ggtitle("Clone A and E - Merged") + gg.theme( legend_key=gg.element_rect(color="black", fill = "white"), strip_background=gg.element_rect(colour="black", fill="#fdfff4") ) ) file = os.path.join("figures", "umap", "cloneAE_umap") for extension in save_file_extensions: clone_ae_umap_gg.save( filename='{}{}'.format(file, extension), height=3, width=3.5, dpi=400 )
def syntactic_diversity_plots(): with open('data/external/syntactic_diversity_table.json') as f: rows = json.load(f) parse_df = pd.DataFrame(rows) parse_df['parse_ratio'] = parse_df['unique_parses'] / parse_df['parses'] melt_df = pd.melt( parse_df, id_vars=['dataset', 'depth', 'overlap', 'parses'], value_vars=['parse_ratio', 'unique_parses'], var_name='metric', value_name='y' ) def label_facet(name): if name == 'parse_ratio': return 'Average Unique Parses per Instance' elif name == 'unique_parses': return 'Count of Unique Parses' def label_y(ys): formatted_ys = [] for y in ys: y = str(y) if y.endswith('000.0'): formatted_ys.append(y[:-5] + 'K') else: formatted_ys.append(y) return formatted_ys p = ( ggplot(melt_df) + aes(x='depth', y='y', color='dataset') + facet_wrap('metric', scales='free_y', nrow=2, labeller=label_facet) + geom_line() + geom_point() + xlab('Parse Truncation Depth') + ylab('') + scale_color_discrete(name='Dataset') + scale_y_continuous(labels=label_y) + scale_x_continuous( breaks=list(range(1, 11)), minor_breaks=list(range(1, 11)), limits=[1, 10]) + theme_fs() ) p.save(path.join(output_path, 'syn_div_plot.pdf')) p = ( ggplot(parse_df) + aes(x='depth', y='unique_parses', color='dataset') + geom_line() + geom_point() + xlab('Parse Truncation Depth') + ylab('Count of Unique Parses') + scale_color_discrete(name='Dataset') + scale_x_continuous( breaks=list(range(1, 11)), minor_breaks=list(range(1, 11)), limits=[1, 10]) + theme_fs() ) p.save(path.join(output_path, 'n_unique_parses.pdf')) p = ( ggplot(parse_df) + aes(x='depth', y='parse_ratio', color='dataset') + geom_line() + geom_point() + xlab('Parse Truncation Depth') + ylab('Average Unique Parses per Instance') + scale_color_discrete(name='Dataset') + scale_x_continuous(breaks=list(range(1, 11)), minor_breaks=list(range(1, 11)), limits=[1, 10]) + scale_y_continuous(limits=[0, 1]) + theme_fs() ) p.save(path.join(output_path, 'parse_ratio.pdf'))
import numpy as np import pandas as pd import statsmodels.api as sm import statsmodels.formula.api as smf from itertools import combinations import plotnine as p # read data import ssl ssl._create_default_https_context = ssl._create_unverified_context def read_data(file): return pd.read_stata( "https://raw.github.com/scunning1975/mixtape/master/" + file) tb = pd.DataFrame({ 'd': np.concatenate((np.repeat(0, 20), np.repeat(1, 20))), 'y': (0.22, -0.87, -2.39, -1.79, 0.37, -1.54, 1.28, -0.31, -0.74, 1.72, 0.38, -0.17, -0.62, -1.10, 0.30, 0.15, 2.30, 0.19, -0.50, -0.9, -5.13, -2.19, 2.43, -3.83, 0.5, -3.25, 4.32, 1.63, 5.18, -0.43, 7.11, 4.87, -3.10, -5.81, 3.76, 6.31, 2.58, 0.07, 5.76, 3.50) }) p.ggplot() + p.geom_density(tb, p.aes(x='y', color='factor(d)')) + p.xlim( -7, 8) + p.labs(title="Kolmogorov-Smirnov Test") + p.scale_color_discrete( labels=("Control", "Treatment"))
####################################### # ----------- (5) FIGURES ----------- # di_tt = {'bbd': 'BBD', 'phn': 'HN'} tmp = res_fisher.merge(df_inf[cn_gg + ['pval_rep']]) tmp = tmp.query('pval_rep <= 0.05') tmp.tt = tmp.tt.map(di_tt) # (i) Reported versus actual p-value gg_rep_act = (pn.ggplot(tmp, pn.aes(x='pval_rep', y='pval_bl', color='tt')) + pn.theme_bw() + pn.geom_point() + pn.labs(y="Calculated (Fisher's Exact)", x='Reported') + pn.ggtitle('P-values') + pn.geom_hline(yintercept=alpha, linetype='--') + pn.scale_color_discrete(name='Literature')) gg_rep_act.save(os.path.join(dir_output, 'gg_rep_act.png'), width=8, height=4) di_notes = { 'chi2': 'χ2-correction', 'insig': 'Erroneous', 'specification': 'Specification', 'non-replicable': 'Inconsistent' } # (ii) Breakdown of counts tmp = acc_tt.merge( res_fisher.tt.value_counts().reset_index().rename(columns={ 'index': 'tt', 'tt': 'n_lit' })) tmp = tmp.assign(tt=lambda x: x.tt.map(di_tt),
def mixed_linear_plots(df, x_axis, x_label): plotnine.options.figure_size = (8, 10) md = smf.mixedlm('log_score ~ percent_broken + percent_fail_runs', df, groups=df.index.values) mdf_rul = md.fit() print('#' * 18 + 'Log RUL' + '#' * 18) print(mdf_rul.summary()) md = smf.mixedlm('mse ~ percent_broken + percent_fail_runs', df, groups=df.index.values) mdf_mse = md.fit() print('#' * 18 + 'RMSE' + '#' * 18) print(mdf_mse.summary()) df['percent_broken'] = df['percent_broken'].round().astype(np.int) df['percent_fail_runs'] = df['percent_fail_runs'].round().astype(np.int) gg = (plotnine.ggplot( df, plotnine.aes(x=x_axis, y='log_score', color='method')) + plotnine.geom_jitter(width=2.5, show_legend=False) + plotnine.geom_abline( plotnine.aes(intercept=mdf_rul.params['Intercept'], slope=mdf_rul.params[x_axis])) + plotnine.stat_smooth(method='gls', show_legend=False) + plotnine.xlab(x_label) + plotnine.ylab('Logarithmic RUL-Score') + plotnine.scale_color_discrete(name='Method', labels=['DAAN', 'JAN']) + plotnine.theme_classic(base_size=20)) gg.save('%s_log_rul_by_method.pdf' % x_axis) gg = (plotnine.ggplot( df, plotnine.aes(x=x_axis, y='log_score', color='task')) + plotnine.geom_jitter(width=2.5, show_legend=False) + plotnine.geom_abline( plotnine.aes(intercept=mdf_rul.params['Intercept'], slope=mdf_rul.params[x_axis])) + plotnine.stat_smooth(method='gls', show_legend=False) + plotnine.xlab(x_label) + plotnine.ylab('Logarithmic RUL-Score') + plotnine.scale_color_discrete( name='Task', labels=['4→3', '4→2', '1→3', '1→2', '3→4', '3→1', '2→4', '2→1' ]) + plotnine.theme_classic(base_size=20)) gg.save('%s_log_rul_by_task.pdf' % x_axis) gg = ( plotnine.ggplot(df, plotnine.aes(x=x_axis, y='mse', color='method')) + plotnine.geom_jitter(width=2.5) + plotnine.geom_abline( plotnine.aes(intercept=mdf_mse.params['Intercept'], slope=mdf_mse.params[x_axis])) + plotnine.stat_smooth(method='gls') + plotnine.ylab('RMSE') + plotnine.xlab(x_label) + plotnine.scale_color_discrete(name='Method', labels=['DAAN', 'JAN']) + plotnine.theme_classic(base_size=20)) gg.save('%s_mse_by_method.pdf' % x_axis) gg = (plotnine.ggplot(df, plotnine.aes(x=x_axis, y='mse', color='task')) + plotnine.geom_jitter(width=2.5) + plotnine.geom_abline( plotnine.aes(intercept=mdf_mse.params['Intercept'], slope=mdf_mse.params[x_axis])) + plotnine.stat_smooth(method='gls') + plotnine.ylab('RMSE') + plotnine.scale_color_discrete( name='Task', labels=['4→3', '4→2', '1→3', '1→2', '3→4', '3→1', '2→4', '2→1' ]) + plotnine.theme_classic(base_size=20)) gg.save('%s_mse_by_task.pdf' % x_axis)