def load(dataset, method, datafolder=None): print(dataset, method) method, L = get_methodName_and_L(method) if method == "slang": all_exps = slang.slang_base.get_variant("slang_convergence_final").variants def is_relevant(exp): return (DATASET_VILIB_NAME[dataset] in exp and "L_" + str(L) + "_" in exp) relevant_exps = [exp for exp in all_exps if is_relevant(exp)] results = get_experiment_results(slang.slang_base, "slang_convergence_final", relevant_exps) else: all_exps = bbb.bbb_copy_slang.get_variant("bbb_convergence_final").variants def is_relevant(exp): return DATASET_VILIB_NAME[dataset] in exp relevant_exps = [exp for exp in all_exps if is_relevant(exp)] results = get_experiment_results(bbb.bbb_copy_slang, "bbb_convergence_final", relevant_exps) nlls = [] nlZs = [] accs = [] for r in results: nlZs.append(r['metric_history']['elbo_neg_ave']) nlls.append(r['metric_history']['test_pred_logloss']) accs.append(r['metric_history']['test_pred_accuracy']) x = np.array(range(len(nlZs[0]))) * MINIBATCH_SIZES[dataset] / Ns[dataset] return x + 1, np.vstack(nlZs), np.vstack(nlls), np.vstack(accs)
def plot_slang_convergence(data_set, L, metrics, prior_prec_indices): if not (data_set == 'australian_presplit' or data_set == 'breastcancer_presplit' or data_set == 'usps_3vs5'): raise ValueError('"' + data_set + '" is not a valid value for the data_set argument.') # possible datasets are: 'australian_presplit', 'breastcancer_presplit', or 'usps_3vs5'. # valid values of L are: 1 and 10. # possible metrics are: 'test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave', 'accuracy' or 'all'. # possible prior precisions are: # prior_precisions = [0.001, 0.01, 0.1, 1, 8, 32, 64, 128, 512] # prior_prec_indices is used to select these values and should be list containing integers between # 0 and 8. if metrics == 'all': metrics = ['test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave'] prior_precisions = [0.001, 0.01, 0.1, 1, 8, 32, 64, 128, 512] seeds = [1, 2, 3] variants_to_plot = list( filter(lambda z: (data_set in z) and (('_' + str(L) + '_') in z), variants)) experiment_base = slang.slang_cv prior_prec_variants = [] for i in prior_prec_indices: prec = prior_precisions[i] prior_string = ('prior_prec_' + str(prec)) v = list(filter(lambda z: prior_string in z, variants_to_plot)) prior_prec_variants.append(v) experiment_results = {} for i, list_of_restarts in enumerate(prior_prec_variants): crashed = False for restart in list_of_restarts: record = slang.slang_cv.get_variant(experiment_name).get_variant( restart).get_latest_record() crashed = crashed or (not record.has_result()) if not crashed: results = record_utils.get_experiment_results( slang.slang_cv, experiment_name, list_of_restarts) results = list(results[0].values()) experiment_results[prior_precisions[ i]] = record_utils.summarize_metric_histories(results) else: experiment_results[ prior_precisions[i]] = None # One of these restarts crashed. if not os.path.exists('plots/prior_selection/'): os.makedirs('plots/prior_selection/') if not os.path.exists('plots/prior_selection/' + data_set + '/'): os.makedirs('plots/prior_selection/' + data_set + '/') ###################### #### Plot Results #### ###################### plt.ioff() if 'test_pred_logloss' in metrics: # Plot test logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(1 + np.arange(5001), item['test_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Test logloss") plt.title("L = " + str(L) + ", Dataset = " + data_set) plt.savefig("plots/prior_selection/" + data_set + "/slang_L_" + str(L) + "_Dataset_" + data_set + "_test_logloss.pdf") plt.close() if 'train_pred_logloss' in metrics: # Plot train logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(1 + np.arange(5001), item['train_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Train logloss") plt.title("L = " + str(L) + ", Dataset = " + data_set) plt.savefig("plots/prior_selection/" + data_set + "/slang_L_" + str(L) + "_Dataset_" + data_set + "_train_logloss.pdf") plt.close() if 'elbo_neg_ave' in metrics: # Plot ELBO plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.loglog(1 + np.arange(5001), item['elbo_neg_ave']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Neg. Average ELBO") plt.title("L = " + str(L) + ", Dataset = " + data_set) plt.savefig("plots/prior_selection/" + data_set + "/slang_L_" + str(L) + "_Dataset_" + data_set + "_elbo.pdf") plt.close()
def plot_bbb_convergence(data_set, metrics, lr_indices): if not (data_set == 'australian_presplit' or data_set == 'breastcancer_presplit' or data_set == 'usps_3vs5'): raise ValueError('"' + data_set + '" is not a valid value for the data_set argument.') # possible datasets are: 'australian_presplit', 'breastcancer_presplit', or 'usps_3vs5'. # possible metrics are: 'test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave', 'accuracy' or 'all'. # possible learning rates are: # [0.0001, 0.00021544, 0.00046416, 0.001, 0.00215443, 0.00464159, 0.01, 0.02154435, 0.04641589, 0.1] # lr_indices is used to select these values and should be list containing integers between # 0 and 9. if metrics == 'all': metrics = [ 'test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave', 'accuracy' ] seeds = [1, 2, 3] variants_to_plot = list(filter(lambda z: (data_set in z), variants)) lrs = np.logspace(-4, -1, 10) lrs = lrs[lr_indices] lr_variants = [] for lr in lrs: lr_string = ('lr_' + str(lr)) v = list(filter(lambda z: lr_string in z, variants_to_plot)) lr_variants.append(v) experiment_results = {} for i, list_of_restarts in enumerate(lr_variants): crashed = False for restart in list_of_restarts: record = bbb.bbb_cv.get_variant(experiment_name).get_variant( restart).get_latest_record() crashed = crashed or (not record.has_result()) if not crashed: results = record_utils.get_experiment_results( bbb.bbb_cv, experiment_name, list_of_restarts) results = list(results[0].values()) experiment_results[ lrs[i]] = record_utils.summarize_metric_histories(results) else: experiment_results[lrs[i]] = None # One of these restarts crashed. if not os.path.exists('plots/lr_selection/'): os.makedirs('plots/lr_selection/') if not os.path.exists('plots/lr_selection/' + data_set + '/'): os.makedirs('plots/lr_selection/' + data_set + '/') ###################### #### Plot Results #### ###################### plt.ioff() if 'test_pred_logloss' in metrics: # Plot test logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(1 + np.arange(1001), item['test_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Test logloss") plt.title("Dataset = " + data_set) plt.savefig("plots/lr_selection/" + data_set + "/bbb_" + "_Dataset_" + data_set + "_test_logloss.pdf") plt.close() if 'train_pred_logloss' in metrics: # Plot train logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(1 + np.arange(1001), item['train_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Train logloss") plt.title("Dataset = " + data_set) plt.savefig("plots/lr_selection/" + data_set + "/bbb_" + "_Dataset_" + data_set + "_train_logloss.pdf") plt.close() if 'elbo_neg_ave' in metrics: # Plot ELBO plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.loglog(1 + np.arange(1001), item['elbo_neg_ave']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Neg. Average ELBO") plt.title("Dataset = " + data_set) plt.savefig("plots/lr_selection/" + data_set + "/bbb_" + "_Dataset_" + data_set + "_elbo.pdf") plt.close()
def plot_convergence(data_set, metrics): if not (data_set == 'australian_presplit' or data_set == 'breastcancer_presplit' or data_set == 'usps_3vs5'): raise ValueError('"' + data_set + '" is not a valid value for the data_set argument.') # possible datasets are: 'australian_presplit', 'breastcancer_presplit', or 'usps_3vs5'. # possible metrics are: 'test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave', 'accuracy' or 'all'. if metrics == 'all': metrics = ['test_pred_logloss', 'train_pred_logloss', 'elbo_neg_ave', 'accuracy'] random_seeds = np.arange(1,11) slang_variants_to_plot = list(filter(lambda z: (data_set in z), slang_final.variants)) bbb_variants_to_plot = list(filter(lambda z: (data_set in z), bbb_final.variants)) experiment_base = slang.slang_base L_variants = [] Ls = [1,8,32,64] for l in Ls: l_string = ('L_' + str(l)) v = list(filter(lambda z: l_string in z, slang_variants_to_plot)) L_variants.append(v) experiment_results = {} for i, list_of_restarts in enumerate(L_variants): crashed = False for restart in list_of_restarts: record = slang.slang_base.get_variant(slang_final.experiment_name).get_variant(restart).get_latest_record() crashed = crashed or (not record.has_result()) if not crashed: results = record_utils.get_experiment_results(slang.slang_base, slang_final.experiment_name, list_of_restarts) experiment_results[Ls[i]] = record_utils.summarize_metric_histories(results) else: experiment_results[Ls[i]] = None # One of these restarts crashed. results = record_utils.get_experiment_results(bbb.bbb_copy_slang, bbb_final.experiment_name, bbb_variants_to_plot) experiment_results["BBB"] = record_utils.summarize_metric_histories(results) # create the necessary directories if they don't exist. if not os.path.exists('plots/final_plots/'): os.makedirs('plots/final_plots/') if not os.path.exists('plots/final_plots/' + data_set + '/'): os.makedirs('plots/final_plots/' + data_set + '/') if data_set == 'usps_3vs5': iters = 6501 else: iters = 6001 ###################### #### Plot Results #### ###################### plt.ioff() if 'test_pred_logloss' in metrics: # Plot test logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(np.arange(iters), item['test_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Test logloss") plt.title("Dataset = " + data_set) plt.savefig("plots/final_plots/" + data_set + "/test_logloss.pdf") plt.close() if 'accuracy' in metrics: # Plot test logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(np.arange(iters), item['test_pred_accuracy']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Test Accuracy") plt.title("Dataset = " + data_set) plt.savefig("plots/final_plots/" + data_set + "/test_accuracy.pdf") plt.close() if 'train_pred_logloss' in metrics: # Plot train logloss plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.plot(np.arange(iters), item['train_pred_logloss']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Train logloss") plt.title("Dataset = " + data_set) plt.savefig("plots/final_plots/" + data_set + "/train_logloss.pdf") plt.close() if 'elbo_neg_ave' in metrics: # Plot ELBO plt.figure() labels = [] for key, item in experiment_results.items(): if not item is None: plt.loglog(np.arange(iters), item['elbo_neg_ave']['mean']) labels.append(str(key)) plt.legend(labels) plt.grid() plt.xlabel("Epoch") plt.ylabel("Neg. Average ELBO") plt.title("Dataset = " + data_set) plt.savefig("plots/final_plots/" + data_set + "/elbo.pdf") plt.close()