def visualizeBOHB(log_dir): # load the example run from the log files result = hpres.logged_results_to_HBS_result(log_dir) # get all executed runs all_runs = result.get_all_runs() # get the 'dict' that translates config ids to the actual configurations id2conf = result.get_id2config_mapping() # Here is how you get he incumbent (best configuration) inc_id = result.get_incumbent_id() # let's grab the run on the highest budget inc_runs = result.get_runs_by_id(inc_id) inc_run = inc_runs[-1] # We have access to all information: the config, the loss observed during # optimization, and all the additional information inc_valid_score = inc_run.loss inc_config = id2conf[inc_id]['config'] print(inc_config) print('Best found configuration:') print(inc_config) #print('It achieved accuracies of %f (validation) and %f (test).' % (-inc_valid_score, inc_test_score)) # Let's plot the observed losses grouped by budget, hpvis.losses_over_time(all_runs) # the number of concurent runs, hpvis.concurrent_runs_over_time(all_runs) # and the number of finished runs. hpvis.finished_runs_over_time(all_runs) # This one visualizes the spearman rank correlation coefficients of the losses # between different budgets. hpvis.correlation_across_budgets(result) # For model based optimizers, one might wonder how much the model actually helped. # The next plot compares the performance of configs picked by the model vs. random ones hpvis.performance_histogram_model_vs_random(all_runs, id2conf) plot_accuracy_over_budget(result) plot_parallel_scatter(result) plt.show()
def generateViz(out_dir, show=False): ''' Generate plots for BOHB (from BOHB_visualizations.py from the documentations) :param out_dir: Directory to save the plots :param show: True/False to display the plots (additionally to saving) :return: void ''' result = hpres.logged_results_to_HBS_result(out_dir) # get all executed runs all_runs = result.get_all_runs() # get the 'dict' that translates config ids to the actual configurations id2conf = result.get_id2config_mapping() # Let's plot the observed losses grouped by budget, hpvis.losses_over_time(all_runs) plt.tight_layout() plt.savefig(out_dir + '/plot_losses_over_time.png', dpi=300) # the number of concurent runs, hpvis.concurrent_runs_over_time(all_runs) plt.tight_layout() plt.savefig(out_dir + '/plot_concurrent_runs_over_time.png', dpi=300) # and the number of finished runs. hpvis.finished_runs_over_time(all_runs) plt.tight_layout() plt.savefig(out_dir + '/plot_finished_runs_over_time.png', dpi=300) # This one visualizes the spearman rank correlation coefficients of the losses # between different budgets. hpvis.correlation_across_budgets(result) figure = plt.gcf() figure.set_size_inches(10, 10) plt.savefig(out_dir + '/plot_correlation_across_budgets.png', dpi=300) # For model based optimizers, one might wonder how much the model actually helped. # The next plot compares the performance of configs picked by the model vs. random ones hpvis.performance_histogram_model_vs_random(all_runs, id2conf) figure = plt.gcf() figure.set_size_inches(10, 10) plt.savefig(out_dir + '/plot_performance_histogram.png', dpi=150) if show: plt.show()
# Each optimizer returns a hpbandster.core.result.Result object. # It holds information about the optimization run like the incumbent (=best) configuration. # For further details about the Result object, see its documentation. # Here we simply print out the best config and some statistics about the performed runs. id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() print('Best found configuration:', id2config[incumbent]['config']) # Plots the performance of the best found validation error over time all_runs = res.get_all_runs() # Let's plot the observed losses grouped by budget, import hpbandster.visualization as hpvis hpvis.losses_over_time(all_runs) import matplotlib.pyplot as plt plt.savefig("random_search.png") # TODO: retrain the best configuration (called incumbent) and compute the test error def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial) def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial) def conv2d(x, W):
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--result', type=str, required=True, help= "Final result Pickle file or master directory (for running experiments)" ) parser.add_argument('--mode', type=str, choices=('save', 'show', 'disable'), default='disable', help="Plot mode") parser.add_argument('--out-path', type=str, default=None, help="Default output path for plots") parser.add_argument('--dpi', type=int, default=150, help="Plot resolution") args = parser.parse_args() # load run results if os.path.isfile(args.result): default_out_path = os.path.dirname(os.path.abspath(args.result)) exp_name = os.path.splitext(os.path.basename(args.result))[0] with open(args.result, 'rb') as fp: result = pickle.load(fp) elif os.path.isdir(args.result): default_out_path = args.result exp_name = 'exp' result = hpres.logged_results_to_HBS_result(args.result) else: print("No input specified. Use --result") return save_figs = args.mode is None or args.mode == 'save' show_figs = args.mode is None or args.mode == 'show' # File path out_path = args.out_path or default_out_path # Get all executed runs all_runs = result.get_all_runs() # Get the 'dict' that translates config ids to the actual configurations id2conf = result.get_id2config_mapping() # Here is how you get he incumbent (best configuration) inc_id = result.get_incumbent_id() # let's grab the run on the highest budget inc_runs = result.get_runs_by_id(inc_id) inc_run = inc_runs[-1] # We have access to all information: the config, the loss observed during # optimization, and all the additional information inc_loss = inc_run.loss inc_config = id2conf[inc_id]['config'] # Each run contains one or more trainings # chosen_accs: list of the chosen best model accuracy (according to the bohb loss) for each single training chosen_accs = [] all_accs = [] for single_info in inc_run.info['single_info']: # All the BOHB losses of this training (one per epoch) bohb_losses = np.array(single_info['bohb_losses']) # Let's find the best one best_index = bohb_losses.argmin() # Add the best model (according to the bohb loss) accuracy to chosen_accs chosen_accs.append(single_info['target_accuracy'][best_index]) # Add all the accuracies of all the epochs of this training all_accs.append(single_info['target_accuracy']) # Get mean accuracy for this run (average the selected models for each training) acc = np.array(chosen_accs).mean() # Matrix containing ALL the target accuracies of all the epochs of all the trainings of this run all_accs = np.array(all_accs) # Print best configuration print('Best found configuration:') for k in inc_config: nice_print(k, inc_config[k]) nice_print('inc_id', '-'.join(map(str, inc_id))) print() print("Performance:") criterion_names = { 'regression': 'Regression', 'target_accuracy': 'Trg accuracy', 'target_entropy_loss': 'Trg entropy', 'target_div_loss': 'Trg diversity', 'target_class_loss': 'Trg class. loss', 'target_silhouette_score': 'Trg Silhouette', 'target_calinski_harabasz_score': 'Trg Calinski-Harabasz' } # Print info cname = inc_run.info['criterion'] nice_print(criterion_names.get(cname, cname), f"{inc_loss:.10f}") nice_print( "Accuracy", f"{acc * 100:.4f} % (mean of each selected model in selected conf run trainings)" ) nice_print( "Accuracy", f"{all_accs.max(initial=-1) * 100:.4f} % (best in selected run, you shouldn't know this)" ) print() print("Resources:") nice_print( "Total time", datetime.timedelta(seconds=all_runs[-1].time_stamps['finished'] - all_runs[0].time_stamps['started'])) durations = list( map(lambda r: r.time_stamps['finished'] - r.time_stamps['started'], all_runs)) nice_print("Number of runs", len(all_runs)) nice_print("Longest run", datetime.timedelta(seconds=max(durations))) nice_print("Shortest run", datetime.timedelta(seconds=min(durations))) gpu_seconds = sum([ r.time_stamps['finished'] - r.time_stamps['started'] for r in all_runs ]) nice_print("GPU time", datetime.timedelta(seconds=gpu_seconds)) if not (save_figs or show_figs): return print() print("Generating plots") # Let's plot the observed losses grouped by budget, hpvis.losses_over_time(all_runs) if save_figs: plt.savefig(os.path.join(out_path, 'loss-over-time_{}.png'.format(exp_name)), dpi=args.dpi) # the number of concurrent runs, hpvis.concurrent_runs_over_time(all_runs) if save_figs: plt.savefig(os.path.join(out_path, 'concurrent-runs_{}.png'.format(exp_name)), dpi=args.dpi) # and the number of finished runs. hpvis.finished_runs_over_time(all_runs) if save_figs: plt.savefig(os.path.join(out_path, 'finished-runs_{}.png'.format(exp_name)), dpi=args.dpi) # This one visualizes the spearman rank correlation coefficients of the losses # between different budgets. hpvis.correlation_across_budgets(result) if save_figs: plt.savefig(os.path.join(out_path, 'correlation_{}.png'.format(exp_name)), dpi=args.dpi) # For model based optimizers, one might wonder how much the model actually helped. # The next plot compares the performance of configs picked by the model vs. random ones hpvis.performance_histogram_model_vs_random(all_runs, id2conf) if save_figs: plt.savefig(os.path.join(out_path, 'model-vs-random_{}.png'.format(exp_name)), dpi=args.dpi) sensitivity_plot(all_runs, id2conf, cvars=('disc.num_fc_layers', 'disc.hidden_size_log', 'disc.dropout', 'net.bottleneck_size_log', 'base.weight_da')) if save_figs: plt.savefig(os.path.join(out_path, 'sensitivity_{}.png'.format(exp_name)), dpi=args.dpi) if show_figs: plt.show()