def bohb_plot(self, bohb_result, filename='bohb_plot.png'):
""" Import plot from bohb """
try:
from hpbandster.core.result import extract_HB_learning_curves
from hpbandster.visualization import interactive_HB_plot, default_tool_tips
except ImportError as e:
raise ImportError(
"To analyze BOHB-data, please install hpbandster (e.g. `pip install hpbandster`)"
)
filename = os.path.join(self.output_dir, filename)
# Hpbandster contains also a visualization tool to plot the
# 'learning curves' of the sampled configurations
incumbent_trajectory = bohb_result.get_incumbent_trajectory()
lcs = bohb_result.get_learning_curves(
lc_extractor=extract_HB_learning_curves)
tool_tips = default_tool_tips(bohb_result, lcs)
fig, ax, check, none_button, all_button = interactive_HB_plot(
lcs, tool_tip_strings=tool_tips, show=False)
ax.set_ylim([0.1 * incumbent_trajectory['losses'][-1], 1])
#ax.set_yscale('log')
fig.savefig(filename)
plt.close(fig)
return filename
hpvis.finished_runs_over_time(all_runs)
# This one visualizes the spearman rank correlation coefficients of the losses
# between different budgets.
hpvis.correlation_across_budgets(result)
# For model based optimizers, one might wonder how much the model actually helped.
# The next plot compares the performance of configs picked by the model vs. random ones
hpvis.performance_histogram_model_vs_random(all_runs, id2conf)
plt.show()
d1 = res.get_pandas_dataframe()[0]
loss = res.get_pandas_dataframe()[1]
d1['loss'] = loss
if False:
result = res
# get all executed runs
all_runs = result.get_all_runs()
# get the 'dict' that translates config ids to the actual configurations
id2conf = result.get_id2config_mapping()
lcs = result.get_learning_curves()
hpvis.interactive_HBS_plot(lcs,
tool_tip_strings=hpvis.default_tool_tips(
result, lcs))
bandwidth_factor=1,
num_samples=32,
min_bandwidth=1e-1
)
res = HB.run(4)
# After the optimizer run, we shutdown the master.
HB.shutdown(shutdown_workers=True)
NS.shutdown()
# BOHB will return a result object.
# It holds informations about the optimization run like the incumbent (=best) configuration.
# For further details about the result-object, see its documentation.
id2config = res.get_id2config_mapping()
print('A total of %i unique configurations where sampled.'%len(id2config.keys()))
print('A total of %i runs where executed.'%len(res.get_all_runs()))
# Hpbandster contains also a visualization tool to plot the
# 'learning curves' of the sampled configurations
incumbent_trajectory = res.get_incumbent_trajectory()
lcs = res.get_learning_curves(lc_extractor=extract_HB_learning_curves)
tool_tips = default_tool_tips(res, lcs)
fig, ax, check, none_button, all_button = interactive_HB_plot(lcs, tool_tip_strings=tool_tips, show=False)
ax.set_ylim([0.1*incumbent_trajectory['losses'][-1], 1])
ax.set_yscale('log')
plt.show()
[(r.budget, r.info['test_error']) for r in sr],
]
return (value)
bohb_logs_dir = '{}/search_space_{}/run{}-seed{}'.format(
args.working_directory, args.space, args.run_id, args.seed)
res = hpres.logged_results_to_HB_result(bohb_logs_dir)
lcs_temp = res.get_learning_curves(lc_extractor=extract_HB_learning_curves)
lcs = dict(lcs_temp)
for key, value in lcs_temp.items():
if value == [[]]:
del lcs[key]
tool_tips = hpvis.default_tool_tips(res, lcs)
#embed()
inc_id = res.get_incumbent_id()
id2conf = res.get_id2config_mapping()
inc_trajectory = res.get_incumbent_trajectory()
print(inc_trajectory)
print(res.get_runs_by_id(inc_id))
all_runs = list(
filter(lambda r: not (r.info is None or r.loss is None),
res.get_all_runs()))
budgets = res.HB_config['budgets']
import hpbandster.visualization as hpvis
# load the example run from the log files
result = hpres.logged_results_to_HBS_result('example_5_run/')
# get all executed runs
all_runs = result.get_all_runs()
# get the 'dict' that translates config ids to the actual configurations
id2conf = result.get_id2config_mapping()
lcs = result.get_learning_curves()
hpvis.interactive_HBS_plot(lcs, tool_tip_strings=hpvis.default_tool_tips(result, lcs))
def realtime_learning_curves(runs):
"""
example how to extract a different kind of learning curve.
The x values are now the time the runs finished, not the budget anymore.
We no longer plot the validation loss on the y axis, but now the test accuracy.
This is just to show how to get different information into the interactive plot.
"""
sr = sorted(runs, key=lambda r: r.budget)