def get_incumbent_trajectory_for_budget(self, result, budget):
"""
Returns the best configurations over time for a single budget
Parameters
----------
budget: string
budget to be considered
result: Result
result object with runs
Returns
-------
dict:
dictionary with all the config IDs, the times the runs
finished, their respective budgets, and corresponding losses
"""
if not budget in result.HB_config['budgets']:
raise ValueError(
"Budget '{}' (type: {}) does not exist. Choose from {}".format(
str(budget), str(type(budget)), "[" + ", ".join([
str(b) + " (type: " + str(type(b)) + ")"
for b in result.HB_config['budgets']
]) + "]"))
return get_incumbent_trajectory(result, [budget])
def test_incumbent_trajectory(self):
""" Load example result and check incumbent_trajectory generation for general errors (whitebox-test)"""
result = logged_results_to_HBS_result(self.result_path)
# All budgets
traj = get_incumbent_trajectory(result,
result.HB_config['budgets'],
mode='racing')
traj = get_incumbent_trajectory(result,
result.HB_config['budgets'],
mode='minimum')
traj = get_incumbent_trajectory(result,
result.HB_config['budgets'],
mode='prefer_higher_budget')
# Single budgets
traj = get_incumbent_trajectory(result,
[result.HB_config['budgets'][0]],
mode='racing')
traj = get_incumbent_trajectory(result,
[result.HB_config['budgets'][0]],
mode='minimum')
traj = get_incumbent_trajectory(result,
[result.HB_config['budgets'][0]],
mode='prefer_higher_budget')
def _get_bohb_line(self, validator, runs, rh, budget=None):
label = self.formatted_budgets[budget] if budget else 'all budgets'
if budget is None:
budgets = self.bohb_results[0].HB_config['budgets']
else:
budgets = [budget]
data = OrderedDict()
for idx, bohb_result in enumerate(self.bohb_results):
data[idx] = {'costs': [], 'times': []}
traj_dict = get_incumbent_trajectory(bohb_result,
budgets,
mode=self.cot_inc_traj)
data[idx]['costs'] = traj_dict['losses']
data[idx]['times'] = traj_dict['times_finished']
# Average over parallel bohb iterations to get final values
f_time, f_config, f_mean, f_std = [], [], [], []
pointer = OrderedDict([(idx, {
'cost': np.nan,
'time': 0
}) for idx in list(data.keys())])
while (len(data) > 0):
next_idx = min({idx: data[idx]['times'][0]
for idx in data.keys()}.items(),
key=lambda x: x[1])[0]
pointer[next_idx] = {
'cost': data[next_idx]['costs'].pop(0),
'time': data[next_idx]['times'].pop(0)
}
f_time.append(pointer[next_idx]['time'])
f_mean.append(
np.nanmean([values['cost'] for values in pointer.values()]))
f_std.append(
np.nanstd([values['cost'] for values in pointer.values()]))
if len(data[next_idx]['times']) == 0:
data.pop(next_idx)
time_double = [t for sub in zip(f_time, f_time) for t in sub][1:]
mean_double = [t for sub in zip(f_mean, f_mean) for t in sub][:-1]
std_double = [t for sub in zip(f_std, f_std) for t in sub][:-1]
configs_double = ['N/A' for _ in time_double]
return Line(str(label), time_double, mean_double,
[x + y for x, y in zip(mean_double, std_double)],
[x - y
for x, y in zip(mean_double, std_double)], configs_double)
def get_trajectory(self, result, output_path, scenario, rh, budget=None):
"""
If budget is specified, get trajectory for only that budget. Else use hpbandster's averaging.
TODO: would like to be rewritten
"""
cs = scenario.cs
if not output_path:
output_path = tempfile.mkdtemp()
traj_logger = TrajLogger(output_path, Stats(scenario))
total_traj_dict = []
if budget:
traj_dict = get_incumbent_trajectory(result, [budget])
else:
traj_dict = result.get_incumbent_trajectory()
id2config_mapping = result.get_id2config_mapping()
for config_id, time, budget, loss in zip(traj_dict['config_ids'],
traj_dict['times_finished'],
traj_dict['budgets'],
traj_dict['losses']):
incumbent = self._get_config(config_id, id2config_mapping, cs)
try:
incumbent_id = rh.config_ids[incumbent]
except KeyError as e:
# This config was not evaluated on this budget, just skip it
continue
except:
raise
total_traj_dict.append({
'config_id': incumbent_id,
'time_finished': time,
'budget': budget,
'loss': loss
})
last_loss = np.inf
for element in sorted(total_traj_dict,
key=lambda x: x['time_finished']):
incumbent_id = element["config_id"]
incumbent = rh.ids_config[incumbent_id]
time = element["time_finished"]
loss = element["loss"]
if loss > last_loss:
continue
ta_runs = -1
ta_time_used = -1
wallclock_time = time
train_perf = loss
# add
traj_logger.trajectory.append({
"cpu_time": ta_time_used,
"total_cpu_time": None, # TODO: fix this
"wallclock_time": wallclock_time,
"evaluations": ta_runs,
"cost": train_perf,
"incumbent": incumbent
})
traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent,
ta_time_used, wallclock_time)
traj_logger._add_in_aclib_format(train_perf, incumbent_id,
incumbent, ta_time_used,
wallclock_time)
return traj_logger.trajectory