def get_trajectory(self, folder2result, output_path, scenario, rh, budget=None):
"""
If budget is specified, get trajectory for only that budget. Else use hpbandster's averaging.
If multiple results are specified, sort by times_finished and only add to combined trajectory if loss is better
"""
cs = scenario.cs
if not output_path:
output_path = tempfile.mkdtemp()
traj_logger = TrajLogger(output_path, Stats(scenario))
total_traj_dict = []
for f, result in folder2result.items():
if budget:
traj_dict = self.get_incumbent_trajectory_for_budget(result, budget)
else:
traj_dict = result.get_incumbent_trajectory()
id2config_mapping = result.get_id2config_mapping()
for config_id, time, budget, loss in zip(traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses']):
incumbent = self._get_config(config_id, id2config_mapping, cs)
try:
incumbent_id = rh.config_ids[incumbent]
except KeyError as e:
# This config was not evaluated on this budget, just skip it
continue
except:
raise
total_traj_dict.append({'config_id' : incumbent_id, 'time_finished' : time, 'budget' : budget, 'loss' : loss})
last_loss = np.inf
for element in sorted(total_traj_dict, key=lambda x: x['time_finished']):
incumbent_id = element["config_id"]
incumbent = rh.ids_config[incumbent_id]
time = element["time_finished"]
loss = element["loss"]
if loss > last_loss:
continue
ta_runs = -1
ta_time_used = -1
wallclock_time = time
train_perf = loss
# add
traj_logger.trajectory.append({"cpu_time": ta_time_used,
"total_cpu_time": None, # TODO: fix this
"wallclock_time": wallclock_time,
"evaluations": ta_runs,
"cost": train_perf,
"incumbent": incumbent
})
traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time)
traj_logger._add_in_aclib_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time)
return traj_logger.trajectory
def hpbandster2smac(self, result, cs: ConfigurationSpace, output_dir: str):
"""Reading hpbandster-result-object and creating RunHistory and
trajectory...
treats each budget as an individual 'smac'-run, creates an
output-directory with subdirectories for each budget.
Parameters
----------
result: hpbandster.core.result.Result
bohb's result-object
cs: ConfigurationSpace
the configuration space
output_dir: str
the output-dir to save the smac-runs to
"""
# Create runhistories (one per budget)
id2config_mapping = result.get_id2config_mapping()
budget2rh = {}
for run in result.get_all_runs():
if not run.budget in budget2rh:
budget2rh[run.budget] = RunHistory(average_cost)
rh = budget2rh[run.budget]
rh.add(config=Configuration(
cs, id2config_mapping[run.config_id]['config']),
cost=run.loss,
time=run.time_stamps['finished'] -
run.time_stamps['started'],
status=StatusType.SUCCESS,
seed=0,
additional_info={'info': run.info})
# Write to disk
budget2path = {} # paths to individual budgets
for b, rh in budget2rh.items():
output_path = os.path.join(output_dir, 'budget_' + str(b))
budget2path[b] = output_path
scenario = Scenario({'run_obj': 'quality', 'cs': cs})
scenario.output_dir_for_this_run = output_path
scenario.write()
rh.save_json(fn=os.path.join(output_path, 'runhistory.json'))
# trajectory
traj_dict = result.get_incumbent_trajectory()
traj_logger = TrajLogger(output_path, Stats(scenario))
for config_id, time, budget, loss in zip(
traj_dict['config_ids'], traj_dict['times_finished'],
traj_dict['budgets'], traj_dict['losses']):
incumbent = Configuration(
cs, id2config_mapping[config_id]['config'])
try:
incumbent_id = rh.config_ids[incumbent]
except KeyError as e:
# This config was not evaluated on this budget, just skip it
continue
except:
raise
ta_runs = -1
ta_time_used = -1
wallclock_time = time
train_perf = loss
# add
traj_logger._add_in_old_format(train_perf, incumbent_id,
incumbent, ta_time_used,
wallclock_time)
traj_logger._add_in_aclib_format(train_perf, incumbent_id,
incumbent, ta_time_used,
wallclock_time)
return budget2path
