def get_trajectory(self, folder2result, output_path, scenario, rh, budget=None): """ If budget is specified, get trajectory for only that budget. Else use hpbandster's averaging. If multiple results are specified, sort by times_finished and only add to combined trajectory if loss is better """ cs = scenario.cs if not output_path: output_path = tempfile.mkdtemp() traj_logger = TrajLogger(output_path, Stats(scenario)) total_traj_dict = [] for f, result in folder2result.items(): if budget: traj_dict = self.get_incumbent_trajectory_for_budget(result, budget) else: traj_dict = result.get_incumbent_trajectory() id2config_mapping = result.get_id2config_mapping() for config_id, time, budget, loss in zip(traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses']): incumbent = self._get_config(config_id, id2config_mapping, cs) try: incumbent_id = rh.config_ids[incumbent] except KeyError as e: # This config was not evaluated on this budget, just skip it continue except: raise total_traj_dict.append({'config_id' : incumbent_id, 'time_finished' : time, 'budget' : budget, 'loss' : loss}) last_loss = np.inf for element in sorted(total_traj_dict, key=lambda x: x['time_finished']): incumbent_id = element["config_id"] incumbent = rh.ids_config[incumbent_id] time = element["time_finished"] loss = element["loss"] if loss > last_loss: continue ta_runs = -1 ta_time_used = -1 wallclock_time = time train_perf = loss # add traj_logger.trajectory.append({"cpu_time": ta_time_used, "total_cpu_time": None, # TODO: fix this "wallclock_time": wallclock_time, "evaluations": ta_runs, "cost": train_perf, "incumbent": incumbent }) traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) traj_logger._add_in_aclib_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) return traj_logger.trajectory
def hpbandster2smac(self, result, cs: ConfigurationSpace, output_dir: str): """Reading hpbandster-result-object and creating RunHistory and trajectory... treats each budget as an individual 'smac'-run, creates an output-directory with subdirectories for each budget. Parameters ---------- result: hpbandster.core.result.Result bohb's result-object cs: ConfigurationSpace the configuration space output_dir: str the output-dir to save the smac-runs to """ # Create runhistories (one per budget) id2config_mapping = result.get_id2config_mapping() budget2rh = {} for run in result.get_all_runs(): if not run.budget in budget2rh: budget2rh[run.budget] = RunHistory(average_cost) rh = budget2rh[run.budget] rh.add(config=Configuration( cs, id2config_mapping[run.config_id]['config']), cost=run.loss, time=run.time_stamps['finished'] - run.time_stamps['started'], status=StatusType.SUCCESS, seed=0, additional_info={'info': run.info}) # Write to disk budget2path = {} # paths to individual budgets for b, rh in budget2rh.items(): output_path = os.path.join(output_dir, 'budget_' + str(b)) budget2path[b] = output_path scenario = Scenario({'run_obj': 'quality', 'cs': cs}) scenario.output_dir_for_this_run = output_path scenario.write() rh.save_json(fn=os.path.join(output_path, 'runhistory.json')) # trajectory traj_dict = result.get_incumbent_trajectory() traj_logger = TrajLogger(output_path, Stats(scenario)) for config_id, time, budget, loss in zip( traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses']): incumbent = Configuration( cs, id2config_mapping[config_id]['config']) try: incumbent_id = rh.config_ids[incumbent] except KeyError as e: # This config was not evaluated on this budget, just skip it continue except: raise ta_runs = -1 ta_time_used = -1 wallclock_time = time train_perf = loss # add traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) traj_logger._add_in_aclib_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) return budget2path