def get_incumbent_trajectory_for_budget(self, result, budget): """ Returns the best configurations over time for a single budget Parameters ---------- budget: string budget to be considered result: Result result object with runs Returns ------- dict: dictionary with all the config IDs, the times the runs finished, their respective budgets, and corresponding losses """ if not budget in result.HB_config['budgets']: raise ValueError( "Budget '{}' (type: {}) does not exist. Choose from {}".format( str(budget), str(type(budget)), "[" + ", ".join([ str(b) + " (type: " + str(type(b)) + ")" for b in result.HB_config['budgets'] ]) + "]")) return get_incumbent_trajectory(result, [budget])
def test_incumbent_trajectory(self): """ Load example result and check incumbent_trajectory generation for general errors (whitebox-test)""" result = logged_results_to_HBS_result(self.result_path) # All budgets traj = get_incumbent_trajectory(result, result.HB_config['budgets'], mode='racing') traj = get_incumbent_trajectory(result, result.HB_config['budgets'], mode='minimum') traj = get_incumbent_trajectory(result, result.HB_config['budgets'], mode='prefer_higher_budget') # Single budgets traj = get_incumbent_trajectory(result, [result.HB_config['budgets'][0]], mode='racing') traj = get_incumbent_trajectory(result, [result.HB_config['budgets'][0]], mode='minimum') traj = get_incumbent_trajectory(result, [result.HB_config['budgets'][0]], mode='prefer_higher_budget')
def _get_bohb_line(self, validator, runs, rh, budget=None): label = self.formatted_budgets[budget] if budget else 'all budgets' if budget is None: budgets = self.bohb_results[0].HB_config['budgets'] else: budgets = [budget] data = OrderedDict() for idx, bohb_result in enumerate(self.bohb_results): data[idx] = {'costs': [], 'times': []} traj_dict = get_incumbent_trajectory(bohb_result, budgets, mode=self.cot_inc_traj) data[idx]['costs'] = traj_dict['losses'] data[idx]['times'] = traj_dict['times_finished'] # Average over parallel bohb iterations to get final values f_time, f_config, f_mean, f_std = [], [], [], [] pointer = OrderedDict([(idx, { 'cost': np.nan, 'time': 0 }) for idx in list(data.keys())]) while (len(data) > 0): next_idx = min({idx: data[idx]['times'][0] for idx in data.keys()}.items(), key=lambda x: x[1])[0] pointer[next_idx] = { 'cost': data[next_idx]['costs'].pop(0), 'time': data[next_idx]['times'].pop(0) } f_time.append(pointer[next_idx]['time']) f_mean.append( np.nanmean([values['cost'] for values in pointer.values()])) f_std.append( np.nanstd([values['cost'] for values in pointer.values()])) if len(data[next_idx]['times']) == 0: data.pop(next_idx) time_double = [t for sub in zip(f_time, f_time) for t in sub][1:] mean_double = [t for sub in zip(f_mean, f_mean) for t in sub][:-1] std_double = [t for sub in zip(f_std, f_std) for t in sub][:-1] configs_double = ['N/A' for _ in time_double] return Line(str(label), time_double, mean_double, [x + y for x, y in zip(mean_double, std_double)], [x - y for x, y in zip(mean_double, std_double)], configs_double)
def get_trajectory(self, result, output_path, scenario, rh, budget=None): """ If budget is specified, get trajectory for only that budget. Else use hpbandster's averaging. TODO: would like to be rewritten """ cs = scenario.cs if not output_path: output_path = tempfile.mkdtemp() traj_logger = TrajLogger(output_path, Stats(scenario)) total_traj_dict = [] if budget: traj_dict = get_incumbent_trajectory(result, [budget]) else: traj_dict = result.get_incumbent_trajectory() id2config_mapping = result.get_id2config_mapping() for config_id, time, budget, loss in zip(traj_dict['config_ids'], traj_dict['times_finished'], traj_dict['budgets'], traj_dict['losses']): incumbent = self._get_config(config_id, id2config_mapping, cs) try: incumbent_id = rh.config_ids[incumbent] except KeyError as e: # This config was not evaluated on this budget, just skip it continue except: raise total_traj_dict.append({ 'config_id': incumbent_id, 'time_finished': time, 'budget': budget, 'loss': loss }) last_loss = np.inf for element in sorted(total_traj_dict, key=lambda x: x['time_finished']): incumbent_id = element["config_id"] incumbent = rh.ids_config[incumbent_id] time = element["time_finished"] loss = element["loss"] if loss > last_loss: continue ta_runs = -1 ta_time_used = -1 wallclock_time = time train_perf = loss # add traj_logger.trajectory.append({ "cpu_time": ta_time_used, "total_cpu_time": None, # TODO: fix this "wallclock_time": wallclock_time, "evaluations": ta_runs, "cost": train_perf, "incumbent": incumbent }) traj_logger._add_in_old_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) traj_logger._add_in_aclib_format(train_perf, incumbent_id, incumbent, ta_time_used, wallclock_time) return traj_logger.trajectory