def test_multiple_budgets(self):
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs, values={'a': 1, 'b': 2})
rh.add(config=config1,
cost=10,
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=1)
self.assertEqual(rh.get_cost(config1), 10)
# only the higher budget gets included in the config cost
rh.add(config=config1,
cost=20,
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=2)
self.assertEqual(rh.get_cost(config1), 20)
self.assertEqual(rh.get_min_cost(config1), 10)
def _compare_configs_across_budgets(
self,
challenger: Configuration,
incumbent: Configuration,
run_history: RunHistory,
log_traj: bool = True) -> typing.Optional[Configuration]:
"""
compares challenger with current incumbent on any budget
Parameters
----------
challenger : Configuration
promising configuration
incumbent : Configuration
best configuration so far
run_history : smac.runhistory.runhistory.RunHistory
stores all runs we ran so far
log_traj : bool
whether to log changes of incumbents in trajectory
Returns
-------
typing.Optional[Configuration]
incumbent configuration
"""
curr_budget = self.all_budgets[self.stage]
# compare challenger and incumbent based on cost
chall_cost = run_history.get_min_cost(challenger)
inc_cost = run_history.get_min_cost(incumbent)
if np.isfinite(chall_cost) and np.isfinite(inc_cost):
if chall_cost < inc_cost:
self.logger.info(
"Challenger (%.4f) is better than incumbent (%.4f) for any budget.",
chall_cost, inc_cost)
self._log_incumbent_changes(incumbent, challenger)
new_incumbent = challenger
if log_traj:
# adding incumbent entry
self.stats.inc_changed += 1 # first incumbent
self.traj_logger.add_entry(
train_perf=chall_cost,
incumbent_id=self.stats.inc_changed,
incumbent=new_incumbent,
budget=curr_budget)
else:
self.logger.debug(
"Incumbent (%.4f) is at least as good as the challenger (%.4f) for any budget.",
inc_cost, chall_cost)
if log_traj and self.stats.inc_changed == 0:
# adding incumbent entry
self.stats.inc_changed += 1 # first incumbent
self.traj_logger.add_entry(
train_perf=inc_cost,
incumbent_id=self.stats.inc_changed,
incumbent=incumbent,
budget=curr_budget)
new_incumbent = incumbent
else:
self.logger.debug('Non-finite costs from run history!')
new_incumbent = incumbent
return new_incumbent
