def test_budgets(self):
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs, values={"a": 1, "b": 2})
config2 = Configuration(cs, values={"a": 2, "b": 3})
rh.add(
config=config1,
cost=[0, 50],
time=5,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=5,
)
rh.add(
config=config1,
cost=[40, 100],
time=10,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=15,
)
# SMAC does not overwrite by default
rh.add(
config=config1,
cost=[50, 100],
time=10,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=15,
)
rh.add(
config=config2,
cost=[0, 150],
time=15,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=5,
)
self.assertEqual(rh.objective_bounds[0], (0, 40))
self.assertEqual(rh.objective_bounds[1], (50, 150))
# Average cost returns us the cost of the latest budget
self.assertEqual(rh.average_cost(config1), 0.75)
self.assertEqual(rh.average_cost(config2), 0.5)
def _compare_configs(self,
incumbent: Configuration,
challenger: Configuration,
run_history: RunHistory,
log_traj: bool = True) -> typing.Optional[Configuration]:
"""
Compare two configuration wrt the runhistory and return the one which
performs better (or None if the decision is not safe)
Decision strategy to return x as being better than y:
1. x has at least as many runs as y
2. x performs better than y on the intersection of runs on x and y
Implicit assumption:
Challenger was evaluated on the same instance-seed pairs as
incumbent
Parameters
----------
incumbent: Configuration
Current incumbent
challenger: Configuration
Challenger configuration
run_history: smac.runhistory.runhistory.RunHistory
Stores all runs we ran so far
log_traj: bool
Whether to log changes of incumbents in trajectory
Returns
-------
None or better of the two configurations x,y
"""
inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True)
chall_runs = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
to_compare_runs = set(inc_runs).intersection(chall_runs)
# performance on challenger runs
chal_perf = run_history.average_cost(challenger, to_compare_runs)
inc_perf = run_history.average_cost(incumbent, to_compare_runs)
# Line 15
if chal_perf > inc_perf and len(chall_runs) >= self.minR:
# Incumbent beats challenger
self.logger.debug("Incumbent (%.4f) is better than challenger "
"(%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Line 16
if not set(inc_runs) - set(chall_runs):
# no plateau walks
if chal_perf >= inc_perf:
self.logger.debug("Incumbent (%.4f) is at least as good as the "
"challenger (%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Challenger is better than incumbent
# and has at least the same runs as inc
# -> change incumbent
n_samples = len(chall_runs)
self.logger.info("Challenger (%.4f) is better than incumbent (%.4f)"
" on %d runs." % (chal_perf, inc_perf, n_samples))
self._log_incumbent_changes(incumbent, challenger)
if log_traj:
self.stats.inc_changed += 1
self.traj_logger.add_entry(train_perf=chal_perf,
incumbent_id=self.stats.inc_changed,
incumbent=challenger)
return challenger
# undecided
return None
def test_instances(self):
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs, values={"a": 1, "b": 2})
config2 = Configuration(cs, values={"a": 2, "b": 3})
rh.add(
config=config1,
cost=[0, 10],
time=5,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=0,
)
rh.add(
config=config1,
cost=[50, 20],
time=10,
status=StatusType.SUCCESS,
instance_id=2,
seed=1,
budget=0,
)
rh.add(
config=config1,
cost=[75, 20],
time=10,
status=StatusType.SUCCESS,
instance_id=3,
seed=1,
budget=0,
)
rh.add(
config=config2,
cost=[100, 30],
time=15,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
budget=0,
)
rh.add(
config=config2,
cost=[0, 30],
time=15,
status=StatusType.SUCCESS,
instance_id=2,
seed=1,
budget=0,
)
self.assertEqual(rh.objective_bounds[0], (0, 100))
self.assertEqual(rh.objective_bounds[1], (10, 30))
# Average cost returns us the cost of the latest budget
self.assertEqual(rh.average_cost(config1), 0.375)
self.assertEqual(rh.average_cost(config2), 0.75)
