def _get_instances_to_run(
self,
challenger: Configuration,
incumbent: Configuration,
N: int,
run_history: RunHistory,
) -> typing.Tuple[typing.List[InstSeedBudgetKey], float]:
"""
Returns the minimum list of <instance, seed> pairs to run the challenger on
before comparing it with the incumbent
Parameters
----------
incumbent: Configuration
incumbent configuration
challenger: Configuration
promising configuration that is presently being evaluated
run_history: RunHistory
Stores all runs we ran so far
N: int
number of <instance, seed> pairs to select
Returns
-------
typing.List[InstSeedBudgetKey]
list of <instance, seed, budget> tuples to run
float
total (runtime) cost of running the incumbent on the instances (used for adaptive capping while racing)
"""
# get next instances left for the challenger
# Line 8
inc_inst_seeds = set(
run_history.get_runs_for_config(incumbent,
only_max_observed_budget=True))
chall_inst_seeds = set(
run_history.get_runs_for_config(challenger,
only_max_observed_budget=True))
# Line 10
missing_runs = sorted(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
if N < 0:
raise ValueError(
'Argument N must not be smaller than zero, but is %s' % str(N))
to_run = missing_runs[:min(N, len(missing_runs))]
missing_runs = missing_runs[min(N, len(missing_runs)):]
# for adaptive capping
# because of efficiency computed here
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = run_history.sum_cost(
config=incumbent,
instance_seed_budget_keys=inst_seed_pairs,
)
return to_run, inc_sum_cost
def _adapt_cutoff(self,
challenger: Configuration,
run_history: RunHistory,
inc_sum_cost: float) -> float:
"""Adaptive capping:
Compute cutoff based on time so far used for incumbent
and reduce cutoff for next run of challenger accordingly
!Only applicable if self.run_obj_time
!runs on incumbent should be superset of the runs performed for the
challenger
Parameters
----------
challenger : Configuration
Configuration which challenges incumbent
run_history : smac.runhistory.runhistory.RunHistory
Stores all runs we ran so far
inc_sum_cost: float
Sum of runtimes of all incumbent runs
Returns
-------
cutoff: float
Adapted cutoff
"""
if not self.run_obj_time:
raise ValueError('This method only works when the run objective is quality')
curr_cutoff = self.cutoff if self.cutoff is not None else np.inf
# cost used by challenger for going over all its runs
# should be subset of runs of incumbent (not checked for efficiency
# reasons)
chall_inst_seeds = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
chal_sum_cost = run_history.sum_cost(
config=challenger,
instance_seed_budget_keys=chall_inst_seeds,
)
cutoff = min(curr_cutoff,
inc_sum_cost * self.adaptive_capping_slackfactor - chal_sum_cost
)
return cutoff
class TestAbstractIntensifier(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_get_next_challenger(self):
"""
test get_next_challenger - pick from list/chooser
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# Error when nothing to choose from
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier.get_next_challenger(challengers=None,
chooser=None,
run_history=self.rh)
# next challenger from a list
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config1)
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config2)
# next challenger from a chooser
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(24 in config.get_dictionary().values())
self.assertTrue(68 in config.get_dictionary().values())
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(95 in config.get_dictionary().values())
self.assertTrue(38 in config.get_dictionary().values())
def test_get_next_challenger_repeat(self):
"""
test get_next_challenger - repeat configurations
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# should not repeat configurations
self.rh.add(self.config1, 1, 1, StatusType.SUCCESS)
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=False)
self.assertEqual(config, self.config2)
# should repeat configurations
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=True)
self.assertEqual(config, self.config1)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
class TestAbstractRacer(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def eval_challenger(self,
challenger: Configuration,
incumbent: typing.Optional[Configuration],
run_history: RunHistory,
time_bound: float = float(MAXINT),
log_traj: bool = True) -> typing.Tuple[Configuration, float]:
"""
Running intensification via successive halving to determine the incumbent configuration.
*Side effect:* adds runs to run_history
Parameters
----------
challenger : Configuration
promising configuration
incumbent : typing.Optional[Configuration]
best configuration so far, None in 1st run
run_history : smac.runhistory.runhistory.RunHistory
stores all runs we ran so far
time_bound : float, optional (default=2 ** 31 - 1)
time in [sec] available to perform intensify
log_traj : bool
whether to log changes of incumbents in trajectory
Returns
-------
typing.Tuple[Configuration, float]
incumbent and incumbent cost
"""
# calculating the incumbent's performance for adaptive capping
# this check is required because:
# - there is no incumbent performance for the first ever 'intensify' run (from initial design)
# - during the 1st intensify run, the incumbent shouldn't be capped after being compared against itself
if incumbent and incumbent != challenger:
inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True)
inc_sum_cost = run_history.sum_cost(config=incumbent, instance_seed_budget_keys=inc_runs)
else:
inc_sum_cost = np.inf
if self.first_run:
self.logger.info("First run, no incumbent provided; challenger is assumed to be the incumbent")
incumbent = challenger
self.first_run = False
# select which instance to run current config on
curr_budget = self.all_budgets[self.stage]
# selecting instance-seed subset for this budget, depending on the kind of budget
if self.instance_as_budget:
prev_budget = int(self.all_budgets[self.stage - 1]) if self.stage > 0 else 0
curr_insts = self.inst_seed_pairs[int(prev_budget):int(curr_budget)]
else:
curr_insts = self.inst_seed_pairs
n_insts_remaining = len(curr_insts) - self.curr_inst_idx - 1
self.logger.debug(" Running challenger - %s" % str(challenger))
# run the next instance-seed pair for the given configuration
instance, seed = curr_insts[self.curr_inst_idx]
# selecting cutoff if running adaptive capping
cutoff = self.cutoff
if self.run_obj_time:
cutoff = self._adapt_cutoff(challenger=challenger,
run_history=run_history,
inc_sum_cost=inc_sum_cost)
if cutoff is not None and cutoff <= 0:
# ran out of time to validate challenger
self.logger.debug("Stop challenger intensification due to adaptive capping.")
self.curr_inst_idx = np.inf
self.logger.debug('Cutoff for challenger: %s' % str(cutoff))
try:
# run target algorithm for each instance-seed pair
self.logger.debug("Execute target algorithm")
try:
status, cost, dur, res = self.tae_runner.start(
config=challenger,
instance=instance,
seed=seed,
cutoff=cutoff,
budget=0.0 if self.instance_as_budget else curr_budget,
instance_specific=self.instance_specifics.get(instance, "0"),
# Cutoff might be None if self.cutoff is None, but then the first if statement prevents
# evaluation of the second if statement
capped=(self.cutoff is not None) and (cutoff < self.cutoff) # type: ignore[operator] # noqa F821
)
self._ta_time += dur
self.num_run += 1
self.curr_inst_idx += 1
except CappedRunException:
# We move on to the next configuration if a configuration is capped
self.logger.debug("Budget exhausted by adaptive capping; "
"Interrupting current challenger and moving on to the next one")
# ignore all pending instances
self.curr_inst_idx = np.inf
n_insts_remaining = 0
status = StatusType.CAPPED
# adding challengers to the list of evaluated challengers
# - Stop: CAPPED/CRASHED/TIMEOUT/MEMOUT (!= SUCCESS & DONOTADVANCE)
# - Advance to next stage: SUCCESS
# curr_challengers is a set, so "at least 1" success can be counted by set addition (no duplicates)
# If a configuration is successful, it is added to curr_challengers.
# if it fails it is added to fail_challengers.
if np.isfinite(self.curr_inst_idx) and status in [StatusType.SUCCESS, StatusType.DONOTADVANCE]:
self.success_challengers.add(challenger) # successful configs
else:
self.fail_challengers.add(challenger) # capped/crashed/do not advance configs
# get incumbent if all instances have been evaluated
if n_insts_remaining <= 0:
incumbent = self._compare_configs(challenger=challenger,
incumbent=incumbent,
run_history=run_history,
log_traj=log_traj)
except BudgetExhaustedException:
# Returning the final incumbent selected so far because we ran out of optimization budget
self.logger.debug("Budget exhausted; "
"Interrupting optimization run and returning current incumbent")
# if all configurations for the current stage have been evaluated, reset stage
num_chal_evaluated = len(self.success_challengers.union(self.fail_challengers)) + self.fail_chal_offset
if num_chal_evaluated == self.n_configs_in_stage[self.stage] and n_insts_remaining <= 0:
self.logger.info('Successive Halving iteration-step: %d-%d with '
'budget [%.2f / %d] - evaluated %d challenger(s)' %
(self.sh_iters + 1, self.stage + 1, self.all_budgets[self.stage], self.max_budget,
self.n_configs_in_stage[self.stage]))
self._update_stage(run_history=run_history)
# get incumbent cost
inc_perf = run_history.get_cost(incumbent)
return incumbent, inc_perf
