def test_adaptive_capping(self):
'''
test _adapt_cutoff()
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=self.config1,
instance_seed_pairs=inst_seed_pairs,
run_history=self.rh)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def _adapt_cutoff(self, challenger: Configuration,
incumbent: Configuration,
run_history: RunHistory,
inc_sum_cost: float):
"""Adaptive capping:
Compute cutoff based on time so far used for incumbent
and reduce cutoff for next run of challenger accordingly
!Only applicable if self.run_obj_time
!runs on incumbent should be superset of the runs performed for the
challenger
Parameters
----------
challenger : Configuration
Configuration which challenges incumbent
incumbent : Configuration
Best configuration so far
run_history : RunHistory
Stores all runs we ran so far
inc_sum_cost: float
Sum of runtimes of all incumbent runs
Returns
-------
cutoff: int
Adapted cutoff
"""
if not self.run_obj_time:
return self.cutoff
# cost used by challenger for going over all its runs
# should be subset of runs of incumbent (not checked for efficiency
# reasons)
chall_inst_seeds = run_history.get_runs_for_config(challenger)
chal_sum_cost = sum_cost(config=challenger,
instance_seed_pairs=chall_inst_seeds,
run_history=run_history)
cutoff = min(self.cutoff,
inc_sum_cost * self.Adaptive_Capping_Slackfactor -
chal_sum_cost
)
return cutoff
def _race_challenger(self, challenger: Configuration,
incumbent: Configuration, run_history: RunHistory,
aggregate_func: typing.Callable):
'''
aggressively race challenger against incumbent
Parameters
----------
challenger : Configuration
configuration which challenges incumbent
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
aggregate_func: typing.Callable
aggregate performance across instances
Returns
-------
new_incumbent: Configuration
either challenger or incumbent
'''
# at least one run of challenger
# to increase chall_indx counter
first_run = False
# Line 8
N = max(1, self.minR)
inc_inst_seeds = set(run_history.get_runs_for_config(incumbent))
# Line 9
while True:
chall_inst_seeds = set(run_history.get_runs_for_config(challenger))
# Line 10
missing_runs = list(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
to_run = missing_runs[:min(N, len(missing_runs))]
# Line 13 (Line 12 comes below...)
missing_runs = missing_runs[min(N, len(missing_runs)):]
# for adaptive capping
# because of efficieny computed here
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=incumbent,
instance_seed_pairs=inst_seed_pairs,
run_history=run_history)
# Line 12
# Run challenger on all <config,seed> to run
for instance, seed in to_run:
cutoff = self._adapt_cutoff(challenger=challenger,
incumbent=incumbent,
run_history=run_history,
inc_sum_cost=inc_sum_cost)
if cutoff is not None and cutoff <= 0: # no time to validate challenger
self.logger.debug(
"Stop challenger itensification due to adaptive capping."
)
# challenger performs worse than incumbent
return incumbent
if not first_run:
first_run = True
self._chall_indx += 1
self.logger.debug("Add run of challenger")
try:
status, cost, dur, res = self.tae_runner.start(
config=challenger,
instance=instance,
seed=seed,
cutoff=cutoff,
instance_specific=self.instance_specifics.get(
instance, "0"),
capped=(self.cutoff is not None)
and (cutoff < self.cutoff))
self._num_run += 1
except CappedRunException:
return incumbent
new_incumbent = self._compare_configs(
incumbent=incumbent,
challenger=challenger,
run_history=run_history,
aggregate_func=aggregate_func)
if new_incumbent == incumbent:
break
elif new_incumbent == challenger:
incumbent = challenger
break
else: # Line 17
# challenger is not worse, continue
N = 2 * N
return incumbent