def test_adaptive_capping(self):
'''
test _adapt_cutoff()
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=self.config1,
instance_seed_pairs=inst_seed_pairs,
run_history=self.rh)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def _adapt_cutoff(self, challenger: Configuration,
incumbent: Configuration, run_history: RunHistory,
inc_sum_cost: float):
'''
adaptive capping:
compute cutoff based on time so far used for incumbent
and reduce cutoff for next run of challenger accordingly
!Only applicable if self.run_obj_time
!runs on incumbent should be superset of the runs performed for the challenger
Parameters
----------
challenger : Configuration
configuration which challenges incumbent
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
inc_sum_cost: float
sum of runtimes of all incumbent runs
Returns
-------
cutoff: int
adapted cutoff
'''
if not self.run_obj_time:
return self.cutoff
# cost used by challenger for going over all its runs
# should be subset of runs of incumbent (not checked for efficiency
# reasons)
chall_inst_seeds = run_history.get_runs_for_config(challenger)
chal_sum_cost = sum_cost(config=challenger,
instance_seed_pairs=chall_inst_seeds,
run_history=run_history)
cutoff = min(
self.cutoff,
inc_sum_cost * self.Adaptive_Capping_Slackfactor - chal_sum_cost)
return cutoff
def _race_challenger(self, challenger: Configuration,
incumbent: Configuration, run_history: RunHistory,
aggregate_func: typing.Callable):
'''
aggressively race challenger against incumbent
Parameters
----------
challenger : Configuration
configuration which challenges incumbent
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
aggregate_func: typing.Callable
aggregate performance across instances
Returns
-------
new_incumbent: Configuration
either challenger or incumbent
'''
# at least one run of challenger
# to increase chall_indx counter
first_run = False
# Line 8
N = max(1, self.minR)
inc_inst_seeds = set(run_history.get_runs_for_config(incumbent))
# Line 9
while True:
chall_inst_seeds = set(run_history.get_runs_for_config(challenger))
# Line 10
missing_runs = list(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
to_run = missing_runs[:min(N, len(missing_runs))]
# Line 13 (Line 12 comes below...)
missing_runs = missing_runs[min(N, len(missing_runs)):]
# for adaptive capping
# because of efficieny computed here
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=incumbent,
instance_seed_pairs=inst_seed_pairs,
run_history=run_history)
# Line 12
# Run challenger on all <config,seed> to run
for instance, seed in to_run:
cutoff = self._adapt_cutoff(challenger=challenger,
incumbent=incumbent,
run_history=run_history,
inc_sum_cost=inc_sum_cost)
if cutoff is not None and cutoff <= 0: # no time to validate challenger
self.logger.debug(
"Stop challenger itensification due to adaptive capping."
)
# challenger performs worse than incumbent
return incumbent
if not first_run:
first_run = True
self._chall_indx += 1
self.logger.debug("Add run of challenger")
status, cost, dur, res = self.tae_runner.start(
config=challenger,
instance=instance,
seed=seed,
cutoff=cutoff,
instance_specific=self.instance_specifics.get(
instance, "0"))
self._num_run += 1
if status == StatusType.ABORT:
self.logger.debug(
"TAE signaled ABORT -- stop intensification on challenger"
)
return incumbent
new_incumbent = self._compare_configs(
incumbent=incumbent,
challenger=challenger,
run_history=run_history,
aggregate_func=aggregate_func)
if new_incumbent == incumbent:
break
elif new_incumbent == challenger:
incumbent = challenger
break
else: # Line 17
# challenger is not worse, continue
N = 2 * N
return incumbent
def intensify(self, challengers, incumbent, run_history, objective,
time_bound=MAXINT):
'''
running intensification to determine the incumbent configuration
Side effect: adds runs to run_history
Implementation of Procedure 2 in Hutter et al. (2011).
Parameters
----------
challengers : list of ConfigSpace.config
promising configurations
incumbent : ConfigSpace.config
best configuration so far
run_history : runhistory
all runs on all instance,seed pairs for incumbent
objective: func
objective function
time_bound : int, optional (default=2 ** 31 - 1)
time in [sec] available to perform intensify
Returns
-------
incumbent: Configuration()
current (maybe new) incumbent configuration
inc_perf: float
empirical performance of incumbent configuration
'''
self.start_time = time.time()
if time_bound < 0.01:
raise ValueError("time_bound must be >= 0.01")
num_run = 0
# Line 1 + 2
for chall_indx, challenger in enumerate(challengers):
if challenger == incumbent:
self.logger.warn(
"Challenger was the same as the current incumbent; Skipping challenger")
continue
self.logger.debug("Intensify on %s", challenger)
if hasattr(challenger, 'origin'):
self.logger.debug(
"Configuration origin: %s", challenger.origin)
inc_runs = run_history.get_runs_for_config(incumbent)
# Line 3
# First evaluate incumbent on a new instance
if len(inc_runs) < self.maxR:
# Line 4
# find all instances that have the most runs on the inc
inc_inst = [s.instance for s in inc_runs]
inc_inst = list(Counter(inc_inst).items())
inc_inst.sort(key=lambda x: x[1], reverse=True)
max_runs = inc_inst[0][1]
inc_inst = set([x[0] for x in inc_inst if x[1] == max_runs])
available_insts = (self.instances - inc_inst)
# if all instances were used n times, we can pick an instances
# from the complete set again
if not self.deterministic and not available_insts:
available_insts = self.instances
# Line 6 (Line 5 is further down...)
if self.deterministic:
next_seed = 0
else:
next_seed = self.rs.randint(low=0, high=MAXINT,
size=1)[0]
if available_insts:
# Line 5 (here for easier code)
next_instance = random.choice(list(available_insts))
# Line 7
status, cost, dur, res = self.tae.start(config=incumbent,
instance=next_instance,
seed=next_seed,
cutoff=self.cutoff,
instance_specific=self.instance_specifics.get(next_instance, "0"))
run_history.add(config=incumbent,
cost=cost, time=dur, status=status,
instance_id=next_instance, seed=next_seed,
additional_info=res)
# TODO update incumbent performance here!
num_run += 1
else:
self.logger.debug(
"No further instance-seed pairs for incumbent available.")
# Line 8
N = 1
inc_inst_seeds = set(run_history.get_runs_for_config(incumbent))
inc_perf = objective(incumbent, run_history, inc_inst_seeds)
run_history.update_cost(incumbent, inc_perf)
# Line 9
while True:
chall_inst_seeds = set(map(lambda x: (
x.instance, x.seed), run_history.get_runs_for_config(challenger)))
# Line 10
missing_runs = list(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
to_run = missing_runs[:min(N, len(missing_runs))]
# Line 13 (Line 12 comes below...)
missing_runs = missing_runs[min(N, len(missing_runs)):]
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
inc_sum_cost = sum_cost(config=incumbent, instance_seed_pairs=inst_seed_pairs,
run_history=run_history)
# Line 12
for instance, seed in to_run:
# Run challenger on all <config,seed> to run
if self.run_obj_time:
chall_inst_seeds = set(map(lambda x: (
x.instance, x.seed), run_history.get_runs_for_config(challenger)))
chal_sum_cost = sum_cost(config=challenger, instance_seed_pairs=chall_inst_seeds,
run_history=run_history)
cutoff = min(self.cutoff,
(inc_sum_cost - chal_sum_cost) *
self.Adaptive_Capping_Slackfactor)
if cutoff < 0: # no time left to validate challenger
self.logger.debug(
"Stop challenger itensification due to adaptive capping.")
break
else:
cutoff = self.cutoff
status, cost, dur, res = self.tae.start(config=challenger,
instance=instance,
seed=seed,
cutoff=cutoff,
instance_specific=self.instance_specifics.get(instance, "0"))
run_history.add(config=challenger,
cost=cost, time=dur, status=status,
instance_id=instance, seed=seed,
additional_info=res)
num_run += 1
# we cannot use inst_seed_pairs here since we could have less runs
# for the challenger due to the adaptive capping
chall_inst_seeds = set(map(lambda x: (
x.instance, x.seed), run_history.get_runs_for_config(challenger)))
if len(chall_inst_seeds) < len(inst_seed_pairs):
#challenger got not enough runs because of exhausted config budget
self.logger.debug("No (or not enough) valid runs for challenger.")
break
else:
chal_perf = objective(challenger, run_history, chall_inst_seeds)
run_history.update_cost(challenger, chal_perf)
inc_perf = objective(incumbent, run_history, chall_inst_seeds)
# Line 15
if chal_perf > inc_perf:
# Incumbent beats challenger
self.logger.debug("Incumbent (%.4f) is better than challenger (%.4f) on %d runs." % (
inc_perf, chal_perf, len(inst_seed_pairs)))
break
# Line 16
if len(chall_inst_seeds) == len(inc_inst_seeds):
# Challenger is as good as incumbent -> change incumbent
n_samples = len(inst_seed_pairs)
self.logger.info("Challenger (%.4f) is better than incumbent (%.4f) on %d runs." % (
chal_perf / n_samples, inc_perf / n_samples, n_samples))
self.logger.info(
"Changing incumbent to challenger: %s" % (challenger))
incumbent = challenger
inc_perf = chal_perf
run_history.update_cost(challenger, chal_perf)
self.stats.inc_changed += 1
self.traj_logger.add_entry(train_perf=inc_perf,
incumbent_id=self.stats.inc_changed,
incumbent=incumbent)
break
# Line 17
else:
# challenger is not worse, continue
N = 2 * N
if chall_indx >= 1 and num_run > self.run_limit:
self.logger.debug(
"Maximum #runs for intensification reached")
break
elif chall_indx >= 1 and time.time() - self.start_time - time_bound >= 0:
self.logger.debug("Timelimit for intensification reached ("
"used: %f sec, available: %f sec)" % (
time.time() - self.start_time, time_bound))
break
# output estimated performance of incumbent
inc_runs = run_history.get_runs_for_config(incumbent)
inc_perf = objective(incumbent, run_history, inc_runs)
self.logger.info("Updated estimated performance of incumbent on %d runs: %.4f" % (
len(inc_runs), inc_perf))
return incumbent, inc_perf