def _get_instances_to_run(
self,
challenger: Configuration,
incumbent: Configuration,
N: int,
run_history: RunHistory,
) -> typing.Tuple[typing.List[InstSeedBudgetKey], float]:
"""
Returns the minimum list of <instance, seed> pairs to run the challenger on
before comparing it with the incumbent
Parameters
----------
incumbent: Configuration
incumbent configuration
challenger: Configuration
promising configuration that is presently being evaluated
run_history: RunHistory
Stores all runs we ran so far
N: int
number of <instance, seed> pairs to select
Returns
-------
typing.List[InstSeedBudgetKey]
list of <instance, seed, budget> tuples to run
float
total (runtime) cost of running the incumbent on the instances (used for adaptive capping while racing)
"""
# get next instances left for the challenger
# Line 8
inc_inst_seeds = set(
run_history.get_runs_for_config(incumbent,
only_max_observed_budget=True))
chall_inst_seeds = set(
run_history.get_runs_for_config(challenger,
only_max_observed_budget=True))
# Line 10
missing_runs = sorted(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
if N < 0:
raise ValueError(
'Argument N must not be smaller than zero, but is %s' % str(N))
to_run = missing_runs[:min(N, len(missing_runs))]
missing_runs = missing_runs[min(N, len(missing_runs)):]
# for adaptive capping
# because of efficiency computed here
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = run_history.sum_cost(
config=incumbent,
instance_seed_budget_keys=inst_seed_pairs,
)
return to_run, inc_sum_cost
def test_get_config_runs(self):
'''
get some config runs from runhistory
'''
# return max observed budget only
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs,
values={'a': 1, 'b': 2})
config2 = Configuration(cs,
values={'a': 1, 'b': 3})
rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=1)
rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=2)
with self.assertRaisesRegex(ValueError, 'This should not happen!'):
rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=2, seed=2, budget=1)
rh.add(config=config2, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=1)
ist = rh.get_runs_for_config(config=config1, only_max_observed_budget=True)
self.assertEqual(len(ist), 2)
self.assertEqual(ist[0].instance, 1)
self.assertEqual(ist[1].instance, 2)
self.assertEqual(ist[0].budget, 2)
self.assertEqual(ist[1].budget, 1)
# multiple budgets (only_max_observed_budget=False)
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs,
values={'a': 1, 'b': 2})
config2 = Configuration(cs,
values={'a': 1, 'b': 3})
rh.add(config=config1, cost=5, time=10, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=1)
rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=2)
rh.add(config=config2, cost=5, time=10, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=1)
rh.add(config=config2, cost=10, time=20, status=StatusType.SUCCESS,
instance_id=1, seed=1, budget=2)
ist = rh.get_runs_for_config(config=config1, only_max_observed_budget=False)
self.assertEqual(len(ist), 2)
self.assertEqual(ist[0].instance, 1)
self.assertEqual(ist[0].budget, 1)
self.assertEqual(ist[1].budget, 2)
def test_get_config_runs(self):
'''
get some config runs from runhistory
'''
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs,
values={'a': 1, 'b': 2})
config2 = Configuration(cs,
values={'a': 1, 'b': 3})
rh.add(config=config1, cost=10, time=20,
status=StatusType.SUCCESS, instance_id=1,
seed=1)
rh.add(config=config2, cost=10, time=20,
status=StatusType.SUCCESS, instance_id=1,
seed=1)
rh.add(config=config1, cost=10, time=20,
status=StatusType.SUCCESS, instance_id=2,
seed=2)
ist = rh.get_runs_for_config(config=config1)
#print(ist)
#print(ist[0])
#print(ist[1])
self.assertEqual(len(ist), 2)
self.assertEqual(ist[0].instance, 1)
self.assertEqual(ist[1].instance, 2)
def get_cost_dict_for_config(rh: RunHistory,
conf: Configuration,
par: int = 1,
cutoff: typing.Union[float, None] = None):
"""
Aggregates loss for configuration on evaluated instances over seeds.
Parameters
----------
rh: RunHistory
runhistory with data
conf: Configuration
configuration to evaluate
par: int
par-factor with which to multiply timeouts
cutoff: float
cutoff of scenario - used to penalize costs if par != 1
Returns
-------
cost: dict(instance->cost)
cost per instance (aggregated or as list per seed)
"""
# Check if config is in runhistory
conf_id = rh.config_ids[conf]
# Map instances to seeds in dict
runs = rh.get_runs_for_config(conf)
instance_to_seeds = dict()
for run in runs:
inst, seed = run
if inst in instance_to_seeds:
instance_to_seeds[inst].append(seed)
else:
instance_to_seeds[inst] = [seed]
# Get loss per instance
instance_costs = {
i: [rh.data[RunKey(conf_id, i, s)].cost for s in instance_to_seeds[i]]
for i in instance_to_seeds
}
# Aggregate:
instance_costs = {i: np.mean(instance_costs[i]) for i in instance_costs}
# TODO: uncomment next line and delete all above after next SMAC dev->master
# instance_costs = rh.get_instance_costs_for_config(conf)
if par != 1:
if cutoff:
instance_costs = {
k: v if v < cutoff else v * par
for k, v in instance_costs.items()
}
else:
raise ValueError(
"To apply penalization of costs, a cutoff needs to be provided."
)
return instance_costs
def _cost(config: Configuration, run_history: RunHistory,
instance_seed_pairs=None):
"""Return array of all costs for the given config for further calculations.
Parameters
----------
config : Configuration
Configuration to calculate objective for
run_history : RunHistory
RunHistory object from which the objective value is computed.
instance_seed_pairs : list, optional (default=None)
List of tuples of instance-seeds pairs. If None, the run_history is
queried for all runs of the given configuration.
Returns
-------
Costs: list
Array of all costs
"""
try:
id_ = run_history.config_ids[config]
except KeyError: # challenger was not running so far
return []
if instance_seed_pairs is None:
instance_seed_pairs = run_history.get_runs_for_config(config)
costs = []
for i, r in instance_seed_pairs:
k = RunKey(id_, i, r)
costs.append(run_history.data[k].cost)
return costs
def _top_k(self, configs: typing.List[Configuration],
run_history: RunHistory, k: int) -> typing.List[Configuration]:
"""
Selects the top 'k' configurations from the given list based on their performance.
This retrieves the performance for each configuration from the runhistory and checks
that the highest budget they've been evaluated on is the same for each of the configurations.
Parameters
----------
configs: typing.List[Configuration]
list of configurations to filter from
run_history: smac.runhistory.runhistory.RunHistory
stores all runs we ran so far
k: int
number of configurations to select
Returns
-------
typing.List[Configuration]
top challenger configurations, sorted in increasing costs
"""
# extracting costs for each given configuration
config_costs = {}
# sample list instance-seed-budget key to act as base
run_key = run_history.get_runs_for_config(
configs[0], only_max_observed_budget=True)
for c in configs:
# ensuring that all configurations being compared are run on the same set of instance, seed & budget
cur_run_key = run_history.get_runs_for_config(
c, only_max_observed_budget=True)
if cur_run_key != run_key:
raise ValueError(
'Cannot compare configs that were run on different instances-seeds-budgets: %s vs %s'
% (run_key, cur_run_key))
config_costs[c] = run_history.get_cost(c)
configs_sorted = sorted(config_costs, key=config_costs.get)
# select top configurations only
top_configs = configs_sorted[:k]
return top_configs
def test_merge_foreign_data(self):
''' test smac.utils.merge_foreign_data '''
scenario = Scenario(self.test_scenario_dict)
scenario_2 = Scenario(self.test_scenario_dict)
scenario_2.feature_dict = {"inst_new": [4]}
# init cs
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter(name='a',
lower=0,
upper=100))
cs.add_hyperparameter(UniformIntegerHyperparameter(name='b',
lower=0,
upper=100))
# build runhistory
rh_merge = RunHistory()
config = Configuration(cs, values={'a': 1, 'b': 2})
rh_merge.add(config=config, instance_id="inst_new", cost=10, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# "d" is an instance in <scenario>
rh_merge.add(config=config, instance_id="d", cost=5, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# build empty rh
rh_base = RunHistory()
merge_foreign_data(scenario=scenario, runhistory=rh_base,
in_scenario_list=[scenario_2], in_runhistory_list=[rh_merge])
# both runs should be in the runhistory
# but we should not use the data to update the cost of config
self.assertTrue(len(rh_base.data) == 2)
self.assertTrue(np.isnan(rh_base.get_cost(config)))
# we should not get direct access to external run data
runs = rh_base.get_runs_for_config(config, only_max_observed_budget=True)
self.assertTrue(len(runs) == 0)
rh_merge.add(config=config, instance_id="inst_new_2", cost=10, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
self.assertRaises(ValueError, merge_foreign_data, **{
"scenario": scenario, "runhistory": rh_base,
"in_scenario_list": [scenario_2], "in_runhistory_list": [rh_merge]})
def test_add_multiple_times(self):
rh = RunHistory()
cs = get_config_space()
config = Configuration(cs, values={'a': 1, 'b': 2})
for i in range(5):
rh.add(config=config, cost=i + 1, time=i + 1,
status=StatusType.SUCCESS, instance_id=None,
seed=12345, additional_info=None)
self.assertEqual(len(rh.data), 1)
self.assertEqual(len(rh.get_runs_for_config(config, only_max_observed_budget=True)), 1)
self.assertEqual(len(rh._configid_to_inst_seed_budget[1]), 1)
self.assertEqual(list(rh.data.values())[0].cost, 1)
def _process_inc_run(
self,
incumbent: Configuration,
run_history: RunHistory,
log_traj: bool = True,
) -> None:
"""Method to process the results of a challenger that races
an incumbent
Parameters
----------
incumbent: Configuration
Either challenger or incumbent
run_history : RunHistory
stores all runs we ran so far
log_traj: bool
Whether to log changes of incumbents in trajectory
"""
# output estimated performance of incumbent
inc_runs = run_history.get_runs_for_config(
incumbent, only_max_observed_budget=True)
inc_perf = run_history.get_cost(incumbent)
format_value = format_array(inc_perf)
self.logger.info(
f"Updated estimated cost of incumbent on {len(inc_runs)} runs: {format_value}"
)
# if running first configuration, go to next stage after 1st run
if self.stage in [
IntensifierStage.RUN_FIRST_CONFIG,
IntensifierStage.PROCESS_FIRST_CONFIG_RUN
]:
self.stage = IntensifierStage.RUN_INCUMBENT
self._next_iteration()
else:
# Termination condition; after each run, this checks
# whether further runs are necessary due to minR
if len(inc_runs) >= self.minR or len(inc_runs) >= self.maxR:
self.stage = IntensifierStage.RUN_CHALLENGER
else:
self.stage = IntensifierStage.RUN_INCUMBENT
self._compare_configs(incumbent=incumbent,
challenger=incumbent,
run_history=run_history,
log_traj=log_traj)
def _adapt_cutoff(self,
challenger: Configuration,
run_history: RunHistory,
inc_sum_cost: float) -> float:
"""Adaptive capping:
Compute cutoff based on time so far used for incumbent
and reduce cutoff for next run of challenger accordingly
!Only applicable if self.run_obj_time
!runs on incumbent should be superset of the runs performed for the
challenger
Parameters
----------
challenger : Configuration
Configuration which challenges incumbent
run_history : smac.runhistory.runhistory.RunHistory
Stores all runs we ran so far
inc_sum_cost: float
Sum of runtimes of all incumbent runs
Returns
-------
cutoff: float
Adapted cutoff
"""
if not self.run_obj_time:
raise ValueError('This method only works when the run objective is quality')
curr_cutoff = self.cutoff if self.cutoff is not None else np.inf
# cost used by challenger for going over all its runs
# should be subset of runs of incumbent (not checked for efficiency
# reasons)
chall_inst_seeds = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
chal_sum_cost = run_history.sum_cost(
config=challenger,
instance_seed_budget_keys=chall_inst_seeds,
)
cutoff = min(curr_cutoff,
inc_sum_cost * self.adaptive_capping_slackfactor - chal_sum_cost
)
return cutoff
def reduce_runhistory(self, rh: RunHistory, max_configs: int, keep=None):
"""
Reduce configs to desired number, by default just drop the configs with the fewest runs.
Parameters
----------
rh: RunHistory
runhistory that is to be reduced
max_configs: int
if > -1 reduce runhistory to at most max_configs
keep: List[Configuration]
list of configs that should be kept for sure (e.g. default, incumbents)
Returns
-------
rh: RunHistory
reduced runhistory
"""
configs = rh.get_all_configs()
if max_configs <= 0 or max_configs > len(configs): # keep all
return rh
runs = [(c,
len(rh.get_runs_for_config(c,
only_max_observed_budget=False)))
for c in configs]
if not keep:
keep = []
runs = sorted(runs, key=lambda x: x[1])[-self.max_plot:]
keep = [r[0] for r in runs] + keep
self.logger.info(
"Reducing number of configs from %d to %d, dropping from the fewest evaluations",
len(configs), len(keep))
new_rh = RunHistory()
for k, v in list(rh.data.items()):
c = rh.ids_config[k.config_id]
if c in keep:
new_rh.add(config=rh.ids_config[k.config_id],
cost=v.cost,
time=v.time,
status=v.status,
instance_id=k.instance_id,
seed=k.seed)
return new_rh
def _adapt_cutoff(self, challenger: Configuration,
incumbent: Configuration,
run_history: RunHistory,
inc_sum_cost: float):
"""Adaptive capping:
Compute cutoff based on time so far used for incumbent
and reduce cutoff for next run of challenger accordingly
!Only applicable if self.run_obj_time
!runs on incumbent should be superset of the runs performed for the
challenger
Parameters
----------
challenger : Configuration
Configuration which challenges incumbent
incumbent : Configuration
Best configuration so far
run_history : RunHistory
Stores all runs we ran so far
inc_sum_cost: float
Sum of runtimes of all incumbent runs
Returns
-------
cutoff: int
Adapted cutoff
"""
if not self.run_obj_time:
return self.cutoff
# cost used by challenger for going over all its runs
# should be subset of runs of incumbent (not checked for efficiency
# reasons)
chall_inst_seeds = run_history.get_runs_for_config(challenger)
chal_sum_cost = sum_cost(config=challenger,
instance_seed_pairs=chall_inst_seeds,
run_history=run_history)
cutoff = min(self.cutoff,
inc_sum_cost * self.Adaptive_Capping_Slackfactor -
chal_sum_cost
)
return cutoff
def _get_inc_available_inst(
self,
incumbent: Configuration,
run_history: RunHistory,
log_traj: bool = True,
) -> typing.List[str]:
"""
Implementation of line 4 of Intensification
This method queries the inc runs in the run history
and return the pending instances if any is available
Parameters
----------
incumbent: Configuration
Either challenger or incumbent
run_history : RunHistory
stores all runs we ran so far
log_traj: bool
Whether to log changes of incumbents in trajectory
"""
# Line 4
# find all instances that have the most runs on the inc
inc_runs = run_history.get_runs_for_config(
incumbent,
only_max_observed_budget=True
)
inc_inst = [s.instance for s in inc_runs]
inc_inst = list(Counter(inc_inst).items())
inc_inst.sort(key=lambda x: x[1], reverse=True)
try:
max_runs = inc_inst[0][1]
except IndexError:
self.logger.debug("No run for incumbent found")
max_runs = 0
inc_inst = [x[0] for x in inc_inst if x[1] == max_runs]
available_insts = list(sorted(set(self.instances) - set(inc_inst)))
# if all instances were used n times, we can pick an instances
# from the complete set again
if not self.deterministic and not available_insts:
available_insts = self.instances
return available_insts
def _get_runs_per_config_quantiled(self, rh, conf_list, quantiles):
"""Returns a list of lists, each sublist representing the current state
at that timestep (quantile). The current state means a list of times
each config was evaluated at that timestep.
Parameters
----------
rh: RunHistory
rh to evaluate
conf_list: list
list of all Configuration objects that appeared in runhistory
quantiles: int
number of fractions to split rh into
Returns:
--------
runs_per_quantile: np.array
numpy array of runs per configuration per quantile
"""
runs_total = len(rh.data)
# Create LINEAR ranges. TODO do we want log? -> this line
ranges = [int(r) for r in np.linspace(0, runs_total, quantiles + 1)]
self.logger.debug(
"Creating %d quantiles with a step of %.2f and a total "
"runs of %d", quantiles, runs_total / quantiles, runs_total)
self.logger.debug("Ranges: %s", str(ranges))
# Iterate over the runhistory's entries in ranges and creating each
# sublist from a "snapshot"-runhistory
r_p_q_p_c = [] # runs per quantile per config
as_list = list(rh.data.items())
tmp_rh = RunHistory(average_cost)
for i, j in zip(ranges[:-1], ranges[1:]):
for idx in range(i, j):
k, v = as_list[idx]
tmp_rh.add(config=rh.ids_config[k.config_id],
cost=v.cost,
time=v.time,
status=v.status,
instance_id=k.instance_id,
seed=k.seed)
r_p_q_p_c.append(
[len(tmp_rh.get_runs_for_config(c)) for c in conf_list])
return r_p_q_p_c
def test_add_multiple_times(self):
rh = RunHistory()
cs = get_config_space()
config = Configuration(cs, values={"a": 1, "b": 2})
for i in range(5):
rh.add(
config=config,
cost=[i + 1, i + 2],
time=i + 1,
status=StatusType.SUCCESS,
instance_id=None,
seed=12345,
additional_info=None,
)
self.assertEqual(len(rh.data), 1)
self.assertEqual(
len(rh.get_runs_for_config(config, only_max_observed_budget=True)),
1)
self.assertEqual(len(rh._configid_to_inst_seed_budget[1]), 1)
# We expect to get 1.0 and 2.0 because runhistory does not overwrite by default
self.assertEqual(list(rh.data.values())[0].cost, [1.0, 2.0])
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory(aggregate_func=average_cost)
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_compare_configs_no_joint_set(self):
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
'''
challenger is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
'''
incumbent is better
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
'''
challenger is better but has less runs;
-> no decision (None)
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh,
aggregate_func=average_cost)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_race_challenger(self):
'''
test _race_challenger without adaptive capping
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
self.assertEqual(inc, self.config2)
def test_race_challenger_2(self):
'''
test _race_challenger with adaptive capping
'''
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_race_challenger_3(self):
'''
test _race_challenger with adaptive capping on a previously capped configuration
'''
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1,
cost=0.5,
time=.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.runs_per_config[2], 2)
def test_race_challenger_large(self):
'''
test _race_challenger using solution_quality
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
def test_race_challenger_large_blocked_seed(self):
'''
test _race_challenger whether seeds are blocked for challenger runs
'''
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
# tie on first instances and then challenger should always win
# and be returned as inc
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
aggregate_func=average_cost)
# self.assertTrue(False)
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1,
self.rh.get_cost(self.config2))
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_add_inc_run_det(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_nondet(self):
'''
test _add_inc_run()
'''
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_adaptive_capping(self):
'''
test _adapt_cutoff()
'''
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(self.config1)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=self.config1,
instance_seed_pairs=inst_seed_pairs,
run_history=self.rh)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def _compare_configs(self, incumbent: Configuration,
challenger: Configuration, run_history: RunHistory,
aggregate_func: typing.Callable):
'''
compare two configuration wrt the runhistory
and return the one which performs better (or None if the decision is not safe)
Decision strategy to return x as being better than y:
1. x has at least as many runs as y
2. x performs better than y on the intersection of runs on x and y
Implicit assumption:
challenger was evaluated on the same instance-seed pairs as incumbent
Parameters
----------
incumbent: Configuration
current incumbent
challenger: Configuration
challenger configuration
run_history: RunHistory
stores all runs we ran so far
aggregate_func: typing.Callable
aggregate performance across instances
Returns
-------
None or better of the two configurations x,y
'''
inc_runs = run_history.get_runs_for_config(incumbent)
chall_runs = run_history.get_runs_for_config(challenger)
to_compare_runs = set(inc_runs).intersection(chall_runs)
# performance on challenger runs
chal_perf = aggregate_func(challenger, run_history, to_compare_runs)
inc_perf = aggregate_func(incumbent, run_history, to_compare_runs)
# Line 15
if chal_perf > inc_perf and len(chall_runs) >= self.minR:
# Incumbent beats challenger
self.logger.debug(
"Incumbent (%.4f) is better than challenger (%.4f) on %d runs."
% (inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Line 16
if not set(inc_runs) - set(chall_runs):
# Challenger is as good as incumbent
# and has at least the same runs as inc
# -> change incumbent
n_samples = len(chall_runs)
self.logger.info(
"Challenger (%.4f) is better than incumbent (%.4f) on %d runs."
% (chal_perf, inc_perf, n_samples))
# Show changes in the configuration
params = sorted([(param, incumbent[param], challenger[param])
for param in challenger.keys()])
self.logger.info("Changes in incumbent:")
for param in params:
if param[1] != param[2]:
self.logger.info(" %s : %r -> %r" % (param))
else:
self.logger.debug(" %s remains unchanged: %r" %
(param[0], param[1]))
self.stats.inc_changed += 1
self.traj_logger.add_entry(train_perf=chal_perf,
incumbent_id=self.stats.inc_changed,
incumbent=challenger)
return challenger
return None # undecided
def _race_challenger(self, challenger: Configuration,
incumbent: Configuration, run_history: RunHistory,
aggregate_func: typing.Callable):
'''
aggressively race challenger against incumbent
Parameters
----------
challenger : Configuration
configuration which challenges incumbent
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
aggregate_func: typing.Callable
aggregate performance across instances
Returns
-------
new_incumbent: Configuration
either challenger or incumbent
'''
# at least one run of challenger
# to increase chall_indx counter
first_run = False
# Line 8
N = max(1, self.minR)
inc_inst_seeds = set(run_history.get_runs_for_config(incumbent))
# Line 9
while True:
chall_inst_seeds = set(run_history.get_runs_for_config(challenger))
# Line 10
missing_runs = list(inc_inst_seeds - chall_inst_seeds)
# Line 11
self.rs.shuffle(missing_runs)
to_run = missing_runs[:min(N, len(missing_runs))]
# Line 13 (Line 12 comes below...)
missing_runs = missing_runs[min(N, len(missing_runs)):]
# for adaptive capping
# because of efficieny computed here
inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = sum_cost(config=incumbent,
instance_seed_pairs=inst_seed_pairs,
run_history=run_history)
# Line 12
# Run challenger on all <config,seed> to run
for instance, seed in to_run:
cutoff = self._adapt_cutoff(challenger=challenger,
incumbent=incumbent,
run_history=run_history,
inc_sum_cost=inc_sum_cost)
if cutoff is not None and cutoff <= 0: # no time to validate challenger
self.logger.debug(
"Stop challenger itensification due to adaptive capping."
)
# challenger performs worse than incumbent
return incumbent
if not first_run:
first_run = True
self._chall_indx += 1
self.logger.debug("Add run of challenger")
try:
status, cost, dur, res = self.tae_runner.start(
config=challenger,
instance=instance,
seed=seed,
cutoff=cutoff,
instance_specific=self.instance_specifics.get(
instance, "0"),
capped=(self.cutoff is not None)
and (cutoff < self.cutoff))
self._num_run += 1
except CappedRunException:
return incumbent
new_incumbent = self._compare_configs(
incumbent=incumbent,
challenger=challenger,
run_history=run_history,
aggregate_func=aggregate_func)
if new_incumbent == incumbent:
break
elif new_incumbent == challenger:
incumbent = challenger
break
else: # Line 17
# challenger is not worse, continue
N = 2 * N
return incumbent
def _add_inc_run(self, incumbent: Configuration, run_history: RunHistory):
'''
add new run for incumbent
Side effect: adds runs to <run_history>
Parameters
----------
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
'''
inc_runs = run_history.get_runs_for_config(incumbent)
# Line 3
# First evaluate incumbent on a new instance
if len(inc_runs) < self.maxR:
while True:
# Line 4
# find all instances that have the most runs on the inc
inc_runs = run_history.get_runs_for_config(incumbent)
inc_inst = [s.instance for s in inc_runs]
inc_inst = list(Counter(inc_inst).items())
inc_inst.sort(key=lambda x: x[1], reverse=True)
try:
max_runs = inc_inst[0][1]
except IndexError:
self.logger.debug("No run for incumbent found")
max_runs = 0
inc_inst = set([x[0] for x in inc_inst if x[1] == max_runs])
available_insts = (self.instances - inc_inst)
# if all instances were used n times, we can pick an instances
# from the complete set again
if not self.deterministic and not available_insts:
available_insts = self.instances
# Line 6 (Line 5 is further down...)
if self.deterministic:
next_seed = 0
else:
next_seed = self.rs.randint(low=0, high=MAXINT, size=1)[0]
if available_insts:
# Line 5 (here for easier code)
next_instance = self.rs.choice(list(available_insts))
# Line 7
self.logger.debug("Add run of incumbent")
status, cost, dur, res = self.tae_runner.start(
config=incumbent,
instance=next_instance,
seed=next_seed,
cutoff=self.cutoff,
instance_specific=self.instance_specifics.get(
next_instance, "0"))
self._num_run += 1
else:
self.logger.debug(
"No further instance-seed pairs for incumbent available."
)
break
inc_runs = run_history.get_runs_for_config(incumbent)
# Termination condition; after exactly one run, this checks
# whether further runs are necessary due to minR
if len(inc_runs) >= self.minR or len(inc_runs) >= self.maxR:
break
def intensify(self,
challengers: typing.List[Configuration],
incumbent: Configuration,
run_history: RunHistory,
aggregate_func: typing.Callable,
time_bound: int = MAXINT):
'''
running intensification to determine the incumbent configuration
Side effect: adds runs to run_history
Implementation of Procedure 2 in Hutter et al. (2011).
Parameters
----------
challengers : typing.List[Configuration]
promising configurations
incumbent : Configuration
best configuration so far
run_history : RunHistory
stores all runs we ran so far
aggregate_func: typing.Callable
aggregate performance across instances
time_bound : int, optional (default=2 ** 31 - 1)
time in [sec] available to perform intensify
Returns
-------
incumbent: Configuration()
current (maybe new) incumbent configuration
inc_perf: float
empirical performance of incumbent configuration
'''
self.start_time = time.time()
if time_bound < 0.01:
raise ValueError("time_bound must be >= 0.01")
self._num_run = 0
self._chall_indx = 0
# Line 1 + 2
for challenger in challengers:
if challenger == incumbent:
self.logger.warning(
"Challenger was the same as the current incumbent; Skipping challenger"
)
continue
self.logger.debug("Intensify on %s", challenger)
if hasattr(challenger, 'origin'):
self.logger.debug("Configuration origin: %s",
challenger.origin)
try:
# Lines 3-7
self._add_inc_run(incumbent=incumbent, run_history=run_history)
# Lines 8-17
incumbent = self._race_challenger(
challenger=challenger,
incumbent=incumbent,
run_history=run_history,
aggregate_func=aggregate_func)
except BudgetExhaustedException:
# We return incumbent, SMBO stops due to its own budget checks
inc_perf = run_history.get_cost(incumbent)
self.logger.debug("Budget exhausted; Return incumbent")
return incumbent, inc_perf
if self._chall_indx > 1 and self._num_run > self.run_limit:
self.logger.debug("Maximum #runs for intensification reached")
break
elif self._chall_indx > 1 and time.time(
) - self.start_time - time_bound >= 0:
self.logger.debug("Timelimit for intensification reached ("
"used: %f sec, available: %f sec)" %
(time.time() - self.start_time, time_bound))
break
# output estimated performance of incumbent
inc_runs = run_history.get_runs_for_config(incumbent)
inc_perf = aggregate_func(incumbent, run_history, inc_runs)
self.logger.info(
"Updated estimated performance of incumbent on %d runs: %.4f" %
(len(inc_runs), inc_perf))
self.stats.update_average_configs_per_intensify(
n_configs=self._chall_indx)
return incumbent, inc_perf
def _get_runs_per_config_quantiled(self, rh, conf_list, quantiles):
"""Returns a list of lists, each sublist representing the current state
at that timestep (quantile). The current state means a list of times
each config was evaluated at that timestep.
Parameters
----------
rh: RunHistory
rh to be split up
conf_list: list
list of all Configuration objects that appear in runhistory
quantiles: int
number of fractions to split rh into
Returns:
--------
labels: List[str]
labels for timeslider (i.e. wallclock-times)
runs_per_quantile: np.array
numpy array of runs per configuration per quantile
"""
runs_total = len(rh.data)
# Iterate over the runhistory's entries in ranges and creating each
# sublist from a "snapshot"-runhistory
labels, last_time_seen = [], -1 # label, means wallclocktime at splitting points
r_p_q_p_c = [] # runs per quantile per config
as_list = list(rh.data.items())
scale = np.geomspace if self.timeslider_log else np.linspace
# Trying to work with timestamps if they are available
timestamps = None
try:
as_list = sorted(as_list, key=lambda x: x[1].additional_info['timestamps']['finished'])
timestamps = [x[1].additional_info['timestamps']['finished'] for x in as_list]
time_ranges = scale(timestamps[0], timestamps[-1], num=quantiles+1, endpoint=True)
ranges = []
idx = 0
for time_idx, time in enumerate(time_ranges):
while len(timestamps) - 1 > idx and (timestamps[idx] < time or idx <= time_idx):
idx += 1
ranges.append(idx)
except (KeyError, TypeError) as err:
self.logger.debug(err)
self.logger.debug("Failed to sort by timestamps... only a reason to worry if this is BOHB-analysis")
ranges = [int(x) for x in scale(1, runs_total, num=quantiles+1)]
# Fix possible wrong values
ranges[0] = 0
ranges[-1] = len(as_list)
self.logger.debug("Creating %d quantiles with a total number of runs of %d", quantiles, runs_total)
self.logger.debug("Ranges: %s", str(ranges))
for r in range(len(ranges))[1:]:
if ranges[r] <= ranges[r-1]:
if ranges[r-1] + 1 >= len(as_list):
raise RuntimeError("There was a problem with the quantiles of the configuration footprint. "
"Please report this Error on \"https://github.com/automl/CAVE/issues\" and provide the debug.txt-file.")
ranges[r] = ranges[r-1] + 1
self.logger.debug("Fixed ranges to: %s", str(ranges))
# Sanity check
if not ranges[0] == 0 or not ranges[-1] == len(as_list) or not len(ranges) == quantiles + 1:
raise RuntimeError("Sanity check on range-creation in configurator footprint went wrong. "
"Please report this Error on \"https://github.com/automl/CAVE/issues\" and provide the debug.txt-file.")
tmp_rh = RunHistory(average_cost)
for i, j in zip(ranges[:-1], ranges[1:]):
for idx in range(i, j):
k, v = as_list[idx]
tmp_rh.add(config=rh.ids_config[k.config_id],
cost=v.cost, time=v.time, status=v.status,
instance_id=k.instance_id, seed=k.seed,
additional_info=v.additional_info)
if timestamps:
labels.append("{0:.2f}".format(timestamps[j - 1]))
r_p_q_p_c.append([len(tmp_rh.get_runs_for_config(c)) for c in conf_list])
self.logger.debug("Labels: " + str(labels))
return labels, r_p_q_p_c
def get_next_run(
self,
challengers: typing.Optional[typing.List[Configuration]],
incumbent: Configuration,
chooser: typing.Optional[EPMChooser],
run_history: RunHistory,
repeat_configs: bool = True,
num_workers: int = 1,
) -> typing.Tuple[RunInfoIntent, RunInfo]:
"""
This procedure is in charge of generating a RunInfo object to comply
with lines 7 (in case stage is stage==RUN_INCUMBENT) or line 12
(In case of stage==RUN_CHALLENGER)
A RunInfo object encapsulates the necessary information for a worker
to execute the job, nevertheless, a challenger is not always available.
This could happen because no more configurations are available or the new
configuration to try was already executed.
To circumvent this, a intent is also returned:
- (intent=RUN) Run the RunInfo object (Normal Execution
- (intent=SKIP) Skip this iteration. No challenger is available, in particular
because challenger is the same as incumbent
Parameters
----------
challengers : typing.List[Configuration]
promising configurations
incumbent: Configuration
incumbent configuration
chooser : smac.optimizer.epm_configuration_chooser.EPMChooser
optimizer that generates next configurations to use for racing
run_history : RunHistory
stores all runs we ran so far
repeat_configs : bool
if False, an evaluated configuration will not be generated again
num_workers: int
the maximum number of workers available
at a given time.
Returns
-------
intent: RunInfoIntent
What should the smbo object do with the runinfo.
run_info: RunInfo
An object that encapsulates necessary information for a config run
"""
if num_workers > 1:
raise ValueError(
"Intensifier does not support more than 1 worker, yet "
"the argument num_workers to get_next_run is {}".format(
num_workers))
# If this function is called, it means the iteration is
# not complete (we can be starting a new iteration, or re-running a
# challenger due to line 17). We evaluate if a iteration is complete or not
# via _process_results
self.iteration_done = False
# In case a crash happens, and FirstRunCrashedException prevents a
# failure, revert back to running the incumbent
# Challenger case is by construction ok, as there is no special
# stage for its processing
if self.stage == IntensifierStage.PROCESS_FIRST_CONFIG_RUN:
self.stage = IntensifierStage.RUN_FIRST_CONFIG
elif self.stage == IntensifierStage.PROCESS_INCUMBENT_RUN:
self.stage = IntensifierStage.RUN_INCUMBENT
# if first ever run, then assume current challenger to be the incumbent
# Because this is the first ever run, we need to sample a new challenger
# This new challenger is also assumed to be the incumbent
if self.stage == IntensifierStage.RUN_FIRST_CONFIG:
if incumbent is None:
self.logger.info("First run, no incumbent provided;"
" challenger is assumed to be the incumbent")
challenger, self.new_challenger = self.get_next_challenger(
challengers=challengers,
chooser=chooser,
)
incumbent = challenger
else:
inc_runs = run_history.get_runs_for_config(
incumbent, only_max_observed_budget=True)
if len(inc_runs) > 0:
self.logger.debug("Skipping RUN_FIRST_CONFIG stage since "
"incumbent has already been ran")
self.stage = IntensifierStage.RUN_INCUMBENT
# LINES 3-7
if self.stage in [
IntensifierStage.RUN_FIRST_CONFIG,
IntensifierStage.RUN_INCUMBENT
]:
# Line 3
# A modified version, that not only checks for maxR
# but also makes sure that there are runnable instances,
# that is, instances has not been exhausted
inc_runs = run_history.get_runs_for_config(
incumbent, only_max_observed_budget=True)
# Line 4
available_insts = self._get_inc_available_inst(
incumbent, run_history)
if available_insts and len(inc_runs) < self.maxR:
# Lines 5-6-7
instance, seed, cutoff = self._get_next_inc_run(
available_insts)
instance_specific = "0"
if instance is not None:
instance_specific = self.instance_specifics.get(
instance, "0")
return RunInfoIntent.RUN, RunInfo(
config=incumbent,
instance=instance,
instance_specific=instance_specific,
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
else:
# This point marks the transitions from lines 3-7
# to 8-18.
self.logger.debug(
"No further instance-seed pairs for incumbent available.")
self.stage = IntensifierStage.RUN_CHALLENGER
# Understand who is the active challenger.
if self.stage == IntensifierStage.RUN_BASIS:
# if in RUN_BASIS stage,
# return the basis configuration (i.e., `always_race_against`)
self.logger.debug(
"Race against basis configuration after incumbent change.")
challenger = self.always_race_against
elif self.current_challenger and self.continue_challenger:
# if the current challenger could not be rejected,
# it is run again on more instances
challenger = self.current_challenger
else:
# Get a new challenger if all instance/pairs have
# been completed. Else return the currently running
# challenger
challenger, self.new_challenger = self.get_next_challenger(
challengers=challengers,
chooser=chooser,
)
# No new challengers are available for this iteration,
# Move to the next iteration. This can only happen
# when all configurations for this iteration are exhausted
# and have been run in all proposed instance/pairs.
if challenger is None:
return RunInfoIntent.SKIP, RunInfo(
config=None,
instance=None,
instance_specific="0",
seed=0,
cutoff=self.cutoff,
capped=False,
budget=0.0,
)
# Skip the iteration if the challenger was previously run
if challenger == incumbent and self.stage == IntensifierStage.RUN_CHALLENGER:
self.challenger_same_as_incumbent = True
self.logger.debug(
"Challenger was the same as the current incumbent; Skipping challenger"
)
return RunInfoIntent.SKIP, RunInfo(
config=None,
instance=None,
instance_specific="0",
seed=0,
cutoff=self.cutoff,
capped=False,
budget=0.0,
)
self.logger.debug("Intensify on %s", challenger)
if hasattr(challenger, 'origin'):
self.logger.debug("Configuration origin: %s", challenger.origin)
if self.stage in [
IntensifierStage.RUN_CHALLENGER, IntensifierStage.RUN_BASIS
]:
if not self.to_run:
self.to_run, self.inc_sum_cost = self._get_instances_to_run(
incumbent=incumbent,
challenger=challenger,
run_history=run_history,
N=self.N)
is_there_time_due_to_adaptive_cap = self._is_there_time_due_to_adaptive_cap(
challenger=challenger,
run_history=run_history,
)
# If there is no more configs to run in this iteration, or no more
# time to do so, change the current stage base on how the current
# challenger performs as compared to the incumbent. This is done
# via _process_racer_results
if len(self.to_run) == 0 or not is_there_time_due_to_adaptive_cap:
# If no more time, stage transition is a must
if not is_there_time_due_to_adaptive_cap:
self.stage = IntensifierStage.RUN_INCUMBENT
self.logger.debug("Stop challenger itensification due "
"to adaptive capping.")
# Nevertheless, if there are no more instances to run,
# we might need to comply with line 17 and keep running the
# same challenger. In this case, if there is not enough information
# to decide if the challenger is better/worst than the incumbent,
# line 17 doubles the number of instances to run.
self.logger.debug("No further runs for challenger possible")
self._process_racer_results(
challenger=challenger,
incumbent=incumbent,
run_history=run_history,
)
# Request SMBO to skip this run. This function will
# be called again, after the _process_racer_results
# has updated the intensifier stage
return RunInfoIntent.SKIP, RunInfo(
config=None,
instance=None,
instance_specific="0",
seed=0,
cutoff=self.cutoff,
capped=False,
budget=0.0,
)
else:
# Lines 8-11
incumbent, instance, seed, cutoff = self._get_next_racer(
challenger=challenger,
incumbent=incumbent,
run_history=run_history,
)
capped = False
if (self.cutoff is not None) and (
cutoff <
self.cutoff): # type: ignore[operator] # noqa F821
capped = True
instance_specific = "0"
if instance is not None:
instance_specific = self.instance_specifics.get(
instance, "0")
# Line 12
return RunInfoIntent.RUN, RunInfo(
config=challenger,
instance=instance,
instance_specific=instance_specific,
seed=seed,
cutoff=cutoff,
capped=capped,
budget=0.0,
)
else:
raise ValueError('No valid stage found!')
def _compare_configs(self,
incumbent: Configuration,
challenger: Configuration,
run_history: RunHistory,
log_traj: bool = True) -> typing.Optional[Configuration]:
"""
Compare two configuration wrt the runhistory and return the one which
performs better (or None if the decision is not safe)
Decision strategy to return x as being better than y:
1. x has at least as many runs as y
2. x performs better than y on the intersection of runs on x and y
Implicit assumption:
Challenger was evaluated on the same instance-seed pairs as
incumbent
Parameters
----------
incumbent: Configuration
Current incumbent
challenger: Configuration
Challenger configuration
run_history: smac.runhistory.runhistory.RunHistory
Stores all runs we ran so far
log_traj: bool
Whether to log changes of incumbents in trajectory
Returns
-------
None or better of the two configurations x,y
"""
inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True)
chall_runs = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
to_compare_runs = set(inc_runs).intersection(chall_runs)
# performance on challenger runs
chal_perf = run_history.average_cost(challenger, to_compare_runs)
inc_perf = run_history.average_cost(incumbent, to_compare_runs)
# Line 15
if chal_perf > inc_perf and len(chall_runs) >= self.minR:
# Incumbent beats challenger
self.logger.debug("Incumbent (%.4f) is better than challenger "
"(%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Line 16
if not set(inc_runs) - set(chall_runs):
# no plateau walks
if chal_perf >= inc_perf:
self.logger.debug("Incumbent (%.4f) is at least as good as the "
"challenger (%.4f) on %d runs." %
(inc_perf, chal_perf, len(chall_runs)))
return incumbent
# Challenger is better than incumbent
# and has at least the same runs as inc
# -> change incumbent
n_samples = len(chall_runs)
self.logger.info("Challenger (%.4f) is better than incumbent (%.4f)"
" on %d runs." % (chal_perf, inc_perf, n_samples))
self._log_incumbent_changes(incumbent, challenger)
if log_traj:
self.stats.inc_changed += 1
self.traj_logger.add_entry(train_perf=chal_perf,
incumbent_id=self.stats.inc_changed,
incumbent=challenger)
return challenger
# undecided
return None
class TestIntensify(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_race_challenger(self):
"""
test _race_challenger without adaptive capping
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
intensifier.N = 1
inc = intensifier._race_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
def test_race_challenger_2(self):
"""
test _race_challenger with adaptive capping
"""
def target(x):
time.sleep(1.5)
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1])
self.rh.add(config=self.config1,
cost=.001,
time=0.001,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
intensifier.N = 1
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
inc = intensifier._race_challenger(
challenger=self.config2,
incumbent=self.config1,
run_history=self.rh,
)
# self.assertTrue(False)
self.assertEqual(inc, self.config1)
def test_race_challenger_3(self):
"""
test _race_challenger with adaptive capping on a previously capped configuration
"""
def target(config: Configuration, seed: int, instance: str):
if instance == 1:
time.sleep(2.1)
else:
time.sleep(0.6)
return (config['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="runtime",
par_factor=1)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
cutoff=2,
instances=[1])
self.rh.add(config=self.config1,
cost=0.5,
time=.5,
status=StatusType.SUCCESS,
instance_id=1,
seed=12345,
additional_info=None)
# config2 should have a timeout (due to adaptive capping)
# and config1 should still be the incumbent
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
# further run for incumbent
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.TIMEOUT,
instance_id=2,
seed=12345,
additional_info=None)
# give config2 a second chance - now it should run on both instances
# run on instance 1
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# run on instance 2
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config2)
self.assertTrue(intensifier.continue_challenger)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# the incumbent should still be config1 because
# config2 should get on inst 1 a full timeout
# such that c(config1) = 1.25 and c(config2) close to 1.3
self.assertEqual(inc, self.config1)
# the capped run should not be counted in runs_perf_config
self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2)
self.assertFalse(intensifier.continue_challenger)
def test_race_challenger_large(self):
"""
test _race_challenger using solution_quality
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=True)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=12345,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
def test_race_challenger_large_blocked_seed(self):
"""
test _race_challenger whether seeds are blocked for challenger runs
"""
def target(x):
return 1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(10)),
run_obj_time=False,
deterministic=False)
for i in range(10):
self.rh.add(config=self.config1,
cost=i + 1,
time=1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# tie on first instances and then challenger should always win
# and be returned as inc
while True:
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3], chooser=None)
inc = intensifier._race_challenger(
challenger=config,
incumbent=self.config1,
run_history=self.rh,
)
# stop when challenger evaluation is over
if not intensifier.stage == IntensifierStage.RUN_CHALLENGER:
break
self.assertEqual(inc, self.config2)
self.assertEqual(self.rh.get_cost(self.config2), 1)
# get data for config2 to check that the correct run was performed
runs = self.rh.get_runs_for_config(self.config2,
only_max_observed_budget=True)
self.assertEqual(len(runs), 10)
seeds = sorted([r.seed for r in runs])
self.assertEqual(seeds, list(range(10)), seeds)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_add_inc_run_nondet(self):
"""
test _add_inc_run()
"""
def target(x):
return (x['a'] + 1) / 1000.
taf = ExecuteTAFuncDict(ta=target,
stats=self.stats,
runhistory=self.rh,
run_obj="solution_quality")
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1,
only_max_observed_budget=True)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
def test_get_next_challenger(self):
"""
test get_next_challenger()
"""
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# get a new challenger to evaluate
config, new = intensifier.get_next_challenger(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertEqual(intensifier.N, 1)
self.assertTrue(new)
# when already evaluating a challenger, return the same challenger
intensifier.to_run = [(1, 1, 0)]
config, new = intensifier.get_next_challenger(
challengers=[self.config2], chooser=None)
self.assertEqual(config, self.config1, intensifier.current_challenger)
self.assertEqual(intensifier._chall_indx, 1)
self.assertFalse(new)
def test_generate_challenger(self):
"""
test generate_challenger()
"""
# test get generator from a list of challengers
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
gen = intensifier._generate_challengers(
challengers=[self.config1, self.config2], chooser=None)
self.assertEqual(next(gen), self.config1)
self.assertEqual(next(gen), self.config2)
self.assertRaises(StopIteration, next, gen)
# test get generator from a chooser - would return only 1 configuration
intensifier = Intensifier(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True)
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
gen = intensifier._generate_challengers(challengers=None,
chooser=chooser)
self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68})
self.assertRaises(StopIteration, next, gen)
# when both are none, raise error
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier._generate_challengers(challengers=None, chooser=None)
def test_eval_challenger(self):
"""
test eval_challenger() - a complete intensification run
"""
def target(x):
return x['a']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Intensifier(tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=False,
always_race_against=self.config3,
run_limit=1)
# run incumbent first if it was not run before
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
# run challenger now that the incumbent has been executed
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# challenger should have a better performance, so incumbent should have changed
self.assertEqual(inc, self.config1)
self.assertEqual(self.stats.inc_changed, 1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_DEFAULT)
self.assertFalse(intensifier.continue_challenger)
# run `always_race_against` now since the incumbent has changed
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config1, self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(inc, self.config1)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(
len(
self.rh.get_runs_for_config(self.config3,
only_max_observed_budget=True)), 1)
self.assertEqual(intensifier.n_iters, 1)
self.assertIsInstance(intensifier.configs_to_run, collections.Iterator)
with self.assertRaises(StopIteration):
next(intensifier.configs_to_run)
class TestAbstractIntensifier(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_get_next_challenger(self):
"""
test get_next_challenger - pick from list/chooser
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# Error when nothing to choose from
with self.assertRaisesRegex(ValueError,
"No configurations/chooser provided"):
intensifier.get_next_challenger(challengers=None,
chooser=None,
run_history=self.rh)
# next challenger from a list
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config1)
config, _ = intensifier.get_next_challenger(
challengers=[self.config2, self.config3],
chooser=None,
run_history=self.rh)
self.assertEqual(config, self.config2)
# next challenger from a chooser
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(24 in config.get_dictionary().values())
self.assertTrue(68 in config.get_dictionary().values())
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=chooser,
run_history=self.rh)
self.assertEqual(len(list(config.get_dictionary().values())), 2)
self.assertTrue(95 in config.get_dictionary().values())
self.assertTrue(38 in config.get_dictionary().values())
def test_get_next_challenger_repeat(self):
"""
test get_next_challenger - repeat configurations
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
cutoff=1,
instances=[1])
# should not repeat configurations
self.rh.add(self.config1, 1, 1, StatusType.SUCCESS)
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=False)
self.assertEqual(config, self.config2)
# should repeat configurations
config, _ = intensifier.get_next_challenger(
challengers=[self.config1, self.config2],
chooser=None,
run_history=self.rh,
repeat_configs=True)
self.assertEqual(config, self.config1)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
class TestAbstractRacer(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100})
self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0})
self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
for i in range(2):
self.rh.add(config=self.config1,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=i,
additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_chall(self):
"""
challenger is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=0,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger has enough runs and is better
self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=2,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))
def test_adaptive_capping(self):
"""
test _adapt_cutoff()
"""
intensifier = AbstractRacer(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
instances=list(range(5)),
deterministic=False)
for i in range(5):
self.rh.add(config=self.config1,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
for i in range(3):
self.rh.add(config=self.config2,
cost=i + 1,
time=i + 1,
status=StatusType.SUCCESS,
instance_id=i,
seed=i,
additional_info=None)
inst_seed_pairs = self.rh.get_runs_for_config(
self.config1, only_max_observed_budget=True)
# cost used by incumbent for going over all runs in inst_seed_pairs
inc_sum_cost = self.rh.sum_cost(
config=self.config1, instance_seed_budget_keys=inst_seed_pairs)
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# 15*1.2 - 6
self.assertEqual(cutoff, 12)
intensifier.cutoff = 5
cutoff = intensifier._adapt_cutoff(challenger=self.config2,
run_history=self.rh,
inc_sum_cost=inc_sum_cost)
# scenario cutoff
self.assertEqual(cutoff, 5)
def __init__(self,
scenario: Scenario,
smac: Union[SMAC, None] = None,
mode: str = 'all',
X: Union[None, List[list], np.ndarray] = None,
y: Union[None, List[list], np.ndarray] = None,
numParams: int = -1,
impute: bool = False,
seed: int = 12345,
run: bool = False,
max_sample_size: int = -1,
fanova_cut_at_default: bool = False,
fANOVA_pairwise: bool = True,
forwardsel_feat_imp: bool = False,
incn_quant_var: bool = True,
marginalize_away_instances: bool = False,
save_folder: str = 'PIMP'):
"""
Interface to be used with SMAC or with X and y matrices.
:param scenario: The scenario object, that knows the configuration space.
:param smac: The smac object that keeps all the run-data
:param mode: The mode with which to run PIMP [ablation, fanova, all, forward-selection]
:param X: Numpy Array that contains parameter arrays
:param y: Numpy array that contains the corresponding performance values
:param numParams: The number of parameters to evaluate
:param impute: Flag to decide if censored data gets imputed or not
:param seed: The random seed
:param run: Flag to immediately compute the importance values after this setup or not.
"""
self.scenario = scenario
self.imp = None
self.mode = mode
self.save_folder = save_folder
if not os.path.exists(self.save_folder): os.mkdir(self.save_folder)
if smac is not None:
self.imp = Importance(scenario=scenario,
runhistory=smac.runhistory,
incumbent=smac.solver.incumbent,
seed=seed,
parameters_to_evaluate=numParams,
save_folder='PIMP',
impute_censored=impute,
max_sample_size=max_sample_size,
fANOVA_cut_at_default=fanova_cut_at_default,
fANOVA_pairwise=fANOVA_pairwise,
forwardsel_feat_imp=forwardsel_feat_imp,
incn_quant_var=incn_quant_var,
preprocess=marginalize_away_instances)
elif X is not None and y is not None:
X = np.array(X)
y = np.array(y)
runHist = RunHistory(average_cost)
if X.shape[0] != y.shape[0]:
raise Exception('Number of samples in X and y dont match!')
n_params = len(scenario.cs.get_hyperparameters())
feats = None
if X.shape[1] > n_params:
feats = X[:, n_params:]
assert feats.shape[1] == scenario.feature_array.shape[1]
X = X[:, :n_params]
for p in range(X.shape[1]): # Normalize the data to fit into [0, 1]
_min, _max = np.min(X[:, p]), np.max(X[:, p])
if _min < 0. or 1 < _max: # if it is not already normalized
for id, v in enumerate(X[:, p]):
X[id, p] = (v - _min) / (_max - _min)
# Add everything to a runhistory such that PIMP can work with it
for x, feat, y_val in zip(X, feats if feats is not None else X, y):
id = None
for inst in scenario.feature_dict: # determine on which instance a configuration was run
if np.all(scenario.feature_dict[inst] == feat):
id = inst
break
runHist.add(Configuration(scenario.cs, vector=x), y_val, 0, StatusType.SUCCESS, id)
self.X = X
self.y = y
best_ = None # Determine incumbent according to the best mean cost in the runhistory
for config in runHist.config_ids:
inst_seed_pairs = runHist.get_runs_for_config(config)
all_ = []
for inst, seed in inst_seed_pairs:
rk = RunKey(runHist.config_ids[config], inst, seed)
all_.append(runHist.data[rk].cost)
mean = np.mean(all_)
if best_ is None or best_[0] > mean:
best_ = (mean, config)
incumbent = best_[1]
self.imp = Importance(scenario=scenario,
runhistory=runHist,
seed=seed,
parameters_to_evaluate=numParams,
save_folder=self.save_folder,
impute_censored=impute,
incumbent=incumbent,
fANOVA_cut_at_default=fanova_cut_at_default,
fANOVA_pairwise=fANOVA_pairwise,
forwardsel_feat_imp=forwardsel_feat_imp,
incn_quant_var=incn_quant_var,
preprocess=marginalize_away_instances
)
else:
raise Exception('Neither X and y matrices nor a SMAC object were specified to compute the importance '
'values from!')
if run:
self.compute_importances()
def _process_racer_results(
self,
challenger: Configuration,
incumbent: Configuration,
run_history: RunHistory,
log_traj: bool = True,
) -> typing.Optional[Configuration]:
"""Process the result of a racing configuration against the
current incumbent. Might propose a new incumbent.
Parameters
----------
challenger : Configuration
Configuration which challenges incumbent
incumbent : Configuration
Best configuration so far
run_history : RunHistory
Stores all runs we ran so far
Returns
-------
new_incumbent: typing.Optional[Configuration]
Either challenger or incumbent
"""
chal_runs = run_history.get_runs_for_config(
challenger, only_max_observed_budget=True)
chal_perf = run_history.get_cost(challenger)
# if all <instance, seed> have been run, compare challenger performance
if not self.to_run:
new_incumbent = self._compare_configs(incumbent=incumbent,
challenger=challenger,
run_history=run_history,
log_traj=log_traj)
# update intensification stage
if new_incumbent == incumbent:
# move on to the next iteration
self.stage = IntensifierStage.RUN_INCUMBENT
self.continue_challenger = False
self.logger.debug(
'Estimated cost of challenger on %d runs: %.4f, but worse than incumbent',
len(chal_runs), chal_perf)
elif new_incumbent == challenger:
# New incumbent found
incumbent = challenger
self.continue_challenger = False
# compare against basis configuration if provided, else go to next iteration
if self.always_race_against and \
self.always_race_against != challenger:
self.stage = IntensifierStage.RUN_BASIS
else:
self.stage = IntensifierStage.RUN_INCUMBENT
self.logger.debug(
'Estimated cost of challenger on %d runs: %.4f, becomes new incumbent',
len(chal_runs), chal_perf)
else: # Line 17
# challenger is not worse, continue
self.N = 2 * self.N
self.continue_challenger = True
self.logger.debug(
'Estimated cost of challenger on %d runs: %.4f, adding %d runs to the queue',
len(chal_runs), chal_perf, self.N / 2)
else:
self.logger.debug(
'Estimated cost of challenger on %d runs: %.4f, still %d runs to go (continue racing)',
len(chal_runs), chal_perf, len(self.to_run))
return incumbent
def __init__(
self,
original_rh: RunHistory,
validated_rh: RunHistory,
validator: Validator,
scenario: Scenario,
default: Configuration,
incumbent: Configuration,
param_imp: Union[None, Dict[str, float]],
params: Union[int, List[str]],
n_configs: int,
pc_sort_by: str,
output_dir: str,
cs: ConfigurationSpace,
runtime: bool = False,
max_runs_epm: int = 3000000,
):
"""This function prepares the data from a SMAC-related
format (using runhistories and parameters) to a more general format
(using a dataframe). The resulting dataframe is passed to the
parallel_coordinates-routine
Parameters
----------
original_rh: RunHistory
runhistory that should contain only runs that were executed during search
validated_rh: RunHistory
runhistory that may contain as many runs as possible, also external runs.
this runhistory will be used to build the EPM
validator: Validator
validator to be used to estimate costs for configurations
scenario: Scenario
scenario object to take instances from
default, incumbent: Configuration
default and incumbent, they will surely be displayed
param_imp: Union[None, Dict[str->float]
if given, maps parameter-names to importance
params: Union[int, List[str]]
either directly the parameters to displayed or the number of parameters (will try to define the most
important ones
n_configs: int
number of configs to be plotted
pc_sort_by: str
defines the pimp-method by which to choose the plotted parameters
max_runs_epm: int
maximum number of runs to train the epm with. this should prevent MemoryErrors
output_dir: str
output directory for plots
cs: ConfigurationSpace
parameter configuration space to be visualized
runtime: boolean
runtime will be on logscale
"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
self.error = None
self.default = default
self.param_imp = param_imp
self.cs = cs
# Sorting by importance, if possible (choose first executed parameter-importance)
self.method, self.importance = "", {}
if pc_sort_by == 'all':
self.logger.debug("Sorting by average importance")
self.method = 'average'
for m, i in self.param_imp.items():
if i:
for p, imp in i.items():
if p in self.importance:
self.importance[p].append(imp)
else:
self.importance[p] = [imp]
self.importance = {
k: sum(v) / len(v)
for k, v in self.importance.items()
}
elif pc_sort_by in self.param_imp:
self.method, self.importance = pc_sort_by, self.param_imp[
pc_sort_by]
else:
self.logger.debug("%s not evaluated.. choosing at random from: %s",
pc_sort_by, str(list(self.param_imp.keys())))
for m, i in self.param_imp.items():
if i:
self.method, self.importance = m, i
break
self.hp_names = sorted(
[hp for hp in self.cs.get_hyperparameter_names()],
key=lambda x: self.importance.get(x, 0),
reverse=True)
self.logger.debug("Sorted hp's by method \'%s\': %s", self.method,
str(self.hp_names))
# To be set
self.plots = []
# Define set of configurations (limiting to max and choosing most interesting ones)
all_configs = original_rh.get_all_configs()
max_runs_epm = 300000 # Maximum total number of runs considered for epm to limit maximum possible number configs
max_configs = int(
max_runs_epm /
(len(scenario.train_insts) + len(scenario.test_insts)))
if len(all_configs) > max_configs:
self.logger.debug(
"Limiting number of configs to train epm from %d to %d (based on max runs %d) and choosing "
"the ones with the most runs (for parallel coordinates)",
len(all_configs), max_configs, max_runs_epm)
all_configs = sorted(
all_configs,
key=lambda c: len(original_rh.get_runs_for_config(c)
))[:max_configs]
if not default in all_configs:
all_configs = [default] + all_configs
if not incumbent in all_configs:
all_configs.append(incumbent)
# Get costs for those configurations
epm_rh = RunHistory(average_cost)
epm_rh.update(validated_rh)
if scenario.feature_dict: # if instances are available
epm_rh.update(
timing(validator.validate_epm)(all_configs,
'train+test',
1,
runhistory=validated_rh))
self.config_to_cost = {c: epm_rh.get_cost(c) for c in all_configs}
self.params = self.get_params(params)
self.n_configs = n_configs
self.pcp = ParallelCoordinatesPlotter(self.config_to_cost, output_dir,
cs, runtime)
def _plot_parallel_coordinates(
self,
original_rh: RunHistory,
validated_rh: RunHistory,
validator: Validator,
scenario: Scenario,
default: Configuration,
incumbent: Configuration,
param_imp: Union[None, Dict[str, float]],
output_dir: str,
cs: ConfigurationSpace,
runtime: bool = False,
):
"""
Parameters:
-----------
original_rh: RunHistory
runhistory that should contain only runs that were executed during search
validated_rh: RunHistory
runhistory that may contain as many runs as possible, also external runs.
this runhistory will be used to build the EPM
validator: Validator
validator to be used to estimate costs for configurations
scenario: Scenario
scenario object to take instances from
default, incumbent: Configuration
default and incumbent, they will surely be displayed
param_imp: Union[None, Dict[str->float]
if given, maps parameter-names to importance
output_dir: str
output directory for plots
cs: ConfigurationSpace
parameter configuration space to be visualized
runtime: boolean
runtime will be on logscale
"""
# Sorting parameters by importance, if possible (choose first executed parameter-importance)
method, importance = "", {}
if self.pc_sort_by == 'all':
self.logger.debug("Sorting by average importance")
method = 'average'
for m, i in param_imp.items():
if i:
for p, imp in i.items():
if p in importance:
importance[p].append(imp)
else:
importance[p] = [imp]
importance = {k: sum(v) / len(v) for k, v in importance.items()}
elif self.pc_sort_by in param_imp:
method, importance = self.pc_sort_by, param_imp[self.pc_sort_by]
else:
self.logger.debug("%s not evaluated.. choosing at random from: %s",
self.pc_sort_by, str(list(param_imp.keys())))
for m, i in param_imp.items():
if i:
method, importance = m, i
self.logger.debug("Chose %s", method)
break
hp_names = sorted([hp for hp in cs.get_hyperparameter_names()],
key=lambda x: importance.get(x, 0),
reverse=True)
self.logger.debug("Sorted hp's by method \'%s\': %s", method,
str(hp_names))
# To be set
self.plots = []
# Define set of configurations (limiting to max and choosing most interesting ones)
all_configs = original_rh.get_all_configs()
max_runs_epm = self.max_runs_epm # Maximum total number of runs considered for epm to limit maximum possible number configs
max_configs = int(
max_runs_epm /
(len(scenario.train_insts) + len(scenario.test_insts)))
if len(all_configs) > max_configs:
self.logger.debug(
"Limiting number of configs to train epm from %d to %d (based on max runs %d) and choosing "
"the ones with the most runs (for parallel coordinates)",
len(all_configs), max_configs, max_runs_epm)
all_configs = sorted(
all_configs,
key=lambda c: len(original_rh.get_runs_for_config(c)
))[:max_configs]
if not default in all_configs:
all_configs = [default] + all_configs
if not incumbent in all_configs:
all_configs.append(incumbent)
# Get costs for those configurations
epm_rh = RunHistory(average_cost)
epm_rh.update(validated_rh)
if scenario.feature_dict: # if instances are available
epm_rh.update(
timing(validator.validate_epm)(all_configs,
'train+test',
1,
runhistory=validated_rh))
config_to_cost = {c: epm_rh.get_cost(c) for c in all_configs}
pcp = ParallelCoordinatesPlotter(config_to_cost, output_dir, cs,
runtime)
try:
plots = [
pcp.plot_n_configs(
self.n_configs,
self.get_params(self.params, importance, hp_names))
]
self.logger.debug("Paths to plot(s): %s", str(plots))
return {'figure': plots}
except ValueError as err:
self.logger.debug("Error: %s", str(err))
return {'else': str(err)}
