def test_update_stage(self):
"""
test initialization of all parameters and tracking variables
"""
intensifier = _Hyperband(
stats=self.stats, traj_logger=None,
rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False,
instances=[1], initial_budget=0.1, max_budget=1, eta=2)
intensifier._update_stage()
self.assertEqual(intensifier.s, 3)
self.assertEqual(intensifier.s_max, 3)
self.assertEqual(intensifier.hb_iters, 0)
self.assertIsInstance(intensifier.sh_intensifier, _SuccessiveHalving)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0])
# next HB stage
intensifier._update_stage()
self.assertEqual(intensifier.s, 2)
self.assertEqual(intensifier.hb_iters, 0)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [4.0, 2.0, 1.0])
intensifier._update_stage() # s = 1
intensifier._update_stage() # s = 0
# HB iteration completed
intensifier._update_stage()
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(intensifier.hb_iters, 1)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0])
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
def test_budget_initialization(self):
"""
Check computing budgets (only for non-instance cases)
"""
intensifier = _Hyperband(stats=None,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=None,
initial_budget=1,
max_budget=81,
eta=3)
self.assertListEqual([1, 3, 9, 27, 81],
intensifier.all_budgets.tolist())
self.assertListEqual([81, 27, 9, 3, 1], intensifier.n_configs_in_stage)
to_check = [
# minb, maxb, eta, n_configs_in_stage, all_budgets
[1, 81, 3, [81, 27, 9, 3, 1], [1, 3, 9, 27, 81]],
[
1, 600, 3, [243, 81, 27, 9, 3, 1],
[2.469135, 7.407407, 22.222222, 66.666666, 200, 600]
],
[1, 100, 10, [100, 10, 1], [1, 10, 100]],
[
0.001, 1, 3, [729, 243, 81, 27, 9, 3, 1],
[
0.001371, 0.004115, 0.012345, 0.037037, 0.111111, 0.333333,
1.0
]
],
[
1, 1000, 3, [729, 243, 81, 27, 9, 3, 1],
[
1.371742, 4.115226, 12.345679, 37.037037, 111.111111,
333.333333, 1000.0
]
],
[
0.001, 100, 10, [100000, 10000, 1000, 100, 10, 1],
[0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
],
]
for minb, maxb, eta, n_configs_in_stage, all_budgets in to_check:
intensifier = _Hyperband(stats=None,
traj_logger=None,
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=None,
initial_budget=minb,
max_budget=maxb,
eta=eta)
intensifier._init_sh_params(
initial_budget=minb,
max_budget=maxb,
eta=eta,
_all_budgets=None,
_n_configs_in_stage=None,
)
for i in range(len(all_budgets) + 10):
intensifier._update_stage()
comp_budgets = intensifier.sh_intensifier.all_budgets.tolist()
comp_configs = intensifier.sh_intensifier.n_configs_in_stage
self.assertIsInstance(comp_configs, list)
for c in comp_configs:
self.assertIsInstance(c, int)
# all_budgets for SH is always a subset of all_budgets of HB
np.testing.assert_array_almost_equal(
all_budgets[i % len(all_budgets):],
comp_budgets,
decimal=5)
# The content of these lists might differ
self.assertEqual(
len(n_configs_in_stage[i % len(n_configs_in_stage):]),
len(comp_configs))
def test_eval_challenger(self):
"""
since hyperband uses eval_challenger and get_next_run of the internal successive halving,
we don't test these method extensively
"""
def target(x):
return 0.1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = _Hyperband(stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[None],
initial_budget=0.5,
max_budget=1,
eta=2)
self.assertFalse(hasattr(intensifier, 's'))
# Testing get_next_run - get next configuration
intent, run_info = intensifier.get_next_run(
challengers=[self.config2, self.config3],
chooser=None,
incumbent=None,
run_history=self.rh)
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(run_info.config, self.config2)
# update to the last SH iteration of the given HB stage
self.assertEqual(intensifier.s, 1)
self.assertEqual(intensifier.s_max, 1)
# We assume now that process results was called with below successes.
# We track closely run execution through run_tracker, so this also
# has to be update -- the fact that the succesive halving inside hyperband
# processed the given configurations
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
seed=0,
budget=1)
intensifier.sh_intensifier.run_tracker[(self.config1, None, 0,
1)] = True
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
intensifier.sh_intensifier.run_tracker[(self.config2, None, 0,
0.5)] = True
self.rh.add(config=self.config3,
cost=3,
time=2,
status=StatusType.SUCCESS,
seed=0,
budget=0.5)
intensifier.sh_intensifier.run_tracker[(self.config3, None, 0,
0.5)] = True
intensifier.sh_intensifier.success_challengers = {
self.config2, self.config3
}
intensifier.sh_intensifier._update_stage(self.rh)
intent, run_info = intensifier.get_next_run(
challengers=[self.config2, self.config3],
chooser=None,
incumbent=None,
run_history=self.rh)
# evaluation should change the incumbent to config2
self.assertIsNotNone(run_info.config)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, inc_value = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.s, 0)
self.assertEqual(inc_value, 0.1)
self.assertEqual(
list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2])
self.assertEqual(self.stats.inc_changed, 1)