def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31})
self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=False,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
n_seeds=2,
initial_budget=2,
max_budget=5,
eta=2,
)
def test_update_stage(self):
"""
test initialization of all parameters and tracking variables
"""
intensifier = Hyperband(
tae_runner=None, stats=self.stats, traj_logger=None,
rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False,
instances=[1], initial_budget=0.1, max_budget=1, eta=2)
intensifier._update_stage()
self.assertEqual(intensifier.s, 3)
self.assertEqual(intensifier.s_max, 3)
self.assertEqual(intensifier.hb_iters, 0)
self.assertIsInstance(intensifier.sh_intensifier, SuccessiveHalving)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0])
# next HB stage
intensifier._update_stage()
self.assertEqual(intensifier.s, 2)
self.assertEqual(intensifier.hb_iters, 0)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [4.0, 2.0, 1.0])
intensifier._update_stage() # s = 1
intensifier._update_stage() # s = 0
# HB iteration completed
intensifier._update_stage()
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(intensifier.hb_iters, 1)
self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125)
self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0])
def test_eval_challenger(self):
"""
since hyperband uses eval_challenger and get_next_challenger of the internal successive halving,
we don't test these method extensively
"""
def target(x):
return 0.1
taf = ExecuteTAFuncDict(ta=target, stats=self.stats)
taf.runhistory = self.rh
intensifier = Hyperband(
tae_runner=taf, stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False,
instances=[None], initial_budget=0.5, max_budget=1, eta=2)
self.assertFalse(hasattr(intensifier, 's'))
# Testing get_next_challenger - get next configuration
config, _ = intensifier.get_next_challenger(challengers=[self.config2, self.config3],
chooser=None, run_history=self.rh)
self.assertEqual(intensifier.s, intensifier.s_max)
self.assertEqual(config, self.config2)
# update to the last SH iteration of the given HB stage
self.assertEqual(intensifier.s, 1)
self.assertEqual(intensifier.s_max, 1)
self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS,
seed=0, budget=1)
self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS,
seed=0, budget=0.5)
self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS,
seed=0, budget=0.5)
intensifier.sh_intensifier.success_challengers = {self.config2, self.config3}
intensifier.sh_intensifier._update_stage(self.rh)
# evaluation should change the incumbent to config2
inc, inc_value = intensifier.eval_challenger(challenger=self.config2,
incumbent=self.config1,
run_history=self.rh)
self.assertEqual(inc, self.config2)
self.assertEqual(intensifier.s, 0)
self.assertEqual(inc_value, 0.1)
self.assertEqual(self.stats.ta_runs, 1)
self.assertEqual(list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2])
self.assertEqual(self.stats.inc_changed, 1)
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)
class TestHyperband(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.rh = RunHistory()
self.cs = get_config_space()
self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11})
self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17})
self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7})
self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31})
self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33})
self.scen = Scenario({
"cutoff_time": 2,
'cs': self.cs,
"run_obj": 'runtime',
"output_dir": ''
})
self.stats = Stats(scenario=self.scen)
self.stats.start_timing()
# Create the base object
self.HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=False,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
n_seeds=2,
initial_budget=2,
max_budget=5,
eta=2,
)
def test_initialization(self):
"""Makes sure that a proper_HB is created"""
# We initialize the HB with zero intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 0)
# Add an instance to check the_HB initialization
self.assertTrue(self.HB._add_new_instance(num_workers=1))
# Some default init
self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0)
self.assertEqual(self.HB.intensifier_instances[0].max_budget, 5)
self.assertEqual(self.HB.intensifier_instances[0].initial_budget, 2)
# Update the stage
(self.HB.intensifier_instances[0]._update_stage(self.rh))
# Parameters properly passed to SH
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.inst_seed_pairs
), 10)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5)
def test_process_results_via_sourceid(self):
"""Makes sure source id is honored when deciding
which_HB instance will consume the result/run_info
"""
# Mock the_HB instance so we can make sure the correct item is passed
for i in range(10):
self.HB.intensifier_instances[i] = mock.Mock()
self.HB.intensifier_instances[i].process_results.return_value = (
self.config1, 0.5)
# make iter false so the mock object is not overwritten
self.HB.intensifier_instances[i].iteration_done = False
# randomly create run_infos and push into HB. Then we will make
# sure they got properly allocated
for i in np.random.choice(list(range(10)), 30):
run_info = RunInfo(
config=self.config1,
instance=0,
instance_specific="0",
cutoff=None,
seed=0,
capped=False,
budget=0.0,
source_id=i,
)
# make sure results aren't messed up via magic variable
# That is we check only the proper_HB instance has this
magic = time.time()
result = RunValue(cost=1,
time=0.5,
status=StatusType.SUCCESS,
starttime=1,
endtime=2,
additional_info=magic)
self.HB.process_results(run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=None,
result=result,
log_traj=False)
# Check the call arguments of each sh instance and make sure
# it is the correct one
# First the expected one
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['run_info'], run_info)
self.assertEqual(
self.HB.intensifier_instances[i].process_results.call_args[1]
['result'], result)
all_other_run_infos, all_other_results = [], []
for j in range(len(self.HB.intensifier_instances)):
# Skip the expected_HB instance
if i == j:
continue
if self.HB.intensifier_instances[
j].process_results.call_args is None:
all_other_run_infos.append(None)
else:
all_other_run_infos.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['run_info'])
all_other_results.append(
self.HB.intensifier_instances[j].process_results.
call_args[1]['result'])
self.assertNotIn(run_info, all_other_run_infos)
self.assertNotIn(result, all_other_results)
def test_get_next_run_single_HB_instance(self):
"""Makes sure that a single_HB instance returns a valid config"""
challengers = [self.config1, self.config2, self.config3, self.config4]
for i in range(30):
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=1,
)
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
if intent == RunInfoIntent.WAIT:
break
# Add the config to self.rh in order to make HB aware that this
# config/instance was launched
self.rh.add(
config=run_info.config,
cost=10,
time=0.0,
status=StatusType.RUNNING,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
# smax==1 (int(np.floor(np.log(self.max_budget / self.initial_budget) / np.log(self. eta))))
self.assertEqual(self.HB.intensifier_instances[0].s_max, 1)
# And we do not even complete 1 iteration, so s has to be 1
self.assertEqual(self.HB.intensifier_instances[0].s, 1)
# We should not create more_HB instance intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 1)
# We are running with:
# 'all_budgets': array([2.5, 5. ]) -> 2 intensifier_instances per config top
# 'n_configs_in_stage': [2.0, 1.0],
# This means we run int(2.5) + 2.0 = 4 runs before waiting
self.assertEqual(i, 4)
# Also, check the internals of the unique sh instance
# sh_initial_budget==2.5 (self.eta ** -self.s * self.max_budget)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
# n_challengers=2 (int(np.floor((self.s_max + 1) / (self.s + 1)) * self.eta ** self.s))
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.
n_configs_in_stage), 2)
def test_get_next_run_multiple_HB_instances(self):
"""Makes sure that two _HB instance can properly coexist and tag
run_info properly"""
# We allow 2_HB instance to be created. This means, we have a newer iteration
# to expect in hyperband
challengers = [self.config1, self.config2, self.config3, self.config4]
run_infos = []
for i in range(30):
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=2,
)
run_infos.append(run_info)
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
# Add the config to self.rh in order to make HB aware that this
# config/instance was launched
if intent == RunInfoIntent.WAIT:
break
self.rh.add(
config=run_info.config,
cost=10,
time=0.0,
status=StatusType.RUNNING,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
# We have not completed an iteration
self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0)
# Because n workers is now 2, we expect 2 sh intensifier_instances
self.assertEqual(len(self.HB.intensifier_instances), 2)
# Each of the intensifier_instances should have s equal to 1
# As no iteration has been completed
self.assertEqual(self.HB.intensifier_instances[0].s_max, 1)
self.assertEqual(self.HB.intensifier_instances[0].s, 1)
self.assertEqual(self.HB.intensifier_instances[1].s_max, 1)
self.assertEqual(self.HB.intensifier_instances[1].s, 1)
# First let us check everything makes sense in_HB-SH-0 HB-SH-0
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5)
self.assertEqual(
len(self.HB.intensifier_instances[0].sh_intensifier.
n_configs_in_stage), 2)
self.assertEqual(
self.HB.intensifier_instances[1].sh_intensifier.initial_budget, 2)
self.assertEqual(
self.HB.intensifier_instances[1].sh_intensifier.max_budget, 5)
self.assertEqual(
len(self.HB.intensifier_instances[1].sh_intensifier.
n_configs_in_stage), 2)
# We are running with:
# + 4 runs for sh instance 0 ('all_budgets': array([2.5, 5. ]), 'n_configs_in_stage': [2.0, 1.0])
# that is, for SH0 we run in stage==0 int(2.5) intensifier_instances * 2.0 configs
# And this times 2 because we have 2_HB intensifier_instances
self.assertEqual(i, 8)
# Adding a new worker is not possible as we already have 2 intensifier_instances
# and n_workers==2
intent, run_info = self.HB.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=self.rh,
num_workers=2,
)
self.assertEqual(intent, RunInfoIntent.WAIT)
def test_add_new_instance(self):
"""Test whether we can add a instance and when we should not"""
# By default we do not create a_HB
# test adding the first instance!
self.assertEqual(len(self.HB.intensifier_instances), 0)
self.assertTrue(self.HB._add_new_instance(num_workers=1))
self.assertEqual(len(self.HB.intensifier_instances), 1)
self.assertIsInstance(self.HB.intensifier_instances[0], _Hyperband)
# A second call should not add a new_HB instance
self.assertFalse(self.HB._add_new_instance(num_workers=1))
# We try with 2_HB instance active
# We effectively return true because we added a new_HB instance
self.assertTrue(self.HB._add_new_instance(num_workers=2))
self.assertEqual(len(self.HB.intensifier_instances), 2)
self.assertIsInstance(self.HB.intensifier_instances[1], _Hyperband)
# Trying to add a third one should return false
self.assertFalse(self.HB._add_new_instance(num_workers=2))
self.assertEqual(len(self.HB.intensifier_instances), 2)
def _exhaust_run_and_get_incumbent(self, sh, rh, num_workers=2):
"""
Runs all provided configs on all intensifier_instances and return the incumbent
as a nice side effect runhistory/stats are properly filled
"""
challengers = [self.config1, self.config2, self.config3, self.config4]
incumbent = None
for i in range(100):
try:
intent, run_info = sh.get_next_run(
challengers=challengers,
incumbent=None,
chooser=None,
run_history=rh,
num_workers=num_workers,
)
except ValueError as e:
# Get configurations until you run out of them
print(e)
break
# Regenerate challenger list
challengers = [c for c in challengers if c != run_info.config]
if intent == RunInfoIntent.WAIT:
break
result = target_from_run_info(run_info)
rh.add(
config=run_info.config,
cost=result.cost,
time=result.time,
status=result.status,
instance_id=run_info.instance,
seed=run_info.seed,
budget=run_info.budget,
)
incumbent, inc_perf = sh.process_results(
run_info=run_info,
incumbent=incumbent,
run_history=rh,
time_bound=100.0,
result=result,
log_traj=False,
)
return incumbent, inc_perf
def test_parallel_same_as_serial_HB(self):
"""Makes sure we behave the same as a serial run at the end"""
# Get the run_history for a_HB instance run:
rh = RunHistory()
stats = Stats(scenario=self.scen)
stats.start_timing()
_HB = _Hyperband(
stats=stats,
traj_logger=TrajLogger(output_dir=None, stats=stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent, inc_perf = self._exhaust_run_and_get_incumbent(
_HB, rh, num_workers=1)
# Just to make sure nothing has changed from the_HB instance side to make
# this check invalid:
# We add config values, so config 3 with 0 and 7 should be the lesser cost
self.assertEqual(incumbent, self.config3)
self.assertEqual(inc_perf, 7.0)
# Do the same for HB, but have multiple_HB instance in there
# This_HB instance will be created via num_workers==2
# in self._exhaust_run_and_get_incumbent
HB = Hyperband(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
deterministic=True,
run_obj_time=False,
instances=[1, 2, 3, 4, 5],
initial_budget=2,
max_budget=5,
eta=2,
)
incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent(
HB, self.rh)
self.assertEqual(incumbent, incumbent_phb)
# This makes sure there is a single incumbent in HB
self.assertEqual(inc_perf, inc_perf_phb)
# We don't want to loose any configuration, and particularly
# we want to make sure the values of_HB instance to HB match
self.assertEqual(len(self.rh.data), len(rh.data))
# Because it is a deterministic run, the run histories must be the
# same on exhaustion
self.assertDictEqual(self.rh.data, rh.data)