def test_suggest_promote_identic_objectives(self, hyperband: Hyperband,
bracket: HyperbandBracket):
"""Test that identic objectives are handled properly"""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
n_trials = 9
resources = 1
results = {}
for param in np.linspace(0, 8, 9):
trial = create_trial_for_hb((resources, param), objective=0)
trial_hash = trial.compute_trial_hash(
trial,
ignore_fidelity=True,
ignore_experiment=True,
)
assert trial.objective is not None
results[trial_hash] = (trial.objective.value, trial)
bracket.rungs[0] = RungDict(n_trials=n_trials,
resources=resources,
results=results)
candidates = hyperband.suggest(2)
assert candidates is not None
assert len(candidates) == 2
assert (sum(1 for trial in candidates
if trial.params[hyperband.fidelity_index] == 3) == 2)
def test_suggest_duplicates_between_calls(self, monkeypatch,
hyperband: Hyperband,
bracket: HyperbandBracket):
"""Test that same trials are not allowed in different suggest call of
the same hyperband execution.
"""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
fidelity = 1
duplicate_trial = create_trial_for_hb((fidelity, 0.0))
new_trial = create_trial_for_hb((fidelity, 0.5))
duplicate_id = hyperband.get_id(duplicate_trial, ignore_fidelity=True)
bracket.rungs[0]["results"] = {
duplicate_id: (0.0, duplicate_trial)
} # type: ignore
hyperband.trial_to_brackets[duplicate_id] = 0
trials = [duplicate_trial, new_trial]
mock_samples(
hyperband,
trials +
[create_trial_for_hb((fidelity, i)) for i in range(10 - 2)],
)
trials = hyperband.suggest(100)
assert trials is not None
assert trials[0].params == new_trial.params
def test_suggest_duplicates_between_execution(self, monkeypatch,
hyperband: Hyperband,
budgets: list[BudgetTuple]):
"""Test that sampling collisions are handled between different hyperband execution."""
hyperband.repetitions = 2
bracket = HyperbandBracket(hyperband, budgets, 1)
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
for i in range(9):
force_observe(hyperband, create_trial_for_hb((1, i), objective=i))
for i in range(3):
force_observe(hyperband, create_trial_for_hb((3, i), objective=i))
force_observe(hyperband, create_trial_for_hb((9, 0), objective=0))
assert not hyperband.is_done
# lr:7 and lr:8 are already sampled in first repetition, they should not be present
# in second repetition. Samples with lr:7 and lr:8 will be ignored.
# (9, 0) already exists
candidates_for_epoch_9_bracket = [(9, 0), (9, 2), (9, 3), (9, 10)]
# (9, 1) -> (3, 1) already promoted in last repetition
# (9, 3) sampled for previous bracket
candidates_for_epoch_3_bracket = [(9, 1), (9, 3), (9, 4), (9, 5),
(9, 11)]
# (9, 0) -> (1, 0) already sampled in last repetition
# (9, 8) -> (1, 8) already sampled in last repetition
candidates_for_epoch_1_bracket = [(9, 0), (9, 8), (9, 12), (9, 13)]
zhe_point = list(
map(
create_trial_for_hb,
candidates_for_epoch_9_bracket +
candidates_for_epoch_3_bracket +
candidates_for_epoch_1_bracket,
))
hyperband._refresh_brackets()
mock_samples(hyperband, zhe_point)
zhe_samples = hyperband.suggest(100)
assert zhe_samples is not None
assert len(zhe_samples) == 8
assert zhe_samples[0].params == {"epoch": 9, "lr": 2}
assert zhe_samples[1].params == {"epoch": 9, "lr": 3}
assert zhe_samples[2].params == {"epoch": 9, "lr": 10}
assert zhe_samples[3].params == {"epoch": 3, "lr": 4}
assert zhe_samples[4].params == {"epoch": 3, "lr": 5}
assert zhe_samples[5].params == {"epoch": 3, "lr": 11}
assert zhe_samples[6].params == {"epoch": 1, "lr": 12}
assert zhe_samples[7].params == {"epoch": 1, "lr": 13}
def test_suggest_promote(self, hyperband: Hyperband,
bracket: HyperbandBracket, rung_0: RungDict):
"""Test that correct point is promoted and returned."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
bracket.rungs[0] = rung_0
points = hyperband.suggest(100)
assert points is not None
assert len(points) == 3
assert points[0].params == {"epoch": 3, "lr": 0}
assert points[1].params == {"epoch": 3, "lr": 1}
assert points[2].params == {"epoch": 3, "lr": 2}
def test_suggest_duplicates_one_call(self, monkeypatch,
hyperband: Hyperband,
bracket: HyperbandBracket):
"""Test that same points are not allowed in the same suggest call ofxs
the same hyperband execution.
"""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
zhe_point = list(
map(create_trial_for_hb, [(1, 0.0), (1, 1.0), (1, 1.0), (1, 2.0)]))
mock_samples(hyperband, zhe_point * 2)
zhe_samples = hyperband.suggest(100)
assert zhe_samples is not None
assert zhe_samples[0].params["lr"] == 0.0
assert zhe_samples[1].params["lr"] == 1.0
assert zhe_samples[2].params["lr"] == 2.0
# zhe_point =
mock_samples(
hyperband,
list(
map(
create_trial_for_hb,
[
(3, 0.0),
(3, 1.0),
(3, 1.0),
(3, 2.0),
(3, 5.0),
(3, 4.0),
],
)),
)
hyperband.trial_to_brackets[hyperband.get_id(create_trial_for_hb(
(1, 0.0)),
ignore_fidelity=True)] = 0
hyperband.trial_to_brackets[hyperband.get_id(create_trial_for_hb(
(1, 0.0)),
ignore_fidelity=True)] = 0
zhe_samples = hyperband.suggest(100)
assert zhe_samples is not None
assert zhe_samples[0].params["lr"] == 5.0
assert zhe_samples[1].params["lr"] == 4.0
def test_suggest_opt_out(
self,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
rung_2: RungDict,
):
"""Test that Hyperband opts out when rungs are not ready."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
bracket.rungs[0] = rung_0
trial_id = next(iter(rung_0["results"].keys()))
objective, point = rung_0["results"][trial_id]
rung_0["results"][trial_id] = (None, point)
assert hyperband.suggest(100) == []
def test_suggest_inf_duplicates(
self,
monkeypatch,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
rung_2: RungDict,
):
"""Test that sampling inf collisions will return None."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
zhe_trial = create_trial_for_hb(("fidelity", 0.0))
hyperband.trial_to_brackets[hyperband.get_id(zhe_trial,
ignore_fidelity=True)] = 0
mock_samples(hyperband, [zhe_trial] * 2)
assert hyperband.suggest(100) == []
def test_suggest_new(
self,
monkeypatch,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
rung_2: RungDict,
):
"""Test that a new point is sampled."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
mock_samples(hyperband,
[create_trial_for_hb(("fidelity", i)) for i in range(10)])
trials = hyperband.suggest(100)
assert trials is not None
assert trials[0].params == {"epoch": 1.0, "lr": 0}
assert trials[1].params == {"epoch": 1.0, "lr": 1}
def test_register(
self,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
):
"""Check that a point is registered inside the bracket."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
bracket.rungs = [rung_0, rung_1]
trial = create_trial_for_hb((1, 0.0), 0.0)
trial_id = hyperband.get_id(trial, ignore_fidelity=True)
hyperband.observe([trial])
assert len(bracket.rungs[0])
assert trial_id in bracket.rungs[0]["results"]
assert bracket.rungs[0]["results"][trial_id][0] == 0.0
assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
def test_is_ready(
self,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
rung_2: RungDict,
):
"""Test that Hyperband bracket detects when rung is ready."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
bracket.rungs[0] = rung_0
rung = bracket.rungs[0]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung[trial_id]
rung[trial_id] = (None, point)
assert not bracket.is_ready()
assert not bracket.is_ready(0)
rung[trial_id] = (objective, point)
assert bracket.is_ready()
assert bracket.is_ready(0)
assert not bracket.is_ready(1)
assert not bracket.is_ready(2)
bracket.rungs[1] = rung_1
rung = bracket.rungs[1]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung[trial_id]
rung[trial_id] = (None, point)
assert not bracket.is_ready(
) # Should depend on last rung that contains trials
assert bracket.is_ready(0)
assert not bracket.is_ready(1)
assert not bracket.is_ready(2)
rung[trial_id] = (objective, point)
assert bracket.is_ready(
) # Should depend on last rung that contains trials
assert bracket.is_ready(0)
assert bracket.is_ready(1)
assert not bracket.is_ready(2)
bracket.rungs[2] = rung_2
rung = bracket.rungs[2]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung[trial_id]
rung[trial_id] = (None, point)
assert not bracket.is_ready(
) # Should depend on last rung that contains trials
assert bracket.is_ready(0)
assert bracket.is_ready(1)
assert not bracket.is_ready(2)
rung[trial_id] = (objective, point)
assert bracket.is_ready(
) # Should depend on last rung that contains trials
assert bracket.is_ready(0)
assert bracket.is_ready(1)
assert bracket.is_ready(2)
def test_is_filled(
self,
hyperband: Hyperband,
bracket: HyperbandBracket,
rung_0: RungDict,
rung_1: RungDict,
rung_2: RungDict,
):
"""Test that Hyperband bracket detects when rung is filled."""
hyperband.brackets = [bracket]
assert bracket.owner is hyperband
bracket.rungs[0] = rung_0
rung = bracket.rungs[0]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung.pop(trial_id)
assert not bracket.is_filled
assert not bracket.has_rung_filled(0)
rung[trial_id] = (objective, point)
assert bracket.is_filled
assert bracket.has_rung_filled(0)
assert not bracket.has_rung_filled(1)
assert not bracket.has_rung_filled(2)
bracket.rungs[1] = rung_1
rung = bracket.rungs[1]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung.pop(trial_id)
assert bracket.is_filled # Should depend first rung only
assert bracket.has_rung_filled(0)
assert not bracket.has_rung_filled(1)
rung[trial_id] = (objective, point)
assert bracket.is_filled # Should depend first rung only
assert bracket.has_rung_filled(0)
assert bracket.has_rung_filled(1)
assert not bracket.has_rung_filled(2)
bracket.rungs[2] = rung_2
rung = bracket.rungs[2]["results"]
trial_id = next(iter(rung.keys()))
objective, point = rung.pop(trial_id)
assert bracket.is_filled # Should depend first rung only
assert bracket.has_rung_filled(0)
assert bracket.has_rung_filled(1)
assert not bracket.has_rung_filled(2)
rung[trial_id] = (objective, point)
assert bracket.is_filled # Should depend first rung only
assert bracket.has_rung_filled(0)
assert bracket.has_rung_filled(1)
assert bracket.has_rung_filled(2)