def test_register_bracket_multi_fidelity(self, space: Space):
"""Check that a point is registered inside the same bracket for diff fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 1
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = hyperband.get_id(trial, ignore_fidelity=True)
force_observe(hyperband, trial)
assert hyperband.brackets is not None
bracket = hyperband.brackets[0]
assert len(bracket.rungs[0])
assert trial_id in bracket.rungs[0]["results"]
assert bracket.rungs[0]["results"][trial_id][0] == 0.0
assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
fidelity = 3
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = hyperband.get_id(trial, ignore_fidelity=True)
force_observe(hyperband, trial)
assert len(bracket.rungs[1])
assert trial_id in bracket.rungs[1]["results"]
assert bracket.rungs[0]["results"][trial_id][1].params != trial.params
assert bracket.rungs[1]["results"][trial_id][0] == 0.0
assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(["whatever", [1, 1]]) == hyperband.get_id(
["is here", [1, 1]]
)
assert hyperband.get_id(["whatever", [1, 1]]) != hyperband.get_id(
["is here", [2, 2]]
)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [2, 2]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) == hyperband.get_id(["same", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["same", [1, 1]])
def test_register_bracket_multi_fidelity(self, space):
"""Check that a point is registered inside the same bracket for diff fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 1
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
hyperband.observe([point], [{'objective': 0.0}])
bracket = hyperband.brackets[0]
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[0]['results']
assert (0.0, point) == bracket.rungs[0]['results'][point_hash]
fidelity = 3
point = [fidelity, value]
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
hyperband.observe([point], [{'objective': 0.0}])
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[1]['results']
assert (0.0, point) != bracket.rungs[0]['results'][point_hash]
assert (0.0, point) == bracket.rungs[1]['results'][point_hash]
def test_register_next_bracket(self, space):
"""Check that a point is registered inside the good bracket when higher fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 3
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
hyperband.observe([point], [{'objective': 0.0}])
assert sum(len(rung['results']) for rung in hyperband.brackets[0].rungs) == 0
assert sum(len(rung['results']) for rung in hyperband.brackets[1].rungs) == 1
assert sum(len(rung['results']) for rung in hyperband.brackets[2].rungs) == 0
assert point_hash in hyperband.brackets[1].rungs[0]['results']
assert (0.0, point) == hyperband.brackets[1].rungs[0]['results'][point_hash]
value = 51
fidelity = 9
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
hyperband.observe([point], [{'objective': 0.0}])
assert sum(len(rung['results']) for rung in hyperband.brackets[0].rungs) == 0
assert sum(len(rung['results']) for rung in hyperband.brackets[1].rungs) == 1
assert sum(len(rung['results']) for rung in hyperband.brackets[2].rungs) == 1
assert point_hash in hyperband.brackets[2].rungs[0]['results']
assert (0.0, point) == hyperband.brackets[2].rungs[0]['results'][point_hash]
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
force_observe(hyperband, (1, i), {"objective": i})
assert hyperband.suggest() is None
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
hyperband.observe([(1, i)], [{'objective': i}])
assert hyperband.suggest() is None
def test_register_invalid_fidelity(self, space):
"""Check that a point cannot registered if fidelity is invalid."""
hyperband = Hyperband(space)
value = 50
fidelity = 2
point = (fidelity, value)
with pytest.raises(ValueError) as ex:
force_observe(hyperband, point, {"objective": 0.0})
assert "No bracket found for point" in str(ex.value)
def test_register_invalid_fidelity(self, space: Space):
"""Check that a point cannot registered if fidelity is invalid."""
hyperband = Hyperband(space)
value = 50
fidelity = 2
trial = create_trial_for_hb((fidelity, value))
hyperband.observe([trial])
assert not hyperband.has_suggested(trial)
assert not hyperband.has_observed(trial)
def test_register_invalid_fidelity(self, space):
"""Check that a point cannot registered if fidelity is invalid."""
hyperband = Hyperband(space)
value = 50
fidelity = 2
point = (fidelity, value)
with pytest.raises(ValueError) as ex:
hyperband.observe([point], [{'objective': 0.0}])
assert 'No bracket found for point' in str(ex.value)
def test_register_not_sampled(self, space, caplog):
"""Check that a point cannot registered if not sampled."""
hyperband = Hyperband(space)
value = 50
fidelity = 2
trial = create_trial_for_hb((fidelity, value))
with caplog.at_level(logging.DEBUG, logger="orion.algo.hyperband"):
hyperband.observe([trial])
assert len(caplog.records) == 1
assert "Ignoring trial" in caplog.records[0].msg
def test_register_not_sampled(self, space, caplog):
"""Check that a point cannot registered if not sampled."""
hyperband = Hyperband(space)
value = 50
fidelity = 2
point = (fidelity, value)
with caplog.at_level(logging.INFO, logger="orion.algo.hyperband"):
hyperband.observe([point], [{"objective": 0.0}])
assert len(caplog.records) == 1
assert "Ignoring point" in caplog.records[0].msg
def test_get_id(self, space):
"""Test valid id of points"""
hyperband = Hyperband(space)
assert hyperband.get_id(['whatever', 1]) == hyperband.get_id(['is here', 1])
assert hyperband.get_id(['whatever', 1]) != hyperband.get_id(['is here', 2])
assert hyperband.get_id(['whatever', 1], ignore_fidelity=False) != \
hyperband.get_id(['is here', 1], ignore_fidelity=False)
assert hyperband.get_id(['whatever', 1], ignore_fidelity=False) != \
hyperband.get_id(['is here', 2], ignore_fidelity=False)
assert hyperband.get_id(['same', 1], ignore_fidelity=False) == \
hyperband.get_id(['same', 1], ignore_fidelity=False)
assert hyperband.get_id(['same', 1], ignore_fidelity=False) != \
hyperband.get_id(['same', 1])
def test_register_corrupted_db(self, caplog, space):
"""Check that a point cannot registered if passed in order diff than fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 3
point = (fidelity, value)
hyperband.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' not in caplog.text
fidelity = 1
point = [fidelity, value]
caplog.clear()
hyperband.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' in caplog.text
def test_register_corrupted_db(self, caplog, space):
"""Check that a point cannot registered if passed in order diff than fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 3
point = (fidelity, value)
force_observe(hyperband, point, {"objective": 0.0})
assert "Point registered to wrong bracket" not in caplog.text
fidelity = 1
point = [fidelity, value]
caplog.clear()
force_observe(hyperband, point, {"objective": 0.0})
assert "Point registered to wrong bracket" in caplog.text
def test_register_corrupted_db(self, caplog, space):
"""Check that a point cannot registered if passed in order diff than fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 3
trial = create_trial_for_hb((fidelity, value))
force_observe(hyperband, trial)
assert "Trial registered to wrong bracket" not in caplog.text
fidelity = 1
trial = create_trial_for_hb((fidelity, value))
caplog.clear()
force_observe(hyperband, trial)
assert "Trial registered to wrong bracket" in caplog.text
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(['whatever', [1, 1]]) == hyperband.get_id(['is here', [1, 1]])
assert hyperband.get_id(['whatever', [1, 1]]) != hyperband.get_id(['is here', [2, 2]])
assert hyperband.get_id(['whatever', [1, 1]], ignore_fidelity=False) != \
hyperband.get_id(['is here', [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(['whatever', [1, 1]], ignore_fidelity=False) != \
hyperband.get_id(['is here', [2, 2]], ignore_fidelity=False)
assert hyperband.get_id(['same', [1, 1]], ignore_fidelity=False) == \
hyperband.get_id(['same', [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(['same', [1, 1]], ignore_fidelity=False) != \
hyperband.get_id(['same', [1, 1]])
def test_get_id(self, space):
"""Test valid id of points"""
hyperband = Hyperband(space)
assert hyperband.get_id(["whatever", 1]) == hyperband.get_id(["is here", 1])
assert hyperband.get_id(["whatever", 1]) != hyperband.get_id(["is here", 2])
assert hyperband.get_id(
["whatever", 1], ignore_fidelity=False
) != hyperband.get_id(["is here", 1], ignore_fidelity=False)
assert hyperband.get_id(
["whatever", 1], ignore_fidelity=False
) != hyperband.get_id(["is here", 2], ignore_fidelity=False)
assert hyperband.get_id(["same", 1], ignore_fidelity=False) == hyperband.get_id(
["same", 1], ignore_fidelity=False
)
assert hyperband.get_id(["same", 1], ignore_fidelity=False) != hyperband.get_id(
["same", 1]
)
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
force_observe(
hyperband,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert hyperband.suggest(100) == []
def test_register_next_bracket(self, space: Space):
"""Check that a point is registered inside the good bracket when higher fidelity."""
hyperband = Hyperband(space)
value = 50
fidelity = 3
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = hyperband.get_id(trial, ignore_fidelity=True)
force_observe(hyperband, trial)
assert hyperband.brackets is not None
assert sum(
len(rung["results"]) for rung in hyperband.brackets[0].rungs) == 0
assert sum(
len(rung["results"]) for rung in hyperband.brackets[1].rungs) == 1
assert sum(
len(rung["results"]) for rung in hyperband.brackets[2].rungs) == 0
assert trial_id in hyperband.brackets[1].rungs[0]["results"]
compare_registered_trial(
hyperband.brackets[1].rungs[0]["results"][trial_id], trial)
value = 51
fidelity = 9
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = hyperband.get_id(trial, ignore_fidelity=True)
force_observe(hyperband, trial)
assert sum(
len(rung["results"]) for rung in hyperband.brackets[0].rungs) == 0
assert sum(
len(rung["results"]) for rung in hyperband.brackets[1].rungs) == 1
assert sum(
len(rung["results"]) for rung in hyperband.brackets[2].rungs) == 1
assert trial_id in hyperband.brackets[2].rungs[0]["results"]
compare_registered_trial(
hyperband.brackets[2].rungs[0]["results"][trial_id], trial)
def hyperband(space):
"""Return an instance of Hyperband."""
return Hyperband(space, repetitions=1)