def test_register_bracket_multi_fidelity(self, space, b_config):
"""Check that a point is registered inside the same bracket for diff fidelity."""
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
value = 50
fidelity = 1
point = (value, fidelity)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
bracket = asha.brackets[0]
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[0][1]
assert (0.0, point) == bracket.rungs[0][1][point_hash]
fidelity = 3
point = [value, fidelity]
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[1][1]
assert (0.0, point) != bracket.rungs[0][1][point_hash]
assert (0.0, point) == bracket.rungs[1][1][point_hash]
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(
asha,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
for i in range(2):
force_observe(
asha,
create_trial(
(3, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert asha.suggest(1) == []
def test_register_bracket_multi_fidelity(self, space, b_config):
"""Check that a point is registered inside the same bracket for diff fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 1
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
bracket = asha.brackets[0]
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[0][1]
assert (0.0, point) == bracket.rungs[0][1][point_hash]
fidelity = 3
point = [fidelity, value]
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
assert len(bracket.rungs[0])
assert point_hash in bracket.rungs[1][1]
assert (0.0, point) != bracket.rungs[0][1][point_hash]
assert (0.0, point) == bracket.rungs[1][1][point_hash]
def test_register_bracket_multi_fidelity(self, space, b_config):
"""Check that a point is registered inside the same bracket for diff fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 1
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = asha.get_id(trial, ignore_fidelity=True)
force_observe(asha, trial)
bracket = asha.brackets[0]
assert len(bracket.rungs[0])
assert trial_id in bracket.rungs[0]["results"]
assert bracket.rungs[0]["results"][trial_id][0] == 0.0
assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
fidelity = 3
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = asha.get_id(trial, ignore_fidelity=True)
force_observe(asha, trial)
assert len(bracket.rungs[1])
assert trial_id in bracket.rungs[1]["results"]
assert bracket.rungs[0]["results"][trial_id][1].params != trial.params
assert bracket.rungs[1]["results"][trial_id][0] == 0.0
assert bracket.rungs[1]["results"][trial_id][1].params == trial.params
def test_register_next_bracket(self, space, b_config):
"""Check that a point is registered inside the good bracket when higher fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 3
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
force_observe(asha, point, {"objective": 0.0})
assert sum(len(rung[1]) for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung[1]) for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung[1]) for rung in asha.brackets[2].rungs) == 0
assert point_hash in asha.brackets[1].rungs[0][1]
assert (0.0, point) == asha.brackets[1].rungs[0][1][point_hash]
value = 51
fidelity = 9
point = (fidelity, value)
point_hash = hashlib.md5(str([value]).encode("utf-8")).hexdigest()
force_observe(asha, point, {"objective": 0.0})
assert sum(len(rung[1]) for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung[1]) for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung[1]) for rung in asha.brackets[2].rungs) == 1
assert point_hash in asha.brackets[2].rungs[0][1]
assert (0.0, point) == asha.brackets[2].rungs[0][1][point_hash]
def test_register_next_bracket(self, space, b_config):
"""Check that a point is registered inside the good bracket when higher fidelity."""
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
value = 50
fidelity = 3
point = (value, fidelity)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
assert sum(len(rung[1]) for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung[1]) for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung[1]) for rung in asha.brackets[2].rungs) == 0
assert point_hash in asha.brackets[1].rungs[0][1]
assert (0.0, point) == asha.brackets[1].rungs[0][1][point_hash]
value = 51
fidelity = 9
point = (value, fidelity)
point_hash = hashlib.md5(str([value]).encode('utf-8')).hexdigest()
asha.observe([point], [{'objective': 0.0}])
assert sum(len(rung[1]) for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung[1]) for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung[1]) for rung in asha.brackets[2].rungs) == 1
assert point_hash in asha.brackets[2].rungs[0][1]
assert (0.0, point) == asha.brackets[2].rungs[0][1][point_hash]
def test_get_id(self, space, b_config):
"""Test valid id of points"""
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
assert asha.get_id([1, 'whatever']) == asha.get_id([1, 'is here'])
assert asha.get_id([1, 'whatever']) != asha.get_id([2, 'is here'])
def test_register_invalid_fidelity(self, space, b_config):
"""Check that a point cannot registered if fidelity is invalid."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 2
point = (fidelity, value)
with pytest.raises(ValueError) as ex:
asha.observe([point], [{'objective': 0.0}])
assert 'No bracket found for point' in str(ex.value)
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(["whatever",
[1, 1]]) == asha.get_id(["is here", [1, 1]])
assert asha.get_id(["whatever", [1, 1]]) != asha.get_id(
["is here", [2, 2]])
def test_register_invalid_fidelity(self, space, b_config):
"""Check that a point cannot registered if fidelity is invalid."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 2
point = (fidelity, value)
with pytest.raises(ValueError) as ex:
force_observe(asha, point, {"objective": 0.0})
assert "No bracket found for point" in str(ex.value)
def test_register_invalid_fidelity(self, space, b_config):
"""Check that a point cannot registered if fidelity is invalid."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 2
trial = create_trial_for_hb((fidelity, value))
asha.observe([trial])
assert not asha.has_suggested(trial)
assert not asha.has_observed(trial)
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(['whatever',
[1, 1]]) == asha.get_id(['is here', [1, 1]])
assert asha.get_id(['whatever', [1, 1]]) != asha.get_id(
['is here', [2, 2]])
def test_register_not_sampled(self, space, b_config, caplog):
"""Check that a point cannot registered if not sampled."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 2
point = (fidelity, value)
with caplog.at_level(logging.INFO, logger="orion.algo.asha"):
asha.observe([point], [{"objective": 0.0}])
assert len(caplog.records) == 1
assert "Ignoring point" in caplog.records[0].msg
def test_register_not_sampled(self, space, b_config, caplog):
"""Check that a point cannot registered if not sampled."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 2
trial = create_trial_for_hb((fidelity, value))
with caplog.at_level(logging.DEBUG, logger="orion.algo.hyperband"):
asha.observe([trial])
assert len(caplog.records) == 1
assert "Ignoring trial" in caplog.records[0].msg
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(asha, (1, i), {"objective": i})
for i in range(2):
force_observe(asha, (3, i), {"objective": i})
assert asha.suggest() is None
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
asha = ASHA(space)
for i in range(6):
asha.observe([(1, i)], [{'objective': i}])
for i in range(2):
asha.observe([(3, i)], [{'objective': i}])
assert asha.suggest() is None
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch'))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
assert asha.get_id(['whatever',
[1, 1]]) == asha.get_id(['is here', [1, 1]])
assert asha.get_id(['whatever', [1, 1]]) != asha.get_id(
['is here', [2, 2]])
def test_register_invalid_fidelity(self, space, b_config):
"""Check that a point cannot registered if fidelity is invalid."""
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
value = 50
fidelity = 2
point = (value, fidelity)
with pytest.raises(ValueError) as ex:
asha.observe([point], [{'objective': 0.0}])
assert 'No bracket found for point' in str(ex.value)
def test_register_corrupted_db(self, caplog, space, b_config):
"""Check that a point cannot registered if passed in order diff than fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 3
trial = create_trial_for_hb((fidelity, value))
force_observe(asha, trial)
assert "Trial registered to wrong bracket" not in caplog.text
fidelity = 1
trial = create_trial_for_hb((fidelity, value), objective=0.0)
caplog.clear()
force_observe(asha, trial)
assert "Trial registered to wrong bracket" in caplog.text
def test_register_corrupted_db(self, caplog, space, b_config):
"""Check that a point cannot registered if passed in order diff than fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 3
point = (fidelity, value)
asha.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' not in caplog.text
fidelity = 1
point = [fidelity, value]
caplog.clear()
asha.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' in caplog.text
def test_register_corrupted_db(self, caplog, space, b_config):
"""Check that a point cannot registered if passed in order diff than fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 3
point = (fidelity, value)
force_observe(asha, point, {"objective": 0.0})
assert "Point registered to wrong bracket" not in caplog.text
fidelity = 1
point = [fidelity, value]
caplog.clear()
force_observe(asha, point, {"objective": 0.0})
assert "Point registered to wrong bracket" in caplog.text
def test_register_corrupted_db(self, caplog, space, b_config):
"""Check that a point cannot registered if passed in order diff than fidelity."""
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
value = 50
fidelity = 3
point = (value, fidelity)
asha.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' not in caplog.text
fidelity = 1
point = [value, fidelity]
caplog.clear()
asha.observe([point], [{'objective': 0.0}])
assert 'Point registered to wrong bracket' in caplog.text
def test_register_next_bracket(self, space, b_config):
"""Check that a point is registered inside the good bracket when higher fidelity."""
asha = ASHA(space, num_brackets=3)
value = 50
fidelity = 3
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = asha.get_id(trial, ignore_fidelity=True)
force_observe(asha, trial)
assert sum(len(rung["results"])
for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung["results"])
for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung["results"])
for rung in asha.brackets[2].rungs) == 0
assert trial_id in asha.brackets[1].rungs[0]["results"]
compare_registered_trial(
asha.brackets[1].rungs[0]["results"][trial_id], trial)
value = 51
fidelity = 9
trial = create_trial_for_hb((fidelity, value), 0.0)
trial_id = asha.get_id(trial, ignore_fidelity=True)
force_observe(asha, trial)
assert sum(len(rung["results"])
for rung in asha.brackets[0].rungs) == 0
assert sum(len(rung["results"])
for rung in asha.brackets[1].rungs) == 1
assert sum(len(rung["results"])
for rung in asha.brackets[2].rungs) == 1
assert trial_id in asha.brackets[2].rungs[0]["results"]
compare_registered_trial(
asha.brackets[2].rungs[0]["results"][trial_id], trial)
def test_get_id(self, space, b_config):
"""Test valid id of points"""
asha = ASHA(space, num_brackets=3)
assert asha.get_id(['whatever', 1]) == asha.get_id(['is here', 1])
assert asha.get_id(['whatever', 1]) != asha.get_id(['is here', 2])
def asha(b_config, space):
"""Return an instance of ASHA."""
return ASHA(space)
def asha(space: Space):
"""Return an instance of ASHA."""
return ASHA(space)
def asha(b_config, space):
"""Return an instance of ASHA."""
return ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'])