def test_local_grid_searcher_numeric():
search_spaces = [
[dict(a=Bool()), [{
'a': 0
}, {
'a': 1
}]],
[dict(a=Int(12, 15)), [{
'a': 12
}, {
'a': 13
}, {
'a': 14
}, {
'a': 15
}]],
[
dict(a=Real(12, 16)),
[{
'a': 12.0
}, {
'a': 13.333333333333334
}, {
'a': 14.666666666666666
}, {
'a': 16.0
}]
],
[
dict(a=Real(12, 16, log=True)),
[{
'a': 12.0
}, {
'a': 13.207708995578509
}, {
'a': 14.536964742657117
}, {
'a': 16.0
}]
],
]
for search_space, expected_values in search_spaces:
searcher = LocalGridSearcher(search_space=search_space)
actual_values = []
while True:
try:
cfg = searcher.get_config()
actual_values.append(cfg)
searcher.update(cfg, reward=0.1)
except AssertionError as e:
assert expected_values == actual_values
break
def test_local_random_searcher():
search_space = dict(
a=Real(0, 1, default=0.2),
b=Real(0.05, 1, default=0.4, log=True),
c=Int(5, 15),
d=Int(7, 23, default=16),
e=Categorical('a', 7, ['hello', 2]),
)
searcher = LocalRandomSearcher(search_space=search_space)
expected_config_1 = {'a': 0.2, 'b': 0.4, 'c': 5, 'd': 16, 'e': 'a'}
expected_config_2 = {'a': 0.5488135039273248, 'b': 0.4260424000595025, 'c': 8, 'd': 10, 'e': 7}
expected_config_3 = {'a': 0.6235636967859723, 'b': 0.15814742875130683, 'c': 12, 'd': 13, 'e': 'a'}
expected_config_4 = {'a': 0.9636627605010293, 'b': 0.1577026248478398, 'c': 11, 'd': 14, 'e': ['hello', 2]}
expected_config_5 = {'a': 0.5680445610939323, 'b': 0.800200824711684, 'c': 13, 'd': 16, 'e': 'a'}
assert searcher.get_best_reward() == float("-inf")
config1 = searcher.get_config()
assert searcher.get_reward(config1) == float("-inf")
assert searcher.get_best_reward() == float("-inf")
searcher.update(config1, accuracy=0.2)
assert searcher.get_reward(config1) == 0.2
assert searcher.get_best_reward() == 0.2
assert searcher.get_best_config() == config1
config2 = searcher.get_config()
config3 = searcher.get_config()
config4 = searcher.get_config()
searcher.update(config4, accuracy=0.1)
assert searcher.get_reward(config4) == 0.1
assert searcher.get_best_reward() == 0.2
assert searcher.get_best_config() == config1
searcher.update(config4, accuracy=0.5)
assert searcher.get_reward(config4) == 0.5
assert searcher.get_best_reward() == 0.5
assert searcher.get_best_config() == config4
config5 = searcher.get_config()
assert expected_config_1 == config1
assert expected_config_2 == config2
assert expected_config_3 == config3
assert expected_config_4 == config4
assert expected_config_5 == config5
assert len(searcher._results) == 5
def test_local_sequential_scheduler():
search_space = dict(
lr=Real(1e-3, 1e-2, log=True),
wd=Real(1e-3, 1e-2),
epochs=10,
)
def train_fn(args, reporter):
for e in range(args['epochs']):
dummy_reward = 1 - np.power(1.8, -np.random.uniform(e, 2 * e))
reporter(epoch=e + 1, reward=dummy_reward, lr=args.lr, wd=args.wd)
scheduler = LocalSequentialScheduler(train_fn,
search_space=search_space,
num_trials=10)
scheduler.run()
scheduler.join_jobs()
best_config = scheduler.get_best_config()
best_task_id = scheduler.get_best_task_id()
assert pickle.dumps(
scheduler.config_history[best_task_id]) == pickle.dumps(best_config)
def test_search_space():
search_space = dict(
a=Real(1e-3, 1e-2, log=True),
b=Real(1e-3, 1e-2),
c=Int(1, 10),
d=Categorical('a', 'b', 'c', 'd'),
)
def train_fn(args, reporter):
a, b, c, d = args['a'], args['b'], args['c'], args['d']
assert 1e-2 >= a >= 1e-3
assert 1e-2 >= b >= 1e-3
assert 10 >= c >= 1
assert d in ['a', 'b', 'c', 'd']
reporter(epoch=1, reward=0)
scheduler = LocalSequentialScheduler(train_fn,
search_space=search_space,
num_trials=10,
time_attr='epoch')
scheduler.run()
scheduler.join_jobs()
def test_timeout_scheduler():
search_space = dict(lr=Real(1E-5, 1E-3), )
def train_fn(args, reporter):
start_tick = time.time()
time.sleep(0.01)
reporter(reward=time.time() - start_tick, time_attr=0)
scheduler = LocalSequentialScheduler(train_fn,
search_space=search_space,
num_trials=7,
time_attr='time_attr',
time_out=0.025)
scheduler.run()
scheduler.join_jobs()
assert len(scheduler.config_history) <= 2
import pytest
from autogluon.core.hpo.space_converter import RaySpaceConverterFactory
from autogluon.core.space import Space, Categorical, Real, Int, Bool
from ray import tune
@pytest.mark.parametrize('space, expected_space', [
(Categorical([1, 2]), tune.choice([1, 2])),
(Real(1, 2, log=True), tune.loguniform(1, 2)),
(Real(1, 2, log=False), tune.uniform(1, 2)),
(Int(1, 2), tune.randint(1, 3)),
(Bool(), tune.randint(0, 2)),
])
def test_space_converter(space, expected_space):
ray_space = RaySpaceConverterFactory.get_space_converter(
space.__class__.__name__).convert(space)
assert type(ray_space) == type(expected_space)