def test_dimensionality_and_warping_ranges():
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeCategorical('categorical1', ('X', 'Y')),
HyperparameterRangeContinuous('integer', 0.1, 10.0, LogScaling()),
HyperparameterRangeCategorical('categorical2', ('a', 'b', 'c')),
HyperparameterRangeContinuous('real', 0.0, 10.0, LinearScaling(), 2.5,
5.0),
HyperparameterRangeCategorical('categorical3', ('X', 'Y')),
)
dim, warping_ranges = dimensionality_and_warping_ranges(hp_ranges)
assert dim == 9
assert warping_ranges == {2: (0.0, 1.0), 6: (0.0, 1.0)}
def test_to_ndarray_name_last_pos():
np.random.seed(123456)
random_state = np.random.RandomState(123456)
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameters([
CSH.UniformFloatHyperparameter('a', lower=0., upper=1.),
CSH.UniformIntegerHyperparameter('b', lower=2, upper=3),
CSH.CategoricalHyperparameter('c', choices=('1', '2', '3')),
CSH.UniformIntegerHyperparameter('d', lower=2, upper=3),
CSH.CategoricalHyperparameter('e', choices=('1', '2'))
])
hp_a = HyperparameterRangeContinuous('a',
lower_bound=0.,
upper_bound=1.,
scaling=LinearScaling())
hp_b = HyperparameterRangeInteger('b',
lower_bound=2,
upper_bound=3,
scaling=LinearScaling())
hp_c = HyperparameterRangeCategorical('c', choices=('1', '2', '3'))
hp_d = HyperparameterRangeInteger('d',
lower_bound=2,
upper_bound=3,
scaling=LinearScaling())
hp_e = HyperparameterRangeCategorical('e', choices=('1', '2'))
for name_last_pos in ['a', 'c', 'd', 'e']:
hp_ranges_cs = HyperparameterRanges_CS(config_space,
name_last_pos=name_last_pos)
if name_last_pos == 'a':
lst = [hp_b, hp_c, hp_d, hp_e, hp_a]
elif name_last_pos == 'c':
lst = [hp_a, hp_b, hp_d, hp_e, hp_c]
elif name_last_pos == 'd':
lst = [hp_a, hp_b, hp_c, hp_e, hp_d]
else:
lst = [hp_a, hp_b, hp_c, hp_d, hp_e]
hp_ranges = HyperparameterRanges_Impl(*lst)
names = [hp.name for hp in hp_ranges.hp_ranges]
config_cs = hp_ranges_cs.random_candidate(random_state)
_config = config_cs.get_dictionary()
config = (_config[name] for name in names)
ndarr_cs = hp_ranges_cs.to_ndarray(config_cs)
ndarr = hp_ranges.to_ndarray(config)
assert_allclose(ndarr_cs, ndarr, rtol=1e-4)
def test_get_internal_candidate_evaluations():
"""we do not test the case with no evaluations, as it is assumed
that there will be always some evaluations generated in the beginning
of the BO loop."""
candidates = [
CandidateEvaluation((2, 3.3, 'X'), dictionarize_objective(5.3)),
CandidateEvaluation((1, 9.9, 'Y'), dictionarize_objective(10.9)),
CandidateEvaluation((7, 6.1, 'X'), dictionarize_objective(13.1)),
]
state = TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeInteger('integer', 0, 10, LinearScaling()),
HyperparameterRangeContinuous('real', 0, 10, LinearScaling()),
HyperparameterRangeCategorical('categorical', ('X', 'Y')),
),
candidate_evaluations=candidates,
failed_candidates=[candidates[0].candidate
], # these should be ignored by the model
pending_evaluations=[])
result = get_internal_candidate_evaluations(state,
DEFAULT_METRIC,
normalize_targets=True,
num_fantasize_samples=20)
assert len(result.X.shape) == 2, "Input should be a matrix"
assert len(result.y.shape) == 2, "Output should be a matrix"
assert result.X.shape[0] == len(candidates)
assert result.y.shape[
-1] == 1, "Only single output value per row is suppored"
assert np.abs(np.mean(
result.y)) < 1e-8, "Mean of the normalized outputs is not 0.0"
assert np.abs(np.std(result.y) -
1.0) < 1e-8, "Std. of the normalized outputs is not 1.0"
np.testing.assert_almost_equal(result.mean, 9.766666666666666)
np.testing.assert_almost_equal(result.std, 3.283629428273267)
def tuning_job_state():
return {'algo-1': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a1_hp_1', -5.0, 5.0, LinearScaling()),
HyperparameterRangeCategorical('a1_hp_2', ('a', 'b', 'c'))),
candidate_evaluations=[CandidateEvaluation(candidate=(-3.0, 'a'), value=1.0),
CandidateEvaluation(candidate=(-1.9, 'c'), value=2.0),
CandidateEvaluation(candidate=(-3.5, 'a'), value=0.3)],
failed_candidates=[],
pending_evaluations=[]
),
'algo-2': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a2_hp_1', -5.0, 5.0, LinearScaling()),
HyperparameterRangeInteger('a2_hp_2', -5, 5, LinearScaling(), -5, 5)),
candidate_evaluations=[CandidateEvaluation(candidate=(-1.9, -1), value=0.0),
CandidateEvaluation(candidate=(-3.5, 3), value=2.0)],
failed_candidates=[],
pending_evaluations=[]
)
}
def test_to_ndarray():
np.random.seed(123456)
random_state = np.random.RandomState(123456)
prob_categ = 0.3
for iter in range(20):
# Create ConfigurationSpace
num_hps = np.random.randint(low=1, high=20)
if iter == 0:
_prob_categ = 0.
elif iter == 1:
_prob_categ = 1.
else:
_prob_categ = prob_categ
config_space = CS.ConfigurationSpace()
ndarray_size = 0
_hp_ranges = dict()
for hp_it in range(num_hps):
name = str(hp_it)
if np.random.random() < _prob_categ:
num_choices = np.random.randint(low=2, high=11)
choices = tuple([str(i) for i in range(num_choices)])
hp = CSH.CategoricalHyperparameter(name, choices=choices)
hp2 = HyperparameterRangeCategorical(name, choices)
ndarray_size += num_choices
else:
ndarray_size += 1
rand_coin = np.random.random()
if rand_coin < 0.5:
log_scaling = (rand_coin < 0.25)
hp = CSH.UniformFloatHyperparameter(name=name,
lower=0.5,
upper=5.,
log=log_scaling)
hp2 = HyperparameterRangeContinuous(
name,
lower_bound=0.5,
upper_bound=5.,
scaling=LogScaling()
if log_scaling else LinearScaling())
else:
log_scaling = (rand_coin < 0.75)
hp = CSH.UniformIntegerHyperparameter(name=name,
lower=2,
upper=10,
log=log_scaling)
hp2 = HyperparameterRangeInteger(
name=name,
lower_bound=2,
upper_bound=10,
scaling=LogScaling()
if log_scaling else LinearScaling())
config_space.add_hyperparameter(hp)
_hp_ranges[name] = hp2
hp_ranges_cs = HyperparameterRanges_CS(config_space)
hp_ranges = HyperparameterRanges_Impl(
*[_hp_ranges[x] for x in config_space.get_hyperparameter_names()])
# Compare ndarrays created by both codes
for cmp_it in range(5):
config_cs = hp_ranges_cs.random_candidate(random_state)
_config = config_cs.get_dictionary()
config = (_config[name]
for name in config_space.get_hyperparameter_names())
ndarr_cs = hp_ranges_cs.to_ndarray(config_cs)
ndarr = hp_ranges.to_ndarray(config)
assert_allclose(ndarr_cs, ndarr, rtol=1e-4)
def hp_ranges():
return HyperparameterRanges_Impl(
HyperparameterRangeInteger('hp1', 0, 200, LinearScaling()),
HyperparameterRangeCategorical('hp2', ('a', 'b', 'c')))
import pytest
from autogluon.core.searcher.bayesopt.datatypes.hp_ranges import \
HyperparameterRanges_Impl, HyperparameterRangeInteger, \
HyperparameterRangeContinuous, HyperparameterRangeCategorical
from autogluon.core.searcher.bayesopt.datatypes.scaling import LinearScaling
from autogluon.core.searcher.bayesopt.utils.duplicate_detector import \
DuplicateDetectorEpsilon, DuplicateDetectorIdentical, \
DuplicateDetectorNoDetection
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeInteger('hp1', 0, 1000000000, scaling=LinearScaling()),
HyperparameterRangeContinuous('hp2', -10.0, 10.0, scaling=LinearScaling()),
HyperparameterRangeCategorical('hp3', ('a', 'b', 'c')),
)
duplicate_detector_epsilon = DuplicateDetectorEpsilon(hp_ranges)
@pytest.mark.parametrize('existing, new, contained', [
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10000, 3.0, 'c'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.000001, 'a'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (20, 2.000001, 'b'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (25, 1.0, 'a'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (20, 2.0, 'b'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (19, 1.0, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0000001, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'c'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'b'), False),
def test_categorical_to_and_from_ndarray(choices, external_hp,
internal_ndarray):
hp_range = HyperparameterRangeCategorical('hp', choices)
assert_allclose(hp_range.to_ndarray(external_hp),
np.array(internal_ndarray))
assert hp_range.from_ndarray(np.array(internal_ndarray)) == external_hp
def test_distribution_of_random_candidates():
random_state = np.random.RandomState(0)
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('0',
1.0,
1000.0,
scaling=LinearScaling()),
HyperparameterRangeContinuous('1', 1.0, 1000.0, scaling=LogScaling()),
HyperparameterRangeContinuous('2',
0.9,
0.9999,
scaling=ReverseLogScaling()),
HyperparameterRangeInteger('3', 1, 1000, scaling=LinearScaling()),
HyperparameterRangeInteger('4', 1, 1000, scaling=LogScaling()),
HyperparameterRangeCategorical('5', ('a', 'b', 'c')),
)
num_random_candidates = 600
random_candidates = [
hp_ranges.random_candidate(random_state)
for _ in range(num_random_candidates)
]
# check converting back gets to the same candidate
for cand in random_candidates[2:]:
ndarray_candidate = hp_ranges.to_ndarray(cand)
converted_back = hp_ranges.from_ndarray(ndarray_candidate)
for hp, hp_converted_back in zip(cand, converted_back):
if isinstance(hp, str):
assert hp == hp_converted_back
else:
assert_almost_equal(hp, hp_converted_back)
hps0, hps1, hps2, hps3, hps4, hps5 = zip(*random_candidates)
assert 200 < np.percentile(hps0, 25) < 300
assert 450 < np.percentile(hps0, 50) < 550
assert 700 < np.percentile(hps0, 75) < 800
# same bounds as the previous but log scaling
assert 3 < np.percentile(hps1, 25) < 10
assert 20 < np.percentile(hps1, 50) < 40
assert 100 < np.percentile(hps1, 75) < 200
# reverse log
assert 0.9 < np.percentile(hps2, 25) < 0.99
assert 0.99 < np.percentile(hps2, 50) < 0.999
assert 0.999 < np.percentile(hps2, 75) < 0.9999
# integer
assert 200 < np.percentile(hps3, 25) < 300
assert 450 < np.percentile(hps3, 50) < 550
assert 700 < np.percentile(hps3, 75) < 800
# same bounds as the previous but log scaling
assert 3 < np.percentile(hps4, 25) < 10
assert 20 < np.percentile(hps4, 50) < 40
assert 100 < np.percentile(hps4, 75) < 200
counter = Counter(hps5)
assert len(counter) == 3
assert 150 < counter['a'] < 250 # should be about 200
assert 150 < counter['b'] < 250 # should be about 200
assert 150 < counter['c'] < 250 # should be about 200
