def _construct_models(X, Y, metric, do_mcmc, with_pending):
pending_candidates = []
if with_pending:
pending_candidates = [PendingEvaluation((0.5, 0.5)), PendingEvaluation((0.2, 0.2))]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
),
[
CandidateEvaluation(x, y) for x, y in zip(X, Y)
],
[],
pending_candidates
)
random_seed = 0
gpmodel = default_gpmodel(
state, random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
result = [GaussProcSurrogateModel(
state, metric, random_seed, gpmodel, fit_parameters=True,
num_fantasy_samples=20)]
if do_mcmc:
gpmodel_mcmc = default_gpmodel_mcmc(
state, random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
result.append(
GaussProcSurrogateModel(
state, metric, random_seed, gpmodel_mcmc,
fit_parameters=True,num_fantasy_samples=20))
return result
def test_continuous_to_and_from_ndarray(lower, upper, external_hp,
internal_ndarray, scaling):
hp_range = HyperparameterRangeContinuous('hp', lower, upper, scaling)
assert_allclose(hp_range.to_ndarray(external_hp),
np.array([internal_ndarray]))
assert_allclose(hp_range.from_ndarray(np.array([internal_ndarray])),
external_hp)
def test_dimensionality_and_warping_ranges():
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeCategorical('categorical1', ('X', 'Y')),
HyperparameterRangeContinuous('integer', 0.1, 10.0, LogScaling()),
HyperparameterRangeCategorical('categorical2', ('a', 'b', 'c')),
HyperparameterRangeContinuous('real', 0.0, 10.0, LinearScaling(), 2.5,
5.0),
HyperparameterRangeCategorical('categorical3', ('X', 'Y')),
)
dim, warping_ranges = dimensionality_and_warping_ranges(hp_ranges)
assert dim == 9
assert warping_ranges == {2: (0.0, 1.0), 6: (0.0, 1.0)}
def default_models() -> List[GaussProcSurrogateModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0
), # same evals are added multiple times to force GP to unlearn prior
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
[],
[],
)
random_seed = 0
gpmodel = default_gpmodel(state,
random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
gpmodel_mcmc = default_gpmodel_mcmc(state,
random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
return [
GaussProcSurrogateModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=20),
GaussProcSurrogateModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel_mcmc,
fit_parameters=True,
num_fantasy_samples=20)
]
def tuning_job_state() -> TuningJobState:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
return TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def tuning_job_state_mcmc(X, Y) -> TuningJobState:
Y = [dictionarize_objective(y) for y in Y]
return TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', -4., 4., LinearScaling())),
[CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def default_models(metric, do_mcmc=True) -> List[GaussProcSurrogateModel]:
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
if metric == DEFAULT_METRIC:
Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X]
elif metric == DEFAULT_COST_METRIC:
# Increasing the first hp increases cost
Y = [{metric: 1.0 + x[0] * 2.0}
for x in X]
else:
raise ValueError(f"{metric} is not a valid metric")
state = TuningJobState(
HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()),
),
[
CandidateEvaluation(x, y) for x, y in zip(X, Y)
],
[], []
)
random_seed = 0
gpmodel = default_gpmodel(
state, random_seed=random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
result = [GaussProcSurrogateModel(
state, metric, random_seed, gpmodel, fit_parameters=True,
num_fantasy_samples=20)]
if do_mcmc:
gpmodel_mcmc = default_gpmodel_mcmc(
state, random_seed=random_seed,
mcmc_config=DEFAULT_MCMC_CONFIG)
result.append(
GaussProcSurrogateModel(
state, metric, random_seed, gpmodel_mcmc,
fit_parameters=True,num_fantasy_samples=20))
return result
def test_pick_from_locally_optimized():
duplicate_detector1 = DuplicateDetectorIdentical()
duplicate_detector2 = DuplicateDetectorEpsilon(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('hp1', -10.0, 10.0, scaling=LinearScaling()),
HyperparameterRangeContinuous('hp2', -10.0, 10.0, scaling=LinearScaling()),
)
)
for duplicate_detector in (duplicate_detector1, duplicate_detector2):
got = _pick_from_locally_optimized(
candidates_with_optimization=[
# original, optimized
((0.1, 1.0), (0.1, 1.0)),
((0.1, 1.0), (0.6, 1.0)), # not a duplicate
((0.2, 1.0), (0.1, 1.0)), # duplicate optimized; Resolved by the original
((0.1, 1.0), (0.1, 1.0)), # complete duplicate
((0.3, 1.0), (0.1, 1.0)), # blacklisted original
((0.4, 3.0), (0.3, 1.0)), # blacklisted all
((1.0, 2.0), (1.0, 1.0)), # final candidate to be selected into a batch
((0.0, 2.0), (1.0, 0.0)), # skipped
((0.0, 2.0), (1.0, 0.0)), # skipped
],
blacklisted_candidates={
(0.3, 1.0),
(0.4, 3.0),
(0.0, 0.0), # blacklisted candidate, not present in candidates
},
num_candidates=4,
duplicate_detector=duplicate_detector,
)
expected = [
(0.1, 1.0),
(0.6, 1.0),
(0.2, 1.0),
(1.0, 1.0)
]
# order of the candidates should be preserved
assert len(expected) == len(got)
assert all(a == b for a, b in zip(got, expected))
def tuning_job_state():
return {'algo-1': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a1_hp_1', -5.0, 5.0, LinearScaling()),
HyperparameterRangeCategorical('a1_hp_2', ('a', 'b', 'c'))),
candidate_evaluations=[CandidateEvaluation(candidate=(-3.0, 'a'), value=1.0),
CandidateEvaluation(candidate=(-1.9, 'c'), value=2.0),
CandidateEvaluation(candidate=(-3.5, 'a'), value=0.3)],
failed_candidates=[],
pending_evaluations=[]
),
'algo-2': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a2_hp_1', -5.0, 5.0, LinearScaling()),
HyperparameterRangeInteger('a2_hp_2', -5, 5, LinearScaling(), -5, 5)),
candidate_evaluations=[CandidateEvaluation(candidate=(-1.9, -1), value=0.0),
CandidateEvaluation(candidate=(-3.5, 3), value=2.0)],
failed_candidates=[],
pending_evaluations=[]
)
}
def test_to_ndarray_name_last_pos():
np.random.seed(123456)
random_state = np.random.RandomState(123456)
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameters([
CSH.UniformFloatHyperparameter('a', lower=0., upper=1.),
CSH.UniformIntegerHyperparameter('b', lower=2, upper=3),
CSH.CategoricalHyperparameter('c', choices=('1', '2', '3')),
CSH.UniformIntegerHyperparameter('d', lower=2, upper=3),
CSH.CategoricalHyperparameter('e', choices=('1', '2'))
])
hp_a = HyperparameterRangeContinuous('a',
lower_bound=0.,
upper_bound=1.,
scaling=LinearScaling())
hp_b = HyperparameterRangeInteger('b',
lower_bound=2,
upper_bound=3,
scaling=LinearScaling())
hp_c = HyperparameterRangeCategorical('c', choices=('1', '2', '3'))
hp_d = HyperparameterRangeInteger('d',
lower_bound=2,
upper_bound=3,
scaling=LinearScaling())
hp_e = HyperparameterRangeCategorical('e', choices=('1', '2'))
for name_last_pos in ['a', 'c', 'd', 'e']:
hp_ranges_cs = HyperparameterRanges_CS(config_space,
name_last_pos=name_last_pos)
if name_last_pos == 'a':
lst = [hp_b, hp_c, hp_d, hp_e, hp_a]
elif name_last_pos == 'c':
lst = [hp_a, hp_b, hp_d, hp_e, hp_c]
elif name_last_pos == 'd':
lst = [hp_a, hp_b, hp_c, hp_e, hp_d]
else:
lst = [hp_a, hp_b, hp_c, hp_d, hp_e]
hp_ranges = HyperparameterRanges_Impl(*lst)
names = [hp.name for hp in hp_ranges.hp_ranges]
config_cs = hp_ranges_cs.random_candidate(random_state)
_config = config_cs.get_dictionary()
config = (_config[name] for name in names)
ndarr_cs = hp_ranges_cs.to_ndarray(config_cs)
ndarr = hp_ranges.to_ndarray(config)
assert_allclose(ndarr_cs, ndarr, rtol=1e-4)
def multi_algo_state():
def _candidate_evaluations(num):
return [
CandidateEvaluation(
candidate=(i,),
metrics=dictionarize_objective(float(i)))
for i in range(num)]
return {
'0': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeContinuous('a1_hp_1', -5.0, 5.0, LinearScaling(), -5.0, 5.0)),
candidate_evaluations=_candidate_evaluations(2),
failed_candidates=[(i,) for i in range(3)],
pending_evaluations=[PendingEvaluation((i,)) for i in range(100)]
),
'1': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(5),
failed_candidates=[],
pending_evaluations=[]
),
'2': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(3),
failed_candidates=[(i,) for i in range(10)],
pending_evaluations=[PendingEvaluation((i,)) for i in range(1)]
),
'3': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(6),
failed_candidates=[],
pending_evaluations=[]
),
'4': TuningJobState(
hp_ranges=HyperparameterRanges_Impl(),
candidate_evaluations=_candidate_evaluations(120),
failed_candidates=[],
pending_evaluations=[]
),
}
def test_get_internal_candidate_evaluations():
"""we do not test the case with no evaluations, as it is assumed
that there will be always some evaluations generated in the beginning
of the BO loop."""
candidates = [
CandidateEvaluation((2, 3.3, 'X'), dictionarize_objective(5.3)),
CandidateEvaluation((1, 9.9, 'Y'), dictionarize_objective(10.9)),
CandidateEvaluation((7, 6.1, 'X'), dictionarize_objective(13.1)),
]
state = TuningJobState(
hp_ranges=HyperparameterRanges_Impl(
HyperparameterRangeInteger('integer', 0, 10, LinearScaling()),
HyperparameterRangeContinuous('real', 0, 10, LinearScaling()),
HyperparameterRangeCategorical('categorical', ('X', 'Y')),
),
candidate_evaluations=candidates,
failed_candidates=[candidates[0].candidate
], # these should be ignored by the model
pending_evaluations=[])
result = get_internal_candidate_evaluations(state,
DEFAULT_METRIC,
normalize_targets=True,
num_fantasize_samples=20)
assert len(result.X.shape) == 2, "Input should be a matrix"
assert len(result.y.shape) == 2, "Output should be a matrix"
assert result.X.shape[0] == len(candidates)
assert result.y.shape[
-1] == 1, "Only single output value per row is suppored"
assert np.abs(np.mean(
result.y)) < 1e-8, "Mean of the normalized outputs is not 0.0"
assert np.abs(np.std(result.y) -
1.0) < 1e-8, "Std. of the normalized outputs is not 1.0"
np.testing.assert_almost_equal(result.mean, 9.766666666666666)
np.testing.assert_almost_equal(result.std, 3.283629428273267)
def test_gp_fantasizing():
"""
Compare whether acquisition function evaluations (values, gradients) with
fantasizing are the same as averaging them by hand.
"""
random_seed = 4567
_set_seeds(random_seed)
num_fantasy_samples = 10
num_pending = 5
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()),
HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()))
X = [
(0.0, 0.0),
(1.0, 0.0),
(0.0, 1.0),
(1.0, 1.0),
]
num_data = len(X)
Y = [
dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data)
]
# Draw fantasies. This is done for a number of fixed pending candidates
# The model parameters are fit in the first iteration, when there are
# no pending candidates
# Note: It is important to not normalize targets, because this would be
# done on the observed targets only, not the fantasized ones, so it
# would be hard to compare below.
pending_evaluations = []
for _ in range(num_pending):
pending_cand = tuple(np.random.rand(2, ))
pending_evaluations.append(PendingEvaluation(pending_cand))
state = TuningJobState(hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X, Y)],
failed_candidates=[],
pending_evaluations=pending_evaluations)
gpmodel = default_gpmodel(state,
random_seed,
optimization_config=DEFAULT_OPTIMIZATION_CONFIG)
model = GaussProcSurrogateModel(state,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=True,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
fantasy_samples = model.fantasy_samples
# Evaluate acquisition function and gradients with fantasizing
num_test = 50
X_test = [
hp_ranges.to_ndarray(tuple(np.random.rand(2, )))
for _ in range(num_test)
]
acq_func = EIAcquisitionFunction(model)
fvals, grads = _compute_acq_with_gradient_many(acq_func, X_test)
# Do the same computation by averaging by hand
fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape)
grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape)
X_full = X + state.pending_candidates
for it in range(num_fantasy_samples):
Y_full = Y + [
dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it])
for eval in fantasy_samples
]
state2 = TuningJobState(
hp_ranges,
[CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)],
failed_candidates=[],
pending_evaluations=[])
# We have to skip parameter optimization here
model2 = GaussProcSurrogateModel(
state2,
DEFAULT_METRIC,
random_seed,
gpmodel,
fit_parameters=False,
num_fantasy_samples=num_fantasy_samples,
normalize_targets=False)
acq_func2 = EIAcquisitionFunction(model2)
fvals_, grads_ = _compute_acq_with_gradient_many(acq_func2, X_test)
fvals_cmp[it, :] = fvals_
grads_cmp[it, :] = grads_
# Comparison
fvals2 = np.mean(fvals_cmp, axis=0)
grads2 = np.mean(grads_cmp, axis=0)
assert np.allclose(fvals, fvals2)
assert np.allclose(grads, grads2)
def _test_continuous_to_ndarray_and_back(lower, upper, external_hp, scaling):
hp_range = HyperparameterRangeContinuous('hp', lower, upper, scaling)
assert hp_range.from_ndarray(
hp_range.to_ndarray(external_hp)) == approx(external_hp)
def test_to_ndarray():
np.random.seed(123456)
random_state = np.random.RandomState(123456)
prob_categ = 0.3
for iter in range(20):
# Create ConfigurationSpace
num_hps = np.random.randint(low=1, high=20)
if iter == 0:
_prob_categ = 0.
elif iter == 1:
_prob_categ = 1.
else:
_prob_categ = prob_categ
config_space = CS.ConfigurationSpace()
ndarray_size = 0
_hp_ranges = dict()
for hp_it in range(num_hps):
name = str(hp_it)
if np.random.random() < _prob_categ:
num_choices = np.random.randint(low=2, high=11)
choices = tuple([str(i) for i in range(num_choices)])
hp = CSH.CategoricalHyperparameter(name, choices=choices)
hp2 = HyperparameterRangeCategorical(name, choices)
ndarray_size += num_choices
else:
ndarray_size += 1
rand_coin = np.random.random()
if rand_coin < 0.5:
log_scaling = (rand_coin < 0.25)
hp = CSH.UniformFloatHyperparameter(name=name,
lower=0.5,
upper=5.,
log=log_scaling)
hp2 = HyperparameterRangeContinuous(
name,
lower_bound=0.5,
upper_bound=5.,
scaling=LogScaling()
if log_scaling else LinearScaling())
else:
log_scaling = (rand_coin < 0.75)
hp = CSH.UniformIntegerHyperparameter(name=name,
lower=2,
upper=10,
log=log_scaling)
hp2 = HyperparameterRangeInteger(
name=name,
lower_bound=2,
upper_bound=10,
scaling=LogScaling()
if log_scaling else LinearScaling())
config_space.add_hyperparameter(hp)
_hp_ranges[name] = hp2
hp_ranges_cs = HyperparameterRanges_CS(config_space)
hp_ranges = HyperparameterRanges_Impl(
*[_hp_ranges[x] for x in config_space.get_hyperparameter_names()])
# Compare ndarrays created by both codes
for cmp_it in range(5):
config_cs = hp_ranges_cs.random_candidate(random_state)
_config = config_cs.get_dictionary()
config = (_config[name]
for name in config_space.get_hyperparameter_names())
ndarr_cs = hp_ranges_cs.to_ndarray(config_cs)
ndarr = hp_ranges.to_ndarray(config)
assert_allclose(ndarr_cs, ndarr, rtol=1e-4)
import pytest
from autogluon.core.searcher.bayesopt.datatypes.hp_ranges import \
HyperparameterRanges_Impl, HyperparameterRangeInteger, \
HyperparameterRangeContinuous, HyperparameterRangeCategorical
from autogluon.core.searcher.bayesopt.datatypes.scaling import LinearScaling
from autogluon.core.searcher.bayesopt.utils.duplicate_detector import \
DuplicateDetectorEpsilon, DuplicateDetectorIdentical, \
DuplicateDetectorNoDetection
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeInteger('hp1', 0, 1000000000, scaling=LinearScaling()),
HyperparameterRangeContinuous('hp2', -10.0, 10.0, scaling=LinearScaling()),
HyperparameterRangeCategorical('hp3', ('a', 'b', 'c')),
)
duplicate_detector_epsilon = DuplicateDetectorEpsilon(hp_ranges)
@pytest.mark.parametrize('existing, new, contained', [
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10000, 3.0, 'c'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.000001, 'a'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (20, 2.000001, 'b'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (25, 1.0, 'a'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (20, 2.0, 'b'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (19, 1.0, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0000001, 'a'), True),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'c'), False),
({(10, 1.0, 'a'), (20, 2.0, 'b')}, (10, 1.0, 'b'), False),
def test_distribution_of_random_candidates():
random_state = np.random.RandomState(0)
hp_ranges = HyperparameterRanges_Impl(
HyperparameterRangeContinuous('0',
1.0,
1000.0,
scaling=LinearScaling()),
HyperparameterRangeContinuous('1', 1.0, 1000.0, scaling=LogScaling()),
HyperparameterRangeContinuous('2',
0.9,
0.9999,
scaling=ReverseLogScaling()),
HyperparameterRangeInteger('3', 1, 1000, scaling=LinearScaling()),
HyperparameterRangeInteger('4', 1, 1000, scaling=LogScaling()),
HyperparameterRangeCategorical('5', ('a', 'b', 'c')),
)
num_random_candidates = 600
random_candidates = [
hp_ranges.random_candidate(random_state)
for _ in range(num_random_candidates)
]
# check converting back gets to the same candidate
for cand in random_candidates[2:]:
ndarray_candidate = hp_ranges.to_ndarray(cand)
converted_back = hp_ranges.from_ndarray(ndarray_candidate)
for hp, hp_converted_back in zip(cand, converted_back):
if isinstance(hp, str):
assert hp == hp_converted_back
else:
assert_almost_equal(hp, hp_converted_back)
hps0, hps1, hps2, hps3, hps4, hps5 = zip(*random_candidates)
assert 200 < np.percentile(hps0, 25) < 300
assert 450 < np.percentile(hps0, 50) < 550
assert 700 < np.percentile(hps0, 75) < 800
# same bounds as the previous but log scaling
assert 3 < np.percentile(hps1, 25) < 10
assert 20 < np.percentile(hps1, 50) < 40
assert 100 < np.percentile(hps1, 75) < 200
# reverse log
assert 0.9 < np.percentile(hps2, 25) < 0.99
assert 0.99 < np.percentile(hps2, 50) < 0.999
assert 0.999 < np.percentile(hps2, 75) < 0.9999
# integer
assert 200 < np.percentile(hps3, 25) < 300
assert 450 < np.percentile(hps3, 50) < 550
assert 700 < np.percentile(hps3, 75) < 800
# same bounds as the previous but log scaling
assert 3 < np.percentile(hps4, 25) < 10
assert 20 < np.percentile(hps4, 50) < 40
assert 100 < np.percentile(hps4, 75) < 200
counter = Counter(hps5)
assert len(counter) == 3
assert 150 < counter['a'] < 250 # should be about 200
assert 150 < counter['b'] < 250 # should be about 200
assert 150 < counter['c'] < 250 # should be about 200
