def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(
ValueError,
match=
"TPE now only supports uniform, loguniform, uniform discrete and choices",
):
tpe = TPE(space=build_required_space(
space, shape_requirement=TPE.requires_shape))
def test_split_trials(self):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6)
space.register(dim1)
tpe = TPE(space, seed=1)
rng = np.random.RandomState(1)
points = numpy.linspace(-3, 3, num=10, endpoint=False).reshape(-1, 1)
objectives = numpy.linspace(0, 1, num=10, endpoint=False)
point_objectives = list(zip(points, objectives))
rng.shuffle(point_objectives)
points, objectives = zip(*point_objectives)
for point, objective in zip(points, objectives):
trial = _array_to_trial(point, space=tpe.space, y=objective)
tpe.observe([trial])
tpe.gamma = 0.25
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_trials) == 7
tpe.gamma = 0.2
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4]]
assert len(above_trials) == 8
def test_sample_real_dimension(self):
"""Test sample values for a real dimension"""
space = Space()
dim1 = Real('yolo1', 'uniform', -10, 20)
space.register(dim1)
dim2 = Real('yolo2', 'uniform', -5, 10, shape=(2))
space.register(dim2)
dim3 = Real('yolo3', 'reciprocal', 1, 20)
space.register(dim3)
tpe = TPE(space)
points = numpy.random.uniform(-10, 10, 20)
below_points = [points[:8]]
above_points = [points[8:]]
points = tpe._sample_real_dimension(dim1, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 10)
points = numpy.random.uniform(1, 20, 20)
below_points = [points[:8]]
above_points = [points[8:]]
points = tpe._sample_real_dimension(dim3, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= 1)
assert all(points < 20)
below_points = numpy.random.uniform(-10, 0, 25).reshape(1, 25)
above_points = numpy.random.uniform(0, 10, 75).reshape(1, 75)
points = tpe._sample_real_dimension(dim1, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 0)
points = numpy.random.uniform(-5, 5, 32)
below_points = [points[:8], points[8:16]]
above_points = [points[16:24], points[24:]]
points = tpe._sample_real_dimension(dim2, 2, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 2
assert all(points >= -10)
assert all(points < 10)
tpe.n_ei_candidates = 0
points = tpe._sample_real_dimension(dim2, 2, below_points,
above_points)
assert len(points) == 0
def test_suggest(self, tpe: TPE):
"""Test suggest with no shape dimensions"""
tpe.n_initial_points = 10
results = numpy.random.random(10)
for i in range(10):
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 3
assert not isinstance(points[0][0], tuple)
trials[0] = _add_result(trials[0], results[i])
tpe.observe(trials)
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 3
assert not isinstance(points[0][0], tuple)
def test_suggest_initial_points(self, tpe: TPE, monkeypatch):
"""Test that initial points can be sampled correctly"""
_points = [(i, i - 6, "c") for i in range(1, 12)]
_trials = [
format_trials.tuple_to_trial(point, space=tpe.space)
for point in _points
]
index = 0
def sample(num: int = 1, seed=None) -> list[Trial]:
nonlocal index
result = _trials[index:index + num]
index += num
return result
monkeypatch.setattr(tpe.space, "sample", sample)
tpe.n_initial_points = 10
results = numpy.random.random(10)
for i in range(1, 11):
trials = tpe.suggest(1)
assert trials is not None
trial = trials[0]
assert trial.params == _trials[i]
point = format_trials.trial_to_tuple(trial, space=tpe.space)
assert point == (i, i - 6, "c")
trial.results = [
Trial.Result(name="objective",
type="objective",
value=results[i - 1])
]
tpe.observe([trial])
trials = tpe.suggest(1)
assert trials is not None
trial = trials[0]
assert trial == _trials[-1]
# BUG: This is failing. We expect this trial to be sampled from the model, not from the
# search space.
assert format_trials.trial_to_tuple(trial,
space=tpe.space) != (11, 5, "c")
def test_cat_sampler_creation(self, tpe: TPE):
"""Test CategoricalSampler creation"""
obs: list | numpy.ndarray = [0, 3, 9]
choices = list(range(-5, 5))
cat_sampler = CategoricalSampler(tpe, obs, choices)
assert len(cat_sampler.weights) == len(choices)
obs = numpy.array([0, 3, 9])
choices = ["a", "b", 11, 15, 17, 18, 19, 20, 25, "c"]
cat_sampler = CategoricalSampler(tpe, obs, choices)
assert len(cat_sampler.weights) == len(choices)
tpe.equal_weight = True
tpe.prior_weight = 1.0
obs = numpy.random.randint(0, 10, 100)
cat_sampler = CategoricalSampler(tpe, obs, choices)
counts_obs = numpy.bincount(obs) + 1.0
weights = counts_obs / counts_obs.sum()
assert numpy.all(cat_sampler.weights == weights)
tpe.equal_weight = False
tpe.prior_weight = 0.5
tpe.full_weight_num = 30
obs = numpy.random.randint(0, 10, 100)
cat_sampler = CategoricalSampler(tpe, obs, choices)
ramp = numpy.linspace(1.0 / 100, 1.0, num=100 - 30)
full = numpy.ones(30)
ramp_weights = numpy.concatenate([ramp, full])
counts_obs = numpy.bincount(obs, weights=ramp_weights) + 0.5
weights = counts_obs / counts_obs.sum()
assert numpy.all(cat_sampler.weights == weights)
def test_sample_categorical_dimension(self):
"""Test sample values for a categorical dimension"""
space = Space()
categories = ["a", "b", 11, 15, 17, 18, 19, 20, 25, "c"]
dim1 = Categorical("yolo1", categories)
space.register(dim1)
dim2 = Categorical("yolo2", categories, shape=(2))
space.register(dim2)
tpe = TPE(space)
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25]]
above_points = [obs_points[25:]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories
obs_points_below = numpy.random.randint(0, 3, 25)
obs_points_above = numpy.random.randint(3, 10, 75)
below_points = [[categories[point] for point in obs_points_below]]
above_points = [[categories[point] for point in obs_points_above]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories[:3]
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25], obs_points[25:50]]
above_points = [obs_points[50:75], obs_points[75:]]
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 2
assert points[0] in categories
assert points[1] in categories
tpe.n_ei_candidates = 0
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 0
def test_sample_int_dimension(self):
"""Test sample values for a integer dimension"""
space = Space()
dim1 = Integer("yolo1", "uniform", -10, 20)
space.register(dim1)
dim2 = Integer("yolo2", "uniform", -5, 10, shape=(2))
space.register(dim2)
tpe = TPE(space)
obs_points = numpy.random.randint(-10, 10, 100)
below_points = [obs_points[:25]]
above_points = [obs_points[25:]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 10)
obs_points_below = numpy.random.randint(-10, 0, 25).reshape(1, 25)
obs_points_above = numpy.random.randint(0, 10, 75).reshape(1, 75)
points = tpe.sample_one_dimension(
dim1, 1, obs_points_below, obs_points_above, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 0)
obs_points = numpy.random.randint(-5, 5, 100)
below_points = [obs_points[:25], obs_points[25:50]]
above_points = [obs_points[50:75], obs_points[75:]]
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 2
assert all(points >= -10)
assert all(points < 10)
tpe.n_ei_candidates = 0
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_int_point
)
assert len(points) == 0
def test_suggest_ei_candidates(self, tpe: TPE):
"""Test suggest with no shape dimensions"""
tpe.n_initial_points = 2
tpe.n_ei_candidates = 0
results = numpy.random.random(2)
for i in range(2):
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [format_trials.trial_to_tuple(trials[0], space=tpe.space)]
assert len(points[0]) == 3
assert not isinstance(points[0][0], tuple)
trials[0] = _add_result(trials[0], results[i])
tpe.observe(trials)
trials = tpe.suggest(1)
assert not trials
tpe.n_ei_candidates = 24
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) > 0
def tpe(space):
"""Return an instance of TPE."""
return TPE(space, seed=1)