def select_new_region_of_interest(
factorial_importance_analysis: pd.DataFrame,
space: Space,
threshold: float,
n_levels: int,
) -> Tuple[Space, dict]:
"""Select new region of interest and frozen parameter values based on factorial analysis.
Parameters
----------
factorial_importance_analysis: dict
Marginal variance ratios on best levels of the factorial performance analysis.
Should have format {'dim-name': <marginal variance ratio>, ...}
space: ``orion.algo.space.Space``
Space object representing the current region of interest.
threshold: float
Threshold of marginal variance ratio below which we should freeze a dimension.
"""
frozen_param_values = {}
new_space = Space()
for key, dim in space.items():
dim_importance_analysis = factorial_importance_analysis[
factorial_importance_analysis["param"] == key]
if float(dim_importance_analysis["importance"]) < threshold:
frozen_param_values[key] = sum(dim.interval()) / 2.0
else:
level = int(dim_importance_analysis["best_level"])
low, high = dim.interval()
intervals = (high - low) / n_levels
new_low = low + intervals * (level - 1)
new_space.register(Real(dim.name, "uniform", new_low, intervals))
return new_space, frozen_param_values
def test_split_trials(self):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6)
space.register(dim1)
tpe = TPE(space, seed=1)
rng = np.random.RandomState(1)
points = numpy.linspace(-3, 3, num=10, endpoint=False).reshape(-1, 1)
objectives = numpy.linspace(0, 1, num=10, endpoint=False)
point_objectives = list(zip(points, objectives))
rng.shuffle(point_objectives)
points, objectives = zip(*point_objectives)
for point, objective in zip(points, objectives):
trial = _array_to_trial(point, space=tpe.space, y=objective)
tpe.observe([trial])
tpe.gamma = 0.25
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_trials) == 7
tpe.gamma = 0.2
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4]]
assert len(above_trials) == 8
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
tpe = TPE(space)
tpe.space = build_required_space(
space, shape_requirement=TPE.requires_shape
)
assert (
"TPE now only supports uniform, loguniform, uniform discrete and choices"
in str(ex.value)
)
def test_1d_shape(self):
"""Test suggest with 1D shape dimensions"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6, shape=(2))
space.register(dim1)
dim2 = Real("yolo2", "uniform", -2, 4)
space.register(dim2)
tpe = create_algo(space=space,
algo_type=TPE,
seed=1,
n_initial_points=10)
results = numpy.random.random(10)
for i in range(10):
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 2
assert len(points[0][0]) == 2
trials[0] = _add_result(trials[0], results[i])
tpe.observe(trials)
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 2
assert len(points[0][0]) == 2
def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Integer("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert algo.is_done
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert not algo.is_done
def space():
"""Create a Space with a real dimension and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
# NOTE: Slightly different value than HyperBand (which has (1, 9, 3))
space.register(Fidelity("epoch", 1, 9, 1))
return space
def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, 'is_done')
space = Space()
space.register(Integer('yolo1', 'uniform', 1, 4))
algo = dumbalgo(space)
algo.suggest()
for i in range(1, 6):
algo.observe([[i]], [{'objective': 3}])
assert len(algo.state_dict['_trials_info']) == 5
assert algo.is_done
space = Space()
space.register(Real('yolo1', 'uniform', 1, 4))
algo = dumbalgo(space)
algo.suggest()
for i in range(1, 6):
algo.observe([[i]], [{'objective': 3}])
assert len(algo.state_dict['_trials_info']) == 5
assert not algo.is_done
def test_split_trials(self, tpe):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real('yolo1', 'uniform', -3, 6)
space.register(dim1)
tpe.space = space
points = numpy.linspace(-3, 3, num=10, endpoint=False)
results = numpy.linspace(0, 1, num=10, endpoint=False)
points_results = list(zip(points, results))
numpy.random.shuffle(points_results)
points, results = zip(*points_results)
for point, result in zip(points, results):
tpe.observe([[point]], [{'objective': result}])
tpe.gamma = 0.25
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_points) == 7
tpe.gamma = 0.2
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4]]
assert len(above_points) == 8
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(["whatever", [1, 1]]) == hyperband.get_id(
["is here", [1, 1]]
)
assert hyperband.get_id(["whatever", [1, 1]]) != hyperband.get_id(
["is here", [2, 2]]
)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [2, 2]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) == hyperband.get_id(["same", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["same", [1, 1]])
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(
asha,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
for i in range(2):
force_observe(
asha,
create_trial(
(3, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert asha.suggest(1) == []
def space(dim, dim2, dim3):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(dim3)
return space
def build_space(dims=None):
"""Create an example `orion.algo.space.Space`."""
if dims is None:
dims = [dim1(), dim2(), dim3(), dim4()]
space = Space()
for dim in dims:
space.register(dim)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Real('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'reciprocal', 1, 10)
space.register(dim2)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer("yolo1", "uniform", -3, 6)
space.register(dim1)
dim2 = Real("yolo2", "uniform", 0, 1)
space.register(dim2)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'norm', 0.9)
space.register(dim2)
return space
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(['whatever', [1, 1]]) == hyperband.get_id(['is here', [1, 1]])
assert hyperband.get_id(['whatever', [1, 1]]) != hyperband.get_id(['is here', [2, 2]])
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
force_observe(hyperband, (1, i), {"objective": i})
assert hyperband.suggest() is None
def test_repr(self):
"""Test str/repr."""
space = Space()
dim = Integer('yolo2', 'uniform', -3, 6, shape=(2, ))
space.register(dim)
dim = Real('yolo3', 'norm', 0.9)
space.register(dim)
assert str(space) == "Space(["\
"Integer(name=yolo2, prior={uniform: (-3, 6), {}}, shape=(2,)),\n" \
" Real(name=yolo3, prior={norm: (0.9,), {}}, shape=())])"
def test_suggest_unique():
"""Verify that RandomSearch do not sample duplicates"""
space = Space()
space.register(Integer('yolo1', 'uniform', -3, 6))
random_search = Random(space)
n_samples = 6
values = sum(random_search.suggest(n_samples), tuple())
assert len(values) == n_samples
assert len(set(values)) == n_samples
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
hyperband.observe([(1, i)], [{'objective': i}])
assert hyperband.suggest() is None
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(["whatever",
[1, 1]]) == asha.get_id(["is here", [1, 1]])
assert asha.get_id(["whatever", [1, 1]]) != asha.get_id(
["is here", [2, 2]])
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(['whatever',
[1, 1]]) == asha.get_id(['is here', [1, 1]])
assert asha.get_id(['whatever', [1, 1]]) != asha.get_id(
['is here', [2, 2]])
def test_build_grid():
"""Test that grid search builds the proper grid"""
dim1 = Real("dim1", "uniform", 0, 1)
dim2 = Integer("dim2", "uniform", 0, 10)
dim3 = Categorical("dim3", "abcde")
space = Space()
space.register(dim1)
space.register(dim2)
space.register(dim3)
grid = GridSearch.build_grid(space, {"dim1": 3, "dim2": 4, "dim3": 1})
assert len(grid) == 3 * 4 * 5
def test_cardinality(self, dim2):
"""Check cardinality of reshaped space"""
space = Space()
space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(space, shape_requirement="flattened")
assert rspace.cardinality == numpy.inf
rspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert rspace.cardinality == (3**(2 * 2)) * 4
def test_repr(self):
"""Test str/repr."""
space = Space()
dim = Integer("yolo2", "uniform", -3, 6, shape=(2, ))
space.register(dim)
dim = Real("yolo3", "norm", 0.9)
space.register(dim)
assert (
str(space) == "Space(["
"Integer(name=yolo2, prior={uniform: (-3, 6), {}}, shape=(2,), "
"default value=None),\n"
" Real(name=yolo3, prior={norm: (0.9,), {}}, shape=(), "
"default value=None)])")
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(asha, (1, i), {"objective": i})
for i in range(2):
force_observe(asha, (3, i), {"objective": i})
assert asha.suggest() is None
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
asha = ASHA(space)
for i in range(6):
asha.observe([(1, i)], [{'objective': i}])
for i in range(2):
asha.observe([(3, i)], [{'objective': i}])
assert asha.suggest() is None
def test_build_grid_limit_size(caplog):
"""Test that grid search reduces the n_values when grid is too large"""
dim1 = Real("dim1", "uniform", 0, 1)
dim2 = Integer("dim2", "uniform", 0, 10)
dim3 = Categorical("dim3", "abcde")
space = Space()
space.register(dim1)
space.register(dim2)
space.register(dim3)
with caplog.at_level(logging.WARNING, logger="orion.algo.gridsearch"):
grid = GridSearch.build_grid(space, {k: 5 for k in space}, 100)
assert len(grid) == 4 * 4 * 5
assert "`n_values` reduced by 1 to limit number of trials below 100" in caplog.text
def test_sample_categorical_dimension(self):
"""Test sample values for a categorical dimension"""
space = Space()
categories = ["a", "b", 11, 15, 17, 18, 19, 20, 25, "c"]
dim1 = Categorical("yolo1", categories)
space.register(dim1)
dim2 = Categorical("yolo2", categories, shape=(2))
space.register(dim2)
tpe = TPE(space)
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25]]
above_points = [obs_points[25:]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories
obs_points_below = numpy.random.randint(0, 3, 25)
obs_points_above = numpy.random.randint(3, 10, 75)
below_points = [[categories[point] for point in obs_points_below]]
above_points = [[categories[point] for point in obs_points_above]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories[:3]
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25], obs_points[25:50]]
above_points = [obs_points[50:75], obs_points[75:]]
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 2
assert points[0] in categories
assert points[1] in categories
tpe.n_ei_candidates = 0
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 0
def test_sample_int_dimension(self):
"""Test sample values for a integer dimension"""
space = Space()
dim1 = Integer("yolo1", "uniform", -10, 20)
space.register(dim1)
dim2 = Integer("yolo2", "uniform", -5, 10, shape=(2))
space.register(dim2)
tpe = TPE(space)
obs_points = numpy.random.randint(-10, 10, 100)
below_points = [obs_points[:25]]
above_points = [obs_points[25:]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 10)
obs_points_below = numpy.random.randint(-10, 0, 25).reshape(1, 25)
obs_points_above = numpy.random.randint(0, 10, 75).reshape(1, 75)
points = tpe.sample_one_dimension(
dim1, 1, obs_points_below, obs_points_above, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 0)
obs_points = numpy.random.randint(-5, 5, 100)
below_points = [obs_points[:25], obs_points[25:50]]
above_points = [obs_points[50:75], obs_points[75:]]
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_int_point
)
points = numpy.asarray(points)
assert len(points) == 2
assert all(points >= -10)
assert all(points < 10)
tpe.n_ei_candidates = 0
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_int_point
)
assert len(points) == 0