def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(
ValueError,
match=
"TPE now only supports uniform, loguniform, uniform discrete and choices",
):
tpe = TPE(space=build_required_space(
space, shape_requirement=TPE.requires_shape))
def test_1d_shape(self):
"""Test suggest with 1D shape dimensions"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6, shape=(2))
space.register(dim1)
dim2 = Real("yolo2", "uniform", -2, 4)
space.register(dim2)
tpe = create_algo(space=space,
algo_type=TPE,
seed=1,
n_initial_points=10)
results = numpy.random.random(10)
for i in range(10):
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 2
assert len(points[0][0]) == 2
trials[0] = _add_result(trials[0], results[i])
tpe.observe(trials)
trials = tpe.suggest(1)
assert trials is not None
assert len(trials) == 1
points = [_trial_to_array(t, space=tpe.space) for t in trials]
assert len(points[0]) == 2
assert len(points[0][0]) == 2
def space(dim, dim2, dim3):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(dim3)
return space
def build_from(self, cmd_args):
"""Create a definition of the problem's search space, using information
from the user's script configuration (if provided) and command line arguments.
:param cmd_args: A list of command line arguments provided for the user's script.
:rtype: `orion.algo.space.Space`
.. note:: A template configuration file complementing user's script can be
provided either by explicitly using the prefix '--config=' or by being the
first positional argument.
"""
self.userargs_tmpl = None
self.userconfig_tmpl = None
self.space = Space()
self.userconfig, self.is_userconfig_an_option = self._build_from_args(
cmd_args)
if self.userconfig:
self._build_from_config(self.userconfig)
log.debug(
"Configuration and command line arguments were parsed and "
"a `Space` object was built successfully:\n%s", self.space)
return self.space
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(["whatever", [1, 1]]) == hyperband.get_id(
["is here", [1, 1]]
)
assert hyperband.get_id(["whatever", [1, 1]]) != hyperband.get_id(
["is here", [2, 2]]
)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [2, 2]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) == hyperband.get_id(["same", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["same", [1, 1]])
def build(self, configuration):
"""Create a definition of the problem's search space.
Using information from the user's script configuration (if provided) and the
command line arguments, will create a `Space` object defining the problem's
search space.
Parameters
----------
configuration: OrderedDict
An OrderedDict containing the name and the expression of the parameters.
Returns
-------
`orion.algo.space.Space`
The problem's search space definition.
"""
self.space = Space()
for namespace, expression in configuration.items():
if _should_not_be_built(expression):
continue
expression = _remove_marker(expression)
dimension = self.dimbuilder.build(namespace, expression)
try:
self.space.register(dimension)
except ValueError as exc:
error_msg = 'Conflict for name \'{}\' in parameters'.format(
namespace)
raise ValueError(error_msg) from exc
return self.space
def space1():
"""Create a Space with two real dimensions and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
space.register(Real("weight_decay", "uniform", 0, 1))
space.register(Fidelity("epoch", 1, 8, 2))
return space
def test_split_trials(self, tpe):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real('yolo1', 'uniform', -3, 6)
space.register(dim1)
tpe.space = space
points = numpy.linspace(-3, 3, num=10, endpoint=False)
results = numpy.linspace(0, 1, num=10, endpoint=False)
points_results = list(zip(points, results))
numpy.random.shuffle(points_results)
points, results = zip(*points_results)
for point, result in zip(points, results):
tpe.observe([[point]], [{'objective': result}])
tpe.gamma = 0.25
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_points) == 7
tpe.gamma = 0.2
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4]]
assert len(above_points) == 8
def test_hierarchical_register_and_contain(self):
"""Register hierarchical dimensions and check if points/name are in space."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo.nested", categories, shape=2)
space.register(dim)
dim = Integer("yolo2.nested", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "norm", 0.9)
space.register(dim)
trial = Trial(
params=[
{"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"},
{"name": "yolo2.nested", "value": 1, "type": "integer"},
{"name": "yolo3", "value": 0.5, "type": "real"},
]
)
assert "yolo" in trial.params
assert "nested" in trial.params["yolo"]
assert "yolo2" in trial.params
assert "nested" in trial.params["yolo2"]
assert "yolo3" in trial.params
assert trial in space
def test_capacity(self, space_each_type):
"""Check transformer space capacity"""
tspace = build_required_space(space_each_type, type_requirement="real")
assert tspace.cardinality == numpy.inf
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim = Categorical("yolo0",
OrderedDict(zip(categories, probs)),
shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1)
dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1))
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
dist_requirement="linear")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def test_arg_names(pass_to_super: bool):
"""Test that the `_arg_names` can be determined programmatically when the args aren't passed to
`super().__init__(space, **kwargs)`.
Also checks that the auto-generated configuration dict acts the same way.
"""
class SomeAlgo(BaseAlgorithm):
def __init__(self, space, foo: int = 123, bar: str = "heyo"):
if pass_to_super:
super().__init__(space, foo=foo, bar=bar)
else:
super().__init__(space)
self.foo = foo
self.bar = bar
# Param names should be correct, either way.
assert self._param_names == ["foo", "bar"]
# Attributes should be set correctly either way:
assert self.foo == foo
assert self.bar == bar
space = Space(x=Real("yolo1", "uniform", 1, 4))
algo = SomeAlgo(space, foo=111, bar="barry")
assert algo.configuration == {
"somealgo": {
"bar": "barry",
"foo": 111,
}
}
def test_split_trials(self):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6)
space.register(dim1)
tpe = TPE(space, seed=1)
rng = np.random.RandomState(1)
points = numpy.linspace(-3, 3, num=10, endpoint=False).reshape(-1, 1)
objectives = numpy.linspace(0, 1, num=10, endpoint=False)
point_objectives = list(zip(points, objectives))
rng.shuffle(point_objectives)
points, objectives = zip(*point_objectives)
for point, objective in zip(points, objectives):
trial = _array_to_trial(point, space=tpe.space, y=objective)
tpe.observe([trial])
tpe.gamma = 0.25
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_trials) == 7
tpe.gamma = 0.2
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4]]
assert len(above_trials) == 8
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(
asha,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
for i in range(2):
force_observe(
asha,
create_trial(
(3, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert asha.suggest(1) == []
def test_real_requirement(self, space_each_type):
"""Check what is built using 'real' requirement."""
tspace = build_required_space(space_each_type, type_requirement="real")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "real"
assert tspace[2].type == "real"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),
ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\
""") # noqa
def select_new_region_of_interest(
factorial_importance_analysis: pd.DataFrame,
space: Space,
threshold: float,
n_levels: int,
) -> Tuple[Space, dict]:
"""Select new region of interest and frozen parameter values based on factorial analysis.
Parameters
----------
factorial_importance_analysis: dict
Marginal variance ratios on best levels of the factorial performance analysis.
Should have format {'dim-name': <marginal variance ratio>, ...}
space: ``orion.algo.space.Space``
Space object representing the current region of interest.
threshold: float
Threshold of marginal variance ratio below which we should freeze a dimension.
"""
frozen_param_values = {}
new_space = Space()
for key, dim in space.items():
dim_importance_analysis = factorial_importance_analysis[
factorial_importance_analysis["param"] == key]
if float(dim_importance_analysis["importance"]) < threshold:
frozen_param_values[key] = sum(dim.interval()) / 2.0
else:
level = int(dim_importance_analysis["best_level"])
low, high = dim.interval()
intervals = (high - low) / n_levels
new_low = low + intervals * (level - 1)
new_space.register(Real(dim.name, "uniform", new_low, intervals))
return new_space, frozen_param_values
def space():
"""Create a Space with a real dimension and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
# NOTE: Slightly different value than HyperBand (which has (1, 9, 3))
space.register(Fidelity("epoch", 1, 9, 1))
return space
def space_each_type(dim, dim2):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(Integer('yolo3', 'randint', 3, 10))
return space
def test_cardinality(self, dim2):
"""Check cardinality of reshaped space"""
space = Space()
space.register(Real("yolo", "reciprocal", 0.1, 1, precision=1, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(space, shape_requirement="flattened")
assert rspace.cardinality == (10 ** (2 * 2)) * 4
space = Space()
space.register(Real("yolo", "uniform", 0, 2, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(
space, type_requirement="integer", shape_requirement="flattened"
)
assert rspace.cardinality == (3 ** (2 * 2)) * 4
def test_no_requirement(self, space_each_type):
"""Check what is built using 'None' requirement."""
tspace = build_required_space(space_each_type)
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
# NOTE:HEAD
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "integer"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
""") # noqa
def build_space(dims=None):
"""Create an example `orion.algo.space.Space`."""
if dims is None:
dims = [dim1(), dim2(), dim3(), dim4()]
space = Space()
for dim in dims:
space.register(dim)
return space
def test_linear_requirement(self, space_each_type):
"""Check what is built using 'linear' requirement."""
tspace = build_required_space(space_each_type,
dist_requirement="linear")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\
""") # noqa
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'norm', 0.9)
space.register(dim2)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer("yolo1", "uniform", -3, 6)
space.register(dim1)
dim2 = Real("yolo2", "uniform", 0, 1)
space.register(dim2)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Real('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'reciprocal', 1, 10)
space.register(dim2)
return space
def space_each_type(dim, dim2, dim3, logdim, logintdim):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(dim3)
space.register(logdim)
space.register(logintdim)
return space
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(['whatever', [1, 1]]) == hyperband.get_id(['is here', [1, 1]])
assert hyperband.get_id(['whatever', [1, 1]]) != hyperband.get_id(['is here', [2, 2]])
def test_suggest_unique():
"""Verify that RandomSearch do not sample duplicates"""
space = Space()
space.register(Integer('yolo1', 'uniform', -3, 6))
random_search = Random(space)
n_samples = 6
values = sum(random_search.suggest(n_samples), tuple())
assert len(values) == n_samples
assert len(set(values)) == n_samples
def space():
"""Construct a simple space with every possible kind of Dimension."""
space = Space()
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical('yolo', categories, shape=2)
space.register(dim)
dim = Integer('yolo2', 'uniform', -3, 6)
space.register(dim)
dim = Real('yolo3', 'alpha', 0.9)
space.register(dim)
return space
def space():
"""Construct a simple space with every possible kind of Dimension."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo", categories, shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "alpha", 0.9)
space.register(dim)
return space
def hierarchical_space():
"""Construct a space with hierarchical Dimensions."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo.first", categories, shape=2)
space.register(dim)
dim = Integer("yolo.second", "uniform", -3, 6)
space.register(dim)
dim = Real("yoloflat", "alpha", 0.9)
space.register(dim)
return space
