def rung_4(space1: Space):
"""Create duplicated fake points and objectives for rung 4."""
points = np.linspace(1, 4, 4)
keys = list(space1.keys())
types = [dim.type for dim in space1.values()]
results: dict[str, tuple[float, Trial]] = {}
for point in points:
trial = create_trial(
(1, point // 2, point // 2),
names=keys,
results={"objective": point},
types=types,
)
trial_hash = trial.compute_trial_hash(
trial,
ignore_fidelity=True,
ignore_experiment=True,
)
assert trial.objective is not None
results[trial_hash] = (trial.objective.value, trial)
return RungDict(
n_trials=4,
resources=1,
results=results,
)
def build_from(self, cmd_args):
"""Create a definition of the problem's search space, using information
from the user's script configuration (if provided) and command line arguments.
:param cmd_args: A list of command line arguments provided for the user's script.
:rtype: `orion.algo.space.Space`
.. note:: A template configuration file complementing user's script can be
provided either by explicitly using the prefix '--config=' or by being the
first positional argument.
"""
self.userargs_tmpl = None
self.userconfig_tmpl = None
self.space = Space()
self.userconfig, self.is_userconfig_an_option = self._build_from_args(
cmd_args)
if self.userconfig:
self._build_from_config(self.userconfig)
log.debug(
"Configuration and command line arguments were parsed and "
"a `Space` object was built successfully:\n%s", self.space)
return self.space
def test_split_trials(self, tpe):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real('yolo1', 'uniform', -3, 6)
space.register(dim1)
tpe.space = space
points = numpy.linspace(-3, 3, num=10, endpoint=False)
results = numpy.linspace(0, 1, num=10, endpoint=False)
points_results = list(zip(points, results))
numpy.random.shuffle(points_results)
points, results = zip(*points_results)
for point, result in zip(points, results):
tpe.observe([[point]], [{'objective': result}])
tpe.gamma = 0.25
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_points) == 7
tpe.gamma = 0.2
below_points, above_points = tpe.split_trials()
assert below_points == [[-3.0], [-2.4]]
assert len(above_points) == 8
def build(self, configuration):
"""Create a definition of the problem's search space.
Using information from the user's script configuration (if provided) and the
command line arguments, will create a `Space` object defining the problem's
search space.
Parameters
----------
configuration: OrderedDict
An OrderedDict containing the name and the expression of the parameters.
Returns
-------
`orion.algo.space.Space`
The problem's search space definition.
"""
self.space = Space()
for namespace, expression in configuration.items():
if _should_not_be_built(expression):
continue
expression = _remove_marker(expression)
dimension = self.dimbuilder.build(namespace, expression)
try:
self.space.register(dimension)
except ValueError as exc:
error_msg = 'Conflict for name \'{}\' in parameters'.format(
namespace)
raise ValueError(error_msg) from exc
return self.space
def select_new_region_of_interest(
factorial_importance_analysis: pd.DataFrame,
space: Space,
threshold: float,
n_levels: int,
) -> Tuple[Space, dict]:
"""Select new region of interest and frozen parameter values based on factorial analysis.
Parameters
----------
factorial_importance_analysis: dict
Marginal variance ratios on best levels of the factorial performance analysis.
Should have format {'dim-name': <marginal variance ratio>, ...}
space: ``orion.algo.space.Space``
Space object representing the current region of interest.
threshold: float
Threshold of marginal variance ratio below which we should freeze a dimension.
"""
frozen_param_values = {}
new_space = Space()
for key, dim in space.items():
dim_importance_analysis = factorial_importance_analysis[
factorial_importance_analysis["param"] == key]
if float(dim_importance_analysis["importance"]) < threshold:
frozen_param_values[key] = sum(dim.interval()) / 2.0
else:
level = int(dim_importance_analysis["best_level"])
low, high = dim.interval()
intervals = (high - low) / n_levels
new_low = low + intervals * (level - 1)
new_space.register(Real(dim.name, "uniform", new_low, intervals))
return new_space, frozen_param_values
def test_split_trials(self):
"""Test observed trials can be split based on TPE gamma"""
space = Space()
dim1 = Real("yolo1", "uniform", -3, 6)
space.register(dim1)
tpe = TPE(space, seed=1)
rng = np.random.RandomState(1)
points = numpy.linspace(-3, 3, num=10, endpoint=False).reshape(-1, 1)
objectives = numpy.linspace(0, 1, num=10, endpoint=False)
point_objectives = list(zip(points, objectives))
rng.shuffle(point_objectives)
points, objectives = zip(*point_objectives)
for point, objective in zip(points, objectives):
trial = _array_to_trial(point, space=tpe.space, y=objective)
tpe.observe([trial])
tpe.gamma = 0.25
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4], [-1.8]]
assert len(above_trials) == 7
tpe.gamma = 0.2
below_trials, above_trials = tpe.split_trials()
below_points = [
_trial_to_array(t, space=tpe.space) for t in below_trials
]
assert below_points == [[-3.0], [-2.4]]
assert len(above_trials) == 8
def build_space(dims=None):
"""Create an example `orion.algo.space.Space`."""
if dims is None:
dims = [dim1(), dim2(), dim3(), dim4()]
space = Space()
for dim in dims:
space.register(dim)
return space
def space_each_type(dim, dim2, dim3, logdim, logintdim):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(dim3)
space.register(logdim)
space.register(logintdim)
return space
def test_suggest_unique():
"""Verify that RandomSearch do not sample duplicates"""
space = Space()
space.register(Integer('yolo1', 'uniform', -3, 6))
random_search = Random(space)
n_samples = 6
values = sum(random_search.suggest(n_samples), tuple())
assert len(values) == n_samples
assert len(set(values)) == n_samples
def space1():
"""Create a Space with two real dimensions and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
space.register(Real("weight_decay", "uniform", 0, 1))
space.register(Fidelity("epoch", 1, 8, 2))
return space
def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, 'is_done')
space = Space()
space.register(Integer('yolo1', 'uniform', 1, 4))
algo = dumbalgo(space)
algo.suggest()
for i in range(1, 6):
algo.observe([[i]], [{'objective': 3}])
assert len(algo.state_dict['_trials_info']) == 5
assert algo.is_done
space = Space()
space.register(Real('yolo1', 'uniform', 1, 4))
algo = dumbalgo(space)
algo.suggest()
for i in range(1, 6):
algo.observe([[i]], [{'objective': 3}])
assert len(algo.state_dict['_trials_info']) == 5
assert not algo.is_done
def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Integer("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert algo.is_done
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert not algo.is_done
def test_hierarchical_register_and_contain(self):
"""Register hierarchical dimensions and check if points/name are in space."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo.nested", categories, shape=2)
space.register(dim)
dim = Integer("yolo2.nested", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "norm", 0.9)
space.register(dim)
trial = Trial(
params=[
{"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"},
{"name": "yolo2.nested", "value": 1, "type": "integer"},
{"name": "yolo3", "value": 0.5, "type": "real"},
]
)
assert "yolo" in trial.params
assert "nested" in trial.params["yolo"]
assert "yolo2" in trial.params
assert "nested" in trial.params["yolo2"]
assert "yolo3" in trial.params
assert trial in space
def test_capacity(self, space_each_type):
"""Check transformer space capacity"""
tspace = build_required_space(space_each_type, type_requirement="real")
assert tspace.cardinality == numpy.inf
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim = Categorical("yolo0",
OrderedDict(zip(categories, probs)),
shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1)
dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1))
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
dist_requirement="linear")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def space_each_type(dim, dim2):
"""Create an example `Space`."""
space = Space()
space.register(dim)
space.register(dim2)
space.register(Integer('yolo3', 'randint', 3, 10))
return space
def generate_trials(olh_samples: np.array, roi_space: Space) -> list[dict]:
"""
Generates trials from the given normalized orthogonal Latin hypercube samples
Parameters
----------
olh_samples: `numpy.array`
Samples from the orthogonal Latin hypercube
roi_space: orion.algo.space.Space
Parameter space region-of-interest
Returns
-------
A list of trials as `dict` objects, each a list of parameter values in the
original search space
"""
trials = []
for sample in olh_samples:
trial_dict = {}
for j, param_name in enumerate(roi_space.keys()):
interval_min, interval_max = roi_space[param_name].interval()
# TODO: deal with categoricals
trial_dict[param_name] = (sample[j] *
(interval_max - interval_min) +
interval_min)
trials.append(trial_dict)
return trials
def test_register(self, space: Space, transformed_space: TransformedSpace):
"""Tests for the `register` method of the `RegistryMapping` class."""
original_reg = Registry()
transformed_reg = Registry()
mapping = RegistryMapping(original_registry=original_reg,
transformed_registry=transformed_reg)
original_trial = space.sample(1)[0]
transformed_trial = transformed_space.transform(original_trial)
mapping.register(original_trial, transformed_trial)
# NOTE: register doesn't actually register the trial, it just adds it to the mapping.
assert len(mapping) == 1
assert original_trial in mapping
# NOTE: Here since we assume that the trials are supposed to be registered in the registries
# externally, we can't yet iterate over the mapping (e.g. with keys(), values() or items()).
# Now we actually register the trials in the individual registries.
assert original_trial not in original_reg
original_stored_id = original_reg.register(original_trial)
assert transformed_trial not in transformed_reg
transformed_stored_id = transformed_reg.register(transformed_trial)
assert mapping._mapping == {
original_stored_id: {transformed_stored_id}
}
assert list(mapping.keys()) == [original_trial]
assert list(mapping.values()) == [[transformed_trial]]
assert mapping[original_trial] == [transformed_trial]
def test_arg_names(pass_to_super: bool):
"""Test that the `_arg_names` can be determined programmatically when the args aren't passed to
`super().__init__(space, **kwargs)`.
Also checks that the auto-generated configuration dict acts the same way.
"""
class SomeAlgo(BaseAlgorithm):
def __init__(self, space, foo: int = 123, bar: str = "heyo"):
if pass_to_super:
super().__init__(space, foo=foo, bar=bar)
else:
super().__init__(space)
self.foo = foo
self.bar = bar
# Param names should be correct, either way.
assert self._param_names == ["foo", "bar"]
# Attributes should be set correctly either way:
assert self.foo == foo
assert self.bar == bar
space = Space(x=Real("yolo1", "uniform", 1, 4))
algo = SomeAlgo(space, foo=111, bar="barry")
assert algo.configuration == {
"somealgo": {
"bar": "barry",
"foo": 111,
}
}
def test_real_requirement(self, space_each_type):
"""Check what is built using 'real' requirement."""
tspace = build_required_space(space_each_type, type_requirement="real")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "real"
assert tspace[2].type == "real"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),
ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\
""") # noqa
def test_interval(self):
"""Check whether interval is cool."""
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', 2, 3, 4)
dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2)
space.register(dim)
dim = Integer('yolo2', 'uniform', -3, 6)
space.register(dim)
dim = Real('yolo3', 'norm', 0.9)
space.register(dim)
assert space.interval() == [categories, (-3, 3), (-np.inf, np.inf)]
def test_no_requirement(self, space_each_type):
"""Check what is built using 'None' requirement."""
tspace = build_required_space(space_each_type)
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
# NOTE:HEAD
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "integer"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
""") # noqa
def test_is_done_max_trials(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm max_trials check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest()
for i in range(1, 5):
algo.observe([[i]], [{"objective": 3}])
assert len(algo.state_dict["_trials_info"]) == 4
assert not algo.is_done
dumbalgo.max_trials = 4
assert algo.is_done
def test_linear_requirement(self, space_each_type):
"""Check what is built using 'linear' requirement."""
tspace = build_required_space(space_each_type,
dist_requirement="linear")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\
""") # noqa
def test_register(self, space: Space):
"""Tests that appending a trial to a registry works as expected."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
def space():
"""Construct a simple space with every possible kind of Dimension."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo", categories, shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "alpha", 0.9)
space.register(dim)
return space
def hierarchical_space():
"""Construct a space with hierarchical Dimensions."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo.first", categories, shape=2)
space.register(dim)
dim = Integer("yolo.second", "uniform", -3, 6)
space.register(dim)
dim = Real("yoloflat", "alpha", 0.9)
space.register(dim)
return space
def space():
"""Construct a simple space with every possible kind of Dimension."""
space = Space()
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical('yolo', categories, shape=2)
space.register(dim)
dim = Integer('yolo2', 'uniform', -3, 6)
space.register(dim)
dim = Real('yolo3', 'alpha', 0.9)
space.register(dim)
return space
def hierarchical_space():
"""Construct a space with hierarchical Dimensions."""
space = Space()
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical('yolo.first', categories, shape=2)
space.register(dim)
dim = Integer('yolo.second', 'uniform', -3, 6)
space.register(dim)
dim = Real('yoloflat', 'alpha', 0.9)
space.register(dim)
return space
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'uniform', 0, 1)
space.register(dim2)
dim3 = Fidelity('yolo3', 1, 4, 2)
space.register(dim3)
return space
def test_sample_real_dimension(self):
"""Test sample values for a real dimension"""
space = Space()
dim1 = Real('yolo1', 'uniform', -10, 20)
space.register(dim1)
dim2 = Real('yolo2', 'uniform', -5, 10, shape=(2))
space.register(dim2)
dim3 = Real('yolo3', 'reciprocal', 1, 20)
space.register(dim3)
tpe = TPE(space)
points = numpy.random.uniform(-10, 10, 20)
below_points = [points[:8]]
above_points = [points[8:]]
points = tpe._sample_real_dimension(dim1, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 10)
points = numpy.random.uniform(1, 20, 20)
below_points = [points[:8]]
above_points = [points[8:]]
points = tpe._sample_real_dimension(dim3, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= 1)
assert all(points < 20)
below_points = numpy.random.uniform(-10, 0, 25).reshape(1, 25)
above_points = numpy.random.uniform(0, 10, 75).reshape(1, 75)
points = tpe._sample_real_dimension(dim1, 1, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 1
assert all(points >= -10)
assert all(points < 0)
points = numpy.random.uniform(-5, 5, 32)
below_points = [points[:8], points[8:16]]
above_points = [points[16:24], points[24:]]
points = tpe._sample_real_dimension(dim2, 2, below_points,
above_points)
points = numpy.asarray(points)
assert len(points) == 2
assert all(points >= -10)
assert all(points < 10)
tpe.n_ei_candidates = 0
points = tpe._sample_real_dimension(dim2, 2, below_points,
above_points)
assert len(points) == 0
