def rung_4(space1: Space): """Create duplicated fake points and objectives for rung 4.""" points = np.linspace(1, 4, 4) keys = list(space1.keys()) types = [dim.type for dim in space1.values()] results: dict[str, tuple[float, Trial]] = {} for point in points: trial = create_trial( (1, point // 2, point // 2), names=keys, results={"objective": point}, types=types, ) trial_hash = trial.compute_trial_hash( trial, ignore_fidelity=True, ignore_experiment=True, ) assert trial.objective is not None results[trial_hash] = (trial.objective.value, trial) return RungDict( n_trials=4, resources=1, results=results, )
def build_from(self, cmd_args): """Create a definition of the problem's search space, using information from the user's script configuration (if provided) and command line arguments. :param cmd_args: A list of command line arguments provided for the user's script. :rtype: `orion.algo.space.Space` .. note:: A template configuration file complementing user's script can be provided either by explicitly using the prefix '--config=' or by being the first positional argument. """ self.userargs_tmpl = None self.userconfig_tmpl = None self.space = Space() self.userconfig, self.is_userconfig_an_option = self._build_from_args( cmd_args) if self.userconfig: self._build_from_config(self.userconfig) log.debug( "Configuration and command line arguments were parsed and " "a `Space` object was built successfully:\n%s", self.space) return self.space
def test_split_trials(self, tpe): """Test observed trials can be split based on TPE gamma""" space = Space() dim1 = Real('yolo1', 'uniform', -3, 6) space.register(dim1) tpe.space = space points = numpy.linspace(-3, 3, num=10, endpoint=False) results = numpy.linspace(0, 1, num=10, endpoint=False) points_results = list(zip(points, results)) numpy.random.shuffle(points_results) points, results = zip(*points_results) for point, result in zip(points, results): tpe.observe([[point]], [{'objective': result}]) tpe.gamma = 0.25 below_points, above_points = tpe.split_trials() assert below_points == [[-3.0], [-2.4], [-1.8]] assert len(above_points) == 7 tpe.gamma = 0.2 below_points, above_points = tpe.split_trials() assert below_points == [[-3.0], [-2.4]] assert len(above_points) == 8
def build(self, configuration): """Create a definition of the problem's search space. Using information from the user's script configuration (if provided) and the command line arguments, will create a `Space` object defining the problem's search space. Parameters ---------- configuration: OrderedDict An OrderedDict containing the name and the expression of the parameters. Returns ------- `orion.algo.space.Space` The problem's search space definition. """ self.space = Space() for namespace, expression in configuration.items(): if _should_not_be_built(expression): continue expression = _remove_marker(expression) dimension = self.dimbuilder.build(namespace, expression) try: self.space.register(dimension) except ValueError as exc: error_msg = 'Conflict for name \'{}\' in parameters'.format( namespace) raise ValueError(error_msg) from exc return self.space
def select_new_region_of_interest( factorial_importance_analysis: pd.DataFrame, space: Space, threshold: float, n_levels: int, ) -> Tuple[Space, dict]: """Select new region of interest and frozen parameter values based on factorial analysis. Parameters ---------- factorial_importance_analysis: dict Marginal variance ratios on best levels of the factorial performance analysis. Should have format {'dim-name': <marginal variance ratio>, ...} space: ``orion.algo.space.Space`` Space object representing the current region of interest. threshold: float Threshold of marginal variance ratio below which we should freeze a dimension. """ frozen_param_values = {} new_space = Space() for key, dim in space.items(): dim_importance_analysis = factorial_importance_analysis[ factorial_importance_analysis["param"] == key] if float(dim_importance_analysis["importance"]) < threshold: frozen_param_values[key] = sum(dim.interval()) / 2.0 else: level = int(dim_importance_analysis["best_level"]) low, high = dim.interval() intervals = (high - low) / n_levels new_low = low + intervals * (level - 1) new_space.register(Real(dim.name, "uniform", new_low, intervals)) return new_space, frozen_param_values
def test_split_trials(self): """Test observed trials can be split based on TPE gamma""" space = Space() dim1 = Real("yolo1", "uniform", -3, 6) space.register(dim1) tpe = TPE(space, seed=1) rng = np.random.RandomState(1) points = numpy.linspace(-3, 3, num=10, endpoint=False).reshape(-1, 1) objectives = numpy.linspace(0, 1, num=10, endpoint=False) point_objectives = list(zip(points, objectives)) rng.shuffle(point_objectives) points, objectives = zip(*point_objectives) for point, objective in zip(points, objectives): trial = _array_to_trial(point, space=tpe.space, y=objective) tpe.observe([trial]) tpe.gamma = 0.25 below_trials, above_trials = tpe.split_trials() below_points = [ _trial_to_array(t, space=tpe.space) for t in below_trials ] assert below_points == [[-3.0], [-2.4], [-1.8]] assert len(above_trials) == 7 tpe.gamma = 0.2 below_trials, above_trials = tpe.split_trials() below_points = [ _trial_to_array(t, space=tpe.space) for t in below_trials ] assert below_points == [[-3.0], [-2.4]] assert len(above_trials) == 8
def build_space(dims=None): """Create an example `orion.algo.space.Space`.""" if dims is None: dims = [dim1(), dim2(), dim3(), dim4()] space = Space() for dim in dims: space.register(dim) return space
def space_each_type(dim, dim2, dim3, logdim, logintdim): """Create an example `Space`.""" space = Space() space.register(dim) space.register(dim2) space.register(dim3) space.register(logdim) space.register(logintdim) return space
def test_suggest_unique(): """Verify that RandomSearch do not sample duplicates""" space = Space() space.register(Integer('yolo1', 'uniform', -3, 6)) random_search = Random(space) n_samples = 6 values = sum(random_search.suggest(n_samples), tuple()) assert len(values) == n_samples assert len(set(values)) == n_samples
def space1(): """Create a Space with two real dimensions and a fidelity value.""" space = Space() space.register(Real("lr", "uniform", 0, 1)) space.register(Real("weight_decay", "uniform", 0, 1)) space.register(Fidelity("epoch", 1, 8, 2)) return space
def test_is_done_cardinality(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm cardinality check""" monkeypatch.delattr(dumbalgo, 'is_done') space = Space() space.register(Integer('yolo1', 'uniform', 1, 4)) algo = dumbalgo(space) algo.suggest() for i in range(1, 6): algo.observe([[i]], [{'objective': 3}]) assert len(algo.state_dict['_trials_info']) == 5 assert algo.is_done space = Space() space.register(Real('yolo1', 'uniform', 1, 4)) algo = dumbalgo(space) algo.suggest() for i in range(1, 6): algo.observe([[i]], [{'objective': 3}]) assert len(algo.state_dict['_trials_info']) == 5 assert not algo.is_done
def test_is_done_cardinality(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm cardinality check""" monkeypatch.delattr(dumbalgo, "is_done") space = Space() space.register(Integer("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert algo.is_done space = Space() space.register(Real("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert not algo.is_done
def test_hierarchical_register_and_contain(self): """Register hierarchical dimensions and check if points/name are in space.""" space = Space() categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo.nested", categories, shape=2) space.register(dim) dim = Integer("yolo2.nested", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "norm", 0.9) space.register(dim) trial = Trial( params=[ {"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"}, {"name": "yolo2.nested", "value": 1, "type": "integer"}, {"name": "yolo3", "value": 0.5, "type": "real"}, ] ) assert "yolo" in trial.params assert "nested" in trial.params["yolo"] assert "yolo2" in trial.params assert "nested" in trial.params["yolo2"] assert "yolo3" in trial.params assert trial in space
def test_capacity(self, space_each_type): """Check transformer space capacity""" tspace = build_required_space(space_each_type, type_requirement="real") assert tspace.cardinality == numpy.inf space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim = Categorical("yolo0", OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1)) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", shape_requirement="flattened") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", dist_requirement="linear") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def space_each_type(dim, dim2): """Create an example `Space`.""" space = Space() space.register(dim) space.register(dim2) space.register(Integer('yolo3', 'randint', 3, 10)) return space
def generate_trials(olh_samples: np.array, roi_space: Space) -> list[dict]: """ Generates trials from the given normalized orthogonal Latin hypercube samples Parameters ---------- olh_samples: `numpy.array` Samples from the orthogonal Latin hypercube roi_space: orion.algo.space.Space Parameter space region-of-interest Returns ------- A list of trials as `dict` objects, each a list of parameter values in the original search space """ trials = [] for sample in olh_samples: trial_dict = {} for j, param_name in enumerate(roi_space.keys()): interval_min, interval_max = roi_space[param_name].interval() # TODO: deal with categoricals trial_dict[param_name] = (sample[j] * (interval_max - interval_min) + interval_min) trials.append(trial_dict) return trials
def test_register(self, space: Space, transformed_space: TransformedSpace): """Tests for the `register` method of the `RegistryMapping` class.""" original_reg = Registry() transformed_reg = Registry() mapping = RegistryMapping(original_registry=original_reg, transformed_registry=transformed_reg) original_trial = space.sample(1)[0] transformed_trial = transformed_space.transform(original_trial) mapping.register(original_trial, transformed_trial) # NOTE: register doesn't actually register the trial, it just adds it to the mapping. assert len(mapping) == 1 assert original_trial in mapping # NOTE: Here since we assume that the trials are supposed to be registered in the registries # externally, we can't yet iterate over the mapping (e.g. with keys(), values() or items()). # Now we actually register the trials in the individual registries. assert original_trial not in original_reg original_stored_id = original_reg.register(original_trial) assert transformed_trial not in transformed_reg transformed_stored_id = transformed_reg.register(transformed_trial) assert mapping._mapping == { original_stored_id: {transformed_stored_id} } assert list(mapping.keys()) == [original_trial] assert list(mapping.values()) == [[transformed_trial]] assert mapping[original_trial] == [transformed_trial]
def test_arg_names(pass_to_super: bool): """Test that the `_arg_names` can be determined programmatically when the args aren't passed to `super().__init__(space, **kwargs)`. Also checks that the auto-generated configuration dict acts the same way. """ class SomeAlgo(BaseAlgorithm): def __init__(self, space, foo: int = 123, bar: str = "heyo"): if pass_to_super: super().__init__(space, foo=foo, bar=bar) else: super().__init__(space) self.foo = foo self.bar = bar # Param names should be correct, either way. assert self._param_names == ["foo", "bar"] # Attributes should be set correctly either way: assert self.foo == foo assert self.bar == bar space = Space(x=Real("yolo1", "uniform", 1, 4)) algo = SomeAlgo(space, foo=111, bar="barry") assert algo.configuration == { "somealgo": { "bar": "barry", "foo": 111, } }
def test_real_requirement(self, space_each_type): """Check what is built using 'real' requirement.""" tspace = build_required_space(space_each_type, type_requirement="real") assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "real" assert tspace[2].type == "real" assert tspace[3].type == "real" assert tspace[4].type == "real" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))), ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)), Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)), ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\ """) # noqa
def test_interval(self): """Check whether interval is cool.""" space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', 2, 3, 4) dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer('yolo2', 'uniform', -3, 6) space.register(dim) dim = Real('yolo3', 'norm', 0.9) space.register(dim) assert space.interval() == [categories, (-3, 3), (-np.inf, np.inf)]
def test_no_requirement(self, space_each_type): """Check what is built using 'None' requirement.""" tspace = build_required_space(space_each_type) assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "categorical" # NOTE:HEAD assert tspace[2].type == "integer" assert tspace[3].type == "real" assert tspace[4].type == "integer" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None), Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None), Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)), Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\ """) # noqa
def test_is_done_max_trials(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm max_trials check""" monkeypatch.delattr(dumbalgo, "is_done") space = Space() space.register(Real("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest() for i in range(1, 5): algo.observe([[i]], [{"objective": 3}]) assert len(algo.state_dict["_trials_info"]) == 4 assert not algo.is_done dumbalgo.max_trials = 4 assert algo.is_done
def test_linear_requirement(self, space_each_type): """Check what is built using 'linear' requirement.""" tspace = build_required_space(space_each_type, dist_requirement="linear") assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "categorical" assert tspace[2].type == "integer" assert tspace[3].type == "real" assert tspace[4].type == "real" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2), Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None), Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))), Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\ """) # noqa
def test_register(self, space: Space): """Tests that appending a trial to a registry works as expected.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial
def space(): """Construct a simple space with every possible kind of Dimension.""" space = Space() categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo", categories, shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "alpha", 0.9) space.register(dim) return space
def hierarchical_space(): """Construct a space with hierarchical Dimensions.""" space = Space() categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo.first", categories, shape=2) space.register(dim) dim = Integer("yolo.second", "uniform", -3, 6) space.register(dim) dim = Real("yoloflat", "alpha", 0.9) space.register(dim) return space
def space(): """Construct a simple space with every possible kind of Dimension.""" space = Space() categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical('yolo', categories, shape=2) space.register(dim) dim = Integer('yolo2', 'uniform', -3, 6) space.register(dim) dim = Real('yolo3', 'alpha', 0.9) space.register(dim) return space
def hierarchical_space(): """Construct a space with hierarchical Dimensions.""" space = Space() categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical('yolo.first', categories, shape=2) space.register(dim) dim = Integer('yolo.second', 'uniform', -3, 6) space.register(dim) dim = Real('yoloflat', 'alpha', 0.9) space.register(dim) return space
def space(): """Return an optimization space""" space = Space() dim1 = Integer('yolo1', 'uniform', -3, 6) space.register(dim1) dim2 = Real('yolo2', 'uniform', 0, 1) space.register(dim2) dim3 = Fidelity('yolo3', 1, 4, 2) space.register(dim3) return space
def test_sample_real_dimension(self): """Test sample values for a real dimension""" space = Space() dim1 = Real('yolo1', 'uniform', -10, 20) space.register(dim1) dim2 = Real('yolo2', 'uniform', -5, 10, shape=(2)) space.register(dim2) dim3 = Real('yolo3', 'reciprocal', 1, 20) space.register(dim3) tpe = TPE(space) points = numpy.random.uniform(-10, 10, 20) below_points = [points[:8]] above_points = [points[8:]] points = tpe._sample_real_dimension(dim1, 1, below_points, above_points) points = numpy.asarray(points) assert len(points) == 1 assert all(points >= -10) assert all(points < 10) points = numpy.random.uniform(1, 20, 20) below_points = [points[:8]] above_points = [points[8:]] points = tpe._sample_real_dimension(dim3, 1, below_points, above_points) points = numpy.asarray(points) assert len(points) == 1 assert all(points >= 1) assert all(points < 20) below_points = numpy.random.uniform(-10, 0, 25).reshape(1, 25) above_points = numpy.random.uniform(0, 10, 75).reshape(1, 75) points = tpe._sample_real_dimension(dim1, 1, below_points, above_points) points = numpy.asarray(points) assert len(points) == 1 assert all(points >= -10) assert all(points < 0) points = numpy.random.uniform(-5, 5, 32) below_points = [points[:8], points[8:16]] above_points = [points[16:24], points[24:]] points = tpe._sample_real_dimension(dim2, 2, below_points, above_points) points = numpy.asarray(points) assert len(points) == 2 assert all(points >= -10) assert all(points < 10) tpe.n_ei_candidates = 0 points = tpe._sample_real_dimension(dim2, 2, below_points, above_points) assert len(points) == 0