def __init__(self, space, **kwargs):
log.debug(
"Creating Algorithm object of %s type with parameters:\n%s",
type(self).__name__,
kwargs,
)
self._space = space
if kwargs:
param_names = list(kwargs)
else:
init_signature = inspect.signature(type(self))
param_names = [
name for name, param in init_signature.parameters.items()
if name != "space" and param.kind != param.VAR_KEYWORD
]
self._param_names = param_names
# Instantiate tunable parameters of an algorithm
for varname, param in kwargs.items():
setattr(self, varname, param)
# TODO: move this inside an initialization function.
if hasattr(self, "seed"):
self.seed_rng(self.seed)
self.registry = Registry()
def __init__(self, space: Space, algorithm: AlgoType):
super().__init__(space=space)
self.algorithm: AlgoType = algorithm
self.registry = Registry()
self.registry_mapping = RegistryMapping(
original_registry=self.registry,
transformed_registry=self.algorithm.registry,
)
self.max_suggest_attempts = 100
def test_register(self, space: Space):
"""Tests that appending a trial to a registry works as expected."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
def test_init(self):
"""Tests that a new RegistryMapping acts as an empty dict."""
original = Registry()
transformed = Registry()
mapping = RegistryMapping(original_registry=original,
transformed_registry=transformed)
assert not mapping
assert len(mapping) == 0
assert not mapping.keys()
assert not mapping.values()
assert not mapping.items()
def test_register_overwrite_with_experiment(self, space: Space):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
experiment = "BLABLABOB" # TODO: Use an experiment fixture of some sort.
same_but_with_experiment = copy.deepcopy(trial)
same_but_with_experiment.experiment = experiment
same_id = registry.register(same_but_with_experiment)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_experiment]
def test_register_overwrite_with_status(self, space: Space, status: str):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
same_but_with_status = copy.deepcopy(trial)
same_but_with_status._status = status
same_id = registry.register(same_but_with_status)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_status]
def test_register_overwrite_with_results(self, space: Space):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
same_but_with_results = copy.deepcopy(trial)
same_but_with_results._results.append(
Trial.Result(name="objective", type="objective", value=1))
same_id = registry.register(same_but_with_results)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_results]
def test_register(self, space: Space, transformed_space: TransformedSpace):
"""Tests for the `register` method of the `RegistryMapping` class."""
original_reg = Registry()
transformed_reg = Registry()
mapping = RegistryMapping(original_registry=original_reg,
transformed_registry=transformed_reg)
original_trial = space.sample(1)[0]
transformed_trial = transformed_space.transform(original_trial)
mapping.register(original_trial, transformed_trial)
# NOTE: register doesn't actually register the trial, it just adds it to the mapping.
assert len(mapping) == 1
assert original_trial in mapping
# NOTE: Here since we assume that the trials are supposed to be registered in the registries
# externally, we can't yet iterate over the mapping (e.g. with keys(), values() or items()).
# Now we actually register the trials in the individual registries.
assert original_trial not in original_reg
original_stored_id = original_reg.register(original_trial)
assert transformed_trial not in transformed_reg
transformed_stored_id = transformed_reg.register(transformed_trial)
assert mapping._mapping == {
original_stored_id: {transformed_stored_id}
}
assert list(mapping.keys()) == [original_trial]
assert list(mapping.values()) == [[transformed_trial]]
assert mapping[original_trial] == [transformed_trial]
def test_extenal_register_doesnt_increase_len(
self, space: Space, transformed_space: TransformedSpace):
"""Test that externally registering trials in the original or transformed registries does
not affect the length of the mapping.
"""
original = Registry()
transformed = Registry()
mapping = RegistryMapping(original_registry=original,
transformed_registry=transformed)
assert not mapping
assert len(mapping) == 0
original_trial = space.sample(1)[0]
original.register(original_trial)
assert not mapping
transformed_trial = transformed_space.sample(1)[0]
transformed.register(transformed_trial)
assert not mapping
class BaseAlgorithm:
"""Base class describing what an algorithm can do.
Parameters
----------
space : `orion.algo.space.Space`
Definition of a problem's parameter space.
kwargs : dict
Tunable elements of a particular algorithm, a dictionary from
hyperparameter names to values.
Notes
-----
We are using the No Free Lunch theorem's [1]_[3]_ formulation of an
`BaseAlgorithm`.
We treat it as a part of a procedure which in each iteration suggests a
sample of the parameter space of the problem as a candidate solution and observes
the results of its evaluation.
**Developer Note**: Each algorithm's complete specification, i.e. implementation of its methods
and parameters of its own, lies in a separate concrete algorithm class, which must be an
**immediate** subclass of `BaseAlgorithm`. [The reason for this is current implementation of
`orion.core.utils.Factory` metaclass which uses `BaseAlgorithm.__subclasses__()`.] Second, one
must declare an algorithm's own parameters (tunable elements which could be set by
configuration). This is done by passing them to `BaseAlgorithm.__init__()` by calling Python's
super with a `Space` object as a positional argument plus algorithm's own parameters as keyword
arguments. The keys of the keyword arguments passed to `BaseAlgorithm.__init__()` are interpreted
as the algorithm's parameter names. So for example, a subclass could be as simple as this
(regarding the logistics, not an actual algorithm's implementation):
Examples
--------
.. code-block:: python
:linenos:
:emphasize-lines: 7
from orion.algo.base import BaseAlgorithm
from orion.algo.space import (Integer, Space)
class MySimpleAlgo(BaseAlgorithm):
def __init__(self, space, multiplier=1, another_param="a string param"):
super().__init__(space, multiplier=multiplier, another_param=another_param)
def suggest(self, num=1):
print(self.another_param)
return list(map(lambda x: tuple(map(lambda y: self.multiplier * y, x)),
self.space.sample(num)))
def observe(self, points, results):
pass
dim = Integer('named_param', 'norm', 3, 2, shape=(2, 3))
s = Space()
s.register(dim)
algo = MySimpleAlgo(s, 2, "I am just sampling!")
algo.suggest()
References
----------
.. [1] D. H. Wolpert and W. G. Macready, “No Free Lunch Theorems for Optimization,”
IEEE Transactions on Evolutionary Computation, vol. 1, no. 1, pp. 67–82, Apr. 1997.
.. [2] W. G. Macready and D. H. Wolpert, “What Makes An Optimization Problem Hard?,”
Complexity, vol. 1, no. 5, pp. 40–46, 1996.
.. [3] D. H. Wolpert and W. G. Macready, “No Free Lunch Theorems for Search,”
Technical Report SFI-TR-95-02-010, Santa Fe Institute, 1995.
"""
requires_type = None
requires_shape = None
requires_dist = None
def __init__(self, space, **kwargs):
log.debug(
"Creating Algorithm object of %s type with parameters:\n%s",
type(self).__name__,
kwargs,
)
self._space = space
if kwargs:
param_names = list(kwargs)
else:
init_signature = inspect.signature(type(self))
param_names = [
name for name, param in init_signature.parameters.items()
if name != "space" and param.kind != param.VAR_KEYWORD
]
self._param_names = param_names
# Instantiate tunable parameters of an algorithm
for varname, param in kwargs.items():
setattr(self, varname, param)
# TODO: move this inside an initialization function.
if hasattr(self, "seed"):
self.seed_rng(self.seed)
self.registry = Registry()
def seed_rng(self, seed):
"""Seed the state of the random number generator.
:param seed: Integer seed for the random number generator.
.. note:: This methods does nothing if the algorithm is deterministic.
"""
pass
@property
def state_dict(self):
"""Return a state dict that can be used to reset the state of the algorithm."""
return {"registry": self.registry.state_dict}
def set_state(self, state_dict):
"""Reset the state of the algorithm based on the given state_dict
:param state_dict: Dictionary representing state of an algorithm
"""
self.registry.set_state(state_dict["registry"])
def get_id(self, trial, ignore_fidelity=False, ignore_parent=False):
"""Return unique hash for a trials based on params
The trial is assumed to be in the transformed space if the algorithm is working in a
transformed space.
Parameters
----------
trial : Trial
trial from a `orion.algo.space.Space`.
ignore_fidelity: bool, optional
If True, the fidelity dimension is ignored when computing a unique hash for
the trial. Defaults to False.
ignore_parent: bool, optional
If True, the parent id is ignored when computing a unique hash for
the trial. Defaults to False.
"""
return trial.compute_trial_hash(
trial,
ignore_fidelity=ignore_fidelity,
ignore_experiment=True,
ignore_lie=True,
ignore_parent=ignore_parent,
)
@property
def fidelity_index(self):
"""Returns the name of the first fidelity dimension if there is one, otherwise `None`."""
fidelity_dims = [
dim for dim in self.space.values() if dim.type == "fidelity"
]
if fidelity_dims:
return fidelity_dims[0].name
return None
@abstractmethod
def suggest(self, num: int) -> list[Trial]:
"""Suggest a `num` of new sets of parameters.
Parameters
----------
num: int
Number of points to suggest. The algorithm may return less than the number of points
requested.
Returns
-------
list of trials or None
A list of trials representing values suggested by the algorithm. The algorithm may opt
out if it cannot make a good suggestion at the moment (it may be waiting for other
trials to complete), in which case it will return None.
Notes
-----
New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
IMPORTANT: Algorithms must call `self.register(trial)` for every trial that is returned by
this method. This is important for the algorithm to be able to keep track of the trials it
has suggested/observed, and for the auto-generated unit-tests to pass.
"""
pass
def observe(self, trials):
"""Observe the `results` of the evaluation of the `trials` in the
process defined in user's script.
Parameters
----------
trials: list of ``orion.core.worker.trial.Trial``
Trials from a `orion.algo.space.Space`.
"""
for trial in trials:
if not self.has_observed(trial):
self.register(trial)
def register(self, trial):
"""Save the trial as one suggested or observed by the algorithm.
Parameters
----------
trial: ``orion.core.worker.trial.Trial``
a Trial from `self.space`.
"""
self.registry.register(trial)
@property
def n_suggested(self):
"""Number of trials suggested by the algorithm"""
return len(self.registry)
@property
def n_observed(self):
"""Number of completed trials observed by the algorithm."""
return sum(self.has_observed(trial) for trial in self.registry)
def has_suggested(self, trial):
"""Whether the algorithm has suggested a given point.
Parameters
----------
trial: ``orion.core.worker.trial.Trial``
Trial from a `orion.algo.space.Space`.
Returns
-------
bool
True if the trial was suggested by the algo, False otherwise.
"""
return self.registry.has_suggested(trial)
def has_observed(self, trial):
"""Whether the algorithm has observed a given point objective.
This only counts observed completed trials.
Parameters
----------
trial: ``orion.core.worker.trial.Trial``
Trial object to retrieve from the database
Returns
-------
bool
True if the trial's objective was observed by the algo, False otherwise.
"""
return self.registry.has_observed(trial)
@property
def is_done(self) -> bool:
"""Whether the algorithm is done and will not make further suggestions.
Return True, if an algorithm holds that there can be no further improvement.
By default, the cardinality of the specified search space will be used to check
if all possible sets of parameters has been tried.
"""
return self.has_completed_max_trials or self.has_suggested_all_possible_values(
)
def has_suggested_all_possible_values(self) -> bool:
"""Returns True if the algorithm has more trials in its registry than the number of possible
values in the search space.
If there is a fidelity dimension in the search space, only the trials with the maximum
fidelity value are counted.
"""
fidelity_index = self.fidelity_index
if fidelity_index is not None:
n_suggested_with_max_fidelity = 0
fidelity_dim = self.space[fidelity_index]
_, max_fidelity_value = fidelity_dim.interval()
for trial in self.registry:
fidelity_value = trial.params[fidelity_index]
if fidelity_value >= max_fidelity_value:
n_suggested_with_max_fidelity += 1
return n_suggested_with_max_fidelity >= self.space.cardinality
return self.n_suggested >= self.space.cardinality
@property
def has_completed_max_trials(self) -> bool:
"""Returns True if the algorithm has a `max_trials` attribute, and has completed more trials
than its value.
"""
if not hasattr(self, "max_trials"):
return False
max_trials = getattr(self, "max_trials")
if max_trials is None:
return False
fidelity_index = self.fidelity_index
def _is_completed(trial: Trial) -> bool:
return trial.status == "completed"
# When a fidelity dimension is present, we only count trials that have the maximum value.
if fidelity_index is not None:
_, max_fidelity_value = self.space[fidelity_index].interval()
def _is_completed(trial: Trial) -> bool:
return (trial.status == "completed"
and trial.params[fidelity_index] >= max_fidelity_value)
return sum(map(_is_completed, self.registry)) >= max_trials
def score(self, trial): # pylint:disable=no-self-use,unused-argument
"""Allow algorithm to evaluate `point` based on a prediction about
this parameter set's performance.
By default, return the same score any parameter (no preference).
Parameters
----------
trial: ``orion.core.worker.trial.Trial``
Trial object to retrieve from the database
Returns
-------
A subjective measure of expected perfomance.
"""
return 0
def judge(self, trial, measurements): # pylint:disable=no-self-use,unused-argument
"""Inform an algorithm about online `measurements` of a running trial.
This method is to be used as a callback in a client-server communication
between user's script and a orion's worker using a `BaseAlgorithm`.
Data returned from this method must be serializable and will be used as
a response to the running environment. Default response is None.
Parameters
----------
trial: ``orion.core.worker.trial.Trial``
Trial object to retrieve from the database
Notes
-----
Calling algorithm to `judge` a `point` based on its online `measurements` will effectively
change a state in the algorithm (like a reinforcement learning agent's hidden state or an
automatic early stopping mechanism's regression), which it may change the value of the
property `should_suspend`.
Returns
-------
None or a serializable dictionary containing named data
"""
return None
def should_suspend(self, trial):
"""Allow algorithm to decide whether a particular running trial is still
worth to complete its evaluation, based on information provided by the
`judge` method.
"""
return False
@property
def configuration(self):
"""Return tunable elements of this algorithm in a dictionary form
appropriate for saving.
"""
dict_form = dict()
for attrname in self._param_names:
if attrname.startswith("_"): # Do not log _space or others in conf
continue
dict_form[attrname] = getattr(self, attrname)
return {self.__class__.__name__.lower(): dict_form}
@property
def space(self):
"""Domain of problem associated with this algorithm's instance."""
return self._space
@space.setter
def space(self, space):
"""Set space."""
self._space = space
def test_init(self):
"""Test that a new registry without trials acts as an empty container."""
registry = Registry()
assert len(registry) == 0
assert not registry
def __init__(self):
self.registry = Registry()
class ParallelStrategy:
"""Strategy to give intermediate results for incomplete trials"""
def __init__(self):
self.registry = Registry()
@property
def state_dict(self) -> dict:
"""Return a state dict that can be used to reset the state of the strategy."""
return {"registry": self.registry.state_dict}
def set_state(self, state_dict: dict) -> None:
self.registry.set_state(state_dict["registry"])
def observe(self, trials: list[Trial]) -> None:
"""Observe completed trials
.. seealso:: `orion.algo.base.BaseAlgorithm.observe` method
Parameters
----------
trials: list of ``orion.core.worker.trial.Trial``
Trials from a `orion.algo.space.Space`.
"""
for trial in trials:
self.registry.register(trial)
def infer(self, trial: Trial) -> Trial | None:
fake_result = self.lie(trial)
if fake_result is None:
return None
fake_trial = copy.deepcopy(trial)
fake_trial._results.append(fake_result)
return fake_trial
# pylint: disable=no-self-use
def lie(self, trial: Trial) -> Trial.Result | None:
"""Construct a fake result for an incomplete trial
Parameters
----------
trial: `orion.core.worker.trial.Trial`
A trial object which is not supposed to be completed.
Returns
-------
``orion.core.worker.trial.Trial.Result``
The fake objective result corresponding to the trial given.
Notes
-----
If the trial has an objective even if not completed, a warning is printed to user
with a pointer to documentation to resolve the database corruption. The result returned is
the corresponding objective instead of the lie.
"""
objective = get_objective(trial)
if objective:
log.warning(CORRUPTED_DB_WARNING, trial.id)
return Trial.Result(name="lie", type="objective", value=objective)
return None
@property
def configuration(self) -> dict:
"""Provide the configuration of the strategy as a dictionary."""
return {"of_type": self.__class__.__name__.lower()}
class SpaceTransformAlgoWrapper(BaseAlgorithm, Generic[AlgoType]):
"""Perform checks on points and transformations. Wrap the primary algorithm.
1. Checks requirements on the parameter space from algorithms and create the
appropriate transformations. Apply transformations before and after methods
of the primary algorithm.
2. Checks whether incoming and outcoming points are compliant with a space.
Parameters
----------
algorithm: instance of `BaseAlgorithm`
Algorithm to be wrapped.
space : `orion.algo.space.Space`
The original definition of a problem's parameters space.
algorithm_config : dict
Configuration for the algorithm.
"""
def __init__(self, space: Space, algorithm: AlgoType):
super().__init__(space=space)
self.algorithm: AlgoType = algorithm
self.registry = Registry()
self.registry_mapping = RegistryMapping(
original_registry=self.registry,
transformed_registry=self.algorithm.registry,
)
self.max_suggest_attempts = 100
@property
def original_space(self) -> Space:
"""The original space (before transformations).
This is exposed to the outside, but not to the wrapped algorithm.
"""
return self._space
@property
def transformed_space(self) -> TransformedSpace:
"""The transformed space (after transformations).
This is only exposed to the wrapped algo, not to classes outside of this.
"""
return self.algorithm.space
def seed_rng(self, seed: int | Sequence[int] | None) -> None:
"""Seed the state of the algorithm's random number generator."""
self.algorithm.seed_rng(seed)
@property
def state_dict(self) -> dict:
"""Return a state dict that can be used to reset the state of the algorithm."""
# TODO: There's currently some duplicates between:
# - self.registry_mapping.original_registry and self.registry
# - self.registry_mapping.transformed_registry and self.algorithm.registry
return copy.deepcopy({
"algorithm":
self.algorithm.state_dict,
"registry":
self.registry.state_dict,
"registry_mapping":
self.registry_mapping.state_dict,
})
def set_state(self, state_dict: dict) -> None:
"""Reset the state of the algorithm based on the given state_dict
:param state_dict: Dictionary representing state of an algorithm
"""
state_dict = copy.deepcopy(state_dict)
self.algorithm.set_state(state_dict["algorithm"])
self.registry.set_state(state_dict["registry"])
self.registry_mapping.set_state(state_dict["registry_mapping"])
def suggest(self, num: int) -> list[Trial]:
"""Suggest a `num` of new sets of parameters.
Parameters
----------
num: int
Number of trials to suggest. The algorithm may return less than the number of trials
requested.
Returns
-------
list of trials
A list of trials representing values suggested by the algorithm. The algorithm may opt
out if it cannot make a good suggestion at the moment (it may be waiting for other
trials to complete), in which case it will return an empty list.
Notes
-----
New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
"""
trials: list[Trial] = []
for suggest_attempt in range(1, self.max_suggest_attempts + 1):
transformed_trials: list[Trial] | None = self.algorithm.suggest(
num)
transformed_trials = transformed_trials or []
for transformed_trial in transformed_trials:
if transformed_trial not in self.transformed_space:
raise ValueError(
f"Trial {transformed_trial.id} not contained in space:\n"
f"Params: {transformed_trial.params}\n"
f"Space: {self.transformed_space}")
original = self.transformed_space.reverse(transformed_trial)
if original in self.registry:
logger.debug(
"Already have a trial that matches %s in the registry.",
original,
)
# We already have a trial that is equivalent to this one.
# Fetch the actual trial (with the status and possibly results)
original = self.registry.get_existing(original)
logger.debug("Matching trial (with results/status): %s",
original)
# Copy over the status and results from the original to the transformed trial
# and observe it.
transformed_trial = _copy_status_and_results(
original_trial=original,
transformed_trial=transformed_trial)
logger.debug("Transformed trial (with results/status): %s",
transformed_trial)
self.algorithm.observe([transformed_trial])
else:
# We haven't seen this trial before. Register it.
self.registry.register(original)
trials.append(original)
# NOTE: Here we DON'T register the transformed trial, we let the algorithm do it
# itself in its `suggest`.
# Register the equivalence between these trials.
self.registry_mapping.register(original, transformed_trial)
if trials:
if suggest_attempt > 1:
logger.debug(
f"Succeeded in suggesting new trials after {suggest_attempt} attempts."
)
return trials
if self.is_done:
logger.debug(
f"Algorithm is done! (after {suggest_attempt} sampling attempts)."
)
break
logger.warning(
f"Unable to sample a new trial from the algorithm, even after "
f"{self.max_suggest_attempts} attempts! Returning an empty list.")
return []
def observe(self, trials: list[Trial]) -> None:
"""Observe evaluated trials.
.. seealso:: `orion.algo.base.BaseAlgorithm.observe`
"""
# For each trial in the original space, find the suggestions from the algo that match it.
# Then, we make the wrapped algo observe each equivalent trial, with the updated status.
for trial in trials:
# Update the status of this trial in the registry (if it was suggested), otherwise set
# it in the registry (for example in testing when we force the algo to observe a trial).
self.registry.register(trial)
# Get the known transformed trials that correspond to this original.
transformed_trials = self.registry_mapping.get_trials(trial)
# Also transfer the status and results of `trial` to the equivalent transformed trials.
transformed_trials = [
_copy_status_and_results(original_trial=trial,
transformed_trial=transformed_trial)
for transformed_trial in transformed_trials
]
if not transformed_trials:
# `trial` is a new, original trial that wasn't suggested by the algorithm. (This
# might happen when an insertion is done according to @bouthilx)
transformed_trial = self.transformed_space.transform(trial)
transformed_trial = _copy_status_and_results(
original_trial=trial, transformed_trial=transformed_trial)
transformed_trials = [transformed_trial]
logger.debug(
f"Observing trial {trial} (transformed as {transformed_trial}), even "
f"though it wasn't suggested by the algorithm.")
# NOTE: @lebrice Here we don't want to store the transformed trial in the
# algo's registry (by either calling `self.algorithm.register(transformed_trial)` or
# `self.algorithm.registry.register(transformed_trial))`, because some algos can't
# observe trials that they haven't suggested. We'd also need to perform all the
# logic that the algo did in `suggest` (e.g. store it in a bracket for HyperBand).
# Therefore we only register it in the wrapper, and store the equivalence between
# these two trials in the registry mapping.
self.registry.register(trial)
self.registry_mapping.register(trial, transformed_trial)
self.algorithm.observe(transformed_trials)
def has_suggested(self, trial: Trial) -> bool:
"""Whether the algorithm has suggested a given trial.
.. seealso:: `orion.algo.base.BaseAlgorithm.has_suggested`
"""
return self.registry.has_suggested(trial)
def has_observed(self, trial: Trial) -> bool:
"""Whether the algorithm has observed a given trial.
.. seealso:: `orion.algo.base.BaseAlgorithm.has_observed`
"""
return self.registry.has_observed(trial)
@property
def n_suggested(self) -> int:
"""Number of trials suggested by the algorithm"""
return len(self.registry)
@property
def n_observed(self) -> int:
"""Number of completed trials observed by the algorithm."""
return sum(self.has_observed(trial) for trial in self.registry)
@property
def is_done(self) -> bool:
"""Return True if the wrapper or the wrapped algorithm is done."""
return super().is_done or self.algorithm.is_done
def score(self, trial: Trial) -> float:
"""Allow algorithm to evaluate `point` based on a prediction about
this parameter set's performance. Return a subjective measure of expected
performance.
By default, return the same score any parameter (no preference).
"""
self._verify_trial(trial)
return self.algorithm.score(self.transformed_space.transform(trial))
def judge(self, trial: Trial, measurements: Any) -> dict | None:
"""Inform an algorithm about online `measurements` of a running trial.
The algorithm can return a dictionary of data which will be provided
as a response to the running environment. Default is None response.
"""
self._verify_trial(trial)
return self.algorithm.judge(self.transformed_space.transform(trial),
measurements)
def should_suspend(self, trial: Trial) -> bool:
"""Allow algorithm to decide whether a particular running trial is still
worth to complete its evaluation, based on information provided by the
`judge` method.
"""
self._verify_trial(trial)
return self.algorithm.should_suspend(trial)
@property
def configuration(self) -> dict:
"""Return tunable elements of this algorithm in a dictionary form
appropriate for saving.
"""
# TODO: Return a dict with the wrapped algo's configuration instead?
# return {
# type(self).__qualname__: {
# "space": self.space.configuration,
# "algorithm": {self.algorithm.configuration},
# }
# }
return self.algorithm.configuration
@property
def space(self) -> Space:
"""Domain of problem associated with this algorithm's instance.
.. note:: Redefining property here without setter, denies base class' setter.
"""
return self._space
@property
def fidelity_index(self) -> str | None:
"""Compute the index of the space where fidelity is.
Returns None if there is no fidelity dimension.
"""
return self.algorithm.fidelity_index
def _verify_trial(self,
trial: Trial,
space: Optional[Space] = None) -> None:
if space is None:
space = self.space
if trial not in space:
raise ValueError(f"Trial {trial.id} not contained in space:"
f"\nParams: {trial.params}\nSpace: {space}")