def test_hierarchical_dict_to_trial(hierarchical_space, hierarchical_trial,
hierarchical_dict_params):
"""Check if hierarchical dict is converted successfully to trial."""
t = dict_to_trial(hierarchical_dict_params, hierarchical_space)
assert len(t._params) == len(hierarchical_trial._params)
for i in range(len(t.params)):
assert t._params[i].to_dict() == hierarchical_trial._params[i].to_dict(
)
def _params_to_trial(self, orion_params: dict) -> Trial:
"""Create a Trial from a dict of hyper-parameters."""
# Need to convert the {name: value} of point_dict into this format for Orion's Trial.
# Add the max value for the Fidelity dimensions, if any.
if self.fidelity_index is not None:
fidelity_dim: Fidelity = self.space[self.fidelity_index]
orion_params[self.fidelity_index] = fidelity_dim.high
trial: Trial = dict_to_trial(orion_params, space=self.space)
return trial
def test_dict_to_trial(space, trial, dict_params):
"""Check if dict is converted successfully to trial."""
t = dict_to_trial(dict_params, space)
assert t.experiment is None
assert t.status == 'new'
assert t.worker is None
assert t.submit_time is None
assert t.start_time is None
assert t.end_time is None
assert t.results == []
assert len(t._params) == len(trial._params)
for i in range(len(t.params)):
assert t._params[i].to_dict() == trial._params[i].to_dict()
def sample_to_trial(self, sample: numpy.ndarray, fidelity: int) -> Trial:
"""Convert a ConfigSpace sample into a trial"""
config = self.dehb.vector_to_configspace(sample)
hps = {}
for k, v in self.space.items():
if v.type == "fidelity":
hps[k] = fidelity
else:
hps[k] = config[k]
return format_trials.dict_to_trial(to_orion(hps), self.space)
def suggest(self, num: int) -> list[Trial]:
"""Suggest a number of new sets of parameters.
Draws points from a prepared set of samples from an orthonal Latin hypercube.
Parameters
----------
num: int, optional
Number of trials to suggest. The algorithm may return less than the number of trials
requested.
Returns
-------
list of trials
A list of trials representing values suggested by the algorithm. The algorithm may opt
out if it cannot make a good suggestion at the moment (it may be waiting for other
trials to complete), in which case it will return an empty list.
Notes
-----
New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
"""
if self.n_trials is None or len(
self.completed_trials) >= self.n_trials:
self._prepare_next_iteration()
trials = []
while (len(trials) < num and len(self.current_trials_params) > 0
and not self.is_done):
trial_params = self.current_trials_params.pop()
trial_params.update(self.frozen_param_values)
trial = dict_to_trial(trial_params, self.space)
if self.has_observed(trial):
similar_trial = self.registry.get_existing(trial)
trial.results = copy.deepcopy(similar_trial.results)
trial.status = similar_trial.status
self.completed_trials.append(trial)
elif self.has_suggested(trial):
self.duplicates[self.get_id(trial)].append(trial)
else:
self.register(trial)
trials.append(trial)
if len(trials) == 0 and len(self.completed_trials) >= self.n_trials:
self.converged = True
return trials
def _ask(self):
suggestion = self.algo.ask()
if suggestion.args:
raise RuntimeError(
"Nevergrad sampled a trial with args but this should never happen."
" Please report this issue at"
" https://github.com/Epistimio/orion.algo.nevergrad/issues")
new_trial = dict_to_trial(suggestion.kwargs, self.space)
if self._associate_trial(new_trial, suggestion):
self.register(new_trial)
return new_trial
else:
logger.debug("Ignoring duplicated trial")
return None
def _suggest_one_bo(self):
params = {}
below_trials, above_trials = self.split_trials()
for dimension in self.space.values():
dim_below_trials = [
trial.params[dimension.name] for trial in below_trials
]
dim_above_trials = [
trial.params[dimension.name] for trial in above_trials
]
if dimension.type == "real":
dim_samples = self._sample_real_dimension(
dimension,
dim_below_trials,
dim_above_trials,
)
elif dimension.type == "integer" and dimension.prior_name in [
"int_uniform",
"int_reciprocal",
]:
dim_samples = self._sample_int_point(
dimension,
dim_below_trials,
dim_above_trials,
)
elif dimension.type == "categorical" and dimension.prior_name == "choices":
dim_samples = self._sample_categorical_point(
dimension,
dim_below_trials,
dim_above_trials,
)
elif dimension.type == "fidelity":
# fidelity dimension
trial = self.space.sample(1)[0]
dim_samples = trial.params[dimension.name]
else:
raise NotImplementedError()
params[dimension.name] = dim_samples
trial = format_trials.dict_to_trial(params, self.space)
return self.format_trial(trial)
def suggest(self, num):
"""Suggest a number of new sets of parameters.
Parameters
----------
num: int
Number of trials to suggest. The algorithm may return less than the number of trials
requested.
Returns
-------
list of trials
A list of trials representing values suggested by the algorithm. The algorithm may opt
out if it cannot make a good suggestion at the moment (it may be waiting for other
trials to complete), in which case it will return None.
Notes
-----
New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
"""
trials = []
with self.get_client() as client:
_trials, _ = client.get_next_trials(num)
for trial_index, parameters in _trials.items():
parameters = AxOptimizer.reverse_params(parameters, self.space)
# Ax does not support Fidelity dimension type so fake it with
# its max
if self.fidelity_index is not None:
# Convert 0-dim arrays into python numbers so their type can
# be validated by Ax
parameters[self.fidelity_index] = float(
self.space[self.fidelity_index].high)
new_trial = format_trials.dict_to_trial(parameters, self.space)
if not self.has_suggested(new_trial):
self.register(new_trial)
trials.append(new_trial)
self._trials_map[self.get_id(
new_trial)] = trial_index # tmp
return trials
def call(self, **kwargs) -> List[Dict]:
"""Get the value of the sampled objective function at the given point (hyper-parameters).
If `self.with_grad` is set, also returns the gradient of the objective function with respect
to the inputs.
Parameters
----------
**kwargs
Dictionary of hyper-parameters.
Returns
-------
List[Dict]
Result dictionaries: objective and optionally gradient.
Raises
------
ValueError
If the input isn't of a supported type.
"""
# A bit of gymnastics to convert the params Dict into a PyTorch tensor.
trial = dict_to_trial(kwargs, self._space)
flattened_trial = self.transformed_space.transform(trial)
flattened_params = flatten(flattened_trial.params)
flattened_point = np.array(
[flattened_params[key] for key in self.transformed_space.keys()])
x_tensor = torch.as_tensor(flattened_point).type_as(self.h_tensor)
if self.with_grad:
x_tensor = x_tensor.requires_grad_(True)
p_tensor = torch.cat([x_tensor, self.h_tensor])
p_tensor = torch.atleast_2d(p_tensor)
devices = [] if self.device.type == "cpu" else [self.device]
# NOTE: Currently no way to locally seed the rng of torch distributions, hence forking the
# rng for torch only here.
with torch.random.fork_rng(devices=devices):
torch.random.manual_seed(self.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(self.seed)
# Forward pass:
out = self.net(p_tensor)
y_mean, y_log_std = out[0, 0], out[0, 1]
y_std = torch.exp(y_log_std)
# NOTE: Here we create a distribution over `y`, and use `rsample()`, so that we get can
# also return the gradients if need be.
y_dist = Normal(loc=y_mean, scale=y_std)
y_sample = y_dist.rsample()
logger.debug(f"y_sample: {y_sample}")
results: List[dict] = [
dict(name=self.name,
type="objective",
value=y_sample.detach().cpu().item())
]
if self.with_grad:
self.net.zero_grad()
y_sample.backward()
assert x_tensor.grad is not None
results.append(
dict(name=self.name,
type="gradient",
value=x_tensor.grad.cpu().numpy()))
return results
def run_to_trial(run):
params = transform(run[1])
params[self.fidelity_index] = run[2]
return dict_to_trial(params, self.space)
def insert(self, params, results=None, reserve=False):
"""Insert a new trial.
Experiment must be in writable ('w') or executable ('x') mode.
Parameters
----------
params: dict
Parameters of the new trial to add to the database. These parameters
must comply with the space definition otherwise a ValueError will be raised.
results: list, optional
Results to be set for the new trial. Results must have the format
{name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise
a ValueError will be raised.
Note that passing results will mark the trial as completed and therefore cannot be
reserved. The returned trial will have status 'completed'.
If the results are invalid, the trial will still be inserted but reservation will be
released.
reserve: bool, optional
If reserve=True, the inserted trial will be reserved. `reserve` cannot be True if
`results` are given.
Defaults to False.
Returns
-------
`orion.core.worker.trial.Trial`
The trial inserted in storage. If `reserve=True` and no results are given, the returned
trial will be in a `reserved` status.
Raises
------
`ValueError`
- If results are given and reserve=True
- If params have invalid format
- If results have invalid format
`orion.core.io.database.DuplicateKeyError`
- If a trial with identical params already exist for the current experiment.
`orion.core.utils.exceptions.UnsupportedOperation`
If the experiment was not loaded in writable mode.
"""
self._check_if_writable()
if results and reserve:
raise ValueError(
"Cannot observe a trial and reserve it. A trial with results has status "
"`completed` and cannot be reserved.")
trial = format_trials.dict_to_trial(params, self.space)
try:
self._experiment.register_trial(trial, status="reserved")
self._maintain_reservation(trial)
except DuplicateKeyError as e:
message = (
"A trial with params {} already exist for experiment {}-v{}".
format(params, self.name, self.version))
raise DuplicateKeyError(message) from e
if results:
try:
self.observe(trial, results)
except ValueError:
self._release_reservation(trial)
raise
return trial
if not reserve:
self.release(trial)
return trial
def to_orion_trial(self):
from orion.core.utils.format_trials import dict_to_trial
from orion.core.worker.trial import Trial
return dict_to_trial(self.to_dict(), self.get_orion_space_dict())
