def test_hierarchical_dict_to_trial(hierarchical_space, hierarchical_trial, hierarchical_dict_params): """Check if hierarchical dict is converted successfully to trial.""" t = dict_to_trial(hierarchical_dict_params, hierarchical_space) assert len(t._params) == len(hierarchical_trial._params) for i in range(len(t.params)): assert t._params[i].to_dict() == hierarchical_trial._params[i].to_dict( )
def _params_to_trial(self, orion_params: dict) -> Trial: """Create a Trial from a dict of hyper-parameters.""" # Need to convert the {name: value} of point_dict into this format for Orion's Trial. # Add the max value for the Fidelity dimensions, if any. if self.fidelity_index is not None: fidelity_dim: Fidelity = self.space[self.fidelity_index] orion_params[self.fidelity_index] = fidelity_dim.high trial: Trial = dict_to_trial(orion_params, space=self.space) return trial
def test_dict_to_trial(space, trial, dict_params): """Check if dict is converted successfully to trial.""" t = dict_to_trial(dict_params, space) assert t.experiment is None assert t.status == 'new' assert t.worker is None assert t.submit_time is None assert t.start_time is None assert t.end_time is None assert t.results == [] assert len(t._params) == len(trial._params) for i in range(len(t.params)): assert t._params[i].to_dict() == trial._params[i].to_dict()
def sample_to_trial(self, sample: numpy.ndarray, fidelity: int) -> Trial: """Convert a ConfigSpace sample into a trial""" config = self.dehb.vector_to_configspace(sample) hps = {} for k, v in self.space.items(): if v.type == "fidelity": hps[k] = fidelity else: hps[k] = config[k] return format_trials.dict_to_trial(to_orion(hps), self.space)
def suggest(self, num: int) -> list[Trial]: """Suggest a number of new sets of parameters. Draws points from a prepared set of samples from an orthonal Latin hypercube. Parameters ---------- num: int, optional Number of trials to suggest. The algorithm may return less than the number of trials requested. Returns ------- list of trials A list of trials representing values suggested by the algorithm. The algorithm may opt out if it cannot make a good suggestion at the moment (it may be waiting for other trials to complete), in which case it will return an empty list. Notes ----- New parameters must be compliant with the problem's domain `orion.algo.space.Space`. """ if self.n_trials is None or len( self.completed_trials) >= self.n_trials: self._prepare_next_iteration() trials = [] while (len(trials) < num and len(self.current_trials_params) > 0 and not self.is_done): trial_params = self.current_trials_params.pop() trial_params.update(self.frozen_param_values) trial = dict_to_trial(trial_params, self.space) if self.has_observed(trial): similar_trial = self.registry.get_existing(trial) trial.results = copy.deepcopy(similar_trial.results) trial.status = similar_trial.status self.completed_trials.append(trial) elif self.has_suggested(trial): self.duplicates[self.get_id(trial)].append(trial) else: self.register(trial) trials.append(trial) if len(trials) == 0 and len(self.completed_trials) >= self.n_trials: self.converged = True return trials
def _ask(self): suggestion = self.algo.ask() if suggestion.args: raise RuntimeError( "Nevergrad sampled a trial with args but this should never happen." " Please report this issue at" " https://github.com/Epistimio/orion.algo.nevergrad/issues") new_trial = dict_to_trial(suggestion.kwargs, self.space) if self._associate_trial(new_trial, suggestion): self.register(new_trial) return new_trial else: logger.debug("Ignoring duplicated trial") return None
def _suggest_one_bo(self): params = {} below_trials, above_trials = self.split_trials() for dimension in self.space.values(): dim_below_trials = [ trial.params[dimension.name] for trial in below_trials ] dim_above_trials = [ trial.params[dimension.name] for trial in above_trials ] if dimension.type == "real": dim_samples = self._sample_real_dimension( dimension, dim_below_trials, dim_above_trials, ) elif dimension.type == "integer" and dimension.prior_name in [ "int_uniform", "int_reciprocal", ]: dim_samples = self._sample_int_point( dimension, dim_below_trials, dim_above_trials, ) elif dimension.type == "categorical" and dimension.prior_name == "choices": dim_samples = self._sample_categorical_point( dimension, dim_below_trials, dim_above_trials, ) elif dimension.type == "fidelity": # fidelity dimension trial = self.space.sample(1)[0] dim_samples = trial.params[dimension.name] else: raise NotImplementedError() params[dimension.name] = dim_samples trial = format_trials.dict_to_trial(params, self.space) return self.format_trial(trial)
def suggest(self, num): """Suggest a number of new sets of parameters. Parameters ---------- num: int Number of trials to suggest. The algorithm may return less than the number of trials requested. Returns ------- list of trials A list of trials representing values suggested by the algorithm. The algorithm may opt out if it cannot make a good suggestion at the moment (it may be waiting for other trials to complete), in which case it will return None. Notes ----- New parameters must be compliant with the problem's domain `orion.algo.space.Space`. """ trials = [] with self.get_client() as client: _trials, _ = client.get_next_trials(num) for trial_index, parameters in _trials.items(): parameters = AxOptimizer.reverse_params(parameters, self.space) # Ax does not support Fidelity dimension type so fake it with # its max if self.fidelity_index is not None: # Convert 0-dim arrays into python numbers so their type can # be validated by Ax parameters[self.fidelity_index] = float( self.space[self.fidelity_index].high) new_trial = format_trials.dict_to_trial(parameters, self.space) if not self.has_suggested(new_trial): self.register(new_trial) trials.append(new_trial) self._trials_map[self.get_id( new_trial)] = trial_index # tmp return trials
def call(self, **kwargs) -> List[Dict]: """Get the value of the sampled objective function at the given point (hyper-parameters). If `self.with_grad` is set, also returns the gradient of the objective function with respect to the inputs. Parameters ---------- **kwargs Dictionary of hyper-parameters. Returns ------- List[Dict] Result dictionaries: objective and optionally gradient. Raises ------ ValueError If the input isn't of a supported type. """ # A bit of gymnastics to convert the params Dict into a PyTorch tensor. trial = dict_to_trial(kwargs, self._space) flattened_trial = self.transformed_space.transform(trial) flattened_params = flatten(flattened_trial.params) flattened_point = np.array( [flattened_params[key] for key in self.transformed_space.keys()]) x_tensor = torch.as_tensor(flattened_point).type_as(self.h_tensor) if self.with_grad: x_tensor = x_tensor.requires_grad_(True) p_tensor = torch.cat([x_tensor, self.h_tensor]) p_tensor = torch.atleast_2d(p_tensor) devices = [] if self.device.type == "cpu" else [self.device] # NOTE: Currently no way to locally seed the rng of torch distributions, hence forking the # rng for torch only here. with torch.random.fork_rng(devices=devices): torch.random.manual_seed(self.seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(self.seed) # Forward pass: out = self.net(p_tensor) y_mean, y_log_std = out[0, 0], out[0, 1] y_std = torch.exp(y_log_std) # NOTE: Here we create a distribution over `y`, and use `rsample()`, so that we get can # also return the gradients if need be. y_dist = Normal(loc=y_mean, scale=y_std) y_sample = y_dist.rsample() logger.debug(f"y_sample: {y_sample}") results: List[dict] = [ dict(name=self.name, type="objective", value=y_sample.detach().cpu().item()) ] if self.with_grad: self.net.zero_grad() y_sample.backward() assert x_tensor.grad is not None results.append( dict(name=self.name, type="gradient", value=x_tensor.grad.cpu().numpy())) return results
def run_to_trial(run): params = transform(run[1]) params[self.fidelity_index] = run[2] return dict_to_trial(params, self.space)
def insert(self, params, results=None, reserve=False): """Insert a new trial. Experiment must be in writable ('w') or executable ('x') mode. Parameters ---------- params: dict Parameters of the new trial to add to the database. These parameters must comply with the space definition otherwise a ValueError will be raised. results: list, optional Results to be set for the new trial. Results must have the format {name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise a ValueError will be raised. Note that passing results will mark the trial as completed and therefore cannot be reserved. The returned trial will have status 'completed'. If the results are invalid, the trial will still be inserted but reservation will be released. reserve: bool, optional If reserve=True, the inserted trial will be reserved. `reserve` cannot be True if `results` are given. Defaults to False. Returns ------- `orion.core.worker.trial.Trial` The trial inserted in storage. If `reserve=True` and no results are given, the returned trial will be in a `reserved` status. Raises ------ `ValueError` - If results are given and reserve=True - If params have invalid format - If results have invalid format `orion.core.io.database.DuplicateKeyError` - If a trial with identical params already exist for the current experiment. `orion.core.utils.exceptions.UnsupportedOperation` If the experiment was not loaded in writable mode. """ self._check_if_writable() if results and reserve: raise ValueError( "Cannot observe a trial and reserve it. A trial with results has status " "`completed` and cannot be reserved.") trial = format_trials.dict_to_trial(params, self.space) try: self._experiment.register_trial(trial, status="reserved") self._maintain_reservation(trial) except DuplicateKeyError as e: message = ( "A trial with params {} already exist for experiment {}-v{}". format(params, self.name, self.version)) raise DuplicateKeyError(message) from e if results: try: self.observe(trial, results) except ValueError: self._release_reservation(trial) raise return trial if not reserve: self.release(trial) return trial
def to_orion_trial(self): from orion.core.utils.format_trials import dict_to_trial from orion.core.worker.trial import Trial return dict_to_trial(self.to_dict(), self.get_orion_space_dict())