def tuning_job_state_mcmc(X, Y) -> TuningJobState: Y = [dictionarize_objective(y) for y in Y] return TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', -4., 4., LinearScaling())), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def data_to_state(data: dict) -> TuningJobState: cs = CS.ConfigurationSpace() cs_names = ['x{}'.format(i) for i in range(len(data['ss_limits']))] cs.add_hyperparameters([ CSH.UniformFloatHyperparameter( name=name, lower=lims['min'], upper=lims['max']) for name, lims in zip(cs_names, data['ss_limits'])]) _evaluations = [] x_mult = [] x_add = [] for lim in data['ss_limits']: mn, mx = lim['min'], lim['max'] x_mult.append(mx - mn) x_add.append(mn) x_mult = np.array(x_mult) x_add = np.array(x_add) for x, y in zip(data['train_inputs'], data['train_targets']): x_decoded = x * x_mult + x_add config_dct = dict(zip(cs_names, x_decoded)) config = CS.Configuration(cs, values=config_dct) _evaluations.append(CandidateEvaluation( config, dictionarize_objective(y))) return TuningJobState( hp_ranges=HyperparameterRanges_CS(cs), candidate_evaluations=_evaluations, failed_candidates=[], pending_evaluations=[])
def test_get_internal_candidate_evaluations(): """we do not test the case with no evaluations, as it is assumed that there will be always some evaluations generated in the beginning of the BO loop.""" candidates = [ CandidateEvaluation((2, 3.3, 'X'), dictionarize_objective(5.3)), CandidateEvaluation((1, 9.9, 'Y'), dictionarize_objective(10.9)), CandidateEvaluation((7, 6.1, 'X'), dictionarize_objective(13.1)), ] state = TuningJobState( hp_ranges=HyperparameterRanges_Impl( HyperparameterRangeInteger('integer', 0, 10, LinearScaling()), HyperparameterRangeContinuous('real', 0, 10, LinearScaling()), HyperparameterRangeCategorical('categorical', ('X', 'Y')), ), candidate_evaluations=candidates, failed_candidates=[candidates[0].candidate ], # these should be ignored by the model pending_evaluations=[]) result = get_internal_candidate_evaluations(state, DEFAULT_METRIC, normalize_targets=True, num_fantasize_samples=20) assert len(result.X.shape) == 2, "Input should be a matrix" assert len(result.y.shape) == 2, "Output should be a matrix" assert result.X.shape[0] == len(candidates) assert result.y.shape[ -1] == 1, "Only single output value per row is suppored" assert np.abs(np.mean( result.y)) < 1e-8, "Mean of the normalized outputs is not 0.0" assert np.abs(np.std(result.y) - 1.0) < 1e-8, "Std. of the normalized outputs is not 1.0" np.testing.assert_almost_equal(result.mean, 9.766666666666666) np.testing.assert_almost_equal(result.std, 3.283629428273267)
def default_models() -> List[GPMXNetModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0 ), # same evals are added multiple times to force GP to unlearn prior (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [], ) random_seed = 0 gpmodel = default_gpmodel(state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) gpmodel_mcmc = default_gpmodel_mcmc(state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) return [ GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20), GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel_mcmc, fit_parameters=True, num_fantasy_samples=20) ]
def tuning_job_state() -> TuningJobState: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] return TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [])
def update(self, config: Candidate, reward: float): """ Registers new datapoint at config, with reward reward. Note that in general, config should previously have been registered as pending (register_pending). If so, it is switched from pending to labeled. If not, it is considered directly labeled. :param config: :param reward: """ crit_val = self.map_reward(reward) self.state_transformer.label_candidate(CandidateEvaluation( candidate=copy.deepcopy(config), metrics=dictionarize_objective(crit_val))) if self.debug_log is not None: config_id = self.debug_log.config_id(config) msg = "Update for config_id {}: reward = {}, crit_val = {}".format( config_id, reward, crit_val) logger.info(msg)
def default_models(do_mcmc=True) -> List[GPMXNetModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], []) random_seed = 0 gpmodel = default_gpmodel(state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) result = [ GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20) ] if do_mcmc: gpmodel_mcmc = default_gpmodel_mcmc(state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) result.append( GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel_mcmc, fit_parameters=True, num_fantasy_samples=20)) return result
def _candidate_evaluations(num): return [ CandidateEvaluation(candidate=(i, ), metrics=dictionarize_objective(float(i))) for i in range(num) ]
def __call__(self, candidate): p = np.array([float(hp) for hp in candidate]) p[0] = np.log10(p[0]) return dictionarize_objective(np.sum((self.local_minima - p)**2))
def test_gp_fantasizing(): """ Compare whether acquisition function evaluations (values, gradients) with fantasizing are the same as averaging them by hand. """ random_seed = 4567 _set_seeds(random_seed) num_fantasy_samples = 10 num_pending = 5 hp_ranges = HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling())) X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] num_data = len(X) Y = [ dictionarize_objective(np.random.randn(1, 1)) for _ in range(num_data) ] # Draw fantasies. This is done for a number of fixed pending candidates # The model parameters are fit in the first iteration, when there are # no pending candidates # Note: It is important to not normalize targets, because this would be # done on the observed targets only, not the fantasized ones, so it # would be hard to compare below. pending_evaluations = [] for _ in range(num_pending): pending_cand = tuple(np.random.rand(2, )) pending_evaluations.append(PendingEvaluation(pending_cand)) state = TuningJobState(hp_ranges, [CandidateEvaluation(x, y) for x, y in zip(X, Y)], failed_candidates=[], pending_evaluations=pending_evaluations) gpmodel = default_gpmodel(state, random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) model = GPMXNetModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=num_fantasy_samples, normalize_targets=False) fantasy_samples = model.fantasy_samples # Evaluate acquisition function and gradients with fantasizing num_test = 50 X_test = np.vstack([ hp_ranges.to_ndarray(tuple(np.random.rand(2, ))) for _ in range(num_test) ]) acq_func = EIAcquisitionFunction(model) fvals, grads = acq_func.compute_acq_with_gradients(X_test) # Do the same computation by averaging by hand fvals_cmp = np.empty((num_fantasy_samples, ) + fvals.shape) grads_cmp = np.empty((num_fantasy_samples, ) + grads.shape) X_full = X + state.pending_candidates for it in range(num_fantasy_samples): Y_full = Y + [ dictionarize_objective(eval.fantasies[DEFAULT_METRIC][:, it]) for eval in fantasy_samples ] state2 = TuningJobState( hp_ranges, [CandidateEvaluation(x, y) for x, y in zip(X_full, Y_full)], failed_candidates=[], pending_evaluations=[]) # We have to skip parameter optimization here model2 = GPMXNetModel(state2, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=False, num_fantasy_samples=num_fantasy_samples, normalize_targets=False) acq_func2 = EIAcquisitionFunction(model2) fvals_, grads_ = acq_func2.compute_acq_with_gradients(X_test) fvals_cmp[it, :] = fvals_ grads_cmp[it, :] = grads_ # Comparison fvals2 = np.mean(fvals_cmp, axis=0) grads2 = np.mean(grads_cmp, axis=0) assert np.allclose(fvals, fvals2) assert np.allclose(grads, grads2)