def _construct_models(X, Y, metric, do_mcmc, with_pending): pending_candidates = [] if with_pending: pending_candidates = [PendingEvaluation((0.5, 0.5)), PendingEvaluation((0.2, 0.2))] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [ CandidateEvaluation(x, y) for x, y in zip(X, Y) ], [], pending_candidates ) random_seed = 0 gpmodel = default_gpmodel( state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) result = [GaussProcSurrogateModel( state, metric, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20)] if do_mcmc: gpmodel_mcmc = default_gpmodel_mcmc( state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) result.append( GaussProcSurrogateModel( state, metric, random_seed, gpmodel_mcmc, fit_parameters=True,num_fantasy_samples=20)) return result
def test_gp_mcmc_fit(tuning_job_state): def tuning_job_state_mcmc(X, Y) -> TuningJobState: Y = [dictionarize_objective(y) for y in Y] return TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', -4., 4., LinearScaling())), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], []) _set_seeds(0) def f(x): return 0.1 * np.power(x, 3) X = np.concatenate((np.random.uniform(-4., -1., 10), np.random.uniform(1., 4., 10))) Y = f(X) X_test = np.sort(np.random.uniform(-1., 1., 10)) X = [(x, ) for x in X] X_test = [(x, ) for x in X_test] tuning_job_state = tuning_job_state_mcmc(X, Y) # checks if fitting is running random_seed = 0 gpmodel = default_gpmodel_mcmc(tuning_job_state, random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) model = GaussProcSurrogateModel(tuning_job_state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20) X = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X] predictions = model.predict(np.array(X)) Y_std_list = [p['std'] for p in predictions] Y_mean_list = [p['mean'] for p in predictions] Y_mean = np.mean(Y_mean_list, axis=0) Y_std = np.mean(Y_std_list, axis=0) assert np.all(np.abs(Y_mean - Y) < 1e-1), \ "in a noiseless setting, mean of GP should coincide closely with outputs at training points" X_test = [tuning_job_state.hp_ranges.to_ndarray(x) for x in X_test] predictions_test = model.predict(np.array(X_test)) Y_std_test_list = [p['std'] for p in predictions_test] Y_std_test = np.mean(Y_std_test_list, axis=0) assert np.max(Y_std) < np.min(Y_std_test), \ "Standard deviation on un-observed points should be greater than at observed ones"
def default_models() -> List[GaussProcSurrogateModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0 ), # same evals are added multiple times to force GP to unlearn prior (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [CandidateEvaluation(x, y) for x, y in zip(X, Y)], [], [], ) random_seed = 0 gpmodel = default_gpmodel(state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) gpmodel_mcmc = default_gpmodel_mcmc(state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) return [ GaussProcSurrogateModel(state, DEFAULT_METRIC, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20), GaussProcSurrogateModel(state, DEFAULT_METRIC, random_seed, gpmodel_mcmc, fit_parameters=True, num_fantasy_samples=20) ]
def default_models(metric, do_mcmc=True) -> List[GaussProcSurrogateModel]: X = [ (0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0), ] if metric == DEFAULT_METRIC: Y = [dictionarize_objective(np.sum(x) * 10.0) for x in X] elif metric == DEFAULT_COST_METRIC: # Increasing the first hp increases cost Y = [{metric: 1.0 + x[0] * 2.0} for x in X] else: raise ValueError(f"{metric} is not a valid metric") state = TuningJobState( HyperparameterRanges_Impl( HyperparameterRangeContinuous('x', 0.0, 1.0, LinearScaling()), HyperparameterRangeContinuous('y', 0.0, 1.0, LinearScaling()), ), [ CandidateEvaluation(x, y) for x, y in zip(X, Y) ], [], [] ) random_seed = 0 gpmodel = default_gpmodel( state, random_seed=random_seed, optimization_config=DEFAULT_OPTIMIZATION_CONFIG) result = [GaussProcSurrogateModel( state, metric, random_seed, gpmodel, fit_parameters=True, num_fantasy_samples=20)] if do_mcmc: gpmodel_mcmc = default_gpmodel_mcmc( state, random_seed=random_seed, mcmc_config=DEFAULT_MCMC_CONFIG) result.append( GaussProcSurrogateModel( state, metric, random_seed, gpmodel_mcmc, fit_parameters=True,num_fantasy_samples=20)) return result