def bohamiann(objective_function, lower, upper, num_iterations=30, acquisition_func="log_ei", n_init=3, rng=None): """ General interface for Bayesian optimization for global black box optimization problems. Parameters ---------- objective_function: function The objective function that is minimized. This function gets a numpy array (D,) as input and returns the function value (scalar) lower: np.ndarray (D,) The lower bound of the search space upper: np.ndarray (D,) The upper bound of the search space num_iterations: int The number of iterations (initial design + BO) acquisition_func: {"ei", "log_ei", "lcb", "pi"} The acquisition function n_init: int Number of points for the initial design. Make sure that it is <= num_iterations. rng: numpy.random.RandomState Random number generator Returns ------- dict with all results """ assert upper.shape[0] == lower.shape[0] assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations" if rng is None: rng = np.random.RandomState(np.random.randint(0, 10000)) model = BayesianNeuralNetwork(sampling_method="sghmc", l_rate=np.sqrt(1e-4), mdecay=0.05, burn_in=3000, n_iters=50000, precondition=True, normalize_input=True, normalize_output=True) if acquisition_func == "ei": a = EI(model) elif acquisition_func == "log_ei": a = LogEI(model) elif acquisition_func == "pi": a = PI(model) elif acquisition_func == "lcb": a = LCB(model) else: print("ERROR: %s is not a valid acquisition function!" % acquisition_func) return max_func = Direct(a, lower, upper, verbose=False) bo = BayesianOptimization(objective_function, lower, upper, a, model, max_func, initial_points=n_init, rng=rng) x_best, f_min = bo.run(num_iterations) results = dict() results["x_opt"] = x_best results["f_opt"] = f_min results["incumbents"] = [inc for inc in bo.incumbents] results["incumbent_values"] = [val for val in bo.incumbents_values] results["runtime"] = bo.runtime results["overhead"] = bo.time_overhead return results
def bayesian_optimization(objective_function, lower, upper, num_iterations=30, maximizer="random", acquisition_func="log_ei", model_type="gp_mcmc", n_init=3, rng=None, output_path=None): """ General interface for Bayesian optimization for global black box optimization problems. Parameters ---------- objective_function: function The objective function that is minimized. This function gets a numpy array (D,) as input and returns the function value (scalar) lower: np.ndarray (D,) The lower bound of the search space upper: np.ndarray (D,) The upper bound of the search space num_iterations: int The number of iterations (initial design + BO) maximizer: {"random", "scipy", "differential_evolution"} The optimizer for the acquisition function. acquisition_func: {"ei", "log_ei", "lcb", "pi"} The acquisition function model_type: {"gp", "gp_mcmc", "rf", "bohamiann"} The model for the objective function. n_init: int Number of points for the initial design. Make sure that it is <= num_iterations. output_path: string Specifies the path where the intermediate output after each iteration will be saved. If None no output will be saved to disk. rng: numpy.random.RandomState Random number generator Returns ------- dict with all results """ assert upper.shape[0] == lower.shape[0], "Dimension miss match" assert np.all(lower < upper), "Lower bound >= upper bound" assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations" if rng is None: rng = np.random.RandomState(np.random.randint(0, 10000)) cov_amp = 2 n_dims = lower.shape[0] initial_ls = np.ones([n_dims]) exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims) kernel = cov_amp * exp_kernel prior = DefaultPrior(len(kernel) + 1) n_hypers = 3 * len(kernel) if n_hypers % 2 == 1: n_hypers += 1 if model_type == "gp": model = GaussianProcess(kernel, prior=prior, rng=rng, normalize_output=False, normalize_input=True, lower=lower, upper=upper) elif model_type == "gp_mcmc": model = GaussianProcessMCMC(kernel, prior=prior, n_hypers=n_hypers, chain_length=200, burnin_steps=100, normalize_input=True, normalize_output=False, rng=rng, lower=lower, upper=upper) elif model_type == "rf": model = RandomForest(rng=rng) elif model_type == "bohamiann": model = BNNModel() else: raise ValueError("'{}' is not a valid model".format(model_type)) if acquisition_func == "ei": a = EI(model) elif acquisition_func == "log_ei": a = LogEI(model) elif acquisition_func == "pi": a = PI(model) elif acquisition_func == "lcb": a = LCB(model) else: raise ValueError("'{}' is not a valid acquisition function".format( acquisition_func)) if model_type == "gp_mcmc": acquisition_func = MarginalizationGPMCMC(a) else: acquisition_func = a if maximizer == "random": max_func = RandomSampling(acquisition_func, lower, upper, rng=rng) elif maximizer == "scipy": max_func = SciPyOptimizer(acquisition_func, lower, upper, rng=rng) elif maximizer == "differential_evolution": max_func = DifferentialEvolution(acquisition_func, lower, upper, rng=rng) else: raise ValueError("'{}' is not a valid function to maximize the " "acquisition function".format(maximizer)) bo = BayesianOptimization(objective_function, lower, upper, acquisition_func, model, max_func, initial_points=n_init, rng=rng, initial_design=init_latin_hypercube_sampling, output_path=output_path) x_best, f_min = bo.run(num_iterations) results = dict() results["x_opt"] = x_best results["f_opt"] = f_min results["incumbents"] = [inc for inc in bo.incumbents] results["incumbent_values"] = [val for val in bo.incumbents_values] results["runtime"] = bo.runtime results["overhead"] = bo.time_overhead results["X"] = [x.tolist() for x in bo.X] results["y"] = [y for y in bo.y] return results
def build_optimizer(model, maximizer="random", acquisition_func="log_ei", maximizer_seed=1): """ General interface for Bayesian optimization for global black box optimization problems. Parameters ---------- maximizer: {"random", "scipy", "differential_evolution"} The optimizer for the acquisition function. acquisition_func: {"ei", "log_ei", "lcb", "pi"} The acquisition function maximizer_seed: int Seed for random number generator of the acquisition function maximizer Returns ------- Optimizer """ if acquisition_func == "ei": a = EI(model) elif acquisition_func == "log_ei": a = LogEI(model) elif acquisition_func == "pi": a = PI(model) elif acquisition_func == "lcb": a = LCB(model) else: raise ValueError("'{}' is not a valid acquisition function".format( acquisition_func)) if isinstance(model, GaussianProcessMCMC): acquisition_func = MarginalizationGPMCMC(a) else: acquisition_func = a maximizer_rng = numpy.random.RandomState(maximizer_seed) if maximizer == "random": max_func = RandomSampling(acquisition_func, model.lower, model.upper, rng=maximizer_rng) elif maximizer == "scipy": max_func = SciPyOptimizer(acquisition_func, model.lower, model.upper, rng=maximizer_rng) elif maximizer == "differential_evolution": max_func = DifferentialEvolution(acquisition_func, model.lower, model.upper, rng=maximizer_rng) else: raise ValueError("'{}' is not a valid function to maximize the " "acquisition function".format(maximizer)) # NOTE: Internal RNG of BO won't be used. # NOTE: Nb of initial points won't be used within BO, but rather outside bo = BayesianOptimization(lambda: None, model.lower, model.upper, acquisition_func, model, max_func, initial_points=None, rng=None, initial_design=init_latin_hypercube_sampling, output_path=None) return bo
def bayesian_optimization(objective_function, lower, upper, num_iterations=30, maximizer="direct", acquisition_func="log_ei", model="gp_mcmc", n_init=3, rng=None): """ General interface for Bayesian optimization for global black box optimization problems. Parameters ---------- objective_function: function The objective function that is minimized. This function gets a numpy array (D,) as input and returns the function value (scalar) lower: np.ndarray (D,) The lower bound of the search space upper: np.ndarray (D,) The upper bound of the search space num_iterations: int The number of iterations (initial design + BO) maximizer: {"direct", "cmaes"} Defines how the acquisition function is maximized. NOTE: "cmaes" only works in D > 1 dimensions acquisition_func: {"ei", "log_ei", "lcb", "pi"} The acquisition function model: {"gp", "gp_mcmc"} The model for the objective function. n_init: int Number of points for the initial design. Make sure that it is <= num_iterations. rng: numpy.random.RandomState Random number generator Returns ------- dict with all results """ assert upper.shape[0] == lower.shape[0] assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations" if rng is None: rng = np.random.RandomState(np.random.randint(0, 10000)) cov_amp = 2 n_dims = lower.shape[0] initial_ls = np.ones([n_dims]) exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims) kernel = cov_amp * exp_kernel prior = DefaultPrior(len(kernel) + 1) n_hypers = 3 * len(kernel) if n_hypers % 2 == 1: n_hypers += 1 if model == "gp": gp = GaussianProcess(kernel, prior=prior, rng=rng, normalize_output=True, normalize_input=True, lower=lower, upper=upper) elif model == "gp_mcmc": gp = GaussianProcessMCMC(kernel, prior=prior, n_hypers=n_hypers, chain_length=200, burnin_steps=100, normalize_input=True, normalize_output=True, rng=rng, lower=lower, upper=upper) else: print("ERROR: %s is not a valid model!" % model) return if acquisition_func == "ei": a = EI(gp) elif acquisition_func == "log_ei": a = LogEI(gp) elif acquisition_func == "pi": a = PI(gp) elif acquisition_func == "lcb": a = LCB(gp) else: print("ERROR: %s is not a valid acquisition function!" % acquisition_func) return if model == "gp": acquisition_func = a elif model == "gp_mcmc": acquisition_func = MarginalizationGPMCMC(a) if maximizer == "cmaes": max_func = CMAES(acquisition_func, lower, upper, verbose=False, rng=rng) elif maximizer == "direct": max_func = Direct(acquisition_func, lower, upper, verbose=False) else: print( "ERROR: %s is not a valid function to maximize the acquisition function!" % maximizer) return bo = BayesianOptimization(objective_function, lower, upper, acquisition_func, gp, max_func, initial_points=n_init, rng=rng) x_best, f_min = bo.run(num_iterations) results = dict() results["x_opt"] = x_best results["f_opt"] = f_min results["incumbents"] = [inc for inc in bo.incumbents] results["incumbent_values"] = [val for val in bo.incumbents_values] results["runtime"] = bo.runtime results["overhead"] = bo.time_overhead return results
def bohamiann(objective_function, lower, upper, cs, num_iterations=30, maximizer="direct", acquisition_func="log_ei", n_init=3, output_path=None, rng=None): """ Bohamiann uses Bayesian neural networks to model the objective function [1] inside Bayesian optimization. Bayesian neural networks usually scale better with the number of function evaluations and the number of dimensions than Gaussian processes. [1] Bayesian optimization with robust Bayesian neural networks J. T. Springenberg and A. Klein and S. Falkner and F. Hutter Advances in Neural Information Processing Systems 29 Parameters ---------- objective_function: function The objective function that is minimized. This function gets a numpy array (D,) as input and returns the function value (scalar) lower: np.ndarray (D,) The lower bound of the search space upper: np.ndarray (D,) The upper bound of the search space num_iterations: int The number of iterations (initial design + BO) acquisition_func: {"ei", "log_ei", "lcb", "pi"} The acquisition function maximizer: {"direct", "cmaes", "random", "scipy", "differential_evolution"} The optimizer for the acquisition function. NOTE: "cmaes" only works in D > 1 dimensions n_init: int Number of points for the initial design. Make sure that it is <= num_iterations. output_path: string Specifies the path where the intermediate output after each iteration will be saved. If None no output will be saved to disk. rng: numpy.random.RandomState Random number generator Returns ------- dict with all results """ assert upper.shape[0] == lower.shape[0] assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations" if rng is None: rng = np.random.RandomState(np.random.randint(0, 10000)) model = DG(num_epochs=20000, learning_rate=0.01, momentum=0.9, adapt_epoch=5000, prior=None, H=50, D=10, alpha=1.0, beta=1000) if acquisition_func == "ei": a = EI(model) elif acquisition_func == "log_ei": a = LogEI(model) elif acquisition_func == "pi": a = PI(model) elif acquisition_func == "lcb": a = LCB(model) else: print("ERROR: %s is not a valid acquisition function!" % acquisition_func) return if maximizer == "cmaes": max_func = CMAES(a, lower, upper, verbose=True, rng=rng) elif maximizer == "direct": max_func = Direct(a, lower, upper, verbose=True) elif maximizer == "random": max_func = RandomSampling(a, lower, upper, rng=rng) elif maximizer == "scipy": max_func = SciPyOptimizer(a, lower, upper, rng=rng) elif maximizer == "differential_evolution": max_func = DifferentialEvolution(a, lower, upper, rng=rng) bo = BayesianOptimization(objective_function, lower, upper, a, model, max_func, cs, initial_points=n_init, output_path=output_path, rng=rng, min_budget=2, max_budget=5) x_best, f_min = bo.run(num_iterations) results = dict() results["x_opt"] = get_config_dictionary(x_best, cs) results["f_opt"] = f_min results["incumbents"] = [ get_config_dictionary(inc, cs) for inc in bo.incumbents ] results["incumbent_values"] = [val for val in bo.incumbents_values] results["runtime"] = bo.runtime results["overhead"] = bo.time_overhead results["X"] = [get_config_dictionary(x.tolist(), cs) for x in bo.X] results["y"] = [y for y in bo.y] return results