def test_gbrt_gaussian(): # estimate quantiles of the normal distribution rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) rgr = GradientBoostingQuantileRegressor() rgr.fit(X, y) estimates = rgr.predict(X) assert_almost_equal(stats.norm.ppf(rgr.quantiles), np.mean(estimates, axis=0), decimal=2)
def test_gbrt_gaussian(): # estimate quantiles of the normal distribution rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) rgr = GradientBoostingQuantileRegressor() rgr.fit(X, y) estimates = rgr.predict(X) assert_almost_equal(stats.norm.ppf(rgr.quantiles), np.mean(estimates, axis=0), decimal=2)
def test_gbrt_with_std(): # simple test of the interface rng = np.random.RandomState(1) X = rng.uniform(0, 5, 500)[:, np.newaxis] noise_level = 0.5 y = truth(X) + rng.normal(0, noise_level, len(X)) X_ = np.linspace(0, 5, 1000)[:, np.newaxis] model = GradientBoostingQuantileRegressor() model.fit(X, y) # three quantiles, so three numbers per sample assert_array_equal( model.predict(X_, return_quantiles=True).shape, (len(X_), 3)) # "traditional" API which returns one number per sample, in this case # just the median/mean assert_array_equal(model.predict(X_).shape, (len(X_))) l, c, h = model.predict(X_, return_quantiles=True).T assert_equal(l.shape, c.shape) assert_equal(c.shape, h.shape) assert_equal(l.shape[0], X_.shape[0]) mean, std = model.predict(X_, return_std=True) assert_array_equal(mean, c) assert_array_equal(std, (h - l) / 2.0)
def __init__( self, problem, num_workers, surrogate_model="RF", acq_func="gp_hedge", acq_kappa=1.96, acq_xi=None, liar_strategy="cl_max", n_jobs=1, **kwargs, ): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY", ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = (inf if surrogate_model == "DUMMY" else max( num_workers, len(self.starting_points))) # Set acq_func_kwargs parameters acq_func_kwargs = {} if type(acq_kappa) is float: acq_func_kwargs["kappa"] = acq_kappa if type(acq_xi) is float: acq_func_kwargs["xi"] = acq_xi self._optimizer = SkOptimizer( dimensions=self.space, base_estimator=base_estimator, acq_optimizer="sampling", acq_func=acq_func, acq_func_kwargs=acq_func_kwargs, random_state=self.SEED, n_initial_points=n_init, ) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def test_gbrt_in_parallel(): # check estimate quantiles with parallel rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) rgr = GradientBoostingQuantileRegressor( n_jobs=1, random_state=np.random.RandomState(1)) rgr.fit(X, y) estimates = rgr.predict(X) rgr.set_params(n_jobs=2, random_state=np.random.RandomState(1)) rgr.fit(X, y) estimates_parallel = rgr.predict(X) assert_array_equal(estimates, estimates_parallel)
def test_gbrt_with_std(): # simple test of the interface rng = np.random.RandomState(1) X = rng.uniform(0, 5, 500)[:, np.newaxis] noise_level = 0.5 y = truth(X) + rng.normal(0, noise_level, len(X)) X_ = np.linspace(0, 5, 1000)[:, np.newaxis] model = GradientBoostingQuantileRegressor() model.fit(X, y) # three quantiles, so three numbers per sample assert_array_equal(model.predict(X_, return_quantiles=True).shape, (len(X_), 3)) # "traditional" API which returns one number per sample, in this case # just the median/mean assert_array_equal(model.predict(X_).shape, (len(X_))) l, c, h = model.predict(X_, return_quantiles=True).T assert_equal(l.shape, c.shape, h.shape) assert_equal(l.shape[0], X_.shape[0]) mean, std = model.predict(X_, return_std=True) assert_array_equal(mean, c) assert_array_equal(std, (h - l) / 2.0)
def test_gbrt_in_parallel(): # check estimate quantiles with parallel rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) rgr = GradientBoostingQuantileRegressor( n_jobs=1, random_state=np.random.RandomState(1)) rgr.fit(X, y) estimates = rgr.predict(X) rgr.set_params(n_jobs=2, random_state=np.random.RandomState(1)) rgr.fit(X, y) estimates_parallel = rgr.predict(X) assert_array_equal(estimates, estimates_parallel)
def __init__(self, problem, num_workers, surrogate_model='RF', acq_func='gp_hedge', acq_kappa=1.96, liar_strategy='cl_max', n_jobs=-1, **kwargs): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space cs_kwargs = self.space['create_search_space'].get('kwargs') if cs_kwargs is None: search_space = self.space['create_search_space']['func']() else: search_space = self.space['create_search_space']['func']( **cs_kwargs) # // queue of remaining starting points # // self.starting_points = problem.starting_point n_init = np.inf if surrogate_model == 'DUMMY' else num_workers self.starting_points = [] # ! EMPTY for now TODO # Building search space for SkOptimizer skopt_space = [(0, vnode.num_ops - 1) for vnode in search_space.variable_nodes] self._optimizer = SkOptimizer(skopt_space, base_estimator=base_estimator, acq_optimizer='sampling', acq_func=acq_func, acq_func_kwargs={'kappa': acq_kappa}, random_state=self.SEED, n_initial_points=n_init) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % surrogate_model)
def cook_estimator(base_estimator, space=None, **kwargs): if isinstance(base_estimator, str): base_estimator = base_estimator.upper() allowed_estimators = ['GP', 'ET', 'RF', 'GBRT', 'DUMMY'] if base_estimator not in allowed_estimators: raise ValueError( 'invalid estimator, should be in {}, got {}'.format( allowed_estimators, base_estimator)) elif not is_regressor(base_estimator): raise ValueError('base estimator should be a regressor, got {}'.format( base_estimator)) if base_estimator == 'GP': if space is not None: # space = Space(space) space = Space(normalize_param_space(space)) n_params = space.transformed_n_params is_cat = space.is_categorical else: raise ValueError('expected a space instance, got None') cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_params)) else: other_kernel = Matern(length_scale=np.ones(n_params), length_scale_bounds=[(0.01, 100)] * n_params, nu=2.5) base_estimator = GaussianProcessRegressor(kernel=cov_amplitude * other_kernel, normalize_y=True, noise='gaussian', n_restarts_optimizer=2) elif base_estimator == 'RF': base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == 'ET': base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == 'GRBT': grbt = GradientBoostingRegressor(n_estimators=30, loss='quantile') base_estimator = GradientBoostingQuantileRegressor(base_estimator=grbt) elif base_estimator == 'DUMMY': return None base_estimator.set_params(**kwargs) return base_estimator
def test_gbrt_base_estimator(): rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) base = RandomForestRegressor() rgr = GradientBoostingQuantileRegressor(base_estimator=base) assert_raise_message(ValueError, 'type GradientBoostingRegressor', rgr.fit, X, y) base = GradientBoostingRegressor() rgr = GradientBoostingQuantileRegressor(base_estimator=base) assert_raise_message(ValueError, 'quantile loss', rgr.fit, X, y) base = GradientBoostingRegressor(loss='quantile', n_estimators=20) rgr = GradientBoostingQuantileRegressor(base_estimator=base) rgr.fit(X, y) estimates = rgr.predict(X) assert_almost_equal(stats.norm.ppf(rgr.quantiles), np.mean(estimates, axis=0), decimal=2)
def __init__(self, problem, num_workers, surrogate_model='RF', acq_func='gp_hedge', acq_kappa=1.96, liar_strategy='cl_max', n_jobs=1, **kwargs): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = inf if surrogate_model == 'DUMMY' else max( num_workers, len(self.starting_points)) self._optimizer = SkOptimizer(self.space.values(), base_estimator=base_estimator, acq_optimizer='sampling', acq_func=acq_func, acq_func_kwargs={'kappa': acq_kappa}, random_state=self.SEED, n_initial_points=n_init) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def test_gbrt_with_std(): # simple test of the interface rng = np.random.RandomState(1) X = rng.uniform(0, 5, 500)[:, np.newaxis] noise_level = 0.5 y = truth(X) + rng.normal(0, noise_level, len(X)) X_ = np.linspace(0, 5, 1000)[:, np.newaxis] model = GradientBoostingQuantileRegressor() model.fit(X, y) assert_array_equal(model.predict(X_).shape, (len(X_), 3)) l, c, h = model.predict(X_).T assert_equal(l.shape, c.shape, h.shape) assert_equal(l.shape[0], X_.shape[0]) mean, std = model.predict(X_, return_std=True) assert_array_equal(mean, c) assert_array_equal(std, (h - l) / 2.0)
def test_gbrt_with_std(): # simple test of the interface rng = np.random.RandomState(1) X = rng.uniform(0, 5, 500)[:, np.newaxis] noise_level = 0.5 y = truth(X) + sample_noise(X, noise_level, random_state=rng) X_ = np.linspace(0, 5, 1000)[:, np.newaxis] model = GradientBoostingQuantileRegressor() model.fit(X, y) assert_array_equal(model.predict(X_).shape, (len(X_), 3)) l, c, h = model.predict(X_).T assert_equal(l.shape, c.shape, h.shape) assert_equal(l.shape[0], X_.shape[0])
def test_gbrt_with_std(): # simple test of the interface rng = np.random.RandomState(1) X = rng.uniform(0, 5, 500)[:, np.newaxis] noise_level = 0.5 y = truth(X) + sample_noise(X, noise_level, random_state=rng) X_ = np.linspace(0, 5, 1000)[:, np.newaxis] model = GradientBoostingQuantileRegressor() model.fit(X, y) assert_array_equal(model.predict(X_).shape, (len(X_), 3)) l, c, h = model.predict(X_).T assert_equal(l.shape, c.shape, h.shape) assert_equal(l.shape[0], X_.shape[0]) mean, std = model.predict(X_, return_std=True) assert_array_equal(mean, c) assert_array_equal(std, (h - l) / 2.0)
def test_gbrt_base_estimator(): rng = np.random.RandomState(1) N = 10000 X = np.ones((N, 1)) y = rng.normal(size=N) base = RandomForestRegressor() rgr = GradientBoostingQuantileRegressor(base_estimator=base) assert_raise_message(ValueError, 'type GradientBoostingRegressor', rgr.fit, X, y) base = GradientBoostingRegressor() rgr = GradientBoostingQuantileRegressor(base_estimator=base) assert_raise_message(ValueError, 'quantile loss', rgr.fit, X, y) base = GradientBoostingRegressor(loss='quantile', n_estimators=20) rgr = GradientBoostingQuantileRegressor(base_estimator=base) rgr.fit(X, y) estimates = rgr.predict(X) assert_almost_equal(stats.norm.ppf(rgr.quantiles), np.mean(estimates, axis=0), decimal=2)
from numpy.testing import assert_array_equal from numpy.testing import assert_equal from numpy.testing import assert_raises from skopt import gp_minimize from skopt import forest_minimize from skopt.benchmarks import bench1, bench1_with_time from skopt.benchmarks import branin from skopt.learning import ExtraTreesRegressor, RandomForestRegressor from skopt.learning import GradientBoostingQuantileRegressor from skopt.optimizer import Optimizer from scipy.optimize import OptimizeResult TREE_REGRESSORS = (ExtraTreesRegressor(random_state=2), RandomForestRegressor(random_state=2), GradientBoostingQuantileRegressor(random_state=2)) ACQ_FUNCS_PS = ["EIps", "PIps"] ACQ_FUNCS_MIXED = ["EI", "EIps"] ESTIMATOR_STRINGS = [ "GP", "RF", "ET", "GBRT", "DUMMY", "gp", "rf", "et", "gbrt", "dummy" ] @pytest.mark.fast_test def test_multiple_asks(): # calling ask() multiple times without a tell() inbetween should # be a "no op" base_estimator = ExtraTreesRegressor(random_state=2) opt = Optimizer([(-2.0, 2.0)], base_estimator, n_initial_points=1,
def cook_estimator(base_estimator, space=None, **kwargs): """Cook a default estimator For the special `base_estimator` called "DUMMY", the return value is None. This corresponds to sampling points at random, hence there is no need for an estimator Parameters ---------- base_estimator: {SKLearn Regressor, "GP", "RF", "ET", "GBRT", "DUMMY"}, default="GP" If not string, should inherit from `sklearn.base.RegressorMixin`. In addition, the `predict` method should have an optional `return_std` argument, which returns `std(Y | x)`, along with `E[Y | x]`. If `base_estimator` is a string in {"GP", "RF", "ET", "GBRT", "DUMMY"}, a surrogate model corresponding to the relevant `X_minimize` function is created space: `hyperparameter_hunter.space.space_core.Space` Required only if the `base_estimator` is a Gaussian Process. Ignored otherwise **kwargs: Dict Extra parameters provided to the `base_estimator` at initialization time Returns ------- SKLearn Regressor Regressor instance cooked up according to `base_estimator` and `kwargs`""" #################### Validate `base_estimator` #################### str_estimators = ["GP", "ET", "RF", "GBRT", "DUMMY"] if isinstance(base_estimator, str): if base_estimator.upper() not in str_estimators: raise ValueError( f"Expected `base_estimator` in {str_estimators}. Got {base_estimator}" ) # Convert to upper after error check, so above error shows actual given `base_estimator` base_estimator = base_estimator.upper() elif not is_regressor(base_estimator): raise ValueError("`base_estimator` must be a regressor") #################### Get Cooking #################### if base_estimator == "GP": if space is not None: space = Space(space) # NOTE: Below `normalize_dimensions` is NOT an unnecessary duplicate of the call in # `Optimizer` - `Optimizer` calls `cook_estimator` before its `dimensions` have been # normalized, so `normalize_dimensions` must also be called here space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a `Space` instance, not None") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # Only special if *all* dimensions are `Categorical` if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern(length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2, ) elif base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt) elif base_estimator == "DUMMY": return None base_estimator.set_params(**kwargs) return base_estimator
def __init__(self, dimensions, base_estimator="GP", maximize=True, n_random_starts=10, acq_func="LCB", acq_optimizer="lbfgs", random_state=None, n_points=10000, n_restarts_optimizer=5, xi=0.01, kappa=1.96, n_jobs=1): if not skopt_available: raise ImportError("skopt is not installed correctly") self.maximize = maximize self.n_params = len(dimensions) rng = check_random_state(random_state) if isinstance(base_estimator, str): if base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=n_jobs, random_state=rng) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=n_jobs, random_state=rng) elif base_estimator == "GP": cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) matern = Matern(length_scale=np.ones(len(dimensions)), length_scale_bounds=[(0.01, 100)] * len(dimensions), nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * matern, normalize_y=True, random_state=rng, alpha=0.0, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor( base_estimator=gbrt, n_jobs=n_jobs, random_state=rng) else: raise ValueError( "Valid strings for the base_estimator parameter" " are: 'RF', 'ET', or 'GP', not '%s'" % base_estimator) acq_func_kwargs = {"xi": xi, "kappa": kappa} acq_optimizer_kwargs = { "n_points": n_points, "n_restarts_optimizer": n_restarts_optimizer, "n_jobs": n_jobs } self.optimizer = _SkOptOptimizer( dimensions=dimensions, base_estimator=base_estimator, n_initial_points=n_random_starts, acq_func=acq_func, acq_optimizer=acq_optimizer, random_state=random_state, acq_func_kwargs=acq_func_kwargs, acq_optimizer_kwargs=acq_optimizer_kwargs)
def __init__( self, problem, num_workers, surrogate_model="RF", acq_func="gp_hedge", acq_kappa=1.96, liar_strategy="cl_max", n_jobs=1, **kwargs, ): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY", ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.problem = problem cs_kwargs = self.problem.space["create_search_space"].get("kwargs") if cs_kwargs is None: search_space = self.problem.space["create_search_space"]["func"]() else: search_space = self.problem.space["create_search_space"]["func"]( **cs_kwargs) n_init = np.inf if surrogate_model == "DUMMY" else num_workers self.starting_points = [] # ! EMPTY for now TODO # Building search space for SkOptimizer using ConfigSpace skopt_space = cs.ConfigurationSpace(seed=self.problem.seed) for i, vnode in enumerate(search_space.variable_nodes): hp = csh.UniformIntegerHyperparameter(name=f"vnode_{i}", lower=0, upper=(vnode.num_ops - 1)) skopt_space.add_hyperparameter(hp) self._optimizer = SkOptimizer( skopt_space, base_estimator=base_estimator, acq_optimizer="sampling", acq_func=acq_func, acq_func_kwargs={"kappa": acq_kappa}, random_state=self.SEED, n_initial_points=n_init, ) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % surrogate_model)
def cook_estimator(base_estimator, space=None, **kwargs): """ Cook a default estimator. For the special base_estimator called "DUMMY" the return value is None. This corresponds to sampling points at random, hence there is no need for an estimator. Parameters ---------- * `base_estimator` ["GP", "RF", "ET", "GBRT", "DUMMY" or sklearn regressor, default="GP"]: Should inherit from `sklearn.base.RegressorMixin`. In addition the `predict` method should have an optional `return_std` argument, which returns `std(Y | x)`` along with `E[Y | x]`. If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a surrogate model corresponding to the relevant `X_minimize` function is created. * `space` [Space instance]: Has to be provided if the base_estimator is a gaussian process. Ignored otherwise. * `kwargs` [dict]: Extra parameters provided to the base_estimator at init time. """ if isinstance(base_estimator, str): base_estimator = base_estimator.upper() if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY", "GPM32", "GPM1", "RBF", "RQ"]: raise ValueError("Valid strings for the base_estimator parameter " " are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not " "%s." % base_estimator) elif not is_regressor(base_estimator): raise ValueError("base_estimator has to be a regressor.") if base_estimator == "GP": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GPM32": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GPM1": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RBF": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) other_kernel = RBF(length_scale=np.ones(n_dims)) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RQ": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) other_kernel = RationalQuadratic(length_scale=np.ones(n_dims), alpha=0.1) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt) elif base_estimator == "DUMMY": return None base_estimator.set_params(**kwargs) return base_estimator