def test_min_variance(): rng = np.random.RandomState(0) X = rng.normal(size=(1000, 1)) y = np.ones(1000) rf = RandomForestRegressor(min_variance=0.1) rf.fit(X, y) mean, std = rf.predict(X, return_std=True) assert_array_almost_equal(mean, y) assert_array_almost_equal(std, np.sqrt(0.1 * np.ones(1000)))
def test_min_variance(): rng = np.random.RandomState(0) X = rng.normal(size=(1000, 1)) y = np.ones(1000) rf = RandomForestRegressor(min_variance=0.1) rf.fit(X, y) mean, std = rf.predict(X, return_std=True) assert_array_almost_equal(mean, y) assert_array_almost_equal(std, np.sqrt(0.1*np.ones(1000)))
def __init__( self, problem, num_workers, surrogate_model="RF", acq_func="gp_hedge", acq_kappa=1.96, acq_xi=None, liar_strategy="cl_max", n_jobs=1, **kwargs, ): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY", ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = (inf if surrogate_model == "DUMMY" else max( num_workers, len(self.starting_points))) # Set acq_func_kwargs parameters acq_func_kwargs = {} if type(acq_kappa) is float: acq_func_kwargs["kappa"] = acq_kappa if type(acq_xi) is float: acq_func_kwargs["xi"] = acq_xi self._optimizer = SkOptimizer( dimensions=self.space, base_estimator=base_estimator, acq_optimizer="sampling", acq_func=acq_func, acq_func_kwargs=acq_func_kwargs, random_state=self.SEED, n_initial_points=n_init, ) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, plot="true", n_jobs=1, **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.do_plot = plot == "true" self.n_jobs = int(n_jobs) # Initialize Hyperaparameter space # self.hp_space = cs.ConfigurationSpace(seed=42) # self.hp_space.add_hyperparameter( # check_hyperparameter( # self.problem.space["hyperparameters"]["learning_rate"], "learning_rate" # ) # ) # self.hp_space.add_hyperparameter( # check_hyperparameter( # self.problem.space["hyperparameters"]["batch_size"], "batch_size" # ) # ) self.hp_space = [] self.hp_space.append( self.problem.space["hyperparameters"]["learning_rate"]) # ploting lr_range = self.problem.space["hyperparameters"]["learning_rate"][:2] self.domain_x = np.linspace(*lr_range, 400).reshape(-1, 1) # Initialize opitmizer of hyperparameter space acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration self.n_initial_points = self.free_workers self.hp_opt = SkOptimizer( dimensions=self.hp_space, base_estimator=RandomForestRegressor(n_jobs=32), # base_estimator=RandomForestRegressor(n_jobs=self.n_jobs), acq_func="LCB", acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, # model_queue_size=100, )
def choose_optimizer(optimizer): """ Choose a surrogate model for Bayesian Optimization :param optimizer: list of setting of the BO experiment :type optimizer: Optimizer :return: surrogate model :rtype: scikit object """ params_space_list = dimensions_aslist(optimizer.search_space) estimator = None # Choice of the surrogate model # Random forest if optimizer.surrogate_model == "RF": estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, random_state=optimizer.random_state) # Extra Tree elif optimizer.surrogate_model == "ET": estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3, random_state=optimizer.random_state) # GP Minimize elif optimizer.surrogate_model == "GP": estimator = GaussianProcessRegressor( kernel=optimizer.kernel, random_state=optimizer.random_state) # Random Search elif optimizer.surrogate_model == "RS": estimator = "dummy" if estimator == "dummy": opt = skopt_optimizer( params_space_list, base_estimator=estimator, acq_func=optimizer.acq_func, acq_optimizer='sampling', initial_point_generator=optimizer.initial_point_generator, random_state=optimizer.random_state) else: opt = skopt_optimizer( params_space_list, base_estimator=estimator, acq_func=optimizer.acq_func, acq_optimizer='sampling', n_initial_points=optimizer.n_random_starts, initial_point_generator=optimizer.initial_point_generator, # work only for version skopt 8.0!!! acq_optimizer_kwargs={ "n_points": 10000, "n_restarts_optimizer": 5, "n_jobs": 1 }, acq_func_kwargs={ "xi": 0.01, "kappa": 1.96 }, random_state=optimizer.random_state) return opt
def rfbo_cycle(ndim, space, target_f, n_iters=10, acq_function=ei, n_samples=int(1.0e+5), model=None, show_progress=True): xrange = (lambda title, n: tqdm_notebook(range(n), postfix=title) ) if show_progress else (lambda title, n: range(n)) space = np.array(space) if model is None: model = RandomForestRegressor(n_estimators=200, n_jobs=20, min_variance=1.0e-3, random_state=1234) known_points = [] known_values = [] cost = [] for i in xrange('BO iteration', n_iters): acq = acq_function(model, known_points, known_values) candidates = np.random.uniform(size=( n_samples, ndim, )) f = acq(candidates) best = np.argmin(f) suggestion = reverse_transform(candidates[best].reshape(1, -1), space)[0, :] point_cost, observed = target_f(suggestion) known_points.append(suggestion) known_values.append(observed) cost.append(point_cost) model.fit(transform(np.array(known_points), space), np.array(known_values)) yield model, acq, space, known_points, known_values, cost
def _init_optimizer(self, n_calls): return Optimizer( dimensions=dimensions_aslist(search_space=self.search_space), base_estimator=RandomForestRegressor(n_estimators=10), n_initial_points=int(n_calls * self.random_ratio), acq_func="EI", acq_optimizer="sampling", acq_optimizer_kwargs=dict(n_points=1000, n_jobs=-1), acq_func_kwargs=dict(xi=0.01, kappa=1.96))
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, n_jobs=1, kappa=0.001, xi=0.000001, **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.n_jobs = int(n_jobs) # Initialize Hyperaparameter space self.hp_space = [] # add the 'learning_rate' space to the HPO search space self.hp_space.append( self.problem.space["hyperparameters"]["learning_rate"]) # add the 'batch_size' space to the HPO search space self.hp_space.append( self.problem.space["hyperparameters"]["batch_size"]) # add the 'num_ranks_per_node' space to the HPO search space self.hp_space.append( self.problem.space["hyperparameters"]["ranks_per_node"]) # Initialize opitmizer of hyperparameter space acq_func_kwargs = { "xi": float(xi), "kappa": float(kappa) } # tiny exploration # self.free_workers = 128 #! TODO: test self.n_initial_points = self.free_workers self.hp_opt = SkOptimizer( dimensions=self.hp_space, base_estimator=RandomForestRegressor(n_jobs=32), # base_estimator=RandomForestRegressor(n_jobs=4), acq_func="LCB", acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, # model_queue_size=100, )
def __init__(self, problem, num_workers, surrogate_model='RF', acq_func='gp_hedge', acq_kappa=1.96, liar_strategy='cl_max', n_jobs=-1, **kwargs): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space cs_kwargs = self.space['create_search_space'].get('kwargs') if cs_kwargs is None: search_space = self.space['create_search_space']['func']() else: search_space = self.space['create_search_space']['func']( **cs_kwargs) # // queue of remaining starting points # // self.starting_points = problem.starting_point n_init = np.inf if surrogate_model == 'DUMMY' else num_workers self.starting_points = [] # ! EMPTY for now TODO # Building search space for SkOptimizer skopt_space = [(0, vnode.num_ops - 1) for vnode in search_space.variable_nodes] self._optimizer = SkOptimizer(skopt_space, base_estimator=base_estimator, acq_optimizer='sampling', acq_func=acq_func, acq_func_kwargs={'kappa': acq_kappa}, random_state=self.SEED, n_initial_points=n_init) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % surrogate_model)
def __init__(self, problem, run, evaluator, **kwargs): super().__init__(problem=problem, run=run, evaluator=evaluator, **kwargs) self.free_workers = self.evaluator.num_workers dhlogger.info( jm( type="start_infos", alg="bayesian-optimization-for-hpo-nas", nworkers=self.evaluator.num_workers, encoded_space=json.dumps(self.problem.space, cls=Encoder), )) # Setup self.pb_dict = self.problem.space cs_kwargs = self.pb_dict["create_search_space"].get("kwargs") if cs_kwargs is None: search_space = self.pb_dict["create_search_space"]["func"]() else: search_space = self.pb_dict["create_search_space"]["func"]( **cs_kwargs) self.space_list = [(0, vnode.num_ops - 1) for vnode in search_space.variable_nodes] # Initialize Hyperaparameter space self.dimensions = [] self.size_ha = None # Number of algorithm hyperparameters in the dimension list self.add_ha_dimensions() self.add_hm_dimensions() # Initialize opitmizer of hyperparameter space # acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration acq_func_kwargs = {"xi": 0.000001, "kappa": 1.96} # tiny exploration self.n_initial_points = self.free_workers self.opt = SkOptimizer( dimensions=self.dimensions, base_estimator=RandomForestRegressor(n_jobs=32), # base_estimator=RandomForestRegressor(n_jobs=4), acq_func="LCB", acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, # model_queue_size=100, )
def cook_estimator(base_estimator, space=None, **kwargs): if isinstance(base_estimator, str): base_estimator = base_estimator.upper() allowed_estimators = ['GP', 'ET', 'RF', 'GBRT', 'DUMMY'] if base_estimator not in allowed_estimators: raise ValueError( 'invalid estimator, should be in {}, got {}'.format( allowed_estimators, base_estimator)) elif not is_regressor(base_estimator): raise ValueError('base estimator should be a regressor, got {}'.format( base_estimator)) if base_estimator == 'GP': if space is not None: # space = Space(space) space = Space(normalize_param_space(space)) n_params = space.transformed_n_params is_cat = space.is_categorical else: raise ValueError('expected a space instance, got None') cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_params)) else: other_kernel = Matern(length_scale=np.ones(n_params), length_scale_bounds=[(0.01, 100)] * n_params, nu=2.5) base_estimator = GaussianProcessRegressor(kernel=cov_amplitude * other_kernel, normalize_y=True, noise='gaussian', n_restarts_optimizer=2) elif base_estimator == 'RF': base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == 'ET': base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == 'GRBT': grbt = GradientBoostingRegressor(n_estimators=30, loss='quantile') base_estimator = GradientBoostingQuantileRegressor(base_estimator=grbt) elif base_estimator == 'DUMMY': return None base_estimator.set_params(**kwargs) return base_estimator
def __init__(self, problem, num_workers, surrogate_model='RF', acq_func='gp_hedge', acq_kappa=1.96, liar_strategy='cl_max', n_jobs=1, **kwargs): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = inf if surrogate_model == 'DUMMY' else max( num_workers, len(self.starting_points)) self._optimizer = SkOptimizer(self.space.values(), base_estimator=base_estimator, acq_optimizer='sampling', acq_func=acq_func, acq_func_kwargs={'kappa': acq_kappa}, random_state=self.SEED, n_initial_points=n_init) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, n_jobs=1, kappa=0.001, xi=0.000001, acq_func="LCB", **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.n_jobs = int( n_jobs) # parallelism of BO surrogate model estimator # Initialize Hyperaparameter space self.hp_space = self.problem._hp_space # Initialize opitmizer of hyperparameter space acq_func_kwargs = { "xi": float(xi), "kappa": float(kappa) } # tiny exploration self.n_initial_points = self.free_workers self.hp_opt = SkOptimizer( dimensions=self.hp_space._space, base_estimator=RandomForestRegressor(n_jobs=self.n_jobs), acq_func=acq_func, acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, )
from sklearn.multioutput import MultiOutputRegressor from numpy.testing import assert_array_equal from numpy.testing import assert_equal from numpy.testing import assert_raises from skopt import gp_minimize from skopt import forest_minimize from skopt.benchmarks import bench1, bench1_with_time from skopt.benchmarks import branin from skopt.learning import ExtraTreesRegressor, RandomForestRegressor from skopt.learning import GradientBoostingQuantileRegressor from skopt.optimizer import Optimizer from scipy.optimize import OptimizeResult TREE_REGRESSORS = (ExtraTreesRegressor(random_state=2), RandomForestRegressor(random_state=2), GradientBoostingQuantileRegressor(random_state=2)) ACQ_FUNCS_PS = ["EIps", "PIps"] ACQ_FUNCS_MIXED = ["EI", "EIps"] ESTIMATOR_STRINGS = [ "GP", "RF", "ET", "GBRT", "DUMMY", "gp", "rf", "et", "gbrt", "dummy" ] @pytest.mark.fast_test def test_multiple_asks(): # calling ask() multiple times without a tell() inbetween should # be a "no op" base_estimator = ExtraTreesRegressor(random_state=2) opt = Optimizer([(-2.0, 2.0)], base_estimator,
# generating the data X = np.linspace(0, 20, 1000).reshape(-1, 1) y = np.sin(X)/2 - ((10 - X)**2)/50 + 2 with plt.style.context('seaborn-white'): plt.figure(figsize=(10, 5)) plt.plot(X, y, c='k', linewidth=6) plt.title('The function to be optimized') plt.show() # assembling initial training set X_initial, y_initial = X[150].reshape(1, -1), y[150].reshape(1, -1) # defining the kernel for the Gaussian process kernel = Matern(length_scale=1.0) base = RandomForestRegressor() regressor = RandomForestRegressor(max_depth=3, random_state=0)#ExtraTreesRegressor()#GradientBoostingQuantileRegressor(random_state=0) ''' base = RandomForestRegressor() regressor = GradientBoostingQuantileRegressor(base_estimator=base) with pytest.raises(ValueError): # 'type GradientBoostingRegressor', regressor.fit(X, y) ''' #print(regressor) # initializing the optimizer optimizer = BayesianOptimizer( estimator= regressor, X_training=X_initial, y_training=np.ravel(y_initial), query_strategy=max_UCB
def cook_estimator(base_estimator, space=None, **kwargs): """Cook a default estimator For the special `base_estimator` called "DUMMY", the return value is None. This corresponds to sampling points at random, hence there is no need for an estimator Parameters ---------- base_estimator: {SKLearn Regressor, "GP", "RF", "ET", "GBRT", "DUMMY"}, default="GP" If not string, should inherit from `sklearn.base.RegressorMixin`. In addition, the `predict` method should have an optional `return_std` argument, which returns `std(Y | x)`, along with `E[Y | x]`. If `base_estimator` is a string in {"GP", "RF", "ET", "GBRT", "DUMMY"}, a surrogate model corresponding to the relevant `X_minimize` function is created space: `hyperparameter_hunter.space.space_core.Space` Required only if the `base_estimator` is a Gaussian Process. Ignored otherwise **kwargs: Dict Extra parameters provided to the `base_estimator` at initialization time Returns ------- SKLearn Regressor Regressor instance cooked up according to `base_estimator` and `kwargs`""" #################### Validate `base_estimator` #################### str_estimators = ["GP", "ET", "RF", "GBRT", "DUMMY"] if isinstance(base_estimator, str): if base_estimator.upper() not in str_estimators: raise ValueError( f"Expected `base_estimator` in {str_estimators}. Got {base_estimator}" ) # Convert to upper after error check, so above error shows actual given `base_estimator` base_estimator = base_estimator.upper() elif not is_regressor(base_estimator): raise ValueError("`base_estimator` must be a regressor") #################### Get Cooking #################### if base_estimator == "GP": if space is not None: space = Space(space) # NOTE: Below `normalize_dimensions` is NOT an unnecessary duplicate of the call in # `Optimizer` - `Optimizer` calls `cook_estimator` before its `dimensions` have been # normalized, so `normalize_dimensions` must also be called here space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a `Space` instance, not None") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # Only special if *all* dimensions are `Categorical` if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern(length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2, ) elif base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt) elif base_estimator == "DUMMY": return None base_estimator.set_params(**kwargs) return base_estimator
OptimalParams4 = ([]) OptimalParams_NoBatch = ([]) OptimalScore1 = ([]) OptimalScore2 = ([]) OptimalScore3 = ([]) OptimalScore4 = ([]) OptimalScore_NoBatch = ([]) # First network, original network, two dense layers for i in range(0, 3): bo_rf_41 = Optimizer( dimensions=dimensions_4, base_estimator=RandomForestRegressor(n_estimators=100, n_jobs=4, min_variance=1.0e-6), n_initial_points=10, acq_func='EI', ) bo = bo_rf_41 for j in range(100): x = bo.ask() print(x) f = target_function1(x) # Other inputs are automatically set bo.tell(x, f) #plot_convergence(bo)
def cook_estimator(base_estimator, space=None, **kwargs): """ Cook a default estimator. For the special base_estimator called "DUMMY" the return value is None. This corresponds to sampling points at random, hence there is no need for an estimator. Parameters ---------- * `base_estimator` ["GP", "RF", "ET", "GBRT", "DUMMY" or sklearn regressor, default="GP"]: Should inherit from `sklearn.base.RegressorMixin`. In addition the `predict` method should have an optional `return_std` argument, which returns `std(Y | x)`` along with `E[Y | x]`. If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a surrogate model corresponding to the relevant `X_minimize` function is created. * `space` [Space instance]: Has to be provided if the base_estimator is a gaussian process. Ignored otherwise. * `kwargs` [dict]: Extra parameters provided to the base_estimator at init time. """ if isinstance(base_estimator, str): base_estimator = base_estimator.upper() if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY", "GPM32", "GPM1", "RBF", "RQ"]: raise ValueError("Valid strings for the base_estimator parameter " " are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not " "%s." % base_estimator) elif not is_regressor(base_estimator): raise ValueError("base_estimator has to be a regressor.") if base_estimator == "GP": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GPM32": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GPM1": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern( length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RBF": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) other_kernel = RBF(length_scale=np.ones(n_dims)) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RQ": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) other_kernel = RationalQuadratic(length_scale=np.ones(n_dims), alpha=0.1) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt) elif base_estimator == "DUMMY": return None base_estimator.set_params(**kwargs) return base_estimator
def __init__(self, dimensions, base_estimator="GP", maximize=True, n_random_starts=10, acq_func="LCB", acq_optimizer="lbfgs", random_state=None, n_points=10000, n_restarts_optimizer=5, xi=0.01, kappa=1.96, n_jobs=1): if not skopt_available: raise ImportError("skopt is not installed correctly") self.maximize = maximize self.n_params = len(dimensions) rng = check_random_state(random_state) if isinstance(base_estimator, str): if base_estimator == "RF": base_estimator = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=n_jobs, random_state=rng) elif base_estimator == "ET": base_estimator = ExtraTreesRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=n_jobs, random_state=rng) elif base_estimator == "GP": cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) matern = Matern(length_scale=np.ones(len(dimensions)), length_scale_bounds=[(0.01, 100)] * len(dimensions), nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * matern, normalize_y=True, random_state=rng, alpha=0.0, noise="gaussian", n_restarts_optimizer=2) elif base_estimator == "GBRT": gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile") base_estimator = GradientBoostingQuantileRegressor( base_estimator=gbrt, n_jobs=n_jobs, random_state=rng) else: raise ValueError( "Valid strings for the base_estimator parameter" " are: 'RF', 'ET', or 'GP', not '%s'" % base_estimator) acq_func_kwargs = {"xi": xi, "kappa": kappa} acq_optimizer_kwargs = { "n_points": n_points, "n_restarts_optimizer": n_restarts_optimizer, "n_jobs": n_jobs } self.optimizer = _SkOptOptimizer( dimensions=dimensions, base_estimator=base_estimator, n_initial_points=n_random_starts, acq_func=acq_func, acq_optimizer=acq_optimizer, random_state=random_state, acq_func_kwargs=acq_func_kwargs, acq_optimizer_kwargs=acq_optimizer_kwargs)
def test_random_forest(): # toy sample X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] y = [-1, -1, -1, 1, 1, 1] T = [[-1, -1], [2, 2], [3, 2]] true_result = [-1, 1, 1] clf = RandomForestRegressor(n_estimators=10, random_state=1) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf) clf = RandomForestRegressor(n_estimators=10, min_impurity_decrease=0.1, random_state=1) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf) clf = RandomForestRegressor(n_estimators=10, criterion="mse", max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0., max_features="auto", max_leaf_nodes=None, min_impurity_decrease=0., bootstrap=True, oob_score=False, n_jobs=1, random_state=1, verbose=0, warm_start=False) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf) clf = RandomForestRegressor(n_estimators=10, max_features=1, random_state=1) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf) # also test apply leaf_indices = clf.apply(X) assert leaf_indices.shape == (len(X), clf.n_estimators)
def __init__( self, problem, num_workers, surrogate_model="RF", acq_func="gp_hedge", acq_kappa=1.96, liar_strategy="cl_max", n_jobs=1, **kwargs, ): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY", ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.problem = problem cs_kwargs = self.problem.space["create_search_space"].get("kwargs") if cs_kwargs is None: search_space = self.problem.space["create_search_space"]["func"]() else: search_space = self.problem.space["create_search_space"]["func"]( **cs_kwargs) n_init = np.inf if surrogate_model == "DUMMY" else num_workers self.starting_points = [] # ! EMPTY for now TODO # Building search space for SkOptimizer using ConfigSpace skopt_space = cs.ConfigurationSpace(seed=self.problem.seed) for i, vnode in enumerate(search_space.variable_nodes): hp = csh.UniformIntegerHyperparameter(name=f"vnode_{i}", lower=0, upper=(vnode.num_ops - 1)) skopt_space.add_hyperparameter(hp) self._optimizer = SkOptimizer( skopt_space, base_estimator=base_estimator, acq_optimizer="sampling", acq_func=acq_func, acq_func_kwargs={"kappa": acq_kappa}, random_state=self.SEED, n_initial_points=n_init, ) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % surrogate_model)
# The optimization algorithm, `opt`, needs two methods: # # - `opt.tell`, used like `opt.tell([armId], loss)`, to give an observation of a certain "loss" (`loss = - reward`) from arm #`armId` to the algorithm. # - `opt.ask`, used like `asked = opt.ask()`, to ask the algorithm which arm should be sampled first. # # Let use a simple *Black-Box Bayesian* algorithm, implemented in the [scikit-optimize (`skopt`)](https://scikit-optimize.github.io/) package: [`RandomForestRegressor`](https://scikit-optimize.github.io/learning/index.html#skopt.learning.RandomForestRegressor). # In[9]: from skopt.learning import RandomForestRegressor # First, we need to create a model. # In[10]: our_est = RandomForestRegressor() # In[11]: get_ipython().run_line_magic('pinfo', 'our_est') # Then the optimization process is using the [`Optimizer`](https://scikit-optimize.github.io/#skopt.Optimizer) class from [`skopt`](https://scikit-optimize.github.io/). # In[12]: from skopt import Optimizer # In[13]: def arms_optimizer(nbArms, est):