def test_add_starting_points_with_too_many_dim(self): from deephyper.problem import HpProblem pb = HpProblem() pb.add_hyperparameter((-10, 10), "dim0") with pytest.raises(ValueError): pb.add_starting_point(dim0=0, dim1=2)
def test_dim_with_wrong_name(self): from deephyper.core.exceptions.problem import SpaceDimNameOfWrongType from deephyper.problem import HpProblem pb = HpProblem() with pytest.raises(SpaceDimNameOfWrongType): pb.add_hyperparameter((-10, 10), 0)
def test_sample_types_no_cat(self): import numpy as np from deephyper.evaluator import Evaluator from deephyper.problem import HpProblem from deephyper.search.hps import CBO problem = HpProblem() problem.add_hyperparameter((0, 10), "x_int") problem.add_hyperparameter((0.0, 10.0), "x_float") def run(config): assert np.issubdtype(type(config["x_int"]), np.integer) assert np.issubdtype(type(config["x_float"]), float) return 0 create_evaluator = lambda: Evaluator.create(run, method="serial") CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY").search(10) CBO(problem, create_evaluator(), random_state=42, surrogate_model="RF").search(10)
def test_gp(self): from deephyper.evaluator import Evaluator from deephyper.problem import HpProblem from deephyper.search.hps import CBO # test float hyperparameters problem = HpProblem() problem.add_hyperparameter((0.0, 10.0), "x") def run(config): return config["x"] CBO( problem, Evaluator.create(run, method="serial"), random_state=42, surrogate_model="GP", ).search(10) # test int hyperparameters problem = HpProblem() problem.add_hyperparameter((0, 10), "x") def run(config): return config["x"] CBO( problem, Evaluator.create(run, method="serial"), random_state=42, surrogate_model="GP", ).search(10) # test categorical hyperparameters problem = HpProblem() problem.add_hyperparameter([f"{i}" for i in range(10)], "x") def run(config): return int(config["x"]) CBO( problem, Evaluator.create(run, method="serial"), random_state=42, surrogate_model="GP", ).search(10)
def test_config_space_hp(self): import ConfigSpace.hyperparameters as csh from deephyper.problem import HpProblem alpha = csh.UniformFloatHyperparameter(name="alpha", lower=0, upper=1) beta = csh.UniformFloatHyperparameter(name="beta", lower=0, upper=1) pb = HpProblem() pb.add_hyperparameters([alpha, beta])
def __init__( self, problem, run, evaluator, surrogate_model="RF", acq_func="LCB", kappa=1.96, xi=0.001, liar_strategy="cl_min", n_jobs=1, **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, **kwargs, ) self.n_jobs = int( n_jobs) # parallelism of BO surrogate model estimator self.kappa = float(kappa) self.xi = float(xi) self.n_initial_points = self.evaluator.num_workers self.liar_strategy = liar_strategy # Setup na_search_space = self.problem.build_search_space() self.hp_space = self.problem._hp_space #! hyperparameters self.hp_size = len(self.hp_space.space.get_hyperparameter_names()) self.na_space = HpProblem(self.problem.seed) for i, vnode in enumerate(na_search_space.variable_nodes): self.na_space.add_hyperparameter((0, vnode.num_ops - 1), name=f"vnode_{i:05d}") self.space = CS.ConfigurationSpace(seed=self.problem.seed) self.space.add_configuration_space( prefix="1", configuration_space=self.hp_space.space) self.space.add_configuration_space( prefix="2", configuration_space=self.na_space.space) # Initialize opitmizer of hyperparameter space self.opt = SkOptimizer( dimensions=self.space, base_estimator=self.get_surrogate_model(surrogate_model, self.n_jobs), acq_func=acq_func, acq_optimizer="sampling", acq_func_kwargs={ "xi": self.xi, "kappa": self.kappa }, n_initial_points=self.n_initial_points, )
def test_add_good_dim(self): from deephyper.problem import HpProblem pb = HpProblem() p0 = pb.add_hyperparameter((-10, 10), "p0") p0_csh = csh.UniformIntegerHyperparameter( name="p0", lower=-10, upper=10, log=False ) assert p0 == p0_csh p1 = pb.add_hyperparameter((1, 100, "log-uniform"), "p1") p1_csh = csh.UniformIntegerHyperparameter(name="p1", lower=1, upper=100, log=True) assert p1 == p1_csh p2 = pb.add_hyperparameter((-10.0, 10.0), "p2") p2_csh = csh.UniformFloatHyperparameter( name="p2", lower=-10.0, upper=10.0, log=False ) assert p2 == p2_csh p3 = pb.add_hyperparameter((1.0, 100.0, "log-uniform"), "p3") p3_csh = csh.UniformFloatHyperparameter( name="p3", lower=1.0, upper=100.0, log=True ) assert p3 == p3_csh p4 = pb.add_hyperparameter([1, 2, 3, 4], "p4") p4_csh = csh.OrdinalHyperparameter(name="p4", sequence=[1, 2, 3, 4]) assert p4 == p4_csh p5 = pb.add_hyperparameter([1.0, 2.0, 3.0, 4.0], "p5") p5_csh = csh.OrdinalHyperparameter(name="p5", sequence=[1.0, 2.0, 3.0, 4.0]) assert p5 == p5_csh p6 = pb.add_hyperparameter(["cat0", "cat1"], "p6") p6_csh = csh.CategoricalHyperparameter(name="p6", choices=["cat0", "cat1"]) assert p6 == p6_csh p7 = pb.add_hyperparameter({"mu": 0, "sigma": 1}, "p7") p7_csh = csh.NormalIntegerHyperparameter(name="p7", mu=0, sigma=1) assert p7 == p7_csh if cs.__version__ > "0.4.20": p8 = pb.add_hyperparameter( {"mu": 0, "sigma": 1, "lower": -5, "upper": 5}, "p8" ) p8_csh = csh.NormalIntegerHyperparameter( name="p8", mu=0, sigma=1, lower=-5, upper=5 ) assert p8 == p8_csh p9 = pb.add_hyperparameter({"mu": 0.0, "sigma": 1.0}, "p9") p9_csh = csh.NormalFloatHyperparameter(name="p9", mu=0, sigma=1) assert p9 == p9_csh
def test_random_seed(self): import numpy as np from deephyper.evaluator import Evaluator from deephyper.problem import HpProblem from deephyper.search.hps import CBO problem = HpProblem() problem.add_hyperparameter((0.0, 10.0), "x") def run(config): return config["x"] create_evaluator = lambda: Evaluator.create(run, method="serial") search = CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY") res1 = search.search(max_evals=4) res1_array = res1[["x"]].to_numpy() search = CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY") res2 = search.search(max_evals=4) res2_array = res2[["x"]].to_numpy() assert np.array_equal(res1_array, res2_array) # test multi-objective def run(config): return config["x"], config["x"] create_evaluator = lambda: Evaluator.create(run, method="serial") search = CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY") res1 = search.search(max_evals=4) res1_array = res1[["x"]].to_numpy() search = CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY") res2 = search.search(max_evals=4) res2_array = res2[["x"]].to_numpy() assert np.array_equal(res1_array, res2_array)
def test_sample_types_conditional(self): import ConfigSpace as cs import numpy as np from deephyper.evaluator import Evaluator from deephyper.problem import HpProblem from deephyper.search.hps import CBO problem = HpProblem() # choices choice = problem.add_hyperparameter( name="choice", value=["choice1", "choice2"], ) # integers x1_int = problem.add_hyperparameter(name="x1_int", value=(1, 10)) x2_int = problem.add_hyperparameter(name="x2_int", value=(1, 10)) # conditions cond_1 = cs.EqualsCondition(x1_int, choice, "choice1") cond_2 = cs.EqualsCondition(x2_int, choice, "choice2") problem.add_condition(cond_1) problem.add_condition(cond_2) def run(config): if config["choice"] == "choice1": assert np.issubdtype(type(config["x1_int"]), np.integer) else: assert np.issubdtype(type(config["x2_int"]), np.integer) return 0 create_evaluator = lambda: Evaluator.create(run, method="serial") CBO(problem, create_evaluator(), random_state=42, surrogate_model="DUMMY").search(10)
def __init__( self, problem, evaluator, random_state: int = None, log_dir: str = ".", verbose: int = 0, population_size: int = 100, sample_size: int = 10, **kwargs, ): super().__init__( problem, evaluator, random_state, log_dir, verbose, population_size, sample_size, ) # Setup na_search_space = self._problem.build_search_space() self.hp_space = self._problem._hp_space #! hyperparameters self.hp_size = len(self.hp_space.space.get_hyperparameter_names()) self.na_space = HpProblem() self.na_space._space.seed(self._random_state.get_state()[1][0]) for i, (low, high) in enumerate(na_search_space.choices()): self.na_space.add_hyperparameter((low, high), name=f"vnode_{i:05d}") self._space = CS.ConfigurationSpace(seed=self._random_state.get_state()[1][0]) self._space.add_configuration_space( prefix="1", configuration_space=self.hp_space.space ) self._space.add_configuration_space( prefix="2", configuration_space=self.na_space.space ) self._space_size = len(self._space.get_hyperparameter_names())
def test_quickstart(self): from deephyper.problem import HpProblem from deephyper.search.hps import CBO from deephyper.evaluator import Evaluator # define the variable you want to optimize problem = HpProblem() problem.add_hyperparameter((-10.0, 10.0), "x") # define the evaluator to distribute the computation evaluator = Evaluator.create( run, method="subprocess", method_kwargs={ "num_workers": 2, }, ) # define you search and execute it search = CBO(problem, evaluator) results = search.search(max_evals=15)
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, n_jobs=1, **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.n_jobs = int(n_jobs) # Setup na_search_space = self.problem.build_search_space() self.hp_space = self.problem._hp_space #! hyperparameters self.hp_size = len(self.hp_space.space.get_hyperparameter_names()) self.na_space = HpProblem(self.problem.seed) for i, vnode in enumerate(na_search_space.variable_nodes): self.na_space.add_hyperparameter((0, vnode.num_ops - 1), name=f"vnode_{i:05d}") self.space = CS.ConfigurationSpace(seed=self.problem.seed) self.space.add_configuration_space( prefix="1", configuration_space=self.hp_space.space) self.space.add_configuration_space( prefix="2", configuration_space=self.na_space.space)
def test_add_starting_points_not_in_space_def(self): from deephyper.problem import HpProblem pb = HpProblem() pb.add_hyperparameter((-10, 10), "dim0") pb.add_hyperparameter((-10.0, 10.0), "dim1") pb.add_hyperparameter(["a", "b"], "dim2") with pytest.raises(ValueError): pb.add_starting_point(dim0=-11, dim1=0.0, dim2="a") with pytest.raises(ValueError): pb.add_starting_point(dim0=11, dim1=0.0, dim2="a") with pytest.raises(ValueError): pb.add_starting_point(dim0=0, dim1=-11.0, dim2="a") with pytest.raises(ValueError): pb.add_starting_point(dim0=0, dim1=11.0, dim2="a") with pytest.raises(ValueError): pb.add_starting_point(dim0=0, dim1=0.0, dim2="c") pb.add_starting_point(dim0=0, dim1=0.0, dim2="a")
""" Hyperparameter optimization problem to try forbidden clauses by directly using ConfigSpace. Example command line:: python -m deephyper.search.hps.ambsv1 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_cond_1.Problem --run deephyper.benchmark.hps.toy.problem_cond_1.run --max-evals 100 --kappa 0.001 """ import ConfigSpace as cs from deephyper.problem import HpProblem Problem = HpProblem(seed=45) func = Problem.add_hyperparameter(name="func", value=["f1", "f2"]) # f1 variables f1_x = Problem.add_hyperparameter(name="f1_x", value=(0.0, 1.0, "uniform")) f1_y = Problem.add_hyperparameter(name="f1_y", value=(0.0, 1.0, "uniform")) # f2 variables f2_x = Problem.add_hyperparameter(name="f2_x", value=(0.0, 1.0, "uniform")) f2_y = Problem.add_hyperparameter(name="f2_y", value=(0.0, 1.0, "uniform")) cond_f1_x = cs.EqualsCondition(f1_x, func, "f1") cond_f1_y = cs.EqualsCondition(f1_y, func, "f1") Problem.add_condition(cond_f1_x) Problem.add_condition(cond_f1_y) cond_f2_x = cs.EqualsCondition(f2_x, func, "f2") cond_f2_y = cs.EqualsCondition(f2_y, func, "f2") Problem.add_condition(cond_f2_x) Problem.add_condition(cond_f2_y)
""" Hyperparameter optimization problem to try forbidden clauses by directly using ConfigSpace. Example command line:: python -m deephyper.search.hps.ambs2 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_basic_1.Problem --run deephyper.benchmark.hps.toy.problem_basic_1.run --max-evals 100 --kappa 0.001 """ import ConfigSpace.hyperparameters as csh import numpy as np from deephyper.problem import HpProblem # Problem definition Problem = HpProblem() x_hp = csh.UniformIntegerHyperparameter(name="x", lower=0, upper=10, log=False) y_hp = csh.UniformIntegerHyperparameter(name="y", lower=0, upper=10, log=False) Problem.add_hyperparameters([x_hp, y_hp]) Problem.add_starting_point(x=1, y=1) # Definition of the function which runs the model def run(param_dict): x = param_dict["x"] y = param_dict["y"] res = x + y
def __init__(self, seed=None, **kwargs): self._space = OrderedDict() self._hp_space = HpProblem(seed) self.seed = seed
def test_add_good_reference(self): from deephyper.problem import HpProblem pb = HpProblem() pb.add_hyperparameter((-10, 10), "dim0") pb.add_starting_point(dim0=0)
def test_kwargs(self): from deephyper.problem import HpProblem pb = HpProblem() pb.add_hyperparameter(value=(-10, 10), name="dim0")
def __init__(self): self._space = OrderedDict() self._hp_space = HpProblem() self._space["metrics"] = [] self._space["hyperparameters"] = dict(verbose=0)
def test_create(self): from deephyper.problem import HpProblem pb = HpProblem()
def __init__( self, problem, evaluator, random_state=None, log_dir=".", verbose=0, surrogate_model: str = "RF", acq_func: str = "UCB", kappa: float = 1.96, xi: float = 0.001, n_points: int = 10000, liar_strategy: str = "cl_max", n_jobs: int = 1, **kwargs, ): super().__init__(problem, evaluator, random_state, log_dir, verbose) # Setup the search space na_search_space = self._problem.build_search_space() self.hp_space = self._problem._hp_space #! hyperparameters self.hp_size = len(self.hp_space.space.get_hyperparameter_names()) self.na_space = HpProblem() self.na_space._space.seed(self._random_state.get_state()[1][0]) for i, vnode in enumerate(na_search_space.variable_nodes): self.na_space.add_hyperparameter((0, vnode.num_ops - 1), name=f"vnode_{i:05d}") self._space = CS.ConfigurationSpace( seed=self._random_state.get_state()[1][0]) self._space.add_configuration_space( prefix="1", configuration_space=self.hp_space.space) self._space.add_configuration_space( prefix="2", configuration_space=self.na_space.space) # check input parameters surrogate_model_allowed = ["RF", "ET", "GBRT", "DUMMY"] if not (surrogate_model in surrogate_model_allowed): raise ValueError( f"Parameter 'surrogate_model={surrogate_model}' should have a value in {surrogate_model_allowed}!" ) acq_func_allowed = ["UCB", "EI", "PI", "gp_hedge"] if not (acq_func in acq_func_allowed): raise ValueError( f"Parameter 'acq_func={acq_func}' should have a value in {acq_func_allowed}!" ) if not (np.isscalar(kappa)): raise ValueError(f"Parameter 'kappa' should be a scalar value!") if not (np.isscalar(xi)): raise ValueError("Parameter 'xi' should be a scalar value!") if not (type(n_points) is int): raise ValueError("Parameter 'n_points' shoud be an integer value!") liar_strategy_allowed = ["cl_min", "cl_mean", "cl_max"] if not (liar_strategy in liar_strategy_allowed): raise ValueError( f"Parameter 'liar_strategy={liar_strategy}' should have a value in {liar_strategy_allowed}!" ) if not (type(n_jobs) is int): raise ValueError(f"Parameter 'n_jobs' should be an integer value!") self._n_initial_points = self._evaluator.num_workers self._liar_strategy = MAP_liar_strategy.get(liar_strategy, liar_strategy) base_estimator = self._get_surrogate_model( surrogate_model, n_jobs, random_state=self._random_state.get_state()[1][0]) self._opt = None self._opt_kwargs = dict( dimensions=self._space, base_estimator=base_estimator, acq_func=MAP_acq_func.get(acq_func, acq_func), acq_optimizer="sampling", acq_func_kwargs={ "xi": xi, "kappa": kappa, "n_points": n_points }, n_initial_points=self._n_initial_points, random_state=self._random_state, )
("stdscaler", StandardScaler()), ]) return preprocessor REGRESSORS = { "RandomForest": RandomForestRegressor, "Linear": LinearRegression, "AdaBoost": AdaBoostRegressor, "KNeighbors": KNeighborsRegressor, "MLP": MLPRegressor, "SVR": SVR, "XGBoost": XGBRegressor, } problem_autosklearn1 = HpProblem() regressor = problem_autosklearn1.add_hyperparameter( name="regressor", value=[ "RandomForest", "Linear", "AdaBoost", "KNeighbors", "MLP", "SVR", "XGBoost" ], ) # n_estimators n_estimators = problem_autosklearn1.add_hyperparameter(name="n_estimators", value=(1, 2000, "log-uniform")) cond_n_estimators = cs.OrConjunction(
def run(config: dict, N: int) -> float: y = -sum([config[f"x{i}"] ** 2 for i in range(N)]) return y run_small = functools.partial(run, N=1) run_large = functools.partial(run, N=2) # %% # Then, we can define the hyperparameter problem space based on :math:`n` from deephyper.problem import HpProblem N = 1 problem_small = HpProblem() for i in range(N): problem_small.add_hyperparameter((-10.0, 10.0), f"x{i}") problem_small # %% N = 2 problem_large = HpProblem() for i in range(N): problem_large.add_hyperparameter((-10.0, 10.0), f"x{i}") problem_large # %% # Then, we define setup the search and execute it: from deephyper.evaluator import Evaluator from deephyper.evaluator.callback import TqdmCallback
from deephyper.problem import HpProblem config_space = cs.ConfigurationSpace(seed=42) x_hp = csh.UniformIntegerHyperparameter(name="x", lower=0, upper=10, log=False) y_hp = csh.UniformIntegerHyperparameter(name="y", lower=0, upper=10, log=False) config_space.add_hyperparameters([x_hp, y_hp]) not_zero = cs.ForbiddenAndConjunction(cs.ForbiddenEqualsClause(x_hp, 0), cs.ForbiddenEqualsClause(y_hp, 0)) config_space.add_forbidden_clause(not_zero) # Problem definition Problem = HpProblem(config_space) Problem.add_starting_point(x=1, y=1) # Definition of the function which runs the model def run(param_dict): x = param_dict["x"] y = param_dict["y"] res = np.log(1 / (y + x)) return res # the objective