def test_dim_with_wrong_name(self):
from deephyper.core.exceptions.problem import SpaceDimNameOfWrongType
from deephyper.problem import HpProblem
pb = HpProblem()
with pytest.raises(SpaceDimNameOfWrongType):
pb.add_hyperparameter((-10, 10), 0)
def test_config_space_hp(self):
import ConfigSpace.hyperparameters as csh
from deephyper.problem import HpProblem
alpha = csh.UniformFloatHyperparameter(name="alpha", lower=0, upper=1)
beta = csh.UniformFloatHyperparameter(name="beta", lower=0, upper=1)
pb = HpProblem()
pb.add_hyperparameters([alpha, beta])
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
self.kappa = float(kappa)
self.xi = float(xi)
self.n_initial_points = self.evaluator.num_workers
self.liar_strategy = liar_strategy
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# Initialize opitmizer of hyperparameter space
self.opt = SkOptimizer(
dimensions=self.space,
base_estimator=self.get_surrogate_model(surrogate_model,
self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={
"xi": self.xi,
"kappa": self.kappa
},
n_initial_points=self.n_initial_points,
)
def test_random_seed(self):
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
problem.add_hyperparameter((0.0, 10.0), "x")
def run(config):
return config["x"]
create_evaluator = lambda: Evaluator.create(run, method="serial")
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res1 = search.search(max_evals=4)
res1_array = res1[["x"]].to_numpy()
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res2 = search.search(max_evals=4)
res2_array = res2[["x"]].to_numpy()
assert np.array_equal(res1_array, res2_array)
# test multi-objective
def run(config):
return config["x"], config["x"]
create_evaluator = lambda: Evaluator.create(run, method="serial")
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res1 = search.search(max_evals=4)
res1_array = res1[["x"]].to_numpy()
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res2 = search.search(max_evals=4)
res2_array = res2[["x"]].to_numpy()
assert np.array_equal(res1_array, res2_array)
class Problem:
"""Representation of a problem."""
def __init__(self, seed=None, **kwargs):
self._space = OrderedDict()
self._hp_space = HpProblem(seed)
self.seed = seed
def __str__(self):
return repr(self)
def __repr__(self):
return f"Problem\n{pformat({k:v for k,v in self._space.items()}, indent=2)}"
def add_dim(self, p_name, p_space):
"""Add a dimension to the search space.
Args:
p_name (str): name of the parameter/dimension.
p_space (Object): space corresponding to the new dimension.
"""
self._space[p_name] = p_space
@property
def space(self):
dims = list(self._space.keys())
dims.sort()
space = OrderedDict(**{d: self._space[d] for d in dims})
return space
def add_hyperparameter(self,
value,
name: str = None,
default_value=None) -> csh.Hyperparameter:
return self._hp_space.add_hyperparameter(value, name, default_value)
def test_add_starting_points_with_too_many_dim(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=2)
def test_sample_types_no_cat(self):
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
problem.add_hyperparameter((0, 10), "x_int")
problem.add_hyperparameter((0.0, 10.0), "x_float")
def run(config):
assert np.issubdtype(type(config["x_int"]), np.integer)
assert np.issubdtype(type(config["x_float"]), float)
return 0
create_evaluator = lambda: Evaluator.create(run, method="serial")
CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY").search(10)
CBO(problem, create_evaluator(), random_state=42,
surrogate_model="RF").search(10)
def __init__(
self,
problem,
evaluator,
random_state: int = None,
log_dir: str = ".",
verbose: int = 0,
population_size: int = 100,
sample_size: int = 10,
**kwargs,
):
super().__init__(
problem,
evaluator,
random_state,
log_dir,
verbose,
population_size,
sample_size,
)
# Setup
na_search_space = self._problem.build_search_space()
self.hp_space = self._problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem()
self.na_space._space.seed(self._random_state.get_state()[1][0])
for i, (low, high) in enumerate(na_search_space.choices()):
self.na_space.add_hyperparameter((low, high), name=f"vnode_{i:05d}")
self._space = CS.ConfigurationSpace(seed=self._random_state.get_state()[1][0])
self._space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space
)
self._space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space
)
self._space_size = len(self._space.get_hyperparameter_names())
def test_quickstart(self):
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
from deephyper.evaluator import Evaluator
# define the variable you want to optimize
problem = HpProblem()
problem.add_hyperparameter((-10.0, 10.0), "x")
# define the evaluator to distribute the computation
evaluator = Evaluator.create(
run,
method="subprocess",
method_kwargs={
"num_workers": 2,
},
)
# define you search and execute it
search = CBO(problem, evaluator)
results = search.search(max_evals=15)
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(n_jobs)
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
def test_sample_types_conditional(self):
import ConfigSpace as cs
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
# choices
choice = problem.add_hyperparameter(
name="choice",
value=["choice1", "choice2"],
)
# integers
x1_int = problem.add_hyperparameter(name="x1_int", value=(1, 10))
x2_int = problem.add_hyperparameter(name="x2_int", value=(1, 10))
# conditions
cond_1 = cs.EqualsCondition(x1_int, choice, "choice1")
cond_2 = cs.EqualsCondition(x2_int, choice, "choice2")
problem.add_condition(cond_1)
problem.add_condition(cond_2)
def run(config):
if config["choice"] == "choice1":
assert np.issubdtype(type(config["x1_int"]), np.integer)
else:
assert np.issubdtype(type(config["x2_int"]), np.integer)
return 0
create_evaluator = lambda: Evaluator.create(run, method="serial")
CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY").search(10)
def test_create(self):
from deephyper.problem import HpProblem
pb = HpProblem()
def test_add_starting_points_not_in_space_def(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
pb.add_hyperparameter((-10.0, 10.0), "dim1")
pb.add_hyperparameter(["a", "b"], "dim2")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=-11, dim1=0.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=11, dim1=0.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=-11.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=11.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=0.0, dim2="c")
pb.add_starting_point(dim0=0, dim1=0.0, dim2="a")
"""
deephyper hps ambs --problem nas_big_data.albert.tune_random_forest.Problem --run nas_big_data.albert.tune_random_forest.run --max-evals 30 --evaluator subprocess --n-jobs 3
"""
import json
import os
import pandas as pd
from deephyper.problem import HpProblem
from nas_big_data import RANDOM_STATE
from nas_big_data.albert.load_data import load_data
from sklearn.ensemble import RandomForestClassifier
Problem = HpProblem()
Problem.add_hyperparameter((10, 300), "n_estimators")
Problem.add_hyperparameter(["gini", "entropy"], "criterion")
Problem.add_hyperparameter((1, 50), "max_depth")
Problem.add_hyperparameter((2, 10), "min_samples_split")
# We define a starting point with the defaul hyperparameters from sklearn-learn
# that we consider good in average.
Problem.add_starting_point(n_estimators=100,
criterion="gini",
max_depth=50,
min_samples_split=2)
def test_best():
"""Test data with RandomForest
from deephyper.problem import HpProblem
Problem = HpProblem()
Problem.add_dim('units', (1, 100))
Problem.add_dim('activation', ['NA', 'relu', 'sigmoid', 'tanh'])
Problem.add_dim('lr', (0.0001, 1.))
Problem.add_starting_point(
units=10,
activation='relu',
lr=0.01)
if __name__ == '__main__':
print(Problem)
"""Inspired from https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html
"""
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem(seed=45)
classifier = Problem.add_hyperparameter(
name="classifier",
value=["RandomForest", "Logistic", "AdaBoost", "KNeighbors", "MLP", "SVC", "XGBoost"],
)
# n_estimators
n_estimators = Problem.add_hyperparameter(
name="n_estimators", value=(1, 2000, "log-uniform")
)
cond_n_estimators = cs.OrConjunction(
cs.EqualsCondition(n_estimators, classifier, "RandomForest"),
cs.EqualsCondition(n_estimators, classifier, "AdaBoost"),
)
Problem.add_condition(cond_n_estimators)
# max_depth
max_depth = Problem.add_hyperparameter(name="max_depth", value=(2, 100, "log-uniform"))
cond_max_depth = cs.EqualsCondition(max_depth, classifier, "RandomForest")
Problem.add_condition(cond_max_depth)
def test_gp(self):
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
# test float hyperparameters
problem = HpProblem()
problem.add_hyperparameter((0.0, 10.0), "x")
def run(config):
return config["x"]
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
# test int hyperparameters
problem = HpProblem()
problem.add_hyperparameter((0, 10), "x")
def run(config):
return config["x"]
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
# test categorical hyperparameters
problem = HpProblem()
problem.add_hyperparameter([f"{i}" for i in range(10)], "x")
def run(config):
return int(config["x"])
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
def test_add_good_reference(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
pb.add_starting_point(dim0=0)
def test_add_good_dim(self):
from deephyper.problem import HpProblem
pb = HpProblem()
p0 = pb.add_hyperparameter((-10, 10), "p0")
p0_csh = csh.UniformIntegerHyperparameter(
name="p0", lower=-10, upper=10, log=False
)
assert p0 == p0_csh
p1 = pb.add_hyperparameter((1, 100, "log-uniform"), "p1")
p1_csh = csh.UniformIntegerHyperparameter(name="p1", lower=1, upper=100, log=True)
assert p1 == p1_csh
p2 = pb.add_hyperparameter((-10.0, 10.0), "p2")
p2_csh = csh.UniformFloatHyperparameter(
name="p2", lower=-10.0, upper=10.0, log=False
)
assert p2 == p2_csh
p3 = pb.add_hyperparameter((1.0, 100.0, "log-uniform"), "p3")
p3_csh = csh.UniformFloatHyperparameter(
name="p3", lower=1.0, upper=100.0, log=True
)
assert p3 == p3_csh
p4 = pb.add_hyperparameter([1, 2, 3, 4], "p4")
p4_csh = csh.OrdinalHyperparameter(name="p4", sequence=[1, 2, 3, 4])
assert p4 == p4_csh
p5 = pb.add_hyperparameter([1.0, 2.0, 3.0, 4.0], "p5")
p5_csh = csh.OrdinalHyperparameter(name="p5", sequence=[1.0, 2.0, 3.0, 4.0])
assert p5 == p5_csh
p6 = pb.add_hyperparameter(["cat0", "cat1"], "p6")
p6_csh = csh.CategoricalHyperparameter(name="p6", choices=["cat0", "cat1"])
assert p6 == p6_csh
p7 = pb.add_hyperparameter({"mu": 0, "sigma": 1}, "p7")
p7_csh = csh.NormalIntegerHyperparameter(name="p7", mu=0, sigma=1)
assert p7 == p7_csh
if cs.__version__ > "0.4.20":
p8 = pb.add_hyperparameter(
{"mu": 0, "sigma": 1, "lower": -5, "upper": 5}, "p8"
)
p8_csh = csh.NormalIntegerHyperparameter(
name="p8", mu=0, sigma=1, lower=-5, upper=5
)
assert p8 == p8_csh
p9 = pb.add_hyperparameter({"mu": 0.0, "sigma": 1.0}, "p9")
p9_csh = csh.NormalFloatHyperparameter(name="p9", mu=0, sigma=1)
assert p9 == p9_csh
"""
Hyperparameter optimization problem to try forbidden clauses by directly using ConfigSpace.
Example command line::
python -m deephyper.search.hps.ambs2 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_basic_1.Problem --run deephyper.benchmark.hps.toy.problem_basic_1.run --max-evals 100 --kappa 0.001
"""
import ConfigSpace.hyperparameters as csh
import numpy as np
from deephyper.problem import HpProblem
# Problem definition
Problem = HpProblem()
x_hp = csh.UniformIntegerHyperparameter(name="x", lower=0, upper=10, log=False)
y_hp = csh.UniformIntegerHyperparameter(name="y", lower=0, upper=10, log=False)
Problem.add_hyperparameters([x_hp, y_hp])
Problem.add_starting_point(x=1, y=1)
# Definition of the function which runs the model
def run(param_dict):
x = param_dict["x"]
y = param_dict["y"]
res = x + y
class AMBSMixed(NeuralArchitectureSearch):
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
self.kappa = float(kappa)
self.xi = float(xi)
self.n_initial_points = self.evaluator.num_workers
self.liar_strategy = liar_strategy
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# Initialize opitmizer of hyperparameter space
self.opt = SkOptimizer(
dimensions=self.space,
base_estimator=self.get_surrogate_model(surrogate_model,
self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={
"xi": self.xi,
"kappa": self.kappa
},
n_initial_points=self.n_initial_points,
)
@staticmethod
def _extend_parser(parser):
NeuralArchitectureSearch._extend_parser(parser)
add_arguments_from_signature(parser, AMBSMixed)
return parser
def saved_keys(self, val: dict):
res = {"arch_seq": str(val["arch_seq"])}
hp_names = self.hp_space._space.get_hyperparameter_names()
for hp_name in hp_names:
if hp_name == "loss":
res["loss"] = val["loss"]
else:
res[hp_name] = val["hyperparameters"][hp_name]
return res
def main(self):
num_evals_done = 0
# Filling available nodes at start
dhlogger.info(
f"Generating {self.evaluator.num_workers} initial points...")
self.evaluator.add_eval_batch(
self.get_random_batch(size=self.n_initial_points))
# Main loop
while num_evals_done < self.max_evals:
# Collecting finished evaluations
new_results = list(self.evaluator.get_finished_evals())
if len(new_results) > 0:
stats = {
"num_cache_used": self.evaluator.stats["num_cache_used"]
}
dhlogger.info(jm(type="env_stats", **stats))
self.evaluator.dump_evals(saved_keys=self.saved_keys)
num_received = len(new_results)
num_evals_done += num_received
# Transform configurations to list to fit optimizer
opt_X = []
opt_y = []
for cfg, obj in new_results:
arch_seq = cfg["arch_seq"]
hp_val = self.problem.extract_hp_values(cfg)
x = replace_nan(hp_val + arch_seq)
opt_X.append(x)
opt_y.append(-obj) #! maximizing
self.opt.tell(opt_X, opt_y) #! fit: costly
new_X = self.opt.ask(n_points=len(new_results),
strategy=self.liar_strategy)
new_batch = []
for x in new_X:
new_cfg = self.problem.gen_config(x[self.hp_size:],
x[:self.hp_size])
new_batch.append(new_cfg)
# submit_childs
if len(new_results) > 0:
self.evaluator.add_eval_batch(new_batch)
def get_surrogate_model(self, name: str, n_jobs: int = None):
"""Get a surrogate model from Scikit-Optimize.
Args:
name (str): name of the surrogate model.
n_jobs (int): number of parallel processes to distribute the computation of the surrogate model.
Raises:
ValueError: when the name of the surrogate model is unknown.
"""
accepted_names = ["RF", "ET", "GBRT", "GP", "DUMMY"]
if not (name in accepted_names):
raise ValueError(
f"Unknown surrogate model {name}, please choose among {accepted_names}."
)
if name == "RF":
surrogate = skopt.learning.RandomForestRegressor(n_jobs=n_jobs)
elif name == "ET":
surrogate = skopt.learning.ExtraTreesRegressor(n_jobs=n_jobs)
elif name == "GBRT":
surrogate = skopt.learning.GradientBoostingQuantileRegressor(
n_jobs=n_jobs)
else: # for DUMMY and GP
surrogate = name
return surrogate
def get_random_batch(self, size: int) -> list:
batch = []
n_points = max(0, size - len(batch))
if n_points > 0:
points = self.opt.ask(n_points=n_points)
for point in points:
point_as_dict = self.problem.gen_config(
point[self.hp_size:], point[:self.hp_size])
batch.append(point_as_dict)
return batch
def to_dict(self, x: list) -> dict:
hp_x = x[:self.hp_size]
arch_seq = x[self.hp_size:]
cfg = self.problem.space.copy()
cfg["arch_seq"] = arch_seq
return cfg
("stdscaler", StandardScaler()),
])
return preprocessor
REGRESSORS = {
"RandomForest": RandomForestRegressor,
"Linear": LinearRegression,
"AdaBoost": AdaBoostRegressor,
"KNeighbors": KNeighborsRegressor,
"MLP": MLPRegressor,
"SVR": SVR,
"XGBoost": XGBRegressor,
}
problem_autosklearn1 = HpProblem()
regressor = problem_autosklearn1.add_hyperparameter(
name="regressor",
value=[
"RandomForest", "Linear", "AdaBoost", "KNeighbors", "MLP", "SVR",
"XGBoost"
],
)
# n_estimators
n_estimators = problem_autosklearn1.add_hyperparameter(name="n_estimators",
value=(1, 2000,
"log-uniform"))
cond_n_estimators = cs.OrConjunction(
from deephyper.problem import HpProblem
Problem = HpProblem()
Problem.add_dim("lr", [1e-4, 5e-4, .001, .005, .01, .1])
Problem.add_dim("trade_off", [.01, .05, .1, .5, 1])
Problem.add_dim("cycle_length", [2, 4, 5, 8, 10])
Problem.add_dim("weight_decay", [1e-5, 1e-4, 1e-3, 5e-2, .01, .1])
Problem.add_starting_point(lr=1e-4, trade_off=.01, cycle_length=2, weight_decay = 1e-5)
if __name__ == "__main__":
print(Problem)
def __init__(
self,
problem,
evaluator,
random_state=None,
log_dir=".",
verbose=0,
surrogate_model: str = "RF",
acq_func: str = "UCB",
kappa: float = 1.96,
xi: float = 0.001,
n_points: int = 10000,
liar_strategy: str = "cl_max",
n_jobs: int = 1,
**kwargs,
):
super().__init__(problem, evaluator, random_state, log_dir, verbose)
# Setup the search space
na_search_space = self._problem.build_search_space()
self.hp_space = self._problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem()
self.na_space._space.seed(self._random_state.get_state()[1][0])
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self._space = CS.ConfigurationSpace(
seed=self._random_state.get_state()[1][0])
self._space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self._space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# check input parameters
surrogate_model_allowed = ["RF", "ET", "GBRT", "DUMMY"]
if not (surrogate_model in surrogate_model_allowed):
raise ValueError(
f"Parameter 'surrogate_model={surrogate_model}' should have a value in {surrogate_model_allowed}!"
)
acq_func_allowed = ["UCB", "EI", "PI", "gp_hedge"]
if not (acq_func in acq_func_allowed):
raise ValueError(
f"Parameter 'acq_func={acq_func}' should have a value in {acq_func_allowed}!"
)
if not (np.isscalar(kappa)):
raise ValueError(f"Parameter 'kappa' should be a scalar value!")
if not (np.isscalar(xi)):
raise ValueError("Parameter 'xi' should be a scalar value!")
if not (type(n_points) is int):
raise ValueError("Parameter 'n_points' shoud be an integer value!")
liar_strategy_allowed = ["cl_min", "cl_mean", "cl_max"]
if not (liar_strategy in liar_strategy_allowed):
raise ValueError(
f"Parameter 'liar_strategy={liar_strategy}' should have a value in {liar_strategy_allowed}!"
)
if not (type(n_jobs) is int):
raise ValueError(f"Parameter 'n_jobs' should be an integer value!")
self._n_initial_points = self._evaluator.num_workers
self._liar_strategy = MAP_liar_strategy.get(liar_strategy,
liar_strategy)
base_estimator = self._get_surrogate_model(
surrogate_model,
n_jobs,
random_state=self._random_state.get_state()[1][0])
self._opt = None
self._opt_kwargs = dict(
dimensions=self._space,
base_estimator=base_estimator,
acq_func=MAP_acq_func.get(acq_func, acq_func),
acq_optimizer="sampling",
acq_func_kwargs={
"xi": xi,
"kappa": kappa,
"n_points": n_points
},
n_initial_points=self._n_initial_points,
random_state=self._random_state,
)
def test_kwargs(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter(value=(-10, 10), name="dim0")
This example demonstrates how to handle failure of objectives in hyperparameter search. In many cases such as software auto-tuning (where we minimize the run-time of a software application) some configurations can create run-time errors and therefore no scalar objective is returned. A default choice could be to return in this case the worst case objective if known and it can be done inside the ``run``-function. Other possibilites are to ignore these configurations or to replace them with the running mean/min objective. To illustrate such a use-case we define an artificial ``run``-function which will fail when one of its input parameters is greater than 0.5. To define a failure, it is possible to return a "string" value with ``"F"`` as prefix such as:
"""
def run(config: dict) -> float:
if config["y"] > 0.5:
return "F_postfix"
else:
return config["x"]
# %%
# Then, we define the corresponding hyperparameter problem where ``x`` is the value to maximize and ``y`` is a value impact the appearance of failures.
from deephyper.problem import HpProblem
problem = HpProblem()
problem.add_hyperparameter([1, 2, 4, 8, 16, 32], "x")
problem.add_hyperparameter((0.0, 1.0), "y")
print(problem)
# %%
# Then, we define a centralized Bayesian optimization (CBO) search (i.e., master-worker architecture) which uses the Random-Forest regressor as default surrogate model. We will compare the ``ignore`` strategy which filters-out failed configurations, the ``mean`` strategy which replaces a failure by the running mean of collected objectives and the ``min`` strategy which replaces by the running min of collected objectives.
from deephyper.search.hps import CBO
from deephyper.evaluator import Evaluator
from deephyper.evaluator.callback import TqdmCallback
results = {}
max_evals = 30
for failure_strategy in ["ignore", "mean", "min"]:
"""
problem.py
"""
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem()
# call signature: Problem.add_dim(name, value)
Problem.add_dim('units1', (1, 64)) # int in range 1-64
Problem.add_dim('units2', (1, 64)) # int in range 1-64
Problem.add_dim('dropout1', (0.0, 1.0)) # float in range 0-1
Problem.add_dim('dropout2', (0.0, 1.0)) # float in range 0-1
Problem.add_dim('batch_size', (5, 500)) # int in range 5-500
Problem.add_dim('log10_learning_rate', (-5.0, 0.0)) # float lr range from 10^-5 to 1
# one of ['relu', ..., ]
Problem.add_dim('activation', ['relu', 'elu', 'selu', 'tanh'])
optimizer = Problem.add_dim('optimizer', [
'Adam', 'RMSprop', 'SGD', 'Nadam', 'Adagrad'
])
# Only vary momentum if optimizer is SGD
momentum = Problem.add_dim("momentum", (0.5, 0.9))
Problem.add_condition(cs.EqualsCondition(momentum, optimizer, "SGD"))
# Add a starting point to try first
Problem.add_starting_point(
units1=16,
from deephyper.problem import HpProblem
config_space = cs.ConfigurationSpace(seed=42)
x_hp = csh.UniformIntegerHyperparameter(name="x", lower=0, upper=10, log=False)
y_hp = csh.UniformIntegerHyperparameter(name="y", lower=0, upper=10, log=False)
config_space.add_hyperparameters([x_hp, y_hp])
not_zero = cs.ForbiddenAndConjunction(cs.ForbiddenEqualsClause(x_hp, 0),
cs.ForbiddenEqualsClause(y_hp, 0))
config_space.add_forbidden_clause(not_zero)
# Problem definition
Problem = HpProblem(config_space)
Problem.add_starting_point(x=1, y=1)
# Definition of the function which runs the model
def run(param_dict):
x = param_dict["x"]
y = param_dict["y"]
res = np.log(1 / (y + x))
return res # the objective
"""
Hyperparameter optimization problem to try forbidden clauses by directly using ConfigSpace.
Example command line::
python -m deephyper.search.hps.ambsv1 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_cond_1.Problem --run deephyper.benchmark.hps.toy.problem_cond_1.run --max-evals 100 --kappa 0.001
"""
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem(seed=45)
func = Problem.add_hyperparameter(name="func", value=["f1", "f2"])
# f1 variables
f1_x = Problem.add_hyperparameter(name="f1_x", value=(0.0, 1.0, "uniform"))
f1_y = Problem.add_hyperparameter(name="f1_y", value=(0.0, 1.0, "uniform"))
# f2 variables
f2_x = Problem.add_hyperparameter(name="f2_x", value=(0.0, 1.0, "uniform"))
f2_y = Problem.add_hyperparameter(name="f2_y", value=(0.0, 1.0, "uniform"))
cond_f1_x = cs.EqualsCondition(f1_x, func, "f1")
cond_f1_y = cs.EqualsCondition(f1_y, func, "f1")
Problem.add_condition(cond_f1_x)
Problem.add_condition(cond_f1_y)
cond_f2_x = cs.EqualsCondition(f2_x, func, "f2")
cond_f2_y = cs.EqualsCondition(f2_y, func, "f2")
Problem.add_condition(cond_f2_x)
Problem.add_condition(cond_f2_y)
"""
!! CURRENT OPTIMIZER DOES NOT SUPPORT THIS!!
Hyperparameter optimization problem to try forbidden clauses.
Example command line::
python -m deephyper.search.hps.ambs2 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_forbidden_1.Problem --run deephyper.benchmark.hps.toy.problem_forbidden_1.run --max-evals 100 --kappa 0.001
"""
import numpy as np
from deephyper.problem import HpProblem
from deephyper.problem import config_space as cs
# Problem definition
Problem = HpProblem()
x_hp = Problem.add_hyperparameter(
name="x", value=(0, 10)) # or Problem.add_dim("x", (0, 10))
y_hp = Problem.add_hyperparameter(
name="y", value=(0, 10)) # or Problem.add_dim("y", (0, 10))
not_zero = cs.ForbiddenAndConjunction(cs.ForbiddenEqualsClause(x_hp, 0),
cs.ForbiddenEqualsClause(y_hp, 0))
Problem.add_forbidden_clause(not_zero)
Problem.add_starting_point(x=1, y=1)
# Definition of the function which runs the model