def test_sample_types_no_cat(self):
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
problem.add_hyperparameter((0, 10), "x_int")
problem.add_hyperparameter((0.0, 10.0), "x_float")
def run(config):
assert np.issubdtype(type(config["x_int"]), np.integer)
assert np.issubdtype(type(config["x_float"]), float)
return 0
create_evaluator = lambda: Evaluator.create(run, method="serial")
CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY").search(10)
CBO(problem, create_evaluator(), random_state=42,
surrogate_model="RF").search(10)
def test_add_starting_points_with_too_many_dim(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=2)
def test_dim_with_wrong_name(self):
from deephyper.core.exceptions.problem import SpaceDimNameOfWrongType
from deephyper.problem import HpProblem
pb = HpProblem()
with pytest.raises(SpaceDimNameOfWrongType):
pb.add_hyperparameter((-10, 10), 0)
def test_add_good_dim(self):
from deephyper.problem import HpProblem
pb = HpProblem()
p0 = pb.add_hyperparameter((-10, 10), "p0")
p0_csh = csh.UniformIntegerHyperparameter(
name="p0", lower=-10, upper=10, log=False
)
assert p0 == p0_csh
p1 = pb.add_hyperparameter((1, 100, "log-uniform"), "p1")
p1_csh = csh.UniformIntegerHyperparameter(name="p1", lower=1, upper=100, log=True)
assert p1 == p1_csh
p2 = pb.add_hyperparameter((-10.0, 10.0), "p2")
p2_csh = csh.UniformFloatHyperparameter(
name="p2", lower=-10.0, upper=10.0, log=False
)
assert p2 == p2_csh
p3 = pb.add_hyperparameter((1.0, 100.0, "log-uniform"), "p3")
p3_csh = csh.UniformFloatHyperparameter(
name="p3", lower=1.0, upper=100.0, log=True
)
assert p3 == p3_csh
p4 = pb.add_hyperparameter([1, 2, 3, 4], "p4")
p4_csh = csh.OrdinalHyperparameter(name="p4", sequence=[1, 2, 3, 4])
assert p4 == p4_csh
p5 = pb.add_hyperparameter([1.0, 2.0, 3.0, 4.0], "p5")
p5_csh = csh.OrdinalHyperparameter(name="p5", sequence=[1.0, 2.0, 3.0, 4.0])
assert p5 == p5_csh
p6 = pb.add_hyperparameter(["cat0", "cat1"], "p6")
p6_csh = csh.CategoricalHyperparameter(name="p6", choices=["cat0", "cat1"])
assert p6 == p6_csh
p7 = pb.add_hyperparameter({"mu": 0, "sigma": 1}, "p7")
p7_csh = csh.NormalIntegerHyperparameter(name="p7", mu=0, sigma=1)
assert p7 == p7_csh
if cs.__version__ > "0.4.20":
p8 = pb.add_hyperparameter(
{"mu": 0, "sigma": 1, "lower": -5, "upper": 5}, "p8"
)
p8_csh = csh.NormalIntegerHyperparameter(
name="p8", mu=0, sigma=1, lower=-5, upper=5
)
assert p8 == p8_csh
p9 = pb.add_hyperparameter({"mu": 0.0, "sigma": 1.0}, "p9")
p9_csh = csh.NormalFloatHyperparameter(name="p9", mu=0, sigma=1)
assert p9 == p9_csh
def test_random_seed(self):
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
problem.add_hyperparameter((0.0, 10.0), "x")
def run(config):
return config["x"]
create_evaluator = lambda: Evaluator.create(run, method="serial")
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res1 = search.search(max_evals=4)
res1_array = res1[["x"]].to_numpy()
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res2 = search.search(max_evals=4)
res2_array = res2[["x"]].to_numpy()
assert np.array_equal(res1_array, res2_array)
# test multi-objective
def run(config):
return config["x"], config["x"]
create_evaluator = lambda: Evaluator.create(run, method="serial")
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res1 = search.search(max_evals=4)
res1_array = res1[["x"]].to_numpy()
search = CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY")
res2 = search.search(max_evals=4)
res2_array = res2[["x"]].to_numpy()
assert np.array_equal(res1_array, res2_array)
class Problem:
"""Representation of a problem."""
def __init__(self, seed=None, **kwargs):
self._space = OrderedDict()
self._hp_space = HpProblem(seed)
self.seed = seed
def __str__(self):
return repr(self)
def __repr__(self):
return f"Problem\n{pformat({k:v for k,v in self._space.items()}, indent=2)}"
def add_dim(self, p_name, p_space):
"""Add a dimension to the search space.
Args:
p_name (str): name of the parameter/dimension.
p_space (Object): space corresponding to the new dimension.
"""
self._space[p_name] = p_space
@property
def space(self):
dims = list(self._space.keys())
dims.sort()
space = OrderedDict(**{d: self._space[d] for d in dims})
return space
def add_hyperparameter(self,
value,
name: str = None,
default_value=None) -> csh.Hyperparameter:
return self._hp_space.add_hyperparameter(value, name, default_value)
def test_sample_types_conditional(self):
import ConfigSpace as cs
import numpy as np
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
problem = HpProblem()
# choices
choice = problem.add_hyperparameter(
name="choice",
value=["choice1", "choice2"],
)
# integers
x1_int = problem.add_hyperparameter(name="x1_int", value=(1, 10))
x2_int = problem.add_hyperparameter(name="x2_int", value=(1, 10))
# conditions
cond_1 = cs.EqualsCondition(x1_int, choice, "choice1")
cond_2 = cs.EqualsCondition(x2_int, choice, "choice2")
problem.add_condition(cond_1)
problem.add_condition(cond_2)
def run(config):
if config["choice"] == "choice1":
assert np.issubdtype(type(config["x1_int"]), np.integer)
else:
assert np.issubdtype(type(config["x2_int"]), np.integer)
return 0
create_evaluator = lambda: Evaluator.create(run, method="serial")
CBO(problem,
create_evaluator(),
random_state=42,
surrogate_model="DUMMY").search(10)
def test_quickstart(self):
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
from deephyper.evaluator import Evaluator
# define the variable you want to optimize
problem = HpProblem()
problem.add_hyperparameter((-10.0, 10.0), "x")
# define the evaluator to distribute the computation
evaluator = Evaluator.create(
run,
method="subprocess",
method_kwargs={
"num_workers": 2,
},
)
# define you search and execute it
search = CBO(problem, evaluator)
results = search.search(max_evals=15)
def test_gp(self):
from deephyper.evaluator import Evaluator
from deephyper.problem import HpProblem
from deephyper.search.hps import CBO
# test float hyperparameters
problem = HpProblem()
problem.add_hyperparameter((0.0, 10.0), "x")
def run(config):
return config["x"]
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
# test int hyperparameters
problem = HpProblem()
problem.add_hyperparameter((0, 10), "x")
def run(config):
return config["x"]
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
# test categorical hyperparameters
problem = HpProblem()
problem.add_hyperparameter([f"{i}" for i in range(10)], "x")
def run(config):
return int(config["x"])
CBO(
problem,
Evaluator.create(run, method="serial"),
random_state=42,
surrogate_model="GP",
).search(10)
def test_add_starting_points_not_in_space_def(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
pb.add_hyperparameter((-10.0, 10.0), "dim1")
pb.add_hyperparameter(["a", "b"], "dim2")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=-11, dim1=0.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=11, dim1=0.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=-11.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=11.0, dim2="a")
with pytest.raises(ValueError):
pb.add_starting_point(dim0=0, dim1=0.0, dim2="c")
pb.add_starting_point(dim0=0, dim1=0.0, dim2="a")
"""
deephyper hps ambs --problem nas_big_data.albert.tune_random_forest.Problem --run nas_big_data.albert.tune_random_forest.run --max-evals 30 --evaluator subprocess --n-jobs 3
"""
import json
import os
import pandas as pd
from deephyper.problem import HpProblem
from nas_big_data import RANDOM_STATE
from nas_big_data.albert.load_data import load_data
from sklearn.ensemble import RandomForestClassifier
Problem = HpProblem()
Problem.add_hyperparameter((10, 300), "n_estimators")
Problem.add_hyperparameter(["gini", "entropy"], "criterion")
Problem.add_hyperparameter((1, 50), "max_depth")
Problem.add_hyperparameter((2, 10), "min_samples_split")
# We define a starting point with the defaul hyperparameters from sklearn-learn
# that we consider good in average.
Problem.add_starting_point(n_estimators=100,
criterion="gini",
max_depth=50,
min_samples_split=2)
def test_best():
"""Test data with RandomForest
"""Inspired from https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html
"""
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem(seed=45)
classifier = Problem.add_hyperparameter(
name="classifier",
value=["RandomForest", "Logistic", "AdaBoost", "KNeighbors", "MLP", "SVC", "XGBoost"],
)
# n_estimators
n_estimators = Problem.add_hyperparameter(
name="n_estimators", value=(1, 2000, "log-uniform")
)
cond_n_estimators = cs.OrConjunction(
cs.EqualsCondition(n_estimators, classifier, "RandomForest"),
cs.EqualsCondition(n_estimators, classifier, "AdaBoost"),
)
Problem.add_condition(cond_n_estimators)
# max_depth
max_depth = Problem.add_hyperparameter(name="max_depth", value=(2, 100, "log-uniform"))
cond_max_depth = cs.EqualsCondition(max_depth, classifier, "RandomForest")
Problem.add_condition(cond_max_depth)
"""
def run(config: dict) -> float:
if config["y"] > 0.5:
return "F_postfix"
else:
return config["x"]
# %%
# Then, we define the corresponding hyperparameter problem where ``x`` is the value to maximize and ``y`` is a value impact the appearance of failures.
from deephyper.problem import HpProblem
problem = HpProblem()
problem.add_hyperparameter([1, 2, 4, 8, 16, 32], "x")
problem.add_hyperparameter((0.0, 1.0), "y")
print(problem)
# %%
# Then, we define a centralized Bayesian optimization (CBO) search (i.e., master-worker architecture) which uses the Random-Forest regressor as default surrogate model. We will compare the ``ignore`` strategy which filters-out failed configurations, the ``mean`` strategy which replaces a failure by the running mean of collected objectives and the ``min`` strategy which replaces by the running min of collected objectives.
from deephyper.search.hps import CBO
from deephyper.evaluator import Evaluator
from deephyper.evaluator.callback import TqdmCallback
results = {}
max_evals = 30
for failure_strategy in ["ignore", "mean", "min"]:
# for failure_strategy in ["min"]:
Hyperparameter optimization problem to try forbidden clauses.
Example command line::
python -m deephyper.search.hps.ambs2 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_forbidden_1.Problem --run deephyper.benchmark.hps.toy.problem_forbidden_1.run --max-evals 100 --kappa 0.001
"""
import numpy as np
from deephyper.problem import HpProblem
from deephyper.problem import config_space as cs
# Problem definition
Problem = HpProblem()
x_hp = Problem.add_hyperparameter(
name="x", value=(0, 10)) # or Problem.add_dim("x", (0, 10))
y_hp = Problem.add_hyperparameter(
name="y", value=(0, 10)) # or Problem.add_dim("y", (0, 10))
not_zero = cs.ForbiddenAndConjunction(cs.ForbiddenEqualsClause(x_hp, 0),
cs.ForbiddenEqualsClause(y_hp, 0))
Problem.add_forbidden_clause(not_zero)
Problem.add_starting_point(x=1, y=1)
# Definition of the function which runs the model
def run(param_dict):
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem(seed=45)
#! Default value are very important when adding conditional and forbidden clauses
#! Otherwise the creation of the problem can fail if the default configuration is not
#! Acceptable
classifier = Problem.add_hyperparameter(
name="classifier",
value=["RandomForest", "GradientBoosting"],
default_value="RandomForest",
)
# For both
Problem.add_hyperparameter(name="n_estimators", value=(1, 1000, "log-uniform"))
Problem.add_hyperparameter(name="max_depth", value=(1, 50))
Problem.add_hyperparameter(
name="min_samples_split",
value=(2, 10),
)
Problem.add_hyperparameter(name="min_samples_leaf", value=(1, 10))
criterion = Problem.add_hyperparameter(
name="criterion",
value=["friedman_mse", "mse", "mae", "gini", "entropy"],
default_value="gini",
)
# GradientBoosting
loss = Problem.add_hyperparameter(name="loss",
value=["deviance", "exponential"])
from deephyper.problem import HpProblem
Problem = HpProblem()
Problem.add_hyperparameter((5, 500), "epochs")
Problem.add_hyperparameter((1, 1000), "nunits_l1")
Problem.add_hyperparameter((1, 1000), "nunits_l2")
Problem.add_hyperparameter(["relu", "elu", "selu", "tanh"], "activation_l1")
Problem.add_hyperparameter(["relu", "elu", "selu", "tanh"], "activation_l2")
Problem.add_hyperparameter((8, 1024), "batch_size")
Problem.add_hyperparameter((0.0, 1.0), "dropout_l1")
Problem.add_hyperparameter((0.0, 1.0), "dropout_l2")
Problem.add_starting_point(
epochs=5,
nunits_l1=1,
nunits_l2=2,
activation_l1="relu",
activation_l2="relu",
batch_size=8,
dropout_l1=0.0,
dropout_l2=0.0,
)
if __name__ == "__main__":
print(Problem)
def test_kwargs(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter(value=(-10, 10), name="dim0")
:language: python
:emphasize-lines: 19-28
:linenos:
After defining the black-box we can continue with the definition of our main script:
"""
import black_box_util as black_box
# %%
# Then we define the variable(s) we want to optimize. For this problem we optimize Ackley in a 2-dimensional search space, the true minimul is located at ``(0, 0)``.
from deephyper.problem import HpProblem
nb_dim = 2
problem = HpProblem()
for i in range(nb_dim):
problem.add_hyperparameter((-32.768, 32.768), f"x{i}")
problem
# %%
# Then we define a parallel search.
if __name__ == "__main__":
from deephyper.evaluator import Evaluator
from deephyper.evaluator.callback import TqdmCallback
from deephyper.search.hps import CBO
timeout = 20
num_workers = 4
results = {}
evaluator = Evaluator.create(
black_box.run_ackley,
y = -sum([config[f"x{i}"] ** 2 for i in range(N)])
return y
run_small = functools.partial(run, N=1)
run_large = functools.partial(run, N=2)
# %%
# Then, we can define the hyperparameter problem space based on :math:`n`
from deephyper.problem import HpProblem
N = 1
problem_small = HpProblem()
for i in range(N):
problem_small.add_hyperparameter((-10.0, 10.0), f"x{i}")
problem_small
# %%
N = 2
problem_large = HpProblem()
for i in range(N):
problem_large.add_hyperparameter((-10.0, 10.0), f"x{i}")
problem_large
# %%
# Then, we define setup the search and execute it:
from deephyper.evaluator import Evaluator
from deephyper.evaluator.callback import TqdmCallback
from deephyper.search.hps import CBO
class AMBSMixed(NeuralArchitectureSearch):
"""Asynchronous Model-Based Search baised on the `Scikit-Optimized Optimizer <https://scikit-optimize.github.io/stable/modules/generated/deephyper.skopt.Optimizer.html#deephyper.skopt.Optimizer>`_. It is extended to the case of joint hyperparameter and neural architecture search.
Args:
problem (NaProblem): Neural architecture search problem describing the search space to explore.
evaluator (Evaluator): An ``Evaluator`` instance responsible of distributing the tasks.
random_state (int, optional): Random seed. Defaults to None.
log_dir (str, optional): Log directory where search's results are saved. Defaults to ".".
verbose (int, optional): Indicate the verbosity level of the search. Defaults to 0.
surrogate_model (str, optional): Surrogate model used by the Bayesian optimization. Can be a value in ``["RF", "ET", "GBRT", "DUMMY"]``. Defaults to ``"RF"``.
acq_func (str, optional): Acquisition function used by the Bayesian optimization. Can be a value in ``["UCB", "EI", "PI", "gp_hedge"]``. Defaults to ``"UCB"``.
kappa (float, optional): Manage the exploration/exploitation tradeoff for the "UCB" acquisition function. Defaults to ``1.96`` for a balance between exploitation and exploration.
xi (float, optional): Manage the exploration/exploitation tradeoff of ``"EI"`` and ``"PI"`` acquisition function. Defaults to ``0.001`` for a balance between exploitation and exploration.
n_points (int, optional): The number of configurations sampled from the search space to infer each batch of new evaluated configurations. Defaults to ``10000``.
liar_strategy (str, optional): Definition of the constant value use for the Liar strategy. Can be a value in ``["cl_min", "cl_mean", "cl_max"]`` . Defaults to ``"cl_max"``.
n_jobs (int, optional): Number of parallel processes used to fit the surrogate model of the Bayesian optimization. A value of ``-1`` will use all available cores. Defaults to ``1``.
"""
def __init__(
self,
problem,
evaluator,
random_state=None,
log_dir=".",
verbose=0,
surrogate_model: str = "RF",
acq_func: str = "UCB",
kappa: float = 1.96,
xi: float = 0.001,
n_points: int = 10000,
liar_strategy: str = "cl_max",
n_jobs: int = 1,
**kwargs,
):
super().__init__(problem, evaluator, random_state, log_dir, verbose)
# Setup the search space
na_search_space = self._problem.build_search_space()
self.hp_space = self._problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem()
self.na_space._space.seed(self._random_state.get_state()[1][0])
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self._space = CS.ConfigurationSpace(
seed=self._random_state.get_state()[1][0])
self._space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self._space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# check input parameters
surrogate_model_allowed = ["RF", "ET", "GBRT", "DUMMY"]
if not (surrogate_model in surrogate_model_allowed):
raise ValueError(
f"Parameter 'surrogate_model={surrogate_model}' should have a value in {surrogate_model_allowed}!"
)
acq_func_allowed = ["UCB", "EI", "PI", "gp_hedge"]
if not (acq_func in acq_func_allowed):
raise ValueError(
f"Parameter 'acq_func={acq_func}' should have a value in {acq_func_allowed}!"
)
if not (np.isscalar(kappa)):
raise ValueError(f"Parameter 'kappa' should be a scalar value!")
if not (np.isscalar(xi)):
raise ValueError("Parameter 'xi' should be a scalar value!")
if not (type(n_points) is int):
raise ValueError("Parameter 'n_points' shoud be an integer value!")
liar_strategy_allowed = ["cl_min", "cl_mean", "cl_max"]
if not (liar_strategy in liar_strategy_allowed):
raise ValueError(
f"Parameter 'liar_strategy={liar_strategy}' should have a value in {liar_strategy_allowed}!"
)
if not (type(n_jobs) is int):
raise ValueError(f"Parameter 'n_jobs' should be an integer value!")
self._n_initial_points = self._evaluator.num_workers
self._liar_strategy = MAP_liar_strategy.get(liar_strategy,
liar_strategy)
base_estimator = self._get_surrogate_model(
surrogate_model,
n_jobs,
random_state=self._random_state.get_state()[1][0])
self._opt = None
self._opt_kwargs = dict(
dimensions=self._space,
base_estimator=base_estimator,
acq_func=MAP_acq_func.get(acq_func, acq_func),
acq_optimizer="sampling",
acq_func_kwargs={
"xi": xi,
"kappa": kappa,
"n_points": n_points
},
n_initial_points=self._n_initial_points,
random_state=self._random_state,
)
def _setup_optimizer(self):
self._opt = deephyper.skopt.Optimizer(**self._opt_kwargs)
def _saved_keys(self, job):
res = {"arch_seq": str(job.config["arch_seq"])}
hp_names = self._problem._hp_space._space.get_hyperparameter_names()
for hp_name in hp_names:
if hp_name == "loss":
res["loss"] = job.config["loss"]
else:
res[hp_name] = job.config["hyperparameters"][hp_name]
return res
def _search(self, max_evals, timeout):
if self._opt is None:
self._setup_optimizer()
num_evals_done = 0
# Filling available nodes at start
logging.info(
f"Generating {self._evaluator.num_workers} initial points...")
self._evaluator.submit(
self._get_random_batch(size=self._n_initial_points))
# Main loop
while max_evals < 0 or num_evals_done < max_evals:
# Collecting finished evaluations
new_results = list(self._evaluator.gather("BATCH", size=1))
num_received = len(new_results)
if num_received > 0:
self._evaluator.dump_evals(saved_keys=self._saved_keys,
log_dir=self._log_dir)
num_evals_done += num_received
if num_evals_done >= max_evals:
break
# Transform configurations to list to fit optimizer
opt_X = []
opt_y = []
for cfg, obj in new_results:
arch_seq = cfg["arch_seq"]
hp_val = self._problem.extract_hp_values(cfg)
x = replace_nan(hp_val + arch_seq)
opt_X.append(x)
opt_y.append(-obj) #! maximizing
self._opt.tell(opt_X, opt_y) #! fit: costly
new_X = self._opt.ask(n_points=len(new_results),
strategy=self._liar_strategy)
new_batch = []
for x in new_X:
new_cfg = self._problem.gen_config(x[self.hp_size:],
x[:self.hp_size])
new_batch.append(new_cfg)
# submit_childs
if len(new_results) > 0:
self._evaluator.submit(new_batch)
def _get_surrogate_model(self,
name: str,
n_jobs: int = None,
random_state: int = None):
"""Get a surrogate model from Scikit-Optimize.
Args:
name (str): name of the surrogate model.
n_jobs (int): number of parallel processes to distribute the computation of the surrogate model.
Raises:
ValueError: when the name of the surrogate model is unknown.
"""
accepted_names = ["RF", "ET", "GBRT", "DUMMY"]
if not (name in accepted_names):
raise ValueError(
f"Unknown surrogate model {name}, please choose among {accepted_names}."
)
if name == "RF":
surrogate = deephyper.skopt.learning.RandomForestRegressor(
n_jobs=n_jobs, random_state=random_state)
elif name == "ET":
surrogate = deephyper.skopt.learning.ExtraTreesRegressor(
n_jobs=n_jobs, random_state=random_state)
elif name == "GBRT":
surrogate = deephyper.skopt.learning.GradientBoostingQuantileRegressor(
n_jobs=n_jobs, random_state=random_state)
else: # for DUMMY and GP
surrogate = name
return surrogate
def _get_random_batch(self, size: int) -> list:
batch = []
n_points = max(0, size - len(batch))
if n_points > 0:
points = self._opt.ask(n_points=n_points)
for point in points:
point_as_dict = self._problem.gen_config(
point[self.hp_size:], point[:self.hp_size])
batch.append(point_as_dict)
return batch
import pytest
import numpy as np
from deephyper.problem import HpProblem
from deephyper.evaluator import Evaluator
from deephyper.search.hps import AMBS
def run(config):
return config["x"]
problem = HpProblem()
problem.add_hyperparameter((0.0, 10.0), "x")
def test_ambs():
create_evaluator = lambda: Evaluator.create(
run, method="process", method_kwargs={"num_workers": 1})
search = AMBS(
problem,
create_evaluator(),
random_state=42,
)
res1 = search.search(max_evals=4)
res1_array = res1[["x"]].to_numpy()
search.search(max_evals=100, timeout=1)
"""
python -m deephyper.search.hps.ambs2 --evaluator threadPool --problem deephyper.benchmark.hps.polynome2.Problem --run deephyper.benchmark.hps.polynome2.run --max-evals 100 --kappa 0.001
"""
import numpy as np
from deephyper.benchmark.benchmark_functions_wrappers import polynome_2
from deephyper.problem import HpProblem
# Problem definition
Problem = HpProblem()
num_dim = 10
for i in range(num_dim):
Problem.add_hyperparameter((-10.0, 10.0), f"e{i}")
Problem.add_starting_point(**{f"e{i}": 10.0 for i in range(num_dim)})
# Definition of the function which runs the model
def run(param_dict):
f, _, _ = polynome_2()
num_dim = 10
x = np.array([param_dict[f"e{i}"] for i in range(num_dim)])
return f(x) # the objective
if __name__ == "__main__":
print(Problem)
class RegularizedEvolutionMixed(RegularizedEvolution):
"""Extention of the `Regularized evolution <https://arxiv.org/abs/1802.01548>`_ neural architecture search to the case of joint hyperparameter and neural architecture search.
Args:
problem (NaProblem): Neural architecture search problem describing the search space to explore.
evaluator (Evaluator): An ``Evaluator`` instance responsible of distributing the tasks.
random_state (int, optional): Random seed. Defaults to None.
log_dir (str, optional): Log directory where search's results are saved. Defaults to ".".
verbose (int, optional): Indicate the verbosity level of the search. Defaults to 0.
population_size (int, optional): the number of individuals to keep in the population. Defaults to 100.
sample_size (int, optional): the number of individuals that should participate in each tournament. Defaults to 10.
"""
def __init__(
self,
problem,
evaluator,
random_state: int = None,
log_dir: str = ".",
verbose: int = 0,
population_size: int = 100,
sample_size: int = 10,
**kwargs,
):
super().__init__(
problem,
evaluator,
random_state,
log_dir,
verbose,
population_size,
sample_size,
)
# Setup
na_search_space = self._problem.build_search_space()
self.hp_space = self._problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem()
self.na_space._space.seed(self._random_state.get_state()[1][0])
for i, (low, high) in enumerate(na_search_space.choices()):
self.na_space.add_hyperparameter((low, high), name=f"vnode_{i:05d}")
self._space = CS.ConfigurationSpace(seed=self._random_state.get_state()[1][0])
self._space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space
)
self._space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space
)
self._space_size = len(self._space.get_hyperparameter_names())
def _saved_keys(self, job):
res = {"arch_seq": str(job.config["arch_seq"])}
hp_names = self._problem._hp_space._space.get_hyperparameter_names()
for hp_name in hp_names:
if hp_name == "loss":
res["loss"] = job.config["loss"]
else:
res[hp_name] = job.config["hyperparameters"][hp_name]
return res
def _search(self, max_evals, timeout):
num_evals_done = 0
# Filling available nodes at start
self._evaluator.submit(self._gen_random_batch(size=self._evaluator.num_workers))
# Main loop
while max_evals < 0 or num_evals_done < max_evals:
# Collecting finished evaluations
new_results = list(self._evaluator.gather("BATCH", size=1))
num_received = len(new_results)
if num_received > 0:
self._population.extend(new_results)
self._evaluator.dump_evals(
saved_keys=self._saved_keys, log_dir=self._log_dir
)
num_evals_done += num_received
if num_evals_done >= max_evals:
break
# If the population is big enough evolve the population
if len(self._population) == self._population_size:
children_batch = []
# For each new parent/result we create a child from it
for _ in range(num_received):
# select_sample
indexes = self._random_state.choice(
self._population_size, self._sample_size, replace=False
)
sample = [self._population[i] for i in indexes]
# select_parent
parent = self._select_parent(sample)
# copy_mutate_parent
child = self._copy_mutate_arch(parent)
# add child to batch
children_batch.append(child)
# submit_childs
self._evaluator.submit(children_batch)
else: # If the population is too small keep increasing it
new_batch = self._gen_random_batch(size=num_received)
self._evaluator.submit(new_batch)
def _select_parent(self, sample: list) -> dict:
cfg, _ = max(sample, key=lambda x: x[1])
return cfg
def _gen_random_batch(self, size: int) -> list:
sample = lambda hp, size: [hp.sample(self._space.random) for _ in range(size)]
batch = []
iterator = zip(*(sample(hp, size) for hp in self._space.get_hyperparameters()))
for x in iterator:
cfg = self._problem.gen_config(
list(x[self.hp_size :]), list(x[: self.hp_size])
)
batch.append(cfg)
return batch
def _copy_mutate_arch(self, parent_cfg: dict) -> dict:
"""
# ! Time performance is critical because called sequentialy
Args:
parent_arch (list(int)): embedding of the parent's architecture.
Returns:
dict: embedding of the mutated architecture of the child.
"""
hp_x = self._problem.extract_hp_values(parent_cfg)
x = hp_x + parent_cfg["arch_seq"]
i = self._random_state.choice(self._space_size)
hp = self._space.get_hyperparameters()[i]
x[i] = hp.sample(self._space.random)
child_cfg = self._problem.gen_config(x[self.hp_size :], x[: self.hp_size])
return child_cfg
class RegularizedEvolutionMixed(RegularizedEvolution):
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(n_jobs)
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
@staticmethod
def _extend_parser(parser):
RegularizedEvolution._extend_parser(parser)
add_arguments_from_signature(parser, RegularizedEvolutionMixed)
return parser
def saved_keys(self, val: dict):
res = {"arch_seq": str(val["arch_seq"])}
hp_names = self.hp_space._space.get_hyperparameter_names()
for hp_name in hp_names:
if hp_name == "loss":
res["loss"] = val["loss"]
else:
res[hp_name] = val["hyperparameters"][hp_name]
return res
def main(self):
num_evals_done = 0
population = collections.deque(maxlen=self.population_size)
# Filling available nodes at start
self.evaluator.add_eval_batch(
self.gen_random_batch(size=self.free_workers))
# Main loop
while num_evals_done < self.max_evals:
# Collecting finished evaluations
new_results = list(self.evaluator.get_finished_evals())
if len(new_results) > 0:
population.extend(new_results)
stats = {
"num_cache_used": self.evaluator.stats["num_cache_used"]
}
dhlogger.info(jm(type="env_stats", **stats))
self.evaluator.dump_evals(saved_keys=self.saved_keys)
num_received = len(new_results)
num_evals_done += num_received
# If the population is big enough evolve the population
if len(population) == self.population_size:
children_batch = []
# For each new parent/result we create a child from it
for _ in range(len(new_results)):
# select_sample
indexes = np.random.choice(self.population_size,
self.sample_size,
replace=False)
sample = [population[i] for i in indexes]
# select_parent
parent = self.select_parent(sample)
# copy_mutate_parent
child = self.copy_mutate_arch(parent)
# add child to batch
children_batch.append(child)
# submit_childs
if len(new_results) > 0:
self.evaluator.add_eval_batch(children_batch)
else: # If the population is too small keep increasing it
new_batch = self.gen_random_batch(size=len(new_results))
self.evaluator.add_eval_batch(new_batch)
def select_parent(self, sample: list) -> dict:
cfg, _ = max(sample, key=lambda x: x[1])
return cfg
def gen_random_batch(self, size: int) -> list:
sample = lambda hp, size: [
hp.sample(self.space.random) for _ in range(size)
]
batch = []
iterator = zip(*(sample(hp, size)
for hp in self.space.get_hyperparameters()))
for x in iterator:
#! DeepCopy is critical for nested lists or dicts
cfg = self.problem.gen_config(x[self.hp_size:], x[:self.hp_size])
batch.append(cfg)
return batch
def copy_mutate_arch(self, parent_cfg: dict) -> dict:
"""
# ! Time performance is critical because called sequentialy
Args:
parent_arch (list(int)): embedding of the parent's architecture.
Returns:
dict: embedding of the mutated architecture of the child.
"""
hp_x = self.problem.extract_hp_values(parent_cfg)
x = hp_x + parent_cfg["arch_seq"]
i = np.random.choice(self.hp_size)
hp = self.space.get_hyperparameters()[i]
x[i] = hp.sample(self.space.random)
child_cfg = self.problem.gen_config(x[self.hp_size:], x[:self.hp_size])
return child_cfg
REGRESSORS = {
"RandomForest": RandomForestRegressor,
"Linear": LinearRegression,
"AdaBoost": AdaBoostRegressor,
"KNeighbors": KNeighborsRegressor,
"MLP": MLPRegressor,
"SVR": SVR,
"XGBoost": XGBRegressor,
}
problem_autosklearn1 = HpProblem()
regressor = problem_autosklearn1.add_hyperparameter(
name="regressor",
value=[
"RandomForest", "Linear", "AdaBoost", "KNeighbors", "MLP", "SVR",
"XGBoost"
],
)
# n_estimators
n_estimators = problem_autosklearn1.add_hyperparameter(name="n_estimators",
value=(1, 2000,
"log-uniform"))
cond_n_estimators = cs.OrConjunction(
cs.EqualsCondition(n_estimators, regressor, "RandomForest"),
cs.EqualsCondition(n_estimators, regressor, "AdaBoost"),
)
problem_autosklearn1.add_condition(cond_n_estimators)
def test_add_good_reference(self):
from deephyper.problem import HpProblem
pb = HpProblem()
pb.add_hyperparameter((-10, 10), "dim0")
pb.add_starting_point(dim0=0)
class AMBSMixed(NeuralArchitectureSearch):
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
self.kappa = float(kappa)
self.xi = float(xi)
self.n_initial_points = self.evaluator.num_workers
self.liar_strategy = liar_strategy
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# Initialize opitmizer of hyperparameter space
self.opt = SkOptimizer(
dimensions=self.space,
base_estimator=self.get_surrogate_model(surrogate_model,
self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={
"xi": self.xi,
"kappa": self.kappa
},
n_initial_points=self.n_initial_points,
)
@staticmethod
def _extend_parser(parser):
NeuralArchitectureSearch._extend_parser(parser)
add_arguments_from_signature(parser, AMBSMixed)
return parser
def saved_keys(self, val: dict):
res = {"arch_seq": str(val["arch_seq"])}
hp_names = self.hp_space._space.get_hyperparameter_names()
for hp_name in hp_names:
if hp_name == "loss":
res["loss"] = val["loss"]
else:
res[hp_name] = val["hyperparameters"][hp_name]
return res
def main(self):
num_evals_done = 0
# Filling available nodes at start
dhlogger.info(
f"Generating {self.evaluator.num_workers} initial points...")
self.evaluator.add_eval_batch(
self.get_random_batch(size=self.n_initial_points))
# Main loop
while num_evals_done < self.max_evals:
# Collecting finished evaluations
new_results = list(self.evaluator.get_finished_evals())
if len(new_results) > 0:
stats = {
"num_cache_used": self.evaluator.stats["num_cache_used"]
}
dhlogger.info(jm(type="env_stats", **stats))
self.evaluator.dump_evals(saved_keys=self.saved_keys)
num_received = len(new_results)
num_evals_done += num_received
# Transform configurations to list to fit optimizer
opt_X = []
opt_y = []
for cfg, obj in new_results:
arch_seq = cfg["arch_seq"]
hp_val = self.problem.extract_hp_values(cfg)
x = replace_nan(hp_val + arch_seq)
opt_X.append(x)
opt_y.append(-obj) #! maximizing
self.opt.tell(opt_X, opt_y) #! fit: costly
new_X = self.opt.ask(n_points=len(new_results),
strategy=self.liar_strategy)
new_batch = []
for x in new_X:
new_cfg = self.problem.gen_config(x[self.hp_size:],
x[:self.hp_size])
new_batch.append(new_cfg)
# submit_childs
if len(new_results) > 0:
self.evaluator.add_eval_batch(new_batch)
def get_surrogate_model(self, name: str, n_jobs: int = None):
"""Get a surrogate model from Scikit-Optimize.
Args:
name (str): name of the surrogate model.
n_jobs (int): number of parallel processes to distribute the computation of the surrogate model.
Raises:
ValueError: when the name of the surrogate model is unknown.
"""
accepted_names = ["RF", "ET", "GBRT", "GP", "DUMMY"]
if not (name in accepted_names):
raise ValueError(
f"Unknown surrogate model {name}, please choose among {accepted_names}."
)
if name == "RF":
surrogate = skopt.learning.RandomForestRegressor(n_jobs=n_jobs)
elif name == "ET":
surrogate = skopt.learning.ExtraTreesRegressor(n_jobs=n_jobs)
elif name == "GBRT":
surrogate = skopt.learning.GradientBoostingQuantileRegressor(
n_jobs=n_jobs)
else: # for DUMMY and GP
surrogate = name
return surrogate
def get_random_batch(self, size: int) -> list:
batch = []
n_points = max(0, size - len(batch))
if n_points > 0:
points = self.opt.ask(n_points=n_points)
for point in points:
point_as_dict = self.problem.gen_config(
point[self.hp_size:], point[:self.hp_size])
batch.append(point_as_dict)
return batch
def to_dict(self, x: list) -> dict:
hp_x = x[:self.hp_size]
arch_seq = x[self.hp_size:]
cfg = self.problem.space.copy()
cfg["arch_seq"] = arch_seq
return cfg
"""
Hyperparameter optimization problem to try forbidden clauses by directly using ConfigSpace.
Example command line::
python -m deephyper.search.hps.ambsv1 --evaluator threadPool --problem deephyper.benchmark.hps.toy.problem_cond_1.Problem --run deephyper.benchmark.hps.toy.problem_cond_1.run --max-evals 100 --kappa 0.001
"""
import ConfigSpace as cs
from deephyper.problem import HpProblem
Problem = HpProblem(seed=45)
func = Problem.add_hyperparameter(name="func", value=["f1", "f2"])
# f1 variables
f1_x = Problem.add_hyperparameter(name="f1_x", value=(0.0, 1.0, "uniform"))
f1_y = Problem.add_hyperparameter(name="f1_y", value=(0.0, 1.0, "uniform"))
# f2 variables
f2_x = Problem.add_hyperparameter(name="f2_x", value=(0.0, 1.0, "uniform"))
f2_y = Problem.add_hyperparameter(name="f2_y", value=(0.0, 1.0, "uniform"))
cond_f1_x = cs.EqualsCondition(f1_x, func, "f1")
cond_f1_y = cs.EqualsCondition(f1_y, func, "f1")
Problem.add_condition(cond_f1_x)
Problem.add_condition(cond_f1_y)
cond_f2_x = cs.EqualsCondition(f2_x, func, "f2")
cond_f2_y = cs.EqualsCondition(f2_y, func, "f2")
Problem.add_condition(cond_f2_x)
Problem.add_condition(cond_f2_y)
class NaProblem:
"""A Neural Architecture Problem specification for Neural Architecture Search.
>>> from deephyper.problem import NaProblem
>>> from deephyper.benchmark.nas.linearReg.load_data import load_data
>>> from deephyper.nas.preprocessing import minmaxstdscaler
>>> from deepspace.tabular import OneLayerSpace
>>> Problem = NaProblem()
>>> Problem.load_data(load_data)
>>> Problem.preprocessing(minmaxstdscaler)
>>> Problem.search_space(OneLayerSpace)
>>> Problem.hyperparameters(
... batch_size=100,
... learning_rate=0.1,
... optimizer='adam',
... num_epochs=10,
... callbacks=dict(
... EarlyStopping=dict(
... monitor='val_r2',
... mode='max',
... verbose=0,
... patience=5
... )
... )
... )
>>> Problem.loss('mse')
>>> Problem.metrics(['r2'])
>>> Problem.objective('val_r2__last')
"""
def __init__(self):
self._space = OrderedDict()
self._hp_space = HpProblem()
self._space["metrics"] = []
self._space["hyperparameters"] = dict(verbose=0)
def __repr__(self):
preprocessing = (None if self._space.get("preprocessing") is None else
module_location(self._space["preprocessing"]["func"]))
hps = "".join([
f"\n * {h}: {self._space['hyperparameters'][h]}"
for h in self._space["hyperparameters"]
])
if type(self._space["metrics"]) is list:
metrics = "".join(
[f"\n * {m}" for m in self._space["metrics"]])
else:
metrics = "".join([
f"\n * {m[0]}: {m[1]}"
for m in self._space["metrics"].items()
])
objective = self._space["objective"]
if not type(objective) is str:
objective = module_location(objective)
out = (
f"Problem is:\n"
f" - search space : {module_location(self._space['search_space']['class'])}\n"
f" - data loading : {module_location(self._space['load_data']['func'])}\n"
f" - preprocessing : {preprocessing}\n"
f" - hyperparameters: {hps}\n"
f" - loss : {self._space['loss']}\n"
f" - metrics : {metrics}\n"
f" - objective : {objective}\n")
return out
def load_data(self, func: callable, **kwargs):
"""Define the function loading the data.
.. code-block:: python
Problem.load_data(load_data, load_data_kwargs)
This ``load_data`` callable can follow two different interfaces: Numpy arrays or generators.
1. **Numpy arrays**:
In the case of Numpy arrays, the callable passed to ``Problem.load_data(...)`` has to return the following tuple: ``(X_train, y_train), (X_valid, y_valid)``. In the most simple case where the model takes a single input, each of these elements is a Numpy array. Generally, ``X_train`` and ``y_train`` have to be of the same length (i.e., same ``array.shape[0]``) which is also the case for ``X_valid`` and ``y_valid``. Similarly, the shape of the elements of ``X_train`` and ``X_valid`` which is also the case for ``y_train`` and ``y_valid``. An example ``load_data`` function can be
.. code-block:: python
import numpy as np
def load_data(N=100):
X = np.zeros((N, 1))
y = np.zeros((N,1))
return (X, y), (X, y)
It is also possible for the model to take several inputs. In fact, experimentaly it can be notices that separating some inputs with different inputs can significantly help the learning of the model. Also, sometimes different inputs may be of the "types" for example two molecular fingerprints. In this case, it can be very interesting to share the weights of the model to process these two inputs. In the case of multi-inputs models the ``load_data`` function will also return ``(X_train, y_train), (X_valid, y_valid)`` bu where ``X_train`` and ``X_valid`` are two lists of Numpy arrays. For example, the following is correct:
.. code-block:: python
import numpy as np
def load_data(N=100):
X = np.zeros((N, 1))
y = np.zeros((N,1))
return ([X, X], y), ([X, X], y)
2. **Generators**:
Returning generators with a single input:
.. code-block:: python
def load_data(N=100):
tX, ty = np.zeros((N,1)), np.zeros((N,1))
vX, vy = np.zeros((N,1)), np.zeros((N,1))
def train_gen():
for x, y in zip(tX, ty):
yield ({"input_0": x}, y)
def valid_gen():
for x, y in zip(vX, vy):
yield ({"input_0": x}, y)
res = {
"train_gen": train_gen,
"train_size": N,
"valid_gen": valid_gen,
"valid_size": N,
"types": ({"input_0": tf.float64}, tf.float64),
"shapes": ({"input_0": (1, )}, (1, ))
}
return res
Returning generators with multiple inputs:
.. code-block:: python
def load_data(N=100):
tX0, tX1, ty = np.zeros((N,1)), np.zeros((N,1)), np.zeros((N,1)),
vX0, vX1, vy = np.zeros((N,1)), np.zeros((N,1)), np.zeros((N,1)),
def train_gen():
for x0, x1, y in zip(tX0, tX1, ty):
yield ({
"input_0": x0,
"input_1": x1
}, y)
def valid_gen():
for x0, x1, y in zip(vX0, vX1, vy):
yield ({
"input_0": x0,
"input_1": x1
}, y)
res = {
"train_gen": train_gen,
"train_size": N,
"valid_gen": valid_gen,
"valid_size": N,
"types": ({"input_0": tf.float64, "input_1": tf.float64}, tf.float64),
"shapes": ({"input_0": (5, ), "input_1": (5, )}, (1, ))
}
return res
Args:
func (callable): the load data function.
"""
if not callable(func):
raise ProblemLoadDataIsNotCallable(func)
self._space["load_data"] = {"func": func, "kwargs": kwargs}
def augment(self, func: callable, **kwargs):
"""
:meta private:
"""
if not callable(func):
raise ProblemLoadDataIsNotCallable(func)
self._space["augment"] = {"func": func, "kwargs": kwargs}
def search_space(self, space_class, **kwargs):
"""Set a search space for neural architecture search.
Args:
space_class (KSearchSpace): an object of type ``KSearchSpace`` which has to implement the ``build()`` method.
Raises:
SearchSpaceBuilderMissingParameter: raised when either of ``(input_shape, output_shape)`` are missing parameters of ``func``.
"""
sign = signature(space_class)
if not "input_shape" in sign.parameters:
raise SearchSpaceBuilderMissingParameter("input_shape")
if not "output_shape" in sign.parameters:
raise SearchSpaceBuilderMissingParameter("output_shape")
self._space["search_space"] = {"class": space_class, "kwargs": kwargs}
def add_hyperparameter(self,
value,
name: str = None,
default_value=None) -> csh.Hyperparameter:
"""Add hyperparameters to search the neural architecture search problem.
>>> Problem.hyperparameters(
... batch_size=problem.add_hyperparameter((32, 256), "batch_size")
... )
Args:
value: a hyperparameter description.
name: a name of the defined hyperparameter, the same as the current key.
default_value (Optional): a default value of the hyperparameter.
Returns:
Hyperparameter: the defined hyperparameter.
"""
return self._hp_space.add_hyperparameter(value, name, default_value)
def preprocessing(self, func: callable):
"""Define how to preprocess your data.
Args:
func (callable): a function which returns a preprocessing scikit-learn ``Pipeline``.
"""
if not callable(func):
raise ProblemPreprocessingIsNotCallable(func)
self._space["preprocessing"] = {"func": func}
def hyperparameters(self, **kwargs):
"""Define hyperparameters used to evaluate generated architectures.
Hyperparameters can be defined such as:
.. code-block:: python
Problem.hyperparameters(
batch_size=256,
learning_rate=0.01,
optimizer="adam",
num_epochs=20,
verbose=0,
callbacks=dict(...),
)
"""
if self._space.get("hyperparameters") is None:
self._space["hyperparameters"] = dict()
self._space["hyperparameters"].update(kwargs)
def loss(self, loss, loss_weights=None, class_weights=None):
"""Define the loss used to train generated architectures.
It can be a ``str`` corresponding to a Keras loss function:
.. code-block:: python
problem.loss("categorical_crossentropy")
A custom loss function can also be defined:
.. code-block:: python
def NLL(y, rv_y):
return -rv_y.log_prob(y)
problem.loss(NLL)
The loss can be automatically searched:
.. code-block:: python
problem.loss(
problem.add_hyperparameter(
["mae", "mse", "huber_loss", "log_cosh", "mape", "msle"], "loss"
)
)
It is possible to define a different loss for each output:
.. code-block:: python
problem.loss(
loss={"output_0": "mse", "output_1": "mse"},
loss_weights={"output_0": 0.0, "output_1": 1.0},
)
Args:
loss (str or callable orlist): a string indicating a specific loss function.
loss_weights (list): Optional.
class_weights (dict): Optional.
"""
if not (type(loss) is csh.CategoricalHyperparameter):
if not type(loss) is str and not callable(
loss) and not type(loss) is dict:
raise RuntimeError(
f"The loss should be either a str, dict or a callable when it's of type {type(loss)}"
)
if (type(loss) is dict and loss_weights is not None
and len(loss) != len(loss_weights)):
raise RuntimeError(
f"The losses list (len={len(loss)}) and the weights list (len={len(loss_weights)}) should be of same length!"
)
self._space["loss"] = loss
if loss_weights is not None:
self._space["loss_weights"] = loss_weights
if class_weights is not None:
self._space["class_weights"] = class_weights
def metrics(self, metrics=None):
"""Define a list of metrics for the training of generated architectures.
A list of metrics can be defined to be monitored or used as an objective. It can be a keyword or a callable. For example, if it is a keyword:
.. code-block:: python
problem.metrics(["acc"])
In case you need multiple metrics:
.. code-block:: python
problem.metrics["mae", "mse"]
In case you want to use a custom metric:
.. code-block:: python
def sparse_perplexity(y_true, y_pred):
cross_entropy = tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred)
perplexity = tf.pow(2.0, cross_entropy)
return perplexity
problem.metrics([sparse_perplexity])
Args:
metrics (list(str or callable) or dict): If ``str`` the metric should be defined in Keras or in DeepHyper. If ``callable`` it should take 2 arguments ``(y_pred, y_true)`` which are a prediction and a true value respectively.
"""
if metrics is None:
metrics = []
self._space["metrics"] = metrics
def check_objective(self, objective):
"""
:meta private:
"""
if not type(objective) is str and not callable(objective):
raise WrongProblemObjective(objective)
# elif type(objective) is str:
# list_suffix = ["__min", "__max", "__last"]
# for suffix in list_suffix:
# if suffix in objective:
# objective = objective.replace(suffix, "")
# break # only one suffix autorized
# objective = objective.replace("val_", "")
# possible_names = list()
# if type(self._space["metrics"]) is dict:
# metrics = list(self._space["metrics"].values())
# for k in self._space["metrics"].keys():
# objective = objective.replace(f"{k}_", "")
# else: # assuming it s a list
# metrics = self._space["metrics"]
# for val in ["loss"] + metrics:
# if callable(val):
# possible_names.append(val.__name__)
# else:
# possible_names.append(val)
# if not (objective in possible_names):
# raise WrongProblemObjective(objective, possible_names)
def objective(self, objective):
"""Define the objective you want to maximize for the search.
If you want to use the validation accuracy at the last epoch:
.. code-block:: python
problem.objective("val_acc")
.. note:: Be sure to define ``acc`` in the ``problem.metrics(["acc"])``.
It can accept some prefix and suffix such as ``__min, __max, __last``:
.. code-block:: python
problem.objective("-val_acc__max")
It can be a ``callable``:
.. code-block:: python
def myobjective(history: dict) -> float:
return history["val_acc"][-1]
problem.objective(myobjective)
Args:
objective (str or callable): The objective will be maximized. If ``objective`` is ``str`` then it should be either 'loss' or a defined metric. You can use the ``'val_'`` prefix when you want to select the objective on the validation set. You can use one of ``['__min', '__max', '__last']`` which respectively means you want to select the min, max or last value among all epochs. Using '__last' will save a consequent compute time because the evaluation will not compute metrics on validation set for all epochs but the last. If ``objective`` is callable it should return a scalar value (i.e. float) and it will take a ``dict`` parameter. The ``dict`` will contain keys corresponding to loss and metrics such as ``['loss', 'val_loss', 'r2', 'val_r2]``, it will also contains ``'n_parameters'`` which corresponds to the number of trainable parameters of the current model, ``'training_time'`` which corresponds to the time required to train the model, ``'predict_time'`` which corresponds to the time required to make a prediction over the whole validation set. If this callable has a ``'__last'`` suffix then the evaluation will only compute validation loss/metrics for the last epoch. If this callable has contains 'with_pred' in its name then the ``dict`` will have two other keys ``['y_pred', 'y_true']`` where ``'y_pred`` corresponds to prediction of the model on validation set and ``'y_true'`` corresponds to real prediction.
Raise:
WrongProblemObjective: raised when the objective is of a wrong definition.
"""
if (not self._space.get("loss") is None
and not self._space.get("metrics") is None):
self.check_objective(objective)
else:
raise NaProblemError(
".loss and .metrics should be defined before .objective!")
self._space["objective"] = objective
@property
def space(self):
keys = list(self._space.keys())
keys.sort()
space = OrderedDict(**{d: self._space[d] for d in keys})
return space
def build_search_space(self, seed=None):
"""Build and return a search space object using the infered data shapes after loading data.
Returns:
KSearchSpace: A search space instance.
"""
config = self.space
input_shape, output_shape, _ = setup_data(config, add_to_config=False)
search_space = get_search_space(config,
input_shape,
output_shape,
seed=seed)
return search_space
def get_keras_model(self, arch_seq: list) -> tf.keras.Model:
"""Get a keras model object from a set of decisions in the current search space.
Args:
arch_seq (list): a list of int of floats describing a choice of operations for the search space defined in the current problem.
"""
search_space = self.build_search_space()
return search_space.sample(arch_seq)
def gen_config(self, arch_seq: list, hp_values: list) -> dict:
"""Generate a ``dict`` configuration from the ``arch_seq`` and ``hp_values`` passed.
Args:
arch_seq (list): a valid embedding of a neural network described by the search space of the current ``NaProblem``.
hp_values (list): a valid list of hyperparameters corresponding to the defined hyperparameters of the current ``NaProblem``.
"""
config = deepcopy(self.space)
# architecture DNA
config["arch_seq"] = arch_seq
# replace hp values in the config
hp_names = self._hp_space._space.get_hyperparameter_names()
for hp_name, hp_value in zip(hp_names, hp_values):
if hp_name == "loss":
config["loss"] = hp_value
else:
config["hyperparameters"][hp_name] = hp_value
return config
def extract_hp_values(self, config):
"""Extract the value of hyperparameters present in ``config`` based on the defined hyperparameters in the current ``NaProblem``"""
hp_names = self.hyperparameter_names
hp_values = []
for hp_name in hp_names:
if hp_name == "loss":
hp_values.append(config["loss"])
else:
hp_values.append(config["hyperparameters"][hp_name])
return hp_values
@property
def hyperparameter_names(self):
"""The list of hyperparameters names."""
return self._hp_space.hyperparameter_names
@property
def default_hp_configuration(self):
"""The default configuration as a dictionnary."""
return self._hp_space.default_configuration
