def __init__(self,
num_workers: int,
space,
learner,
acq_func,
liar_strategy,
KAPPA=1.96,
SEED=12345,
**kwargs):
assert learner in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {learner}"
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.space = space
self.learner = learner
self.acq_func = acq_func
self.liar_strategy = liar_strategy
self.KAPPA = KAPPA
self.SEED = SEED
n_init = inf if learner == 'DUMMY' else num_workers
if isinstance(self.space, CS.ConfigurationSpace):
# Pass on seed for replicable RNG
self.space.seed(self.SEED)
self._optimizer = SkOptimizer(
dimensions=self.space,
base_estimator=self.learner,
acq_optimizer='sampling',
acq_func=self.acq_func,
acq_func_kwargs={'kappa': self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init)
else:
self._optimizer = SkOptimizer(
dimensions=self.space.dimensions,
base_estimator=self.learner,
acq_optimizer='sampling',
acq_func=self.acq_func,
acq_func_kwargs={'kappa': self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init)
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" %
self.learner)
def __init__(self, num_workers: int, space, learner, acq_func,
liar_strategy, **kwargs):
assert learner in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {learner}"
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.space = space
self.learner = learner
self.acq_func = acq_func
self.liar_strategy = liar_strategy
n_init = inf if learner == 'DUMMY' else num_workers
self._optimizer = SkOptimizer(dimensions=self.space.dimensions,
base_estimator=self.learner,
acq_optimizer='sampling',
acq_func=self.acq_func,
acq_func_kwargs={'kappa': self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init)
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" %
self.learner)
def __init__(
self,
problem,
num_workers,
surrogate_model="RF",
acq_func="gp_hedge",
acq_kappa=1.96,
acq_xi=None,
liar_strategy="cl_max",
n_jobs=1,
**kwargs,
):
assert surrogate_model in [
"RF",
"ET",
"GBRT",
"GP",
"DUMMY",
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.space = problem.space
# queue of remaining starting points
self.starting_points = problem.starting_point
n_init = (inf if surrogate_model == "DUMMY" else max(
num_workers, len(self.starting_points)))
# Set acq_func_kwargs parameters
acq_func_kwargs = {}
if type(acq_kappa) is float:
acq_func_kwargs["kappa"] = acq_kappa
if type(acq_xi) is float:
acq_func_kwargs["xi"] = acq_xi
self._optimizer = SkOptimizer(
dimensions=self.space,
base_estimator=base_estimator,
acq_optimizer="sampling",
acq_func=acq_func,
acq_func_kwargs=acq_func_kwargs,
random_state=self.SEED,
n_initial_points=n_init,
)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info(
f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
plot="true",
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.do_plot = plot == "true"
self.n_jobs = int(n_jobs)
# Initialize Hyperaparameter space
# self.hp_space = cs.ConfigurationSpace(seed=42)
# self.hp_space.add_hyperparameter(
# check_hyperparameter(
# self.problem.space["hyperparameters"]["learning_rate"], "learning_rate"
# )
# )
# self.hp_space.add_hyperparameter(
# check_hyperparameter(
# self.problem.space["hyperparameters"]["batch_size"], "batch_size"
# )
# )
self.hp_space = []
self.hp_space.append(
self.problem.space["hyperparameters"]["learning_rate"])
# ploting
lr_range = self.problem.space["hyperparameters"]["learning_rate"][:2]
self.domain_x = np.linspace(*lr_range, 400).reshape(-1, 1)
# Initialize opitmizer of hyperparameter space
acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration
self.n_initial_points = self.free_workers
self.hp_opt = SkOptimizer(
dimensions=self.hp_space,
base_estimator=RandomForestRegressor(n_jobs=32),
# base_estimator=RandomForestRegressor(n_jobs=self.n_jobs),
acq_func="LCB",
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
# model_queue_size=100,
)
def __init__(
self,
problem,
run,
evaluator,
surrogate_model="RF",
acq_func="LCB",
kappa=1.96,
xi=0.001,
liar_strategy="cl_min",
n_jobs=1,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
self.kappa = float(kappa)
self.xi = float(xi)
self.n_initial_points = self.evaluator.num_workers
self.liar_strategy = liar_strategy
# Setup
na_search_space = self.problem.build_search_space()
self.hp_space = self.problem._hp_space #! hyperparameters
self.hp_size = len(self.hp_space.space.get_hyperparameter_names())
self.na_space = HpProblem(self.problem.seed)
for i, vnode in enumerate(na_search_space.variable_nodes):
self.na_space.add_hyperparameter((0, vnode.num_ops - 1),
name=f"vnode_{i:05d}")
self.space = CS.ConfigurationSpace(seed=self.problem.seed)
self.space.add_configuration_space(
prefix="1", configuration_space=self.hp_space.space)
self.space.add_configuration_space(
prefix="2", configuration_space=self.na_space.space)
# Initialize opitmizer of hyperparameter space
self.opt = SkOptimizer(
dimensions=self.space,
base_estimator=self.get_surrogate_model(surrogate_model,
self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs={
"xi": self.xi,
"kappa": self.kappa
},
n_initial_points=self.n_initial_points,
)
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
kappa=0.001,
xi=0.000001,
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(n_jobs)
# Initialize Hyperaparameter space
self.hp_space = []
# add the 'learning_rate' space to the HPO search space
self.hp_space.append(
self.problem.space["hyperparameters"]["learning_rate"])
# add the 'batch_size' space to the HPO search space
self.hp_space.append(
self.problem.space["hyperparameters"]["batch_size"])
# add the 'num_ranks_per_node' space to the HPO search space
self.hp_space.append(
self.problem.space["hyperparameters"]["ranks_per_node"])
# Initialize opitmizer of hyperparameter space
acq_func_kwargs = {
"xi": float(xi),
"kappa": float(kappa)
} # tiny exploration
# self.free_workers = 128 #! TODO: test
self.n_initial_points = self.free_workers
self.hp_opt = SkOptimizer(
dimensions=self.hp_space,
base_estimator=RandomForestRegressor(n_jobs=32),
# base_estimator=RandomForestRegressor(n_jobs=4),
acq_func="LCB",
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
# model_queue_size=100,
)
def __init__(self,
problem,
num_workers,
surrogate_model='RF',
acq_func='gp_hedge',
acq_kappa=1.96,
liar_strategy='cl_max',
n_jobs=-1,
**kwargs):
assert surrogate_model in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.space = problem.space
cs_kwargs = self.space['create_search_space'].get('kwargs')
if cs_kwargs is None:
search_space = self.space['create_search_space']['func']()
else:
search_space = self.space['create_search_space']['func'](
**cs_kwargs)
# // queue of remaining starting points
# // self.starting_points = problem.starting_point
n_init = np.inf if surrogate_model == 'DUMMY' else num_workers
self.starting_points = [] # ! EMPTY for now TODO
# Building search space for SkOptimizer
skopt_space = [(0, vnode.num_ops - 1)
for vnode in search_space.variable_nodes]
self._optimizer = SkOptimizer(skopt_space,
base_estimator=base_estimator,
acq_optimizer='sampling',
acq_func=acq_func,
acq_func_kwargs={'kappa': acq_kappa},
random_state=self.SEED,
n_initial_points=n_init)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" %
surrogate_model)
def __init__(self, problem, run, evaluator, **kwargs):
super().__init__(problem=problem,
run=run,
evaluator=evaluator,
**kwargs)
self.free_workers = self.evaluator.num_workers
dhlogger.info(
jm(
type="start_infos",
alg="bayesian-optimization-for-hpo-nas",
nworkers=self.evaluator.num_workers,
encoded_space=json.dumps(self.problem.space, cls=Encoder),
))
# Setup
self.pb_dict = self.problem.space
cs_kwargs = self.pb_dict["create_search_space"].get("kwargs")
if cs_kwargs is None:
search_space = self.pb_dict["create_search_space"]["func"]()
else:
search_space = self.pb_dict["create_search_space"]["func"](
**cs_kwargs)
self.space_list = [(0, vnode.num_ops - 1)
for vnode in search_space.variable_nodes]
# Initialize Hyperaparameter space
self.dimensions = []
self.size_ha = None # Number of algorithm hyperparameters in the dimension list
self.add_ha_dimensions()
self.add_hm_dimensions()
# Initialize opitmizer of hyperparameter space
# acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration
acq_func_kwargs = {"xi": 0.000001, "kappa": 1.96} # tiny exploration
self.n_initial_points = self.free_workers
self.opt = SkOptimizer(
dimensions=self.dimensions,
base_estimator=RandomForestRegressor(n_jobs=32),
# base_estimator=RandomForestRegressor(n_jobs=4),
acq_func="LCB",
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
# model_queue_size=100,
)
def __init__(self,
problem,
num_workers,
surrogate_model='RF',
acq_func='gp_hedge',
acq_kappa=1.96,
liar_strategy='cl_max',
n_jobs=1,
**kwargs):
assert surrogate_model in [
"RF", "ET", "GBRT", "GP", "DUMMY"
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.space = problem.space
# queue of remaining starting points
self.starting_points = problem.starting_point
n_init = inf if surrogate_model == 'DUMMY' else max(
num_workers, len(self.starting_points))
self._optimizer = SkOptimizer(self.space.values(),
base_estimator=base_estimator,
acq_optimizer='sampling',
acq_func=acq_func,
acq_func_kwargs={'kappa': acq_kappa},
random_state=self.SEED,
n_initial_points=n_init)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info(
f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__(
self,
problem,
run,
evaluator,
population_size=100,
sample_size=10,
n_jobs=1,
kappa=0.001,
xi=0.000001,
acq_func="LCB",
**kwargs,
):
super().__init__(
problem=problem,
run=run,
evaluator=evaluator,
population_size=population_size,
sample_size=sample_size,
**kwargs,
)
self.n_jobs = int(
n_jobs) # parallelism of BO surrogate model estimator
# Initialize Hyperaparameter space
self.hp_space = self.problem._hp_space
# Initialize opitmizer of hyperparameter space
acq_func_kwargs = {
"xi": float(xi),
"kappa": float(kappa)
} # tiny exploration
self.n_initial_points = self.free_workers
self.hp_opt = SkOptimizer(
dimensions=self.hp_space._space,
base_estimator=RandomForestRegressor(n_jobs=self.n_jobs),
acq_func=acq_func,
acq_optimizer="sampling",
acq_func_kwargs=acq_func_kwargs,
n_initial_points=self.n_initial_points,
)
def __init__(self, problem, num_workers, args):
assert args.learner in ["RF", "ET", "GBRT", "GP", "DUMMY"], f"Unknown scikit-optimize base_estimator: {args.learner}"
self.space = problem.space
n_init = inf if args.learner=='DUMMY' else num_workers
self._optimizer = SkOptimizer(
self.space.values(),
base_estimator=args.learner,
acq_optimizer='sampling',
acq_func=args.acq_func,
acq_func_kwargs={'kappa':self.KAPPA},
random_state=self.SEED,
n_initial_points=n_init
)
assert args.liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = args.liar_strategy
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" % args.learner)
def __init__(
self,
problem,
num_workers,
surrogate_model="RF",
acq_func="gp_hedge",
acq_kappa=1.96,
liar_strategy="cl_max",
n_jobs=1,
**kwargs,
):
assert surrogate_model in [
"RF",
"ET",
"GBRT",
"GP",
"DUMMY",
], f"Unknown scikit-optimize base_estimator: {surrogate_model}"
if surrogate_model == "RF":
base_estimator = RandomForestRegressor(n_jobs=n_jobs)
elif surrogate_model == "ET":
base_estimator = ExtraTreesRegressor(n_jobs=n_jobs)
elif surrogate_model == "GBRT":
base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs)
else:
base_estimator = surrogate_model
self.problem = problem
cs_kwargs = self.problem.space["create_search_space"].get("kwargs")
if cs_kwargs is None:
search_space = self.problem.space["create_search_space"]["func"]()
else:
search_space = self.problem.space["create_search_space"]["func"](
**cs_kwargs)
n_init = np.inf if surrogate_model == "DUMMY" else num_workers
self.starting_points = [] # ! EMPTY for now TODO
# Building search space for SkOptimizer using ConfigSpace
skopt_space = cs.ConfigurationSpace(seed=self.problem.seed)
for i, vnode in enumerate(search_space.variable_nodes):
hp = csh.UniformIntegerHyperparameter(name=f"vnode_{i}",
lower=0,
upper=(vnode.num_ops - 1))
skopt_space.add_hyperparameter(hp)
self._optimizer = SkOptimizer(
skopt_space,
base_estimator=base_estimator,
acq_optimizer="sampling",
acq_func=acq_func,
acq_func_kwargs={"kappa": acq_kappa},
random_state=self.SEED,
n_initial_points=n_init,
)
assert liar_strategy in "cl_min cl_mean cl_max".split()
self.strategy = liar_strategy
self.evals = {}
self.counter = 0
logger.info("Using skopt.Optimizer with %s base_estimator" %
surrogate_model)
