def test_nested():
from flaml import tune
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 8),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
obj = (config["cost_related"]["a"] - 4)**2 \
+ (config["b"] - config["cost_related"]["a"])**2
tune.report(obj=obj)
tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"])
analysis = tune.run(simple_func,
config=search_space,
low_cost_partial_config={"cost_related": {
"a": 1
}},
metric="obj",
mode="min",
metric_constraints=[("ab", "<=", 4)],
local_dir='logs/',
num_samples=-1,
time_budget_s=1)
best_trial = analysis.get_best_trial()
logger.info(f"Best config: {best_trial.config}")
logger.info(f"Best result: {best_trial.last_result}")
def set_basic_conf(self):
space = {
"height": tune.uniform(-100, 100),
"width": tune.randint(0, 100),
}
def cost(param):
tune.report(loss=(param["height"] - 14) ** 2 - abs(param["width"] - 3))
search_alg = CFO(
space=space,
metric="loss",
mode="min",
seed=20,
)
return search_alg, cost
def test_nested():
from flaml import tune
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 8),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
tune.report(metric=(config["cost_related"]["a"] - 4)**2 *
(config["b"] - 0.7)**2)
analysis = tune.run(simple_func,
init_config={"cost_related": {
"a": 1,
}},
metric="metric",
mode="min",
config=search_space,
local_dir='logs/',
num_samples=-1,
time_budget_s=1)
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def test_nested():
from flaml import tune, CFO
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 9),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
obj = (config["cost_related"]["a"] -
4)**2 + (config["b"] - config["cost_related"]["a"])**2
tune.report(obj=obj)
tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"])
analysis = tune.run(
simple_func,
search_alg=CFO(
space=search_space,
metric="obj",
mode="min",
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=[
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
{
"cost_related": {
"a": 8
}
},
],
metric_constraints=[("ab", "<=", 4)],
),
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"CFO best config: {best_trial.config}")
logger.info(f"CFO best result: {best_trial.last_result}")
analysis = tune.run(
simple_func,
search_alg=BlendSearch(
experimental=True,
space=search_space,
metric="obj",
mode="min",
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=[
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
{
"cost_related": {
"a": 8
}
},
],
metric_constraints=[("ab", "<=", 4)],
),
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"BlendSearch exp best config: {best_trial.config}")
logger.info(f"BlendSearch exp best result: {best_trial.last_result}")
points_to_evaluate = [
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
]
analysis = tune.run(
simple_func,
config=search_space,
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=points_to_evaluate,
evaluated_rewards=[(config["cost_related"]["a"] - 4)**2 +
(config["b"] - config["cost_related"]["a"])**2
for config in points_to_evaluate],
metric="obj",
mode="min",
metric_constraints=[("ab", "<=", 4)],
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"BlendSearch best config: {best_trial.config}")
logger.info(f"BlendSearch best result: {best_trial.last_result}")
def evaluate_config(config):
"""evaluate a hyperparameter configuration"""
# we uss a toy example with 2 hyperparameters
metric = (round(config["x"]) - 85000)**2 - config["x"] / config["y"]
# usually the evaluation takes an non-neglible cost
# and the cost could be related to certain hyperparameters
# in this example, we assume it's proportional to x
time.sleep(config["x"] / 100000)
# use tune.report to report the metric to optimize
tune.report(metric=metric)
config_search_space = {
"x": tune.lograndint(lower=1, upper=100000),
"y": tune.randint(lower=1, upper=100000),
}
low_cost_partial_config = {"x": 1}
def setup_searcher(searcher_name):
from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch
if "cfo" in searcher_name:
searcher = CFO(space=config_search_space,
low_cost_partial_config=low_cost_partial_config)
elif searcher_name == "bs":
searcher = BlendSearch(
metric="metric",
mode="min",
def test_record_incumbent(method="BlendSearch"):
if method != "CFOCat":
search_space = {
"x1": tune.randint(1, 9),
"x2": tune.randint(1, 9),
"x3": tune.randint(1, 9),
"x4": tune.randint(1, 9),
"x5": tune.randint(1, 9),
}
else:
search_space = {
"x1": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"x2": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"x3": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"x4": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"x5": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
}
max_iter = 100
num_samples = 128
time_budget_s = 1
n_cpu = 1
if method == "BlendSearch":
tune.run(
evaluation_function=rosenbrock_function,
config=search_space,
verbose=0,
metric="funcLoss",
mode="min",
max_resource=max_iter,
min_resource=1,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_incumbent_result_in_evaluation=True,
)
return
elif method == "CFO":
from flaml import CFO
algo = CFO(
use_incumbent_result_in_evaluation=True,
)
elif method == "CFOCat":
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
use_incumbent_result_in_evaluation=True,
)
else:
raise NotImplementedError
tune.run(
evaluation_function=rosenbrock_function,
metric="funcLoss",
mode="min",
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
search_alg=algo,
)
def test_define_by_run():
from flaml.tune.space import (
unflatten_hierarchical,
normalize,
indexof,
complete_config,
)
space = {
# Sample a float uniformly between -5.0 and -1.0
"uniform": tune.uniform(-5, -1),
# Sample a float uniformly between 3.2 and 5.4,
# rounding to increments of 0.2
"quniform": tune.quniform(3.2, 5.4, 0.2),
# Sample a float uniformly between 0.0001 and 0.01, while
# sampling in log space
"loguniform": tune.loguniform(1e-4, 1e-2),
# Sample a float uniformly between 0.0001 and 0.1, while
# sampling in log space and rounding to increments of 0.00005
"qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5),
# Sample a random float from a normal distribution with
# mean=10 and sd=2
# "randn": tune.randn(10, 2),
# Sample a random float from a normal distribution with
# mean=10 and sd=2, rounding to increments of 0.2
# "qrandn": tune.qrandn(10, 2, 0.2),
# Sample a integer uniformly between -9 (inclusive) and 15 (exclusive)
"randint": tune.randint(-9, 15),
# Sample a random uniformly between -21 (inclusive) and 12 (inclusive (!))
# rounding to increments of 3 (includes 12)
"qrandint": tune.qrandint(-21, 12, 3),
# Sample a integer uniformly between 1 (inclusive) and 10 (exclusive),
# while sampling in log space
"lograndint": tune.lograndint(1, 10),
# Sample a integer uniformly between 2 (inclusive) and 10 (inclusive (!)),
# while sampling in log space and rounding to increments of 2
"qlograndint": tune.qlograndint(2, 10, 2),
# Sample an option uniformly from the specified choices
"choice": tune.choice(["a", "b", "c"]),
"const": 5,
}
choice = {"nested": space}
bs = BlendSearch(
space={"c": tune.choice([choice])},
low_cost_partial_config={"c": choice},
metric="metric",
mode="max",
)
print(indexof(bs._gs.space["c"], choice))
print(indexof(bs._gs.space["c"], {"nested": {"const": 1}}))
config = bs._gs.suggest("t1")
print(config)
config = unflatten_hierarchical(config, bs._gs.space)[0]
print(config)
print(normalize({"c": [choice]}, bs._gs.space, config, {}, False))
space["randn"] = tune.randn(10, 2)
cfo = CFO(
space={"c": tune.choice([0, choice])},
metric="metric",
mode="max",
)
for i in range(5):
cfo.suggest(f"t{i}")
# print(normalize(config, bs._gs.space, config, {}, False))
print(complete_config({}, cfo._ls.space, cfo._ls))
