def test_scheduler(scheduler=None):
from functools import partial
resource_attr = "samplesize"
max_resource = 10000
# specify the objective functions
if scheduler is None:
evaluation_obj = simple_obj
elif scheduler == "flaml":
evaluation_obj = partial(obj_w_suggested_resource, resource_attr)
elif scheduler == "asha" or isinstance(scheduler, TrialScheduler):
evaluation_obj = partial(obj_w_intermediate_report, max_resource)
else:
try:
from ray.tune.schedulers import TrialScheduler as RayTuneTrialScheduler
except ImportError:
print(
"skip this condition, which may require TrialScheduler from ray tune, \
as ray tune cannot be imported.")
return
if isinstance(scheduler, RayTuneTrialScheduler):
evaluation_obj = partial(obj_w_intermediate_report, max_resource)
else:
raise ValueError
analysis = tune.run(
evaluation_obj,
config={
"x": tune.uniform(5, 20),
"y": tune.uniform(0, 10),
"z": tune.uniform(0, 10),
},
metric="sphere_projection",
mode="max",
verbose=1,
resource_attr=resource_attr,
scheduler=scheduler,
max_resource=max_resource,
min_resource=100,
reduction_factor=2,
time_budget_s=1,
num_samples=500,
)
print("Best hyperparameters found were: ", analysis.best_config)
# print(analysis.get_best_trial)
return analysis.best_config
def test_nested():
from flaml import tune
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 8),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
obj = (config["cost_related"]["a"] - 4)**2 \
+ (config["b"] - config["cost_related"]["a"])**2
tune.report(obj=obj)
tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"])
analysis = tune.run(simple_func,
config=search_space,
low_cost_partial_config={"cost_related": {
"a": 1
}},
metric="obj",
mode="min",
metric_constraints=[("ab", "<=", 4)],
local_dir='logs/',
num_samples=-1,
time_budget_s=1)
best_trial = analysis.get_best_trial()
logger.info(f"Best config: {best_trial.config}")
logger.info(f"Best result: {best_trial.last_result}")
def set_basic_conf(self):
space = {
"height": tune.uniform(-100, 100),
"width": tune.randint(0, 100),
}
def cost(param):
tune.report(loss=(param["height"] - 14) ** 2 - abs(param["width"] - 3))
search_alg = CFO(
space=space,
metric="loss",
mode="min",
seed=20,
)
return search_alg, cost
def search_space(cls, **params):
# (1) Get the hps in the original search space
space = LGBMEstimator.search_space(**params)
# (2) Set up the fixed value from hps from the starting point
for hp_name in hps_to_freeze:
# if an hp is specifed to be freezed, use tine value provided in the starting_point
# otherwise use the setting from the original search space
if hp_name in starting_point:
space[hp_name] = {"domain": starting_point[hp_name]}
# (3.1) Configure the search space for hps that are in the original search space
# but you want to change something, for example the range.
revised_hps_to_search = {
"n_estimators": {
"domain":
tune.lograndint(lower=10, upper=32768),
"init_value":
starting_point.get("n_estimators")
or space["n_estimators"].get("init_value", 10),
"low_cost_init_value":
space["n_estimators"].get("low_cost_init_value", 10),
},
"num_leaves": {
"domain":
tune.lograndint(lower=10, upper=3276),
"init_value":
starting_point.get("num_leaves")
or space["num_leaves"].get("init_value", 10),
"low_cost_init_value":
space["num_leaves"].get("low_cost_init_value", 10),
},
# (3.2) Add a new hp which is not in the original search space
"subsample": {
"domain": tune.uniform(lower=0.1, upper=1.0),
"init_value": 0.1,
},
}
space.update(revised_hps_to_search)
return space
def test_nested():
from flaml import tune
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 8),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
tune.report(metric=(config["cost_related"]["a"] - 4)**2 *
(config["b"] - 0.7)**2)
analysis = tune.run(simple_func,
init_config={"cost_related": {
"a": 1,
}},
metric="metric",
mode="min",
config=search_space,
local_dir='logs/',
num_samples=-1,
time_budget_s=1)
def _test_xgboost(method='BlendSearch'):
try:
import ray
except ImportError:
return
if method == 'BlendSearch':
from flaml import tune
else:
from ray import tune
search_space = {
# You can mix constants with search space objects.
"max_depth": tune.randint(1, 8) if method in [
"BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight": tune.choice([1, 2, 3]),
"subsample": tune.uniform(0.5, 1.0),
"eta": tune.loguniform(1e-4, 1e-1)
}
max_iter = 10
for num_samples in [256]:
time_budget_s = 60 #None
for n_cpu in [8]:
start_time = time.time()
ray.init(num_cpus=n_cpu, num_gpus=0)
if method == 'BlendSearch':
analysis = tune.run(
train_breast_cancer,
init_config={
"max_depth": 1,
"min_child_weight": 3,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
report_intermediate_result=True,
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir='logs/',
num_samples=num_samples*n_cpu,
time_budget_s=time_budget_s,
use_ray=True)
else:
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"max_depth": 1,
"min_child_weight": 3,
}], cat_hp_cost={
"min_child_weight": [6, 3, 2],
})
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples*n_cpu)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(
max_t=max_iter,
grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space, local_dir='logs/',
num_samples=num_samples*n_cpu, time_budget_s=time_budget_s,
scheduler=scheduler, search_alg=algo)
ray.shutdown()
# # Load the best model checkpoint
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss","min","all")
accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method="BlendSearch"):
try:
import ray
except ImportError:
return
if method == "BlendSearch":
from flaml import tune
else:
from ray import tune
search_space = {
"max_depth":
tune.randint(1, 9)
if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9),
"min_child_weight":
tune.choice([1, 2, 3]),
"subsample":
tune.uniform(0.5, 1.0),
"eta":
tune.loguniform(1e-4, 1e-1),
}
max_iter = 10
for num_samples in [128]:
time_budget_s = 60
for n_cpu in [2]:
start_time = time.time()
# ray.init(address='auto')
if method == "BlendSearch":
analysis = tune.run(
train_breast_cancer,
config=search_space,
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
metric="eval-logloss",
mode="min",
max_resource=max_iter,
min_resource=1,
scheduler="asha",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
use_ray=True,
)
else:
if "ASHA" == method:
algo = None
elif "BOHB" == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import TuneBOHB
algo = TuneBOHB(max_concurrent=n_cpu)
scheduler = HyperBandForBOHB(max_t=max_iter)
elif "Optuna" == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif "CFO" == method:
from flaml import CFO
algo = CFO(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "CFOCat" == method:
from flaml.searcher.cfo_cat import CFOCat
algo = CFOCat(
low_cost_partial_config={
"max_depth": 1,
},
cat_hp_cost={
"min_child_weight": [6, 3, 2],
},
)
elif "Dragonfly" == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif "SkOpt" == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif "Nevergrad" == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif "ZOOpt" == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples * n_cpu)
elif "Ax" == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif "HyperOpt" == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != "BOHB":
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_iter, grace_period=1)
analysis = tune.run(
train_breast_cancer,
metric="eval-logloss",
mode="min",
# You can add "gpu": 0.1 to allocate GPUs
resources_per_trial={"cpu": 1},
config=search_space,
local_dir="logs/",
num_samples=num_samples * n_cpu,
time_budget_s=time_budget_s,
scheduler=scheduler,
search_alg=algo,
)
# # Load the best model checkpoint
# import os
# best_bst = xgb.Booster()
# best_bst.load_model(os.path.join(analysis.best_checkpoint,
# "model.xgb"))
best_trial = analysis.get_best_trial("eval-logloss", "min", "all")
accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"]
logloss = best_trial.metric_analysis["eval-logloss"]["min"]
logger.info(f"method={method}")
logger.info(f"n_samples={num_samples*n_cpu}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval loss: {logloss:.4f}")
logger.info(f"Best model total accuracy: {accuracy:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def test_nested():
from flaml import tune, CFO
search_space = {
# test nested search space
"cost_related": {
"a": tune.randint(1, 9),
},
"b": tune.uniform(0.5, 1.0),
}
def simple_func(config):
obj = (config["cost_related"]["a"] -
4)**2 + (config["b"] - config["cost_related"]["a"])**2
tune.report(obj=obj)
tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"])
analysis = tune.run(
simple_func,
search_alg=CFO(
space=search_space,
metric="obj",
mode="min",
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=[
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
{
"cost_related": {
"a": 8
}
},
],
metric_constraints=[("ab", "<=", 4)],
),
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"CFO best config: {best_trial.config}")
logger.info(f"CFO best result: {best_trial.last_result}")
analysis = tune.run(
simple_func,
search_alg=BlendSearch(
experimental=True,
space=search_space,
metric="obj",
mode="min",
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=[
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
{
"cost_related": {
"a": 8
}
},
],
metric_constraints=[("ab", "<=", 4)],
),
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"BlendSearch exp best config: {best_trial.config}")
logger.info(f"BlendSearch exp best result: {best_trial.last_result}")
points_to_evaluate = [
{
"b": 0.99,
"cost_related": {
"a": 3
}
},
{
"b": 0.99,
"cost_related": {
"a": 2
}
},
]
analysis = tune.run(
simple_func,
config=search_space,
low_cost_partial_config={"cost_related": {
"a": 1
}},
points_to_evaluate=points_to_evaluate,
evaluated_rewards=[(config["cost_related"]["a"] - 4)**2 +
(config["b"] - config["cost_related"]["a"])**2
for config in points_to_evaluate],
metric="obj",
mode="min",
metric_constraints=[("ab", "<=", 4)],
local_dir="logs/",
num_samples=-1,
time_budget_s=1,
)
best_trial = analysis.get_best_trial()
logger.info(f"BlendSearch best config: {best_trial.config}")
logger.info(f"BlendSearch best result: {best_trial.last_result}")
def test_define_by_run():
from flaml.tune.space import (
unflatten_hierarchical,
normalize,
indexof,
complete_config,
)
space = {
# Sample a float uniformly between -5.0 and -1.0
"uniform": tune.uniform(-5, -1),
# Sample a float uniformly between 3.2 and 5.4,
# rounding to increments of 0.2
"quniform": tune.quniform(3.2, 5.4, 0.2),
# Sample a float uniformly between 0.0001 and 0.01, while
# sampling in log space
"loguniform": tune.loguniform(1e-4, 1e-2),
# Sample a float uniformly between 0.0001 and 0.1, while
# sampling in log space and rounding to increments of 0.00005
"qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5),
# Sample a random float from a normal distribution with
# mean=10 and sd=2
# "randn": tune.randn(10, 2),
# Sample a random float from a normal distribution with
# mean=10 and sd=2, rounding to increments of 0.2
# "qrandn": tune.qrandn(10, 2, 0.2),
# Sample a integer uniformly between -9 (inclusive) and 15 (exclusive)
"randint": tune.randint(-9, 15),
# Sample a random uniformly between -21 (inclusive) and 12 (inclusive (!))
# rounding to increments of 3 (includes 12)
"qrandint": tune.qrandint(-21, 12, 3),
# Sample a integer uniformly between 1 (inclusive) and 10 (exclusive),
# while sampling in log space
"lograndint": tune.lograndint(1, 10),
# Sample a integer uniformly between 2 (inclusive) and 10 (inclusive (!)),
# while sampling in log space and rounding to increments of 2
"qlograndint": tune.qlograndint(2, 10, 2),
# Sample an option uniformly from the specified choices
"choice": tune.choice(["a", "b", "c"]),
"const": 5,
}
choice = {"nested": space}
bs = BlendSearch(
space={"c": tune.choice([choice])},
low_cost_partial_config={"c": choice},
metric="metric",
mode="max",
)
print(indexof(bs._gs.space["c"], choice))
print(indexof(bs._gs.space["c"], {"nested": {"const": 1}}))
config = bs._gs.suggest("t1")
print(config)
config = unflatten_hierarchical(config, bs._gs.space)[0]
print(config)
print(normalize({"c": [choice]}, bs._gs.space, config, {}, False))
space["randn"] = tune.randn(10, 2)
cfo = CFO(
space={"c": tune.choice([0, choice])},
metric="metric",
mode="max",
)
for i in range(5):
cfo.suggest(f"t{i}")
# print(normalize(config, bs._gs.space, config, {}, False))
print(complete_config({}, cfo._ls.space, cfo._ls))
