def _test_roberta(method='BlendSearch'):
max_num_epoch = 100
num_samples = -1
time_budget_s = 3600
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-5, 3e-5),
"weight_decay": flaml.tune.uniform(0, 0.3),
"per_device_train_batch_size": flaml.tune.choice([16, 32, 64, 128]),
"seed": flaml.tune.choice([12, 22, 33, 42]),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(
points_to_evaluate=[{
"num_train_epochs": 1,
"per_device_train_batch_size": 128,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch(max_concurrent=3)
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(train_roberta,
metric=HP_METRIC,
mode=MODE,
resources_per_trial={
"gpu": 4,
"cpu": 4
},
config=search_space,
local_dir='logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def _test_distillbert(method='BlendSearch'):
max_num_epoch = 64
num_samples = -1
time_budget_s = 10800
search_space = {
# You can mix constants with search space objects.
"num_train_epochs": flaml.tune.loguniform(1, max_num_epoch),
"learning_rate": flaml.tune.loguniform(1e-6, 1e-4),
"adam_beta1": flaml.tune.uniform(0.8, 0.99),
"adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4),
"adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7),
}
start_time = time.time()
ray.init(num_cpus=4, num_gpus=4)
if 'ASHA' == method:
algo = None
elif 'BOHB' == method:
from ray.tune.schedulers import HyperBandForBOHB
from ray.tune.suggest.bohb import tuneBOHB
algo = tuneBOHB(max_concurrent=4)
scheduler = HyperBandForBOHB(max_t=max_num_epoch)
elif 'Optuna' == method:
from ray.tune.suggest.optuna import OptunaSearch
algo = OptunaSearch()
elif 'CFO' == method:
from flaml import CFO
algo = CFO(points_to_evaluate=[{
"num_train_epochs": 1,
}])
elif 'BlendSearch' == method:
from flaml import BlendSearch
algo = BlendSearch(points_to_evaluate=[{
"num_train_epochs": 1,
}])
elif 'Dragonfly' == method:
from ray.tune.suggest.dragonfly import DragonflySearch
algo = DragonflySearch()
elif 'SkOpt' == method:
from ray.tune.suggest.skopt import SkOptSearch
algo = SkOptSearch()
elif 'Nevergrad' == method:
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne)
elif 'ZOOpt' == method:
from ray.tune.suggest.zoopt import ZOOptSearch
algo = ZOOptSearch(budget=num_samples)
elif 'Ax' == method:
from ray.tune.suggest.ax import AxSearch
algo = AxSearch()
elif 'HyperOpt' == method:
from ray.tune.suggest.hyperopt import HyperOptSearch
algo = HyperOptSearch()
scheduler = None
if method != 'BOHB':
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(max_t=max_num_epoch, grace_period=1)
scheduler = None
analysis = ray.tune.run(
train_distilbert,
metric=HP_METRIC,
mode=MODE,
# You can add "gpu": 1 to allocate GPUs
resources_per_trial={"gpu": 1},
config=search_space,
local_dir='test/logs/',
num_samples=num_samples,
time_budget_s=time_budget_s,
keep_checkpoints_num=1,
checkpoint_score_attr=HP_METRIC,
scheduler=scheduler,
search_alg=algo)
ray.shutdown()
best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all")
metric = best_trial.metric_analysis[HP_METRIC][MODE]
logger.info(f"method={method}")
logger.info(f"n_trials={len(analysis.trials)}")
logger.info(f"time={time.time()-start_time}")
logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}")
logger.info(f"Best model parameters: {best_trial.config}")
def test_logging_level(self):
from flaml import logger, logger_formatter
with tempfile.TemporaryDirectory() as d:
training_log = os.path.join(d, "training.log")
# Configure logging for the FLAML logger
# and add a handler that outputs to a buffer.
logger.setLevel(logging.INFO)
buf = io.StringIO()
ch = logging.StreamHandler(buf)
ch.setFormatter(logger_formatter)
logger.addHandler(ch)
# Run a simple job.
automl = AutoML()
automl_settings = {
"time_budget": 1,
"metric": "rmse",
"task": "regression",
"log_file_name": training_log,
"log_training_metric": True,
"n_jobs": 1,
"model_history": True,
"keep_search_state": True,
"learner_selector": "roundrobin",
}
X_train, y_train = fetch_california_housing(return_X_y=True)
n = len(y_train) >> 1
print(automl.model, automl.classes_, automl.predict(X_train))
automl.fit(X_train=X_train[:n],
y_train=y_train[:n],
X_val=X_train[n:],
y_val=y_train[n:],
**automl_settings)
logger.info(automl.search_space)
logger.info(automl.low_cost_partial_config)
logger.info(automl.points_to_evaluate)
logger.info(automl.cat_hp_cost)
import optuna as ot
study = ot.create_study()
from flaml.tune.space import define_by_run_func, add_cost_to_space
sample = define_by_run_func(study.ask(), automl.search_space)
logger.info(sample)
logger.info(unflatten_hierarchical(sample, automl.search_space))
add_cost_to_space(automl.search_space,
automl.low_cost_partial_config,
automl.cat_hp_cost)
logger.info(automl.search_space["ml"].categories)
if automl.best_config:
config = automl.best_config.copy()
config["learner"] = automl.best_estimator
automl.trainable({"ml": config})
from flaml import tune, BlendSearch
from flaml.automl import size
from functools import partial
low_cost_partial_config = automl.low_cost_partial_config
search_alg = BlendSearch(
metric="val_loss",
mode="min",
space=automl.search_space,
low_cost_partial_config=low_cost_partial_config,
points_to_evaluate=automl.points_to_evaluate,
cat_hp_cost=automl.cat_hp_cost,
resource_attr=automl.resource_attr,
min_resource=automl.min_resource,
max_resource=automl.max_resource,
config_constraints=[(partial(size, automl._state), "<=",
automl._mem_thres)],
metric_constraints=automl.metric_constraints,
)
analysis = tune.run(
automl.trainable,
search_alg=search_alg, # verbose=2,
time_budget_s=1,
num_samples=-1,
)
print(
min(trial.last_result["val_loss"]
for trial in analysis.trials))
config = analysis.trials[-1].last_result["config"]["ml"]
automl._state._train_with_config(config["learner"], config)
for _ in range(3):
print(
search_alg._ls.complete_config(
low_cost_partial_config,
search_alg._ls_bound_min,
search_alg._ls_bound_max,
))
# Check if the log buffer is populated.
self.assertTrue(len(buf.getvalue()) > 0)
import pickle
with open("automl.pkl", "wb") as f:
pickle.dump(automl, f, pickle.HIGHEST_PROTOCOL)
print(automl.__version__)
pred1 = automl.predict(X_train)
with open("automl.pkl", "rb") as f:
automl = pickle.load(f)
pred2 = automl.predict(X_train)
delta = pred1 - pred2
assert max(delta) == 0 and min(delta) == 0
automl.save_best_config("test/housing.json")
def test_define_by_run():
from flaml.tune.space import (
unflatten_hierarchical,
normalize,
indexof,
complete_config,
)
space = {
# Sample a float uniformly between -5.0 and -1.0
"uniform": tune.uniform(-5, -1),
# Sample a float uniformly between 3.2 and 5.4,
# rounding to increments of 0.2
"quniform": tune.quniform(3.2, 5.4, 0.2),
# Sample a float uniformly between 0.0001 and 0.01, while
# sampling in log space
"loguniform": tune.loguniform(1e-4, 1e-2),
# Sample a float uniformly between 0.0001 and 0.1, while
# sampling in log space and rounding to increments of 0.00005
"qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5),
# Sample a random float from a normal distribution with
# mean=10 and sd=2
# "randn": tune.randn(10, 2),
# Sample a random float from a normal distribution with
# mean=10 and sd=2, rounding to increments of 0.2
# "qrandn": tune.qrandn(10, 2, 0.2),
# Sample a integer uniformly between -9 (inclusive) and 15 (exclusive)
"randint": tune.randint(-9, 15),
# Sample a random uniformly between -21 (inclusive) and 12 (inclusive (!))
# rounding to increments of 3 (includes 12)
"qrandint": tune.qrandint(-21, 12, 3),
# Sample a integer uniformly between 1 (inclusive) and 10 (exclusive),
# while sampling in log space
"lograndint": tune.lograndint(1, 10),
# Sample a integer uniformly between 2 (inclusive) and 10 (inclusive (!)),
# while sampling in log space and rounding to increments of 2
"qlograndint": tune.qlograndint(2, 10, 2),
# Sample an option uniformly from the specified choices
"choice": tune.choice(["a", "b", "c"]),
"const": 5,
}
choice = {"nested": space}
bs = BlendSearch(
space={"c": tune.choice([choice])},
low_cost_partial_config={"c": choice},
metric="metric",
mode="max",
)
print(indexof(bs._gs.space["c"], choice))
print(indexof(bs._gs.space["c"], {"nested": {"const": 1}}))
config = bs._gs.suggest("t1")
print(config)
config = unflatten_hierarchical(config, bs._gs.space)[0]
print(config)
print(normalize({"c": [choice]}, bs._gs.space, config, {}, False))
space["randn"] = tune.randn(10, 2)
cfo = CFO(
space={"c": tune.choice([0, choice])},
metric="metric",
mode="max",
)
for i in range(5):
cfo.suggest(f"t{i}")
# print(normalize(config, bs._gs.space, config, {}, False))
print(complete_config({}, cfo._ls.space, cfo._ls))