def test_scheduler(scheduler=None): from functools import partial resource_attr = "samplesize" max_resource = 10000 # specify the objective functions if scheduler is None: evaluation_obj = simple_obj elif scheduler == "flaml": evaluation_obj = partial(obj_w_suggested_resource, resource_attr) elif scheduler == "asha" or isinstance(scheduler, TrialScheduler): evaluation_obj = partial(obj_w_intermediate_report, max_resource) else: try: from ray.tune.schedulers import TrialScheduler as RayTuneTrialScheduler except ImportError: print( "skip this condition, which may require TrialScheduler from ray tune, \ as ray tune cannot be imported.") return if isinstance(scheduler, RayTuneTrialScheduler): evaluation_obj = partial(obj_w_intermediate_report, max_resource) else: raise ValueError analysis = tune.run( evaluation_obj, config={ "x": tune.uniform(5, 20), "y": tune.uniform(0, 10), "z": tune.uniform(0, 10), }, metric="sphere_projection", mode="max", verbose=1, resource_attr=resource_attr, scheduler=scheduler, max_resource=max_resource, min_resource=100, reduction_factor=2, time_budget_s=1, num_samples=500, ) print("Best hyperparameters found were: ", analysis.best_config) # print(analysis.get_best_trial) return analysis.best_config
def test_nested(): from flaml import tune search_space = { # test nested search space "cost_related": { "a": tune.randint(1, 8), }, "b": tune.uniform(0.5, 1.0), } def simple_func(config): obj = (config["cost_related"]["a"] - 4)**2 \ + (config["b"] - config["cost_related"]["a"])**2 tune.report(obj=obj) tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"]) analysis = tune.run(simple_func, config=search_space, low_cost_partial_config={"cost_related": { "a": 1 }}, metric="obj", mode="min", metric_constraints=[("ab", "<=", 4)], local_dir='logs/', num_samples=-1, time_budget_s=1) best_trial = analysis.get_best_trial() logger.info(f"Best config: {best_trial.config}") logger.info(f"Best result: {best_trial.last_result}")
def set_basic_conf(self): space = { "height": tune.uniform(-100, 100), "width": tune.randint(0, 100), } def cost(param): tune.report(loss=(param["height"] - 14) ** 2 - abs(param["width"] - 3)) search_alg = CFO( space=space, metric="loss", mode="min", seed=20, ) return search_alg, cost
def search_space(cls, **params): # (1) Get the hps in the original search space space = LGBMEstimator.search_space(**params) # (2) Set up the fixed value from hps from the starting point for hp_name in hps_to_freeze: # if an hp is specifed to be freezed, use tine value provided in the starting_point # otherwise use the setting from the original search space if hp_name in starting_point: space[hp_name] = {"domain": starting_point[hp_name]} # (3.1) Configure the search space for hps that are in the original search space # but you want to change something, for example the range. revised_hps_to_search = { "n_estimators": { "domain": tune.lograndint(lower=10, upper=32768), "init_value": starting_point.get("n_estimators") or space["n_estimators"].get("init_value", 10), "low_cost_init_value": space["n_estimators"].get("low_cost_init_value", 10), }, "num_leaves": { "domain": tune.lograndint(lower=10, upper=3276), "init_value": starting_point.get("num_leaves") or space["num_leaves"].get("init_value", 10), "low_cost_init_value": space["num_leaves"].get("low_cost_init_value", 10), }, # (3.2) Add a new hp which is not in the original search space "subsample": { "domain": tune.uniform(lower=0.1, upper=1.0), "init_value": 0.1, }, } space.update(revised_hps_to_search) return space
def test_nested(): from flaml import tune search_space = { # test nested search space "cost_related": { "a": tune.randint(1, 8), }, "b": tune.uniform(0.5, 1.0), } def simple_func(config): tune.report(metric=(config["cost_related"]["a"] - 4)**2 * (config["b"] - 0.7)**2) analysis = tune.run(simple_func, init_config={"cost_related": { "a": 1, }}, metric="metric", mode="min", config=search_space, local_dir='logs/', num_samples=-1, time_budget_s=1)
def _test_xgboost(method='BlendSearch'): try: import ray except ImportError: return if method == 'BlendSearch': from flaml import tune else: from ray import tune search_space = { # You can mix constants with search space objects. "max_depth": tune.randint(1, 8) if method in [ "BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1) } max_iter = 10 for num_samples in [256]: time_budget_s = 60 #None for n_cpu in [8]: start_time = time.time() ray.init(num_cpus=n_cpu, num_gpus=0) if method == 'BlendSearch': analysis = tune.run( train_breast_cancer, init_config={ "max_depth": 1, "min_child_weight": 3, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, report_intermediate_result=True, # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, use_ray=True) else: if 'ASHA' == method: algo = None elif 'BOHB' == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif 'CFO' == method: from flaml import CFO algo = CFO(points_to_evaluate=[{ "max_depth": 1, "min_child_weight": 3, }], cat_hp_cost={ "min_child_weight": [6, 3, 2], }) elif 'Dragonfly' == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif 'SkOpt' == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif 'ZOOpt' == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples*n_cpu) elif 'Ax' == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif 'HyperOpt' == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != 'BOHB': from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler( max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir='logs/', num_samples=num_samples*n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo) ray.shutdown() # # Load the best model checkpoint # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss","min","all") accuracy = 1. - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def _test_xgboost(method="BlendSearch"): try: import ray except ImportError: return if method == "BlendSearch": from flaml import tune else: from ray import tune search_space = { "max_depth": tune.randint(1, 9) if method in ["BlendSearch", "BOHB", "Optuna"] else tune.randint(1, 9), "min_child_weight": tune.choice([1, 2, 3]), "subsample": tune.uniform(0.5, 1.0), "eta": tune.loguniform(1e-4, 1e-1), } max_iter = 10 for num_samples in [128]: time_budget_s = 60 for n_cpu in [2]: start_time = time.time() # ray.init(address='auto') if method == "BlendSearch": analysis = tune.run( train_breast_cancer, config=search_space, low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, metric="eval-logloss", mode="min", max_resource=max_iter, min_resource=1, scheduler="asha", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, use_ray=True, ) else: if "ASHA" == method: algo = None elif "BOHB" == method: from ray.tune.schedulers import HyperBandForBOHB from ray.tune.suggest.bohb import TuneBOHB algo = TuneBOHB(max_concurrent=n_cpu) scheduler = HyperBandForBOHB(max_t=max_iter) elif "Optuna" == method: from ray.tune.suggest.optuna import OptunaSearch algo = OptunaSearch() elif "CFO" == method: from flaml import CFO algo = CFO( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "CFOCat" == method: from flaml.searcher.cfo_cat import CFOCat algo = CFOCat( low_cost_partial_config={ "max_depth": 1, }, cat_hp_cost={ "min_child_weight": [6, 3, 2], }, ) elif "Dragonfly" == method: from ray.tune.suggest.dragonfly import DragonflySearch algo = DragonflySearch() elif "SkOpt" == method: from ray.tune.suggest.skopt import SkOptSearch algo = SkOptSearch() elif "Nevergrad" == method: from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) elif "ZOOpt" == method: from ray.tune.suggest.zoopt import ZOOptSearch algo = ZOOptSearch(budget=num_samples * n_cpu) elif "Ax" == method: from ray.tune.suggest.ax import AxSearch algo = AxSearch() elif "HyperOpt" == method: from ray.tune.suggest.hyperopt import HyperOptSearch algo = HyperOptSearch() scheduler = None if method != "BOHB": from ray.tune.schedulers import ASHAScheduler scheduler = ASHAScheduler(max_t=max_iter, grace_period=1) analysis = tune.run( train_breast_cancer, metric="eval-logloss", mode="min", # You can add "gpu": 0.1 to allocate GPUs resources_per_trial={"cpu": 1}, config=search_space, local_dir="logs/", num_samples=num_samples * n_cpu, time_budget_s=time_budget_s, scheduler=scheduler, search_alg=algo, ) # # Load the best model checkpoint # import os # best_bst = xgb.Booster() # best_bst.load_model(os.path.join(analysis.best_checkpoint, # "model.xgb")) best_trial = analysis.get_best_trial("eval-logloss", "min", "all") accuracy = 1.0 - best_trial.metric_analysis["eval-error"]["min"] logloss = best_trial.metric_analysis["eval-logloss"]["min"] logger.info(f"method={method}") logger.info(f"n_samples={num_samples*n_cpu}") logger.info(f"time={time.time()-start_time}") logger.info(f"Best model eval loss: {logloss:.4f}") logger.info(f"Best model total accuracy: {accuracy:.4f}") logger.info(f"Best model parameters: {best_trial.config}")
def test_nested(): from flaml import tune, CFO search_space = { # test nested search space "cost_related": { "a": tune.randint(1, 9), }, "b": tune.uniform(0.5, 1.0), } def simple_func(config): obj = (config["cost_related"]["a"] - 4)**2 + (config["b"] - config["cost_related"]["a"])**2 tune.report(obj=obj) tune.report(obj=obj, ab=config["cost_related"]["a"] * config["b"]) analysis = tune.run( simple_func, search_alg=CFO( space=search_space, metric="obj", mode="min", low_cost_partial_config={"cost_related": { "a": 1 }}, points_to_evaluate=[ { "b": 0.99, "cost_related": { "a": 3 } }, { "b": 0.99, "cost_related": { "a": 2 } }, { "cost_related": { "a": 8 } }, ], metric_constraints=[("ab", "<=", 4)], ), local_dir="logs/", num_samples=-1, time_budget_s=1, ) best_trial = analysis.get_best_trial() logger.info(f"CFO best config: {best_trial.config}") logger.info(f"CFO best result: {best_trial.last_result}") analysis = tune.run( simple_func, search_alg=BlendSearch( experimental=True, space=search_space, metric="obj", mode="min", low_cost_partial_config={"cost_related": { "a": 1 }}, points_to_evaluate=[ { "b": 0.99, "cost_related": { "a": 3 } }, { "b": 0.99, "cost_related": { "a": 2 } }, { "cost_related": { "a": 8 } }, ], metric_constraints=[("ab", "<=", 4)], ), local_dir="logs/", num_samples=-1, time_budget_s=1, ) best_trial = analysis.get_best_trial() logger.info(f"BlendSearch exp best config: {best_trial.config}") logger.info(f"BlendSearch exp best result: {best_trial.last_result}") points_to_evaluate = [ { "b": 0.99, "cost_related": { "a": 3 } }, { "b": 0.99, "cost_related": { "a": 2 } }, ] analysis = tune.run( simple_func, config=search_space, low_cost_partial_config={"cost_related": { "a": 1 }}, points_to_evaluate=points_to_evaluate, evaluated_rewards=[(config["cost_related"]["a"] - 4)**2 + (config["b"] - config["cost_related"]["a"])**2 for config in points_to_evaluate], metric="obj", mode="min", metric_constraints=[("ab", "<=", 4)], local_dir="logs/", num_samples=-1, time_budget_s=1, ) best_trial = analysis.get_best_trial() logger.info(f"BlendSearch best config: {best_trial.config}") logger.info(f"BlendSearch best result: {best_trial.last_result}")
def test_define_by_run(): from flaml.tune.space import ( unflatten_hierarchical, normalize, indexof, complete_config, ) space = { # Sample a float uniformly between -5.0 and -1.0 "uniform": tune.uniform(-5, -1), # Sample a float uniformly between 3.2 and 5.4, # rounding to increments of 0.2 "quniform": tune.quniform(3.2, 5.4, 0.2), # Sample a float uniformly between 0.0001 and 0.01, while # sampling in log space "loguniform": tune.loguniform(1e-4, 1e-2), # Sample a float uniformly between 0.0001 and 0.1, while # sampling in log space and rounding to increments of 0.00005 "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-5), # Sample a random float from a normal distribution with # mean=10 and sd=2 # "randn": tune.randn(10, 2), # Sample a random float from a normal distribution with # mean=10 and sd=2, rounding to increments of 0.2 # "qrandn": tune.qrandn(10, 2, 0.2), # Sample a integer uniformly between -9 (inclusive) and 15 (exclusive) "randint": tune.randint(-9, 15), # Sample a random uniformly between -21 (inclusive) and 12 (inclusive (!)) # rounding to increments of 3 (includes 12) "qrandint": tune.qrandint(-21, 12, 3), # Sample a integer uniformly between 1 (inclusive) and 10 (exclusive), # while sampling in log space "lograndint": tune.lograndint(1, 10), # Sample a integer uniformly between 2 (inclusive) and 10 (inclusive (!)), # while sampling in log space and rounding to increments of 2 "qlograndint": tune.qlograndint(2, 10, 2), # Sample an option uniformly from the specified choices "choice": tune.choice(["a", "b", "c"]), "const": 5, } choice = {"nested": space} bs = BlendSearch( space={"c": tune.choice([choice])}, low_cost_partial_config={"c": choice}, metric="metric", mode="max", ) print(indexof(bs._gs.space["c"], choice)) print(indexof(bs._gs.space["c"], {"nested": {"const": 1}})) config = bs._gs.suggest("t1") print(config) config = unflatten_hierarchical(config, bs._gs.space)[0] print(config) print(normalize({"c": [choice]}, bs._gs.space, config, {}, False)) space["randn"] = tune.randn(10, 2) cfo = CFO( space={"c": tune.choice([0, choice])}, metric="metric", mode="max", ) for i in range(5): cfo.suggest(f"t{i}") # print(normalize(config, bs._gs.space, config, {}, False)) print(complete_config({}, cfo._ls.space, cfo._ls))