def _suggest(self, trial: optuna.trial.Trial) -> Suggestion: suggestions: Suggestion = dict() for name, config in self.api_config.items(): low, high = config["range"] log = config["space"] == "log" if config["space"] == "logit": assert 0 < low <= high < 1 low = np.log(low / (1 - low)) high = np.log(high / (1 - high)) if config["type"] == "real": param = trial.suggest_float(name, low, high, log=log) elif config["type"] == "int": param = trial.suggest_int(name, low, high, log=log) else: # TODO(xadrianzetx) Support `suggest_categorical` if benchmark is extended. raise RuntimeError("CategoricalDistribution is not supported in bayesmark.") suggestions[name] = param if config["space"] != "logit" else 1 / (1 + np.exp(-param)) return suggestions
def modelExtraTreesClassifier(self, trial: optuna.trial.Trial): opt_params = dict(n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), max_depth=trial.suggest_int("max_depth", 2, 2**4), criterion=trial.suggest_categorical( "criterion", ["gini", "entropy"])) clf = ExtraTreesClassifier(n_estimators=100, criterion="gini", max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0., max_features="auto", max_leaf_nodes=None, min_impurity_decrease=0., min_impurity_split=None, bootstrap=False, oob_score=False, n_jobs=None, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None) clf.set_params(**{**opt_params, **self.params}) return clf
def objective(trial: optuna.trial.Trial) -> float: # We optimize the number of layers, hidden units in each layer and dropouts. n_layers = trial.suggest_int("n_layers", 1, 3) dropout = trial.suggest_float("dropout", 0.2, 0.5) output_dims = [ trial.suggest_int("n_units_l{}".format(i), 4, 128, log=True) for i in range(n_layers) ] model = LightningNet(dropout, output_dims) datamodule = MNISTDataModule(data_dir=DIR, batch_size=BATCHSIZE) trainer = pl.Trainer( logger=True, limit_val_batches=PERCENT_VALID_EXAMPLES, checkpoint_callback=False, max_epochs=EPOCHS, gpus=-1 if torch.cuda.is_available() else None, callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_acc")], ) hyperparameters = dict(n_layers=n_layers, dropout=dropout, output_dims=output_dims) trainer.logger.log_hyperparams(hyperparameters) trainer.fit(model, datamodule=datamodule) return trainer.callback_metrics["val_acc"].item()
def objective(trial: optuna.trial.Trial) -> float: num_units = trial.suggest_int("NUM_UNITS", 16, 32) dropout_rate = trial.suggest_float("DROPOUT_RATE", 0.1, 0.2) optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"]) accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore return accuracy
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float: x = trial.suggest_uniform("x", -x_max, x_max) y = trial.suggest_loguniform("y", 20, 30) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) return (x - 2)**2 + (y - 25)**2 + z
def objective(trial: optuna.trial.Trial, value: float) -> float: trial.set_system_attr( optuna.integration.lightgbm_tuner.optimize._STEP_NAME_KEY, "step{:.0f}".format(value), ) return trial.suggest_uniform("x", value, value)
def objective(trial: optuna.trial.Trial) -> float: for i in range(n_reports): trial.report(1.0 / (i + 1), i) if trial.should_prune(): raise optuna.TrialPruned() return 1.0
def objective( num_steps: int, episode_length: int, trial: optuna.trial.Trial, policy_bounds: List[float], baseline_bounds: List[float], random_seed: int, ): alpha_policy = trial.suggest_loguniform("alpha_policy", policy_bounds[0], policy_bounds[1]) alpha_baseline = trial.suggest_loguniform("alpha_baseline", baseline_bounds[0], baseline_bounds[1]) print( f"Initialising trial {trial.number}, Alpha policy = {alpha_policy}, Alpha baseline = {alpha_baseline}\n" ) return train_policy( alpha_baseline=alpha_baseline, alpha_policy=alpha_policy, num_steps=num_steps, trial=trial, alpha_decay=0.9999, discount_factor=0.999, episode_length=episode_length, random_seed=random_seed, )
def _xgbclassifier_default(trial: optuna.trial.Trial): param = { 'silent': 1, 'objective': 'binary:logistic', 'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']), 'lambda': trial.suggest_loguniform('lambda', 1e-8, 1.0), 'alpha': trial.suggest_loguniform('alpha', 1e-8, 1.0) } if param['booster'] == 'gbtree' or param['booster'] == 'dart': param['max_depth'] = trial.suggest_int('max_depth', 1, 9) param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0) param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0) param['grow_policy'] = trial.suggest_categorical( 'grow_policy', ['depthwise', 'lossguide']) if param['booster'] == 'dart': param['sample_type'] = trial.suggest_categorical( 'sample_type', ['uniform', 'weighted']) param['normalize_type'] = trial.suggest_categorical( 'normalize_type', ['tree', 'forest']) param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0) param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0) return param
def objective(trial: optuna.trial.Trial) -> float: for i in range(N_REPORTS): trial.report(i, step=i) x = trial.suggest_float("x", -100, 100) y = trial.suggest_int("y", -100, 100) return x ** 2 + y ** 2
def __call__(self, trial: optuna.trial.Trial) -> float: data = TreeDataModule( self._filename, batch_size=trial.suggest_int("batch_size", 32, 160, 32), ) kwargs = { "lstm_size": trial.suggest_categorical("lstm_size", [512, 1024, 2048]), "dropout_prob": trial.suggest_float("dropout", 0.1, 0.5, step=0.1), "learning_rate": trial.suggest_float("lr", 1e-3, 1e-1, log=True), "weight_decay": trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True), } model = RouteDistanceModel(**kwargs) gpus = int(torch.cuda.is_available()) pruning_callback = PyTorchLightningPruningCallback( trial, monitor="val_monitor") trainer = Trainer( gpus=gpus, logger=True, # become a tensorboard logger checkpoint_callback=False, callbacks=[pruning_callback], # type: ignore max_epochs=EPOCHS, ) trainer.fit(model, datamodule=data) return trainer.callback_metrics["val_monitor"].item()
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20.0, 30.0) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective(trial: optuna.trial.Trial) -> float: for i in range(N_REPORTS): trial.report(1.0, i) if trial.should_prune(): raise optuna.TrialPruned() return 1.0
def _objective(self, trial: optuna.trial.Trial): cv = trial.suggest_int('cv', 2, 2**4) opt_params = dict( max_depth=trial.suggest_int("max_depth", 2, 2**4), learning_rate=trial.suggest_float('learning_rate', 0.001, 1, step=0.001), # n_estimators=trial.suggest_int("n_estimators", 2, 2 ** 10, log=True), gamma=trial.suggest_float('gamma', 1e-8, 1, log=True), min_child_weight=trial.suggest_float('min_child_weight', 1e-8, 2**10, log=True), subsample=trial.suggest_float('subsample', 0.1, 1), colsample_bytree=trial.suggest_float('colsample_bytree', 0.1, 1), colsample_bylevel=trial.suggest_float('colsample_bylevel', 0.1, 1), reg_alpha=trial.suggest_float('reg_alpha', 1e-8, 10, log=True), reg_lambda=trial.suggest_float('reg_lambda', 1e-8, 10, log=True), ) if self.params is not None: opt_params.update(self.params) clf_oof = XGBClassifierOOF(self.X, self.y, params=opt_params, cv=cv, feval=self.feval) clf_oof.run() return clf_oof.oof_score # todo: f1
def f(trial: optuna.trial.Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) trial.set_user_attr("train_loss", 3) raise ValueError() return x + y # 3.5
def objective2(trial: optuna.trial.Trial) -> float: p1 = trial.suggest_loguniform("p1", 50, 100) # The range has been changed p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3) p5 = trial.suggest_uniform("p5", 0, 1) return p1 + p3 + p5
def objective(trial: optuna.trial.Trial): settings = Settings( learning_rate=trial.suggest_loguniform('learning_rate', 1e-5, 1e-2), hidden1=trial.suggest_int('hidden1', 50, 200), hidden2=trial.suggest_int('hidden2', 10, 50), ) val_err = run_training(settings) return val_err
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", low=-10, high=10) y = trial.suggest_float("y", low=1, high=10, log=True) first_objective = (x - 2)**2 + (y - 25)**2 second_objective = (x - 2)**3 + (y - 25)**3 return first_objective, second_objective
def objective2(trial: optuna.trial.Trial) -> float: p1 = trial.suggest_float("p1", 50, 100, log=True) # The range has been changed p3 = trial.suggest_float("p3", 0, 9, step=3) p5 = trial.suggest_float("p5", 0, 1) return p1 + p3 + p5
def wrapper(trial: optuna.trial.Trial) -> Union[float, Sequence[float]]: study = trial.study self._initialize_experiment(study) with mlflow.start_run(run_name=str(trial.number), nested=self._nest_trials) as run: trial.set_system_attr(RUN_ID_ATTRIBUTE_KEY, run.info.run_id) return func(trial)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float: x = trial.suggest_float("x", -1.0, 1.0) y = trial.suggest_float("y", 20, 30, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) long_str = str(list(range(5000))) trial.set_user_attr("my_user_attr", long_str) return (x - 2)**2 + (y - 25)**2 + z
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_float("x", -1.0, 1.0) y = trial.suggest_float("y", 20, 30, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") mlflow.log_metric(metric_name, metric) return (x - 2)**2 + (y - 25)**2 + z
def objective0(trial: optuna.trial.Trial) -> float: p0 = trial.suggest_float("p0", 0, 10) p1 = trial.suggest_float("p1", 1, 10, log=True) p2 = trial.suggest_int("p2", 0, 10) p3 = trial.suggest_float("p3", 0, 9, step=3) p4 = trial.suggest_categorical("p4", ["10", "20", "30"]) assert isinstance(p4, str) return p0 + p1 + p2 + p3 + int(p4)
def objective1(trial: optuna.trial.Trial) -> float: # p0, p2 and p4 are deleted. p1 = trial.suggest_loguniform("p1", 1, 10) p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3) # p5 is added. p5 = trial.suggest_uniform("p5", 0, 1) return p1 + p3 + p5
def objective1(trial: optuna.trial.Trial) -> float: # p0, p2 and p4 are deleted. p1 = trial.suggest_float("p1", 1, 10, log=True) p3 = trial.suggest_float("p3", 0, 9, step=3) # p5 is added. p5 = trial.suggest_float("p5", 0, 1) return p1 + p3 + p5
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", low=-10, high=10) y = trial.suggest_float("y", low=1, high=10, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) first_objective = (x - 2)**2 + (y - 25)**2 + z second_objective = (x - 2)**3 + (y - 25)**3 - z return first_objective, second_objective
def _objective(self, trial: optuna.trial.Trial): cv = trial.suggest_int('cv', 2, 2**4) opt_params = dict( num_leaves=trial.suggest_int("num_leaves", 2, 2**8), learning_rate=trial.suggest_float('learning_rate', 0.001, 1, step=0.001), min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8), min_child_weight=trial.suggest_float('min_child_weight', 1e-8, 1, log=True), min_split_gain=trial.suggest_float('min_split_gain', 1e-8, 1, log=True), bagging_fraction=trial.suggest_float('bagging_fraction', 0.4, 1), bagging_freq=trial.suggest_int("bagging_freq", 0, 2**4), feature_fraction=trial.suggest_float('feature_fraction', 0.4, 1), lambda_l1=trial.suggest_float('lambda_l1', 1e-8, 10, log=True), lambda_l2=trial.suggest_float('lambda_l2', 1e-8, 10, log=True), ) if self.params is not None: opt_params.update(self.params) cv_result = lgb.cv(opt_params, self.dtrain, num_boost_round=10000, nfold=cv, stratified='reg' not in opt_params.get( 'application', opt_params.get('objective', 'reg')), feval=None, early_stopping_rounds=100, verbose_eval=100, show_stdv=False, seed=0, eval_train_metric=False) score = -1 self.best_num_boost_round = 0 for key in cv_result: if 'mean' in key: _ = cv_result[key] score = _[-1] self.best_num_boost_round = len(_) print( f'CV Score: {score if score != -1 else "cv_result donot contain mean-metric"}' ) return score
def objective(trial: optuna.trial.Trial) -> float: with mock.patch("optuna.pruners._filter_study") as method_mock: for i in range(N_REPORTS): trial.report(i, step=i) if trial.should_prune(): method_mock.assert_not_called() raise optuna.TrialPruned() else: method_mock.assert_not_called() return 1.0
def objective(trial: optuna.trial.Trial): encoder_num_layers = trial.suggest_int("encoder.num_layer", 1, 4) decoder_num_layers = trial.suggest_int("decoder.num_layer", 1, 4) transformer_d_model = trial.suggest_categorical( 'transformer.d_model', [64, 128, 256]) hparams["basic"]["transformer"]["d_model"] = transformer_d_model hparams["decoder"]["num_layer"] = decoder_num_layers hparams["encoder"]["num_layer"] = encoder_num_layers device = torch.device("cuda" if torch.cuda.is_available() else "cpu") task = Task(hparams, device) return task.train(trial)
def objective_test_upgrade(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float. y = trial.suggest_int("y", 0, 10) z = cast(float, trial.suggest_categorical("z", [-5, 0, 5])) trial.set_system_attr("a", 0) trial.set_user_attr("b", 1) trial.report(0.5, step=0) return x ** 2 + y ** 2 + z ** 2