def _suggest(self, trial: optuna.trial.Trial) -> Suggestion:
suggestions: Suggestion = dict()
for name, config in self.api_config.items():
low, high = config["range"]
log = config["space"] == "log"
if config["space"] == "logit":
assert 0 < low <= high < 1
low = np.log(low / (1 - low))
high = np.log(high / (1 - high))
if config["type"] == "real":
param = trial.suggest_float(name, low, high, log=log)
elif config["type"] == "int":
param = trial.suggest_int(name, low, high, log=log)
else:
# TODO(xadrianzetx) Support `suggest_categorical` if benchmark is extended.
raise RuntimeError("CategoricalDistribution is not supported in bayesmark.")
suggestions[name] = param if config["space"] != "logit" else 1 / (1 + np.exp(-param))
return suggestions
def modelExtraTreesClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(n_estimators=trial.suggest_int("n_estimators",
2,
2**10,
log=True),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
max_depth=trial.suggest_int("max_depth", 2, 2**4),
criterion=trial.suggest_categorical(
"criterion", ["gini", "entropy"]))
clf = ExtraTreesClassifier(n_estimators=100,
criterion="gini",
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.,
max_features="auto",
max_leaf_nodes=None,
min_impurity_decrease=0.,
min_impurity_split=None,
bootstrap=False,
oob_score=False,
n_jobs=None,
random_state=None,
verbose=0,
warm_start=False,
class_weight=None,
ccp_alpha=0.0,
max_samples=None)
clf.set_params(**{**opt_params, **self.params})
return clf
def objective(trial: optuna.trial.Trial) -> float:
# We optimize the number of layers, hidden units in each layer and dropouts.
n_layers = trial.suggest_int("n_layers", 1, 3)
dropout = trial.suggest_float("dropout", 0.2, 0.5)
output_dims = [
trial.suggest_int("n_units_l{}".format(i), 4, 128, log=True)
for i in range(n_layers)
]
model = LightningNet(dropout, output_dims)
datamodule = MNISTDataModule(data_dir=DIR, batch_size=BATCHSIZE)
trainer = pl.Trainer(
logger=True,
limit_val_batches=PERCENT_VALID_EXAMPLES,
checkpoint_callback=False,
max_epochs=EPOCHS,
gpus=-1 if torch.cuda.is_available() else None,
callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_acc")],
)
hyperparameters = dict(n_layers=n_layers,
dropout=dropout,
output_dims=output_dims)
trainer.logger.log_hyperparams(hyperparameters)
trainer.fit(model, datamodule=datamodule)
return trainer.callback_metrics["val_acc"].item()
def objective(trial: optuna.trial.Trial) -> float:
num_units = trial.suggest_int("NUM_UNITS", 16, 32)
dropout_rate = trial.suggest_float("DROPOUT_RATE", 0.1, 0.2)
optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"])
accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore
return accuracy
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float:
x = trial.suggest_uniform("x", -x_max, x_max)
y = trial.suggest_loguniform("y", 20, 30)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
return (x - 2)**2 + (y - 25)**2 + z
def objective(trial: optuna.trial.Trial, value: float) -> float:
trial.set_system_attr(
optuna.integration.lightgbm_tuner.optimize._STEP_NAME_KEY,
"step{:.0f}".format(value),
)
return trial.suggest_uniform("x", value, value)
def objective(trial: optuna.trial.Trial) -> float:
for i in range(n_reports):
trial.report(1.0 / (i + 1), i)
if trial.should_prune():
raise optuna.TrialPruned()
return 1.0
def objective(
num_steps: int,
episode_length: int,
trial: optuna.trial.Trial,
policy_bounds: List[float],
baseline_bounds: List[float],
random_seed: int,
):
alpha_policy = trial.suggest_loguniform("alpha_policy", policy_bounds[0],
policy_bounds[1])
alpha_baseline = trial.suggest_loguniform("alpha_baseline",
baseline_bounds[0],
baseline_bounds[1])
print(
f"Initialising trial {trial.number}, Alpha policy = {alpha_policy}, Alpha baseline = {alpha_baseline}\n"
)
return train_policy(
alpha_baseline=alpha_baseline,
alpha_policy=alpha_policy,
num_steps=num_steps,
trial=trial,
alpha_decay=0.9999,
discount_factor=0.999,
episode_length=episode_length,
random_seed=random_seed,
)
def _xgbclassifier_default(trial: optuna.trial.Trial):
param = {
'silent':
1,
'objective':
'binary:logistic',
'booster':
trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),
'lambda':
trial.suggest_loguniform('lambda', 1e-8, 1.0),
'alpha':
trial.suggest_loguniform('alpha', 1e-8, 1.0)
}
if param['booster'] == 'gbtree' or param['booster'] == 'dart':
param['max_depth'] = trial.suggest_int('max_depth', 1, 9)
param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0)
param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0)
param['grow_policy'] = trial.suggest_categorical(
'grow_policy', ['depthwise', 'lossguide'])
if param['booster'] == 'dart':
param['sample_type'] = trial.suggest_categorical(
'sample_type', ['uniform', 'weighted'])
param['normalize_type'] = trial.suggest_categorical(
'normalize_type', ['tree', 'forest'])
param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0)
param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)
return param
def objective(trial: optuna.trial.Trial) -> float:
for i in range(N_REPORTS):
trial.report(i, step=i)
x = trial.suggest_float("x", -100, 100)
y = trial.suggest_int("y", -100, 100)
return x ** 2 + y ** 2
def __call__(self, trial: optuna.trial.Trial) -> float:
data = TreeDataModule(
self._filename,
batch_size=trial.suggest_int("batch_size", 32, 160, 32),
)
kwargs = {
"lstm_size":
trial.suggest_categorical("lstm_size", [512, 1024, 2048]),
"dropout_prob":
trial.suggest_float("dropout", 0.1, 0.5, step=0.1),
"learning_rate":
trial.suggest_float("lr", 1e-3, 1e-1, log=True),
"weight_decay":
trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True),
}
model = RouteDistanceModel(**kwargs)
gpus = int(torch.cuda.is_available())
pruning_callback = PyTorchLightningPruningCallback(
trial, monitor="val_monitor")
trainer = Trainer(
gpus=gpus,
logger=True, # become a tensorboard logger
checkpoint_callback=False,
callbacks=[pruning_callback], # type: ignore
max_epochs=EPOCHS,
)
trainer.fit(model, datamodule=data)
return trainer.callback_metrics["val_monitor"].item()
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective(trial: optuna.trial.Trial) -> float:
for i in range(N_REPORTS):
trial.report(1.0, i)
if trial.should_prune():
raise optuna.TrialPruned()
return 1.0
def _objective(self, trial: optuna.trial.Trial):
cv = trial.suggest_int('cv', 2, 2**4)
opt_params = dict(
max_depth=trial.suggest_int("max_depth", 2, 2**4),
learning_rate=trial.suggest_float('learning_rate',
0.001,
1,
step=0.001),
# n_estimators=trial.suggest_int("n_estimators", 2, 2 ** 10, log=True),
gamma=trial.suggest_float('gamma', 1e-8, 1, log=True),
min_child_weight=trial.suggest_float('min_child_weight',
1e-8,
2**10,
log=True),
subsample=trial.suggest_float('subsample', 0.1, 1),
colsample_bytree=trial.suggest_float('colsample_bytree', 0.1, 1),
colsample_bylevel=trial.suggest_float('colsample_bylevel', 0.1, 1),
reg_alpha=trial.suggest_float('reg_alpha', 1e-8, 10, log=True),
reg_lambda=trial.suggest_float('reg_lambda', 1e-8, 10, log=True),
)
if self.params is not None:
opt_params.update(self.params)
clf_oof = XGBClassifierOOF(self.X,
self.y,
params=opt_params,
cv=cv,
feval=self.feval)
clf_oof.run()
return clf_oof.oof_score # todo: f1
def f(trial: optuna.trial.Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
trial.set_user_attr("train_loss", 3)
raise ValueError()
return x + y # 3.5
def objective2(trial: optuna.trial.Trial) -> float:
p1 = trial.suggest_loguniform("p1", 50,
100) # The range has been changed
p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3)
p5 = trial.suggest_uniform("p5", 0, 1)
return p1 + p3 + p5
def objective(trial: optuna.trial.Trial):
settings = Settings(
learning_rate=trial.suggest_loguniform('learning_rate', 1e-5, 1e-2),
hidden1=trial.suggest_int('hidden1', 50, 200),
hidden2=trial.suggest_int('hidden2', 10, 50),
)
val_err = run_training(settings)
return val_err
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]:
x = trial.suggest_float("x", low=-10, high=10)
y = trial.suggest_float("y", low=1, high=10, log=True)
first_objective = (x - 2)**2 + (y - 25)**2
second_objective = (x - 2)**3 + (y - 25)**3
return first_objective, second_objective
def objective2(trial: optuna.trial.Trial) -> float:
p1 = trial.suggest_float("p1", 50, 100,
log=True) # The range has been changed
p3 = trial.suggest_float("p3", 0, 9, step=3)
p5 = trial.suggest_float("p5", 0, 1)
return p1 + p3 + p5
def wrapper(trial: optuna.trial.Trial) -> Union[float, Sequence[float]]:
study = trial.study
self._initialize_experiment(study)
with mlflow.start_run(run_name=str(trial.number), nested=self._nest_trials) as run:
trial.set_system_attr(RUN_ID_ATTRIBUTE_KEY, run.info.run_id)
return func(trial)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float:
x = trial.suggest_float("x", -1.0, 1.0)
y = trial.suggest_float("y", 20, 30, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
long_str = str(list(range(5000)))
trial.set_user_attr("my_user_attr", long_str)
return (x - 2)**2 + (y - 25)**2 + z
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_float("x", -1.0, 1.0)
y = trial.suggest_float("y", 20, 30, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
mlflow.log_metric(metric_name, metric)
return (x - 2)**2 + (y - 25)**2 + z
def objective0(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_float("p0", 0, 10)
p1 = trial.suggest_float("p1", 1, 10, log=True)
p2 = trial.suggest_int("p2", 0, 10)
p3 = trial.suggest_float("p3", 0, 9, step=3)
p4 = trial.suggest_categorical("p4", ["10", "20", "30"])
assert isinstance(p4, str)
return p0 + p1 + p2 + p3 + int(p4)
def objective1(trial: optuna.trial.Trial) -> float:
# p0, p2 and p4 are deleted.
p1 = trial.suggest_loguniform("p1", 1, 10)
p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3)
# p5 is added.
p5 = trial.suggest_uniform("p5", 0, 1)
return p1 + p3 + p5
def objective1(trial: optuna.trial.Trial) -> float:
# p0, p2 and p4 are deleted.
p1 = trial.suggest_float("p1", 1, 10, log=True)
p3 = trial.suggest_float("p3", 0, 9, step=3)
# p5 is added.
p5 = trial.suggest_float("p5", 0, 1)
return p1 + p3 + p5
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]:
x = trial.suggest_float("x", low=-10, high=10)
y = trial.suggest_float("y", low=1, high=10, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
first_objective = (x - 2)**2 + (y - 25)**2 + z
second_objective = (x - 2)**3 + (y - 25)**3 - z
return first_objective, second_objective
def _objective(self, trial: optuna.trial.Trial):
cv = trial.suggest_int('cv', 2, 2**4)
opt_params = dict(
num_leaves=trial.suggest_int("num_leaves", 2, 2**8),
learning_rate=trial.suggest_float('learning_rate',
0.001,
1,
step=0.001),
min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8),
min_child_weight=trial.suggest_float('min_child_weight',
1e-8,
1,
log=True),
min_split_gain=trial.suggest_float('min_split_gain',
1e-8,
1,
log=True),
bagging_fraction=trial.suggest_float('bagging_fraction', 0.4, 1),
bagging_freq=trial.suggest_int("bagging_freq", 0, 2**4),
feature_fraction=trial.suggest_float('feature_fraction', 0.4, 1),
lambda_l1=trial.suggest_float('lambda_l1', 1e-8, 10, log=True),
lambda_l2=trial.suggest_float('lambda_l2', 1e-8, 10, log=True),
)
if self.params is not None:
opt_params.update(self.params)
cv_result = lgb.cv(opt_params,
self.dtrain,
num_boost_round=10000,
nfold=cv,
stratified='reg' not in opt_params.get(
'application',
opt_params.get('objective', 'reg')),
feval=None,
early_stopping_rounds=100,
verbose_eval=100,
show_stdv=False,
seed=0,
eval_train_metric=False)
score = -1
self.best_num_boost_round = 0
for key in cv_result:
if 'mean' in key:
_ = cv_result[key]
score = _[-1]
self.best_num_boost_round = len(_)
print(
f'CV Score: {score if score != -1 else "cv_result donot contain mean-metric"}'
)
return score
def objective(trial: optuna.trial.Trial) -> float:
with mock.patch("optuna.pruners._filter_study") as method_mock:
for i in range(N_REPORTS):
trial.report(i, step=i)
if trial.should_prune():
method_mock.assert_not_called()
raise optuna.TrialPruned()
else:
method_mock.assert_not_called()
return 1.0
def objective(trial: optuna.trial.Trial):
encoder_num_layers = trial.suggest_int("encoder.num_layer", 1, 4)
decoder_num_layers = trial.suggest_int("decoder.num_layer", 1, 4)
transformer_d_model = trial.suggest_categorical(
'transformer.d_model', [64, 128, 256])
hparams["basic"]["transformer"]["d_model"] = transformer_d_model
hparams["decoder"]["num_layer"] = decoder_num_layers
hparams["encoder"]["num_layer"] = encoder_num_layers
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
task = Task(hparams, device)
return task.train(trial)
def objective_test_upgrade(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float.
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
trial.report(0.5, step=0)
return x ** 2 + y ** 2 + z ** 2
