def objective(trial: optuna.trial.Trial) -> float:
for i in range(n_reports):
trial.report(1.0 / (i + 1), i)
if trial.should_prune():
raise optuna.TrialPruned()
return 1.0
def objective(trial: optuna.trial.Trial) -> float:
for i in range(N_REPORTS):
trial.report(1.0, i)
if trial.should_prune():
raise optuna.TrialPruned()
return 1.0
def objective(trial: optuna.trial.Trial) -> float:
with mock.patch("optuna.pruners._filter_study") as method_mock:
for i in range(N_REPORTS):
trial.report(i, step=i)
if trial.should_prune():
method_mock.assert_not_called()
raise optuna.TrialPruned()
else:
method_mock.assert_not_called()
return 1.0
def evaluate_params(
trial: optuna.trial.Trial,
train_data: lgb.Dataset,
validation_data: lgb.Dataset,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["num_iterations"] = trial.suggest_int(
"num_iterations", 8, 128)
params["learning_rate"] = trial.suggest_uniform(
"learning_rate", 2**-5, 0.5)
params["num_leaves"] = trial.suggest_int("num_leaves", 8, 256)
params["max_depth"] = trial.suggest_int("max_depth", 4, 32)
params["min_data_in_leaf"] = trial.suggest_int(
"min_data_in_leaf", 4, 512)
params["min_sum_hessian_in_leaf"] = trial.suggest_uniform(
"min_sum_hessian_in_leaf", 2**-5, 0.25)
params["bagging_freq"] = trial.suggest_int("bagging_freq", 0, 1)
params["bagging_fraction"] = trial.suggest_uniform(
"bagging_fraction", 0.5, 1)
params["feature_fraction"] = trial.suggest_uniform(
"feature_fraction", 0.5, 1)
params["lambda_l1"] = trial.suggest_uniform("lambda_l1", 0, 64)
params["lambda_l2"] = trial.suggest_uniform("lambda_l2", 0, 64)
params["min_gain_to_split"] = trial.suggest_uniform(
"min_gain_to_split", 0, 0.25)
params["min_data_per_group"] = trial.suggest_int(
"min_data_per_group", 1, 512)
params["max_cat_threshold"] = trial.suggest_int(
"max_cat_threshold", 1, 512)
params["cat_l2"] = trial.suggest_uniform("cat_l2", 0, 64)
params["cat_smooth"] = trial.suggest_uniform("cat_smooth", 0, 2048)
params["max_cat_to_onehot"] = trial.suggest_int(
"max_cat_to_onehot", 1, 64)
params["max_bin"] = trial.suggest_int("max_bin", 32, 1024)
params["min_data_in_bin"] = trial.suggest_int(
"min_data_in_bin", 1, 64)
params["objective"] = self.objective
params["num_class"] = self.num_class
params["verbosity"] = -1
booster = lgb.Booster(params=params, train_set=train_data)
booster.add_valid(validation_data, "validation_set")
for step in range(params["num_iterations"]):
booster.update()
validation_loss = booster.eval_valid()[0][2]
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def objective(trial: optuna.trial.Trial) -> float:
device = "cuda" if torch.cuda.is_available() else "cpu"
model = Net(trial).to(device)
optimizer = get_optimizer(trial, model)
for step in range(epochs):
train(model, device, train_loader, optimizer)
error_rate = test(model, device, test_loader)
trial.report(error_rate, step)
if trial.should_prune(step):
raise optuna.exceptions.TrialPruned()
return error_rate
def evaluate_params(
trial: optuna.trial.Trial,
x_train: List[np.array],
y_train: np.array,
x_valid: List[np.array],
y_valid: np.array,
max_epochs: int,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["BATCH_SIZE"] = trial.suggest_int(
"BATCH_SIZE", min(32, x_train[0].shape[0]),
x_train[0].shape[0])
params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3)
params["DROPOUT_SHARE"] = trial.suggest_uniform(
"DROPOUT_SHARE", 0, 0.5)
params["EMBED_EXPONENT"] = trial.suggest_uniform(
"EMBED_EXPONENT", 0, 0.25)
params["EMBED_L2_REG"] = trial.suggest_uniform(
"EMBED_L2_REG", 0, 16.0)
if self.categorical_features:
params["POST_FREEZE_EPOCHS"] = trial.suggest_int(
"POST_FREEZE_EPOCHS", 4, max_epochs)
else:
params["POST_FREEZE_EPOCHS"] = 0
max_pre_freeze_epochs = max_epochs
params["PRE_FREEZE_EPOCHS"] = trial.suggest_int(
"PRE_FREEZE_EPOCHS", 4, max_epochs)
params["NODES_PER_DENSE_LAYER"] = trial.suggest_int(
"NODES_PER_DENSE_LAYER", 16, 1024)
construction_args = getfullargspec(
self.construct_embedding_network).args
self.model = self.construct_embedding_network(**{
k.lower(): v
for k, v in params.items() if k in construction_args
})
self.data[self.numeric_features] = self.data[
self.numeric_features].fillna(
self.config.get("NON_CAT_MISSING_VALUE", -1))
model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in self.config.items() if k in construction_args
})
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["PRE_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
model = freeze_embedding_layers(model)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["POST_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step + max_pre_freeze_epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def objective(trial: optuna.trial.Trial):
"""
Optuna objective function.
"""
# Parse CLI arguments.
args = parse_cli_args()
config = HyperparamConfig()
# Prepare dataset class.
train_ds = Dataset(args.train_path)
valid_ds = Dataset(args.valid_path)
# Suggestions.
# >>> TODO >>>
optimizer_name = trial.suggest_categorical("optimizer",
["Adam", "RMSProp", "SGD"])
learning_rate = trial.suggest_float("learning_rate",
low=1e-6,
hight=1e-1,
log=True)
n_layer = trial.suggest_int("n_layer", 32, 1024, step=32)
# <<< TODO <<<
model = make_model(n_layer=n_layer)
optimizer = getattr(optim, optimizer_name)(model.parameters(),
lr=learning_rate)
training = Training(
train_ds,
valid_ds,
model,
optimizer,
config.batch_size,
config.epochs,
)
accuracy_list = []
with mlflow.start_run(run_name=study.study_name):
mlflow.log_params(trial.params)
for epoch in range(config.epochs):
# Training phase.
training.model.train()
train_loss, train_accuracy = training.train_epoch()
mlflow.log_metrics(
{
"train_loss": train_loss,
"train_accuracy": train_accuracy
},
step=epoch + 1,
)
# Validation phase.
training.model.eval()
valid_loss, valid_accuracy = training.validate()
mlflow.log_metrics(
{
"valid_loss": valid_loss,
"valid_accuracy": valid_accuracy
},
step=epoch + 1,
)
accuracy_list.append(valid_accuracy.item())
trial.report(valid_accuracy.item(), epoch)
if trial.should_prune():
print("Pruned with epoch {}".format(epoch))
raise optuna.exceptions.TrialPruned()
print(
"Epoch {}: TrainLoss: {:.5f}, ValidLoss: {:.5f}, ValidAcc: {:.5f}"
.format(epoch + 1, train_loss, valid_loss, valid_accuracy))
accuracy_list.sort()
accuracy = np.mean(accuracy_list[-10:])
mlflow.log_metrics({"top10_avg_accuracy": accuracy})
return accuracy
