def _preprocess(self, trial: optuna.trial.Trial) -> None:
if self.pbar is not None:
self.pbar.set_description(self.pbar_fmt.format(self.step_name, self.best_score))
if "lambda_l1" in self.target_param_names:
self.lgbm_params["lambda_l1"] = trial.suggest_loguniform("lambda_l1", 1e-8, 10.0)
if "lambda_l2" in self.target_param_names:
self.lgbm_params["lambda_l2"] = trial.suggest_loguniform("lambda_l2", 1e-8, 10.0)
if "num_leaves" in self.target_param_names:
tree_depth = self.lgbm_params.get("max_depth", _DEFAULT_TUNER_TREE_DEPTH)
max_num_leaves = 2 ** tree_depth if tree_depth > 0 else 2 ** _DEFAULT_TUNER_TREE_DEPTH
self.lgbm_params["num_leaves"] = trial.suggest_int("num_leaves", 2, max_num_leaves)
if "feature_fraction" in self.target_param_names:
# `GridSampler` is used for sampling feature_fraction value.
# The value 1.0 for the hyperparameter is always sampled.
param_value = min(trial.suggest_uniform("feature_fraction", 0.4, 1.0 + _EPS), 1.0)
self.lgbm_params["feature_fraction"] = param_value
if "bagging_fraction" in self.target_param_names:
# `TPESampler` is used for sampling bagging_fraction value.
# The value 1.0 for the hyperparameter might by sampled.
param_value = min(trial.suggest_uniform("bagging_fraction", 0.4, 1.0 + _EPS), 1.0)
self.lgbm_params["bagging_fraction"] = param_value
if "bagging_freq" in self.target_param_names:
self.lgbm_params["bagging_freq"] = trial.suggest_int("bagging_freq", 1, 7)
if "min_child_samples" in self.target_param_names:
# `GridSampler` is used for sampling min_child_samples value.
# The value 1.0 for the hyperparameter is always sampled.
param_value = int(trial.suggest_uniform("min_child_samples", 5, 100 + _EPS))
self.lgbm_params["min_child_samples"] = param_value
def __init__(self, trial: optuna.trial.Trial) -> None:
super(Net, self).__init__()
self.activation = getattr(
F, trial.suggest_categorical('activation', ['relu', 'elu']))
self.conv1 = nn.Conv2d(1, 32, 3, 1)
self.conv2 = nn.Conv2d(32, 64, 3, 1)
self.dropout1 = nn.Dropout2d(
p=trial.suggest_uniform("dropout_prob1", 0.1, 0.9))
self.dropout2 = nn.Dropout2d(
p=trial.suggest_uniform("dropout_prob2", 0.1, 0.9))
self.fc1 = nn.Linear(9216, 128)
self.fc2 = nn.Linear(128, 10)
def optimize(trial: optuna.trial.Trial, data_dict: dict):
p = {
'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1),
'max_depth': trial.suggest_int('max_depth', 5, 30),
'max_leaves': trial.suggest_int('max_leaves', 5, 50),
'subsample': trial.suggest_uniform('subsample', 0.3, 1.0),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3,
1.0),
'min_child_weight': trial.suggest_int('min_child_weight', 5, 100),
'lambda': trial.suggest_uniform('lambda', 0.05, 0.2),
'alpha': trial.suggest_uniform('alpha', 0.05, 0.2),
'objective': 'binary:logistic',
'booster': 'gbtree',
'tree_method': 'gpu_hist',
'verbosity': 1,
'n_jobs': 10,
'eval_metric': 'auc'
}
print('Choosing parameters:', p)
scores = []
sizes = []
# gts = GroupTimeSeriesSplit()
data = data_dict['data']
target = data_dict['target']
date = data_dict['date']
gts = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10)
for i, (tr_idx, val_idx) in enumerate(gts.split(data, groups=date)):
sizes.append(len(tr_idx))
x_tr, x_val = copy.deepcopy(data.iloc[tr_idx]), copy.deepcopy(
data.iloc[val_idx])
y_tr, y_val = copy.deepcopy(target[tr_idx]), copy.deepcopy(
target[val_idx])
x_tr, x_val = calc_data_mean(x_tr, cache_dir='cache/', fold=i, train=True), \
calc_data_mean(x_val, cache_dir='cache/', fold=i, train=False)
d_tr = xgb.DMatrix(x_tr, label=y_tr)
d_val = xgb.DMatrix(x_val, label=y_val)
clf = xgb.train(p,
d_tr,
500, [(d_val, 'eval')],
early_stopping_rounds=50,
verbose_eval=True)
val_pred = clf.predict(d_val)
score = roc_auc_score(y_val, val_pred)
print(f'Fold {i} ROC AUC:\t', score)
scores.append(score)
del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score
rubbish = gc.collect()
print(scores)
avg_score = weighted_mean(scores, sizes)
print('Avg Score:', avg_score)
return avg_score
def _objective(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_uniform("p0", -3.3, 5.2)
p1 = trial.suggest_uniform("p1", 2.0, 2.0)
p2 = trial.suggest_loguniform("p2", 0.0001, 0.3)
p3 = trial.suggest_loguniform("p3", 1.1, 1.1)
p4 = trial.suggest_int("p4", -100, 8)
p5 = trial.suggest_int("p5", -20, -20)
p6 = trial.suggest_discrete_uniform("p6", 10, 20, 2)
p7 = trial.suggest_discrete_uniform("p7", 0.1, 1.0, 0.1)
p8 = trial.suggest_discrete_uniform("p8", 2.2, 2.2, 0.5)
p9 = trial.suggest_categorical("p9", ["9", "3", "0", "8"])
assert isinstance(p9, str)
return p0 + p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + int(p9)
def objective(trial: optuna.trial.Trial, value: float) -> float:
trial.set_system_attr(
optuna.integration.lightgbm_tuner.optimize._STEP_NAME_KEY,
"step{:.0f}".format(value),
)
return trial.suggest_uniform("x", value, value)
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float:
x = trial.suggest_uniform("x", -x_max, x_max)
y = trial.suggest_loguniform("y", 20, 30)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
return (x - 2)**2 + (y - 25)**2 + z
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective(trial: optuna.trial.Trial) -> float:
num_units = trial.suggest_int("NUM_UNITS", 16, 32)
dropout_rate = trial.suggest_uniform("DROPOUT_RATE", 0.1, 0.2)
optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"])
accuracy = train_test_model(num_units, dropout_rate,
optimizer) # type: ignore
return accuracy
def objective2(trial: optuna.trial.Trial) -> float:
p1 = trial.suggest_loguniform("p1", 50,
100) # The range has been changed
p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3)
p5 = trial.suggest_uniform("p5", 0, 1)
return p1 + p3 + p5
def objective_test_upgrade(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float.
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
trial.report(0.5, step=0)
return x ** 2 + y ** 2 + z ** 2
def optimize(trial: optuna.trial.Trial, data_dict: dict):
p = {
'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1),
'max_depth': trial.suggest_int('max_depth', 5, 30),
'max_leaves': trial.suggest_int('max_leaves', 5, 50),
'subsample': trial.suggest_uniform('subsample', 0.3, 1.0),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3,
1.0),
'min_child_weight': trial.suggest_int('min_child_weight', 5, 100),
'lambda': trial.suggest_uniform('lambda', 0.05, 0.2),
'alpha': trial.suggest_uniform('alpha', 0.05, 0.2),
'objective': 'reg:squarederror',
'booster': 'gbtree',
'tree_method': 'gpu_hist',
'verbosity': 1,
'n_jobs': 10,
'eval_metric': 'rmse'
}
print('Choosing parameters:', p)
scores = []
sizes = []
# gts = GroupTimeSeriesSplit()']
gts = pgs.PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10)
for i, (tr_idx, val_idx) in enumerate(
gts.split(data_dict['data'], groups=data_dict['era'])):
x_tr, x_val = data_dict['data'][tr_idx], data_dict['data'][val_idx]
y_tr, y_val = data_dict['target'][tr_idx], data_dict['target'][val_idx]
d_tr = xgb.DMatrix(x_tr, label=y_tr)
d_val = xgb.DMatrix(x_val, label=y_val)
clf = xgb.train(p,
d_tr,
500, [(d_val, 'eval')],
early_stopping_rounds=50,
verbose_eval=True)
val_pred = clf.predict(d_val)
score = mean_squared_error(y_val, val_pred)
scores.append(score)
sizes.append(len(tr_idx) + len(val_idx))
del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score
rubbish = gc.collect()
print(scores)
avg_score = utils.weighted_mean(scores, sizes)
print('Avg Score:', avg_score)
return avg_score
def objective0(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_uniform("p0", 0, 10)
p1 = trial.suggest_loguniform("p1", 1, 10)
p2 = trial.suggest_int("p2", 0, 10)
p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3)
p4 = trial.suggest_categorical("p4", ["10", "20", "30"])
assert isinstance(p4, str)
return p0 + p1 + p2 + p3 + int(p4)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20, 30)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
long_str = str(list(range(5000)))
trial.set_user_attr("my_user_attr", long_str)
return (x - 2)**2 + (y - 25)**2 + z
def _lgbmclassifier_default(trial: optuna.trial.Trial):
# TODO: using LightGBMTuner
params = {
'boosting_type': trial.suggest_categorical('boosting', ['gbdt', 'dart', 'goss']),
'objective': 'binary',
'metric': ['binary', 'binary_error', 'auc'],
'num_leaves': trial.suggest_int("num_leaves", 10, 500),
'learning_rate': trial.suggest_loguniform("learning_rate", 1e-5, 1),
'feature_fraction': trial.suggest_uniform("feature_fraction", 0.0, 1.0),
}
if params['boosting_type'] == 'dart':
params['drop_rate'] = trial.suggest_loguniform('drop_rate', 1e-8, 1.0)
params['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)
if params['boosting_type'] == 'goss':
params['top_rate'] = trial.suggest_uniform('top_rate', 0.0, 1.0)
params['other_rate'] = trial.suggest_uniform('other_rate', 0.0, 1.0 - params['top_rate'])
return params
def objective(trial: optuna.trial.Trial) -> float:
dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0000..0062}.tar").shuffle(20000).decode()
dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0000..0010}.tar").shuffle(20000).decode()
test_dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0063..0072}.tar").decode()
test_dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0011..0013}.tar").decode()
dataset = SampleEqually([dataset, dataset_2])
test_dataset = SampleEqually([test_dataset_2, test_dataset])
train_loader = DataLoader(dataset, num_workers=16, batch_size=4, pin_memory=True)
test_loader = DataLoader(test_dataset, num_workers=4, batch_size=1, pin_memory=True)
# We optimize the number of layers, hidden units in each layer and dropouts.
config = {
"sample_ratio_one": trial.suggest_uniform("sample_ratio_one", 0.1, 0.9),
"sample_radius_one": trial.suggest_uniform("sample_radius_one", 0.1, 0.9),
"sample_max_neighbor": trial.suggest_int("sample_max_neighbor", 8, 72),
"sample_ratio_two": trial.suggest_uniform("sample_ratio_two", 0.1, 0.9),
"sample_radius_two": trial.suggest_uniform("sample_radius_two", 0.1, 0.9),
"fc_1": trial.suggest_int("fc_1", 128, 256),
"fc_1_out": trial.suggest_int("fc_1_out", 32, 128),
"fc_2_out": trial.suggest_int("fc_2_out", 16, 96),
"dropout": trial.suggest_uniform("dropout", 0.1, 0.9),
}
num_classes = 2
import pytorch_lightning as pl
model = LitPointNet2(num_classes, lr=0.0001, config=config)
trainer = pl.Trainer(
logger=True,
limit_val_batches=10000,
limit_train_batches=10000,
checkpoint_callback=False,
auto_lr_find=True,
max_epochs=20,
gpus=1,
callbacks=[PyTorchLightningPruningCallback(trial, monitor="val/loss")],
)
trainer.logger.log_hyperparams(config)
trainer.tune(model=model, train_dataloader=train_loader, val_dataloaders=test_loader)
trainer.fit(model=model, train_dataloader=train_loader, val_dataloaders=test_loader)
return trainer.callback_metrics["val/loss"].item()
def objective1(trial: optuna.trial.Trial) -> float:
# p0, p2 and p4 are deleted.
p1 = trial.suggest_loguniform("p1", 1, 10)
p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3)
# p5 is added.
p5 = trial.suggest_uniform("p5", 0, 1)
return p1 + p3 + p5
def modelCatBoostClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(
num_leaves=trial.suggest_int("num_leaves", 2, 2**8),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True),
min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8),
min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8,
1),
min_split_gain=trial.suggest_loguniform('min_split_gain', 1e-8, 1),
subsample=trial.suggest_uniform('subsample', 0.4, 1),
subsample_freq=trial.suggest_int("subsample_freq", 0, 2**4),
colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.4, 1),
reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10),
reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10),
)
clf = CatBoostClassifier()
clf.set_params(**{**opt_params, **self.params})
return clf
def _objective_func(trial: optuna.trial.Trial) -> float:
u = trial.suggest_int("u", 0, 10, step=2)
v = trial.suggest_int("v", 1, 10, log=True)
w = trial.suggest_float("w", -1.0, 1.0, step=0.1)
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return u + v + w + (x - 2)**2 + (y - 25)**2 + z
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
p0 = trial.suggest_float("p0", -10, 10)
p1 = trial.suggest_uniform("p1", 3, 5)
p2 = trial.suggest_loguniform("p2", 0.00001, 0.1)
p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5)
p4 = trial.suggest_int("p4", -20, -15)
p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100]))
p6 = trial.suggest_float("p6", -10, 10, step=1.0)
p7 = trial.suggest_int("p7", 1, 7, log=True)
return (
p0 + p1 + p2,
p3 + p4 + p5 + p6 + p7,
)
def evaluate_params(
trial: optuna.trial.Trial,
train_data: lgb.Dataset,
validation_data: lgb.Dataset,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["num_iterations"] = trial.suggest_int(
"num_iterations", 8, 128)
params["learning_rate"] = trial.suggest_uniform(
"learning_rate", 2**-5, 0.5)
params["num_leaves"] = trial.suggest_int("num_leaves", 8, 256)
params["max_depth"] = trial.suggest_int("max_depth", 4, 32)
params["min_data_in_leaf"] = trial.suggest_int(
"min_data_in_leaf", 4, 512)
params["min_sum_hessian_in_leaf"] = trial.suggest_uniform(
"min_sum_hessian_in_leaf", 2**-5, 0.25)
params["bagging_freq"] = trial.suggest_int("bagging_freq", 0, 1)
params["bagging_fraction"] = trial.suggest_uniform(
"bagging_fraction", 0.5, 1)
params["feature_fraction"] = trial.suggest_uniform(
"feature_fraction", 0.5, 1)
params["lambda_l1"] = trial.suggest_uniform("lambda_l1", 0, 64)
params["lambda_l2"] = trial.suggest_uniform("lambda_l2", 0, 64)
params["min_gain_to_split"] = trial.suggest_uniform(
"min_gain_to_split", 0, 0.25)
params["min_data_per_group"] = trial.suggest_int(
"min_data_per_group", 1, 512)
params["max_cat_threshold"] = trial.suggest_int(
"max_cat_threshold", 1, 512)
params["cat_l2"] = trial.suggest_uniform("cat_l2", 0, 64)
params["cat_smooth"] = trial.suggest_uniform("cat_smooth", 0, 2048)
params["max_cat_to_onehot"] = trial.suggest_int(
"max_cat_to_onehot", 1, 64)
params["max_bin"] = trial.suggest_int("max_bin", 32, 1024)
params["min_data_in_bin"] = trial.suggest_int(
"min_data_in_bin", 1, 64)
params["objective"] = self.objective
params["num_class"] = self.num_class
params["verbosity"] = -1
booster = lgb.Booster(params=params, train_set=train_data)
booster.add_valid(validation_data, "validation_set")
for step in range(params["num_iterations"]):
booster.update()
validation_loss = booster.eval_valid()[0][2]
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def modelLGBMClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(
num_leaves=trial.suggest_int("num_leaves", 2, 2**8),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True),
min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8),
min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8,
1),
min_split_gain=trial.suggest_loguniform('min_split_gain', 1e-8, 1),
subsample=trial.suggest_uniform('subsample', 0.4, 1),
subsample_freq=trial.suggest_int("subsample_freq", 0, 2**4),
colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.4, 1),
reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10),
reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10),
)
clf = LGBMClassifier(boosting_type='gbdt',
num_leaves=31,
max_depth=-1,
learning_rate=0.1,
n_estimators=100,
subsample_for_bin=200000,
objective=None,
class_weight=None,
min_split_gain=0.,
min_child_weight=1e-3,
min_child_samples=20,
subsample=1.,
subsample_freq=0,
colsample_bytree=1.,
reg_alpha=0.,
reg_lambda=0.,
random_state=None,
n_jobs=-1,
silent=True,
importance_type='split')
clf.set_params(**{**opt_params, **self.params})
return clf
def modelXGBClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(
max_depth=trial.suggest_int("max_depth", 2, 2**4),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True),
gamma=trial.suggest_loguniform('gamma', 1e-8, 1),
min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8,
2**10),
subsample=trial.suggest_uniform('subsample', 0.1, 1),
colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.1, 1),
colsample_bylevel=trial.suggest_uniform('colsample_bylevel', 0.1,
1),
reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10),
reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10),
)
clf = XGBClassifier(max_depth=3,
learning_rate=0.1,
n_estimators=100,
silent=True,
objective="binary:logistic",
booster='gbtree',
n_jobs=1,
gamma=0,
min_child_weight=1,
max_delta_step=0,
subsample=1,
colsample_bytree=1,
colsample_bylevel=1,
reg_alpha=0,
reg_lambda=1,
scale_pos_weight=1,
base_score=0.5,
random_state=0,
missing=None)
clf.set_params(**{**opt_params, **self.params})
return clf
def objective_for_binary_unet(args, trial: optuna.trial.Trial):
args.lr = trial.suggest_loguniform("lr", low=1e-5, high=1e-2)
args.edge_weight = trial.suggest_uniform("edge_weight", low=1, high=5)
args.wf = trial.suggest_int("wf", low=2, high=4)
args.depth = trial.suggest_int("depth", low=4, high=6)
pl_pruning_callback = PyTorchLightningPruningCallback(
trial, "val/f1_score")
ckpt_callback = train_binary_unet_model(args,
callbacks=[pl_pruning_callback])
best_f1_score = ckpt_callback.best_model_score.detach().cpu().numpy().item(
)
trial.set_user_attr("best_val_f1", best_f1_score)
trial.set_user_attr("best_model_path", ckpt_callback.best_model_path)
return best_f1_score
def modelBaggingClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(
n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True),
max_samples=trial.suggest_uniform('max_samples', 0.1, 1),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
)
clf = BaggingClassifier(base_estimator=None,
n_estimators=10,
max_samples=1.0,
max_features=1.0,
bootstrap=True,
bootstrap_features=False,
oob_score=False,
warm_start=False,
n_jobs=-1,
random_state=None,
verbose=0)
clf.set_params(**{**opt_params, **self.params})
return clf
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", low=-10, high=10)
y = trial.suggest_loguniform("y", low=1, high=10)
return (x - 2)**2 + (y - 25)**2
def objective(trial: optuna.trial.Trial) -> float:
return trial.suggest_uniform("value", 0.0, 1.0)
def objective(trial: optuna.trial.Trial, value: float) -> float:
trial.set_user_attr("lightgbm_tuner:step_name", "step{:.0f}".format(value))
return trial.suggest_uniform("x", value, value)
def evaluate_params(
trial: optuna.trial.Trial,
x_train: List[np.array],
y_train: np.array,
x_valid: List[np.array],
y_valid: np.array,
max_epochs: int,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["BATCH_SIZE"] = trial.suggest_int(
"BATCH_SIZE", min(32, x_train[0].shape[0]),
x_train[0].shape[0])
params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3)
params["DROPOUT_SHARE"] = trial.suggest_uniform(
"DROPOUT_SHARE", 0, 0.5)
params["EMBED_EXPONENT"] = trial.suggest_uniform(
"EMBED_EXPONENT", 0, 0.25)
params["EMBED_L2_REG"] = trial.suggest_uniform(
"EMBED_L2_REG", 0, 16.0)
if self.categorical_features:
params["POST_FREEZE_EPOCHS"] = trial.suggest_int(
"POST_FREEZE_EPOCHS", 4, max_epochs)
else:
params["POST_FREEZE_EPOCHS"] = 0
max_pre_freeze_epochs = max_epochs
params["PRE_FREEZE_EPOCHS"] = trial.suggest_int(
"PRE_FREEZE_EPOCHS", 4, max_epochs)
params["NODES_PER_DENSE_LAYER"] = trial.suggest_int(
"NODES_PER_DENSE_LAYER", 16, 1024)
construction_args = getfullargspec(
self.construct_embedding_network).args
self.model = self.construct_embedding_network(**{
k.lower(): v
for k, v in params.items() if k in construction_args
})
self.data[self.numeric_features] = self.data[
self.numeric_features].fillna(
self.config.get("NON_CAT_MISSING_VALUE", -1))
model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in self.config.items() if k in construction_args
})
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["PRE_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
model = freeze_embedding_layers(model)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["POST_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step + max_pre_freeze_epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def objective(trial: optuna.trial.Trial) -> List[float]:
return [
trial.suggest_uniform("v{}".format(i), 0, 5)
for i in range(n_objectives + 1)
]
