def _preprocess(self, trial: optuna.trial.Trial) -> None: if self.pbar is not None: self.pbar.set_description(self.pbar_fmt.format(self.step_name, self.best_score)) if "lambda_l1" in self.target_param_names: self.lgbm_params["lambda_l1"] = trial.suggest_loguniform("lambda_l1", 1e-8, 10.0) if "lambda_l2" in self.target_param_names: self.lgbm_params["lambda_l2"] = trial.suggest_loguniform("lambda_l2", 1e-8, 10.0) if "num_leaves" in self.target_param_names: tree_depth = self.lgbm_params.get("max_depth", _DEFAULT_TUNER_TREE_DEPTH) max_num_leaves = 2 ** tree_depth if tree_depth > 0 else 2 ** _DEFAULT_TUNER_TREE_DEPTH self.lgbm_params["num_leaves"] = trial.suggest_int("num_leaves", 2, max_num_leaves) if "feature_fraction" in self.target_param_names: # `GridSampler` is used for sampling feature_fraction value. # The value 1.0 for the hyperparameter is always sampled. param_value = min(trial.suggest_uniform("feature_fraction", 0.4, 1.0 + _EPS), 1.0) self.lgbm_params["feature_fraction"] = param_value if "bagging_fraction" in self.target_param_names: # `TPESampler` is used for sampling bagging_fraction value. # The value 1.0 for the hyperparameter might by sampled. param_value = min(trial.suggest_uniform("bagging_fraction", 0.4, 1.0 + _EPS), 1.0) self.lgbm_params["bagging_fraction"] = param_value if "bagging_freq" in self.target_param_names: self.lgbm_params["bagging_freq"] = trial.suggest_int("bagging_freq", 1, 7) if "min_child_samples" in self.target_param_names: # `GridSampler` is used for sampling min_child_samples value. # The value 1.0 for the hyperparameter is always sampled. param_value = int(trial.suggest_uniform("min_child_samples", 5, 100 + _EPS)) self.lgbm_params["min_child_samples"] = param_value
def __init__(self, trial: optuna.trial.Trial) -> None: super(Net, self).__init__() self.activation = getattr( F, trial.suggest_categorical('activation', ['relu', 'elu'])) self.conv1 = nn.Conv2d(1, 32, 3, 1) self.conv2 = nn.Conv2d(32, 64, 3, 1) self.dropout1 = nn.Dropout2d( p=trial.suggest_uniform("dropout_prob1", 0.1, 0.9)) self.dropout2 = nn.Dropout2d( p=trial.suggest_uniform("dropout_prob2", 0.1, 0.9)) self.fc1 = nn.Linear(9216, 128) self.fc2 = nn.Linear(128, 10)
def optimize(trial: optuna.trial.Trial, data_dict: dict): p = { 'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1), 'max_depth': trial.suggest_int('max_depth', 5, 30), 'max_leaves': trial.suggest_int('max_leaves', 5, 50), 'subsample': trial.suggest_uniform('subsample', 0.3, 1.0), 'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3, 1.0), 'min_child_weight': trial.suggest_int('min_child_weight', 5, 100), 'lambda': trial.suggest_uniform('lambda', 0.05, 0.2), 'alpha': trial.suggest_uniform('alpha', 0.05, 0.2), 'objective': 'binary:logistic', 'booster': 'gbtree', 'tree_method': 'gpu_hist', 'verbosity': 1, 'n_jobs': 10, 'eval_metric': 'auc' } print('Choosing parameters:', p) scores = [] sizes = [] # gts = GroupTimeSeriesSplit() data = data_dict['data'] target = data_dict['target'] date = data_dict['date'] gts = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10) for i, (tr_idx, val_idx) in enumerate(gts.split(data, groups=date)): sizes.append(len(tr_idx)) x_tr, x_val = copy.deepcopy(data.iloc[tr_idx]), copy.deepcopy( data.iloc[val_idx]) y_tr, y_val = copy.deepcopy(target[tr_idx]), copy.deepcopy( target[val_idx]) x_tr, x_val = calc_data_mean(x_tr, cache_dir='cache/', fold=i, train=True), \ calc_data_mean(x_val, cache_dir='cache/', fold=i, train=False) d_tr = xgb.DMatrix(x_tr, label=y_tr) d_val = xgb.DMatrix(x_val, label=y_val) clf = xgb.train(p, d_tr, 500, [(d_val, 'eval')], early_stopping_rounds=50, verbose_eval=True) val_pred = clf.predict(d_val) score = roc_auc_score(y_val, val_pred) print(f'Fold {i} ROC AUC:\t', score) scores.append(score) del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score rubbish = gc.collect() print(scores) avg_score = weighted_mean(scores, sizes) print('Avg Score:', avg_score) return avg_score
def _objective(trial: optuna.trial.Trial) -> float: p0 = trial.suggest_uniform("p0", -3.3, 5.2) p1 = trial.suggest_uniform("p1", 2.0, 2.0) p2 = trial.suggest_loguniform("p2", 0.0001, 0.3) p3 = trial.suggest_loguniform("p3", 1.1, 1.1) p4 = trial.suggest_int("p4", -100, 8) p5 = trial.suggest_int("p5", -20, -20) p6 = trial.suggest_discrete_uniform("p6", 10, 20, 2) p7 = trial.suggest_discrete_uniform("p7", 0.1, 1.0, 0.1) p8 = trial.suggest_discrete_uniform("p8", 2.2, 2.2, 0.5) p9 = trial.suggest_categorical("p9", ["9", "3", "0", "8"]) assert isinstance(p9, str) return p0 + p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + int(p9)
def objective(trial: optuna.trial.Trial, value: float) -> float: trial.set_system_attr( optuna.integration.lightgbm_tuner.optimize._STEP_NAME_KEY, "step{:.0f}".format(value), ) return trial.suggest_uniform("x", value, value)
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float: x = trial.suggest_uniform("x", -x_max, x_max) y = trial.suggest_loguniform("y", 20, 30) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) return (x - 2)**2 + (y - 25)**2 + z
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20.0, 30.0) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective(trial: optuna.trial.Trial) -> float: num_units = trial.suggest_int("NUM_UNITS", 16, 32) dropout_rate = trial.suggest_uniform("DROPOUT_RATE", 0.1, 0.2) optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"]) accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore return accuracy
def objective2(trial: optuna.trial.Trial) -> float: p1 = trial.suggest_loguniform("p1", 50, 100) # The range has been changed p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3) p5 = trial.suggest_uniform("p5", 0, 1) return p1 + p3 + p5
def objective_test_upgrade(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float. y = trial.suggest_int("y", 0, 10) z = cast(float, trial.suggest_categorical("z", [-5, 0, 5])) trial.set_system_attr("a", 0) trial.set_user_attr("b", 1) trial.report(0.5, step=0) return x ** 2 + y ** 2 + z ** 2
def optimize(trial: optuna.trial.Trial, data_dict: dict): p = { 'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1), 'max_depth': trial.suggest_int('max_depth', 5, 30), 'max_leaves': trial.suggest_int('max_leaves', 5, 50), 'subsample': trial.suggest_uniform('subsample', 0.3, 1.0), 'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3, 1.0), 'min_child_weight': trial.suggest_int('min_child_weight', 5, 100), 'lambda': trial.suggest_uniform('lambda', 0.05, 0.2), 'alpha': trial.suggest_uniform('alpha', 0.05, 0.2), 'objective': 'reg:squarederror', 'booster': 'gbtree', 'tree_method': 'gpu_hist', 'verbosity': 1, 'n_jobs': 10, 'eval_metric': 'rmse' } print('Choosing parameters:', p) scores = [] sizes = [] # gts = GroupTimeSeriesSplit()'] gts = pgs.PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10) for i, (tr_idx, val_idx) in enumerate( gts.split(data_dict['data'], groups=data_dict['era'])): x_tr, x_val = data_dict['data'][tr_idx], data_dict['data'][val_idx] y_tr, y_val = data_dict['target'][tr_idx], data_dict['target'][val_idx] d_tr = xgb.DMatrix(x_tr, label=y_tr) d_val = xgb.DMatrix(x_val, label=y_val) clf = xgb.train(p, d_tr, 500, [(d_val, 'eval')], early_stopping_rounds=50, verbose_eval=True) val_pred = clf.predict(d_val) score = mean_squared_error(y_val, val_pred) scores.append(score) sizes.append(len(tr_idx) + len(val_idx)) del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score rubbish = gc.collect() print(scores) avg_score = utils.weighted_mean(scores, sizes) print('Avg Score:', avg_score) return avg_score
def objective0(trial: optuna.trial.Trial) -> float: p0 = trial.suggest_uniform("p0", 0, 10) p1 = trial.suggest_loguniform("p1", 1, 10) p2 = trial.suggest_int("p2", 0, 10) p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3) p4 = trial.suggest_categorical("p4", ["10", "20", "30"]) assert isinstance(p4, str) return p0 + p1 + p2 + p3 + int(p4)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20, 30) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) long_str = str(list(range(5000))) trial.set_user_attr("my_user_attr", long_str) return (x - 2)**2 + (y - 25)**2 + z
def _lgbmclassifier_default(trial: optuna.trial.Trial): # TODO: using LightGBMTuner params = { 'boosting_type': trial.suggest_categorical('boosting', ['gbdt', 'dart', 'goss']), 'objective': 'binary', 'metric': ['binary', 'binary_error', 'auc'], 'num_leaves': trial.suggest_int("num_leaves", 10, 500), 'learning_rate': trial.suggest_loguniform("learning_rate", 1e-5, 1), 'feature_fraction': trial.suggest_uniform("feature_fraction", 0.0, 1.0), } if params['boosting_type'] == 'dart': params['drop_rate'] = trial.suggest_loguniform('drop_rate', 1e-8, 1.0) params['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0) if params['boosting_type'] == 'goss': params['top_rate'] = trial.suggest_uniform('top_rate', 0.0, 1.0) params['other_rate'] = trial.suggest_uniform('other_rate', 0.0, 1.0 - params['top_rate']) return params
def objective(trial: optuna.trial.Trial) -> float: dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0000..0062}.tar").shuffle(20000).decode() dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0000..0010}.tar").shuffle(20000).decode() test_dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0063..0072}.tar").decode() test_dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0011..0013}.tar").decode() dataset = SampleEqually([dataset, dataset_2]) test_dataset = SampleEqually([test_dataset_2, test_dataset]) train_loader = DataLoader(dataset, num_workers=16, batch_size=4, pin_memory=True) test_loader = DataLoader(test_dataset, num_workers=4, batch_size=1, pin_memory=True) # We optimize the number of layers, hidden units in each layer and dropouts. config = { "sample_ratio_one": trial.suggest_uniform("sample_ratio_one", 0.1, 0.9), "sample_radius_one": trial.suggest_uniform("sample_radius_one", 0.1, 0.9), "sample_max_neighbor": trial.suggest_int("sample_max_neighbor", 8, 72), "sample_ratio_two": trial.suggest_uniform("sample_ratio_two", 0.1, 0.9), "sample_radius_two": trial.suggest_uniform("sample_radius_two", 0.1, 0.9), "fc_1": trial.suggest_int("fc_1", 128, 256), "fc_1_out": trial.suggest_int("fc_1_out", 32, 128), "fc_2_out": trial.suggest_int("fc_2_out", 16, 96), "dropout": trial.suggest_uniform("dropout", 0.1, 0.9), } num_classes = 2 import pytorch_lightning as pl model = LitPointNet2(num_classes, lr=0.0001, config=config) trainer = pl.Trainer( logger=True, limit_val_batches=10000, limit_train_batches=10000, checkpoint_callback=False, auto_lr_find=True, max_epochs=20, gpus=1, callbacks=[PyTorchLightningPruningCallback(trial, monitor="val/loss")], ) trainer.logger.log_hyperparams(config) trainer.tune(model=model, train_dataloader=train_loader, val_dataloaders=test_loader) trainer.fit(model=model, train_dataloader=train_loader, val_dataloaders=test_loader) return trainer.callback_metrics["val/loss"].item()
def objective1(trial: optuna.trial.Trial) -> float: # p0, p2 and p4 are deleted. p1 = trial.suggest_loguniform("p1", 1, 10) p3 = trial.suggest_discrete_uniform("p3", 0, 9, 3) # p5 is added. p5 = trial.suggest_uniform("p5", 0, 1) return p1 + p3 + p5
def modelCatBoostClassifier(self, trial: optuna.trial.Trial): opt_params = dict( num_leaves=trial.suggest_int("num_leaves", 2, 2**8), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8), min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8, 1), min_split_gain=trial.suggest_loguniform('min_split_gain', 1e-8, 1), subsample=trial.suggest_uniform('subsample', 0.4, 1), subsample_freq=trial.suggest_int("subsample_freq", 0, 2**4), colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.4, 1), reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10), reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10), ) clf = CatBoostClassifier() clf.set_params(**{**opt_params, **self.params}) return clf
def _objective_func(trial: optuna.trial.Trial) -> float: u = trial.suggest_int("u", 0, 10, step=2) v = trial.suggest_int("v", 1, 10, log=True) w = trial.suggest_float("w", -1.0, 1.0, step=0.1) x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20.0, 30.0) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") return u + v + w + (x - 2)**2 + (y - 25)**2 + z
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]: p0 = trial.suggest_float("p0", -10, 10) p1 = trial.suggest_uniform("p1", 3, 5) p2 = trial.suggest_loguniform("p2", 0.00001, 0.1) p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5) p4 = trial.suggest_int("p4", -20, -15) p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100])) p6 = trial.suggest_float("p6", -10, 10, step=1.0) p7 = trial.suggest_int("p7", 1, 7, log=True) return ( p0 + p1 + p2, p3 + p4 + p5 + p6 + p7, )
def evaluate_params( trial: optuna.trial.Trial, train_data: lgb.Dataset, validation_data: lgb.Dataset, ) -> Union[None, dict]: """Compute out-of-sample performance for a parameter set.""" params = {} params["num_iterations"] = trial.suggest_int( "num_iterations", 8, 128) params["learning_rate"] = trial.suggest_uniform( "learning_rate", 2**-5, 0.5) params["num_leaves"] = trial.suggest_int("num_leaves", 8, 256) params["max_depth"] = trial.suggest_int("max_depth", 4, 32) params["min_data_in_leaf"] = trial.suggest_int( "min_data_in_leaf", 4, 512) params["min_sum_hessian_in_leaf"] = trial.suggest_uniform( "min_sum_hessian_in_leaf", 2**-5, 0.25) params["bagging_freq"] = trial.suggest_int("bagging_freq", 0, 1) params["bagging_fraction"] = trial.suggest_uniform( "bagging_fraction", 0.5, 1) params["feature_fraction"] = trial.suggest_uniform( "feature_fraction", 0.5, 1) params["lambda_l1"] = trial.suggest_uniform("lambda_l1", 0, 64) params["lambda_l2"] = trial.suggest_uniform("lambda_l2", 0, 64) params["min_gain_to_split"] = trial.suggest_uniform( "min_gain_to_split", 0, 0.25) params["min_data_per_group"] = trial.suggest_int( "min_data_per_group", 1, 512) params["max_cat_threshold"] = trial.suggest_int( "max_cat_threshold", 1, 512) params["cat_l2"] = trial.suggest_uniform("cat_l2", 0, 64) params["cat_smooth"] = trial.suggest_uniform("cat_smooth", 0, 2048) params["max_cat_to_onehot"] = trial.suggest_int( "max_cat_to_onehot", 1, 64) params["max_bin"] = trial.suggest_int("max_bin", 32, 1024) params["min_data_in_bin"] = trial.suggest_int( "min_data_in_bin", 1, 64) params["objective"] = self.objective params["num_class"] = self.num_class params["verbosity"] = -1 booster = lgb.Booster(params=params, train_set=train_data) booster.add_valid(validation_data, "validation_set") for step in range(params["num_iterations"]): booster.update() validation_loss = booster.eval_valid()[0][2] trial.report(validation_loss, step) if trial.should_prune(): raise optuna.exceptions.TrialPruned() return validation_loss
def modelLGBMClassifier(self, trial: optuna.trial.Trial): opt_params = dict( num_leaves=trial.suggest_int("num_leaves", 2, 2**8), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8), min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8, 1), min_split_gain=trial.suggest_loguniform('min_split_gain', 1e-8, 1), subsample=trial.suggest_uniform('subsample', 0.4, 1), subsample_freq=trial.suggest_int("subsample_freq", 0, 2**4), colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.4, 1), reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10), reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10), ) clf = LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=100, subsample_for_bin=200000, objective=None, class_weight=None, min_split_gain=0., min_child_weight=1e-3, min_child_samples=20, subsample=1., subsample_freq=0, colsample_bytree=1., reg_alpha=0., reg_lambda=0., random_state=None, n_jobs=-1, silent=True, importance_type='split') clf.set_params(**{**opt_params, **self.params}) return clf
def modelXGBClassifier(self, trial: optuna.trial.Trial): opt_params = dict( max_depth=trial.suggest_int("max_depth", 2, 2**4), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), gamma=trial.suggest_loguniform('gamma', 1e-8, 1), min_child_weight=trial.suggest_loguniform('min_child_weight', 1e-8, 2**10), subsample=trial.suggest_uniform('subsample', 0.1, 1), colsample_bytree=trial.suggest_uniform('colsample_bytree', 0.1, 1), colsample_bylevel=trial.suggest_uniform('colsample_bylevel', 0.1, 1), reg_alpha=trial.suggest_loguniform('reg_alpha', 1e-8, 10), reg_lambda=trial.suggest_loguniform('reg_lambda', 1e-8, 10), ) clf = XGBClassifier(max_depth=3, learning_rate=0.1, n_estimators=100, silent=True, objective="binary:logistic", booster='gbtree', n_jobs=1, gamma=0, min_child_weight=1, max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1, reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, random_state=0, missing=None) clf.set_params(**{**opt_params, **self.params}) return clf
def objective_for_binary_unet(args, trial: optuna.trial.Trial): args.lr = trial.suggest_loguniform("lr", low=1e-5, high=1e-2) args.edge_weight = trial.suggest_uniform("edge_weight", low=1, high=5) args.wf = trial.suggest_int("wf", low=2, high=4) args.depth = trial.suggest_int("depth", low=4, high=6) pl_pruning_callback = PyTorchLightningPruningCallback( trial, "val/f1_score") ckpt_callback = train_binary_unet_model(args, callbacks=[pl_pruning_callback]) best_f1_score = ckpt_callback.best_model_score.detach().cpu().numpy().item( ) trial.set_user_attr("best_val_f1", best_f1_score) trial.set_user_attr("best_model_path", ckpt_callback.best_model_path) return best_f1_score
def modelBaggingClassifier(self, trial: optuna.trial.Trial): opt_params = dict( n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), max_samples=trial.suggest_uniform('max_samples', 0.1, 1), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), ) clf = BaggingClassifier(base_estimator=None, n_estimators=10, max_samples=1.0, max_features=1.0, bootstrap=True, bootstrap_features=False, oob_score=False, warm_start=False, n_jobs=-1, random_state=None, verbose=0) clf.set_params(**{**opt_params, **self.params}) return clf
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", low=-10, high=10) y = trial.suggest_loguniform("y", low=1, high=10) return (x - 2)**2 + (y - 25)**2
def objective(trial: optuna.trial.Trial) -> float: return trial.suggest_uniform("value", 0.0, 1.0)
def objective(trial: optuna.trial.Trial, value: float) -> float: trial.set_user_attr("lightgbm_tuner:step_name", "step{:.0f}".format(value)) return trial.suggest_uniform("x", value, value)
def evaluate_params( trial: optuna.trial.Trial, x_train: List[np.array], y_train: np.array, x_valid: List[np.array], y_valid: np.array, max_epochs: int, ) -> Union[None, dict]: """Compute out-of-sample performance for a parameter set.""" params = {} params["BATCH_SIZE"] = trial.suggest_int( "BATCH_SIZE", min(32, x_train[0].shape[0]), x_train[0].shape[0]) params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3) params["DROPOUT_SHARE"] = trial.suggest_uniform( "DROPOUT_SHARE", 0, 0.5) params["EMBED_EXPONENT"] = trial.suggest_uniform( "EMBED_EXPONENT", 0, 0.25) params["EMBED_L2_REG"] = trial.suggest_uniform( "EMBED_L2_REG", 0, 16.0) if self.categorical_features: params["POST_FREEZE_EPOCHS"] = trial.suggest_int( "POST_FREEZE_EPOCHS", 4, max_epochs) else: params["POST_FREEZE_EPOCHS"] = 0 max_pre_freeze_epochs = max_epochs params["PRE_FREEZE_EPOCHS"] = trial.suggest_int( "PRE_FREEZE_EPOCHS", 4, max_epochs) params["NODES_PER_DENSE_LAYER"] = trial.suggest_int( "NODES_PER_DENSE_LAYER", 16, 1024) construction_args = getfullargspec( self.construct_embedding_network).args self.model = self.construct_embedding_network(**{ k.lower(): v for k, v in params.items() if k in construction_args }) self.data[self.numeric_features] = self.data[ self.numeric_features].fillna( self.config.get("NON_CAT_MISSING_VALUE", -1)) model = self.construct_embedding_network( **{ k.lower(): v for k, v in self.config.items() if k in construction_args }) model.compile(loss=surv_likelihood(self.n_intervals), optimizer=Adam(amsgrad=True)) for step in range(params["PRE_FREEZE_EPOCHS"]): model.fit(x_train, y_train, batch_size=params["BATCH_SIZE"], epochs=1) validation_loss = model.evaluate(x_valid, y_valid) trial.report(validation_loss, step) if trial.should_prune(): raise optuna.exceptions.TrialPruned() model = freeze_embedding_layers(model) model.compile(loss=surv_likelihood(self.n_intervals), optimizer=Adam(amsgrad=True)) for step in range(params["POST_FREEZE_EPOCHS"]): model.fit(x_train, y_train, batch_size=params["BATCH_SIZE"], epochs=1) validation_loss = model.evaluate(x_valid, y_valid) trial.report(validation_loss, step + max_pre_freeze_epochs) if trial.should_prune(): raise optuna.exceptions.TrialPruned() return validation_loss
def objective(trial: optuna.trial.Trial) -> List[float]: return [ trial.suggest_uniform("v{}".format(i), 0, 5) for i in range(n_objectives + 1) ]