def objective(trial: optuna.trial.Trial) -> float: for i in range(n_reports): trial.report(1.0 / (i + 1), i) if trial.should_prune(): raise optuna.TrialPruned() return 1.0
def objective(trial: optuna.trial.Trial) -> float: for i in range(N_REPORTS): trial.report(1.0, i) if trial.should_prune(): raise optuna.TrialPruned() return 1.0
def objective(trial: optuna.trial.Trial) -> float: with mock.patch("optuna.pruners._filter_study") as method_mock: for i in range(N_REPORTS): trial.report(i, step=i) if trial.should_prune(): method_mock.assert_not_called() raise optuna.TrialPruned() else: method_mock.assert_not_called() return 1.0
def evaluate_params( trial: optuna.trial.Trial, train_data: lgb.Dataset, validation_data: lgb.Dataset, ) -> Union[None, dict]: """Compute out-of-sample performance for a parameter set.""" params = {} params["num_iterations"] = trial.suggest_int( "num_iterations", 8, 128) params["learning_rate"] = trial.suggest_uniform( "learning_rate", 2**-5, 0.5) params["num_leaves"] = trial.suggest_int("num_leaves", 8, 256) params["max_depth"] = trial.suggest_int("max_depth", 4, 32) params["min_data_in_leaf"] = trial.suggest_int( "min_data_in_leaf", 4, 512) params["min_sum_hessian_in_leaf"] = trial.suggest_uniform( "min_sum_hessian_in_leaf", 2**-5, 0.25) params["bagging_freq"] = trial.suggest_int("bagging_freq", 0, 1) params["bagging_fraction"] = trial.suggest_uniform( "bagging_fraction", 0.5, 1) params["feature_fraction"] = trial.suggest_uniform( "feature_fraction", 0.5, 1) params["lambda_l1"] = trial.suggest_uniform("lambda_l1", 0, 64) params["lambda_l2"] = trial.suggest_uniform("lambda_l2", 0, 64) params["min_gain_to_split"] = trial.suggest_uniform( "min_gain_to_split", 0, 0.25) params["min_data_per_group"] = trial.suggest_int( "min_data_per_group", 1, 512) params["max_cat_threshold"] = trial.suggest_int( "max_cat_threshold", 1, 512) params["cat_l2"] = trial.suggest_uniform("cat_l2", 0, 64) params["cat_smooth"] = trial.suggest_uniform("cat_smooth", 0, 2048) params["max_cat_to_onehot"] = trial.suggest_int( "max_cat_to_onehot", 1, 64) params["max_bin"] = trial.suggest_int("max_bin", 32, 1024) params["min_data_in_bin"] = trial.suggest_int( "min_data_in_bin", 1, 64) params["objective"] = self.objective params["num_class"] = self.num_class params["verbosity"] = -1 booster = lgb.Booster(params=params, train_set=train_data) booster.add_valid(validation_data, "validation_set") for step in range(params["num_iterations"]): booster.update() validation_loss = booster.eval_valid()[0][2] trial.report(validation_loss, step) if trial.should_prune(): raise optuna.exceptions.TrialPruned() return validation_loss
def objective(trial: optuna.trial.Trial) -> float: device = "cuda" if torch.cuda.is_available() else "cpu" model = Net(trial).to(device) optimizer = get_optimizer(trial, model) for step in range(epochs): train(model, device, train_loader, optimizer) error_rate = test(model, device, test_loader) trial.report(error_rate, step) if trial.should_prune(step): raise optuna.exceptions.TrialPruned() return error_rate
def evaluate_params( trial: optuna.trial.Trial, x_train: List[np.array], y_train: np.array, x_valid: List[np.array], y_valid: np.array, max_epochs: int, ) -> Union[None, dict]: """Compute out-of-sample performance for a parameter set.""" params = {} params["BATCH_SIZE"] = trial.suggest_int( "BATCH_SIZE", min(32, x_train[0].shape[0]), x_train[0].shape[0]) params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3) params["DROPOUT_SHARE"] = trial.suggest_uniform( "DROPOUT_SHARE", 0, 0.5) params["EMBED_EXPONENT"] = trial.suggest_uniform( "EMBED_EXPONENT", 0, 0.25) params["EMBED_L2_REG"] = trial.suggest_uniform( "EMBED_L2_REG", 0, 16.0) if self.categorical_features: params["POST_FREEZE_EPOCHS"] = trial.suggest_int( "POST_FREEZE_EPOCHS", 4, max_epochs) else: params["POST_FREEZE_EPOCHS"] = 0 max_pre_freeze_epochs = max_epochs params["PRE_FREEZE_EPOCHS"] = trial.suggest_int( "PRE_FREEZE_EPOCHS", 4, max_epochs) params["NODES_PER_DENSE_LAYER"] = trial.suggest_int( "NODES_PER_DENSE_LAYER", 16, 1024) construction_args = getfullargspec( self.construct_embedding_network).args self.model = self.construct_embedding_network(**{ k.lower(): v for k, v in params.items() if k in construction_args }) self.data[self.numeric_features] = self.data[ self.numeric_features].fillna( self.config.get("NON_CAT_MISSING_VALUE", -1)) model = self.construct_embedding_network( **{ k.lower(): v for k, v in self.config.items() if k in construction_args }) model.compile(loss=surv_likelihood(self.n_intervals), optimizer=Adam(amsgrad=True)) for step in range(params["PRE_FREEZE_EPOCHS"]): model.fit(x_train, y_train, batch_size=params["BATCH_SIZE"], epochs=1) validation_loss = model.evaluate(x_valid, y_valid) trial.report(validation_loss, step) if trial.should_prune(): raise optuna.exceptions.TrialPruned() model = freeze_embedding_layers(model) model.compile(loss=surv_likelihood(self.n_intervals), optimizer=Adam(amsgrad=True)) for step in range(params["POST_FREEZE_EPOCHS"]): model.fit(x_train, y_train, batch_size=params["BATCH_SIZE"], epochs=1) validation_loss = model.evaluate(x_valid, y_valid) trial.report(validation_loss, step + max_pre_freeze_epochs) if trial.should_prune(): raise optuna.exceptions.TrialPruned() return validation_loss
def objective(trial: optuna.trial.Trial): """ Optuna objective function. """ # Parse CLI arguments. args = parse_cli_args() config = HyperparamConfig() # Prepare dataset class. train_ds = Dataset(args.train_path) valid_ds = Dataset(args.valid_path) # Suggestions. # >>> TODO >>> optimizer_name = trial.suggest_categorical("optimizer", ["Adam", "RMSProp", "SGD"]) learning_rate = trial.suggest_float("learning_rate", low=1e-6, hight=1e-1, log=True) n_layer = trial.suggest_int("n_layer", 32, 1024, step=32) # <<< TODO <<< model = make_model(n_layer=n_layer) optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=learning_rate) training = Training( train_ds, valid_ds, model, optimizer, config.batch_size, config.epochs, ) accuracy_list = [] with mlflow.start_run(run_name=study.study_name): mlflow.log_params(trial.params) for epoch in range(config.epochs): # Training phase. training.model.train() train_loss, train_accuracy = training.train_epoch() mlflow.log_metrics( { "train_loss": train_loss, "train_accuracy": train_accuracy }, step=epoch + 1, ) # Validation phase. training.model.eval() valid_loss, valid_accuracy = training.validate() mlflow.log_metrics( { "valid_loss": valid_loss, "valid_accuracy": valid_accuracy }, step=epoch + 1, ) accuracy_list.append(valid_accuracy.item()) trial.report(valid_accuracy.item(), epoch) if trial.should_prune(): print("Pruned with epoch {}".format(epoch)) raise optuna.exceptions.TrialPruned() print( "Epoch {}: TrainLoss: {:.5f}, ValidLoss: {:.5f}, ValidAcc: {:.5f}" .format(epoch + 1, train_loss, valid_loss, valid_accuracy)) accuracy_list.sort() accuracy = np.mean(accuracy_list[-10:]) mlflow.log_metrics({"top10_avg_accuracy": accuracy}) return accuracy