def objective_2 (trial: Trial, param_dic = param_dic): new_params = {} for item in (param_dic): if type(param_dic[str(item)]) == int: new_params[str(item)] = trial.suggest_int(str(item), param_dic[str(item)]*lower_bound, param_dic[str(item)]*upper_bound) elif type(param_dic[str(item)]) == float: if item == 'colsample_bytree' or item == 'colsample_bylevel' or item == 'colsample_bynode' : if param_dic[str(item)]*1.25 >= 1: new_params[str(item)] = trial.suggest_float(str(item), param_dic[str(item)]*lower_bound, 1) else: new_params[str(item)] = trial.suggest_float(str(item), param_dic[str(item)]*lower_bound, param_dic[str(item)]*upper_bound) else: new_params[str(item)] = trial.suggest_float(str(item), param_dic[str(item)]*lower_bound, param_dic[str(item)]*upper_bound) elif type(param_dic[str(item)]) == str: new_params[str(item)] = trial.suggest_categorical(str(item), [(param_dic[str(item)])]) else: print('error, skipped ' + str(item)) continue #print(new_params) if forrest == False: xgb_r = xg.XGBRegressor(**new_params) else: xgb_r = xg.XGBRFRegressor(**new_params) xgb_r.fit(x_train,y_train) score = model_selection.cross_val_score(xgb_r, x_train, y_train, n_jobs=-1, cv=5) accuracy = score.mean() return accuracy
def objective(trial: optuna.Trial) -> float: trial = optuna.trial.Trial( study, study._storage.create_new_trial(study._study_id)) trial.suggest_float("DROPOUT", 0.0, 0.5) executor = optuna.integration.AllenNLPExecutor( trial, input_config_file, tmp_dir) return executor.run()
def objective(trial:Trial, data, target): train_x, valid_x, train_y, valid_y = train_test_split(data, target, test_size=0.25) dtrain = xgb.DMatrix(train_x, label=train_y) dvalid = xgb.DMatrix(valid_x, label=valid_y) param = { "verbosity": 0, "objective": "binary:logistic", "booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]), "lambda": trial.suggest_float("lambda", 1e-8, 1.0, log=True), "alpha": trial.suggest_float("alpha", 1e-8, 1.0, log=True), } if param["booster"] == "gbtree" or param["booster"] == "dart": param["max_depth"] = trial.suggest_int("max_depth", 1, 9) param["eta"] = trial.suggest_float("eta", 1e-8, 1.0, log=True) param["gamma"] = trial.suggest_float("gamma", 1e-8, 1.0, log=True) param["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"]) if param["booster"] == "dart": param["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"]) param["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"]) param["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0, log=True) param["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True) bst = xgb.train(param, dtrain) preds = bst.predict(dvalid) pred_labels = np.rint(preds) precision = sklearn.metrics.f1_score(valid_y, pred_labels) return precision
def evaluate_trial(self, trial: optuna.Trial) -> float: self.prob_corte = trial.suggest_float('prob_corte', self.prob_corte_min, self.prob_corte_max) variable_params = { 'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.2), 'feature_fraction': trial.suggest_float('feature_fraction', 0.5, 0.7), 'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 5000), 'num_leaves': trial.suggest_int('num_leaves', 16, 512), 'max_bin': trial.suggest_int('max_bin', 15, 50), 'lambda_l1': trial.suggest_float('lambda_l1', 0, 5), 'lambda_l2': trial.suggest_float('lambda_l2', 0, 100), 'is_unbalance': trial.suggest_categorical('is_unbalance', [True, False]) } params = {**self.fixed_params, **variable_params} booster = lgb.train(params, lgb.Dataset(self.X, label=self.y, weight=self.weights, feature_name=self.X.names), num_boost_round=5000, feval=self._evaluate, early_stopping_rounds=100, valid_sets=[lgb.Dataset(self.X_val, label=self.y_val, weight=self.weights_val)], valid_names=['validation'], verbose_eval=False) self.models.append(LightGBMModel(booster)) return booster.best_score['validation']['ganancia']
def func(trial: Trial, x_max: float = 1.0) -> float: x = trial.suggest_float("x", -x_max, x_max) y = trial.suggest_float("y", 20, 30, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) return (x - 2) ** 2 + (y - 25) ** 2 + z
def multi_objective_function(trial: Trial) -> Tuple[float, float]: x1: float = trial.suggest_float("x1", 0.1, 3) x2: float = trial.suggest_float("x2", 0.1, 3, log=True) x3: int = trial.suggest_int("x3", 2, 4, log=True) x4 = trial.suggest_categorical("x4", [0.1, 1.0, 10.0]) assert isinstance(x4, float) return (x1 + x2 * x3 + x4, x1 * x4)
def objective(trial: Trial) -> float: x1: float = trial.suggest_float("x1", 0.1, 3) x2: float = trial.suggest_float("x2", 0.1, 3, log=True) x3: int = trial.suggest_int("x3", 2, 4, log=True) x4 = trial.suggest_categorical("x4", [0.1, 1.0, 10.0]) assert isinstance(x4, float) return x1 + x2 * x3 + x4
def objective(trial: optuna.Trial) -> float: trial.suggest_float("DROPOUT", dropout, dropout) trial.suggest_float("LEARNING_RATE", 1e-2, 1e-1) executor = optuna.integration.AllenNLPExecutor( trial, input_config_file, tmp_dir, include_package= "tests.integration_tests.allennlp_tests.tiny_single_id", ) return executor.run()
def bb(trial: optuna.Trial): # our previous naive bitbots approach thresh_gyro_front = trial.suggest_float("thresh_gyro_front", 0, 20) thresh_gyro_side = trial.suggest_float("thresh_gyro_side", 0, 20) thresh_orient_front = trial.suggest_float("thresh_orient_front", 0, math.pi) thresh_orient_side = trial.suggest_float("thresh_orient_side", 0, math.pi) classifier = FallChecker(thresh_gyro_front=thresh_gyro_front, thresh_gyro_side=thresh_gyro_side, thresh_orient_front=thresh_orient_front, thresh_orient_side=thresh_orient_side) return evaluate_classifier(classifier)
def objective(trial: Trial): trial.suggest_int("char_embedding_dim", 16, 128) trial.suggest_int("lstm_hidden_size", 64, 256) trial.suggest_float("lr", 5e-3, 5e-1, log=True) executor = AllenNLPExecutor( trial=trial, config_file=config_file, serialization_dir=f"result/{trial.number}", metrics="best_validation_f1-measure-overall", include_package="allennlp_models", ) return executor.run()
def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]: category = trial.suggest_categorical("category", ["foo", "bar"]) if category == "foo": x = trial.suggest_float("x1", 0, 5) y = trial.suggest_float("y1", 0, 3) v0 = 4 * x**2 + 4 * y**2 v1 = (x - 5)**2 + (y - 5)**2 return v0, v1 else: x = trial.suggest_float("x2", 0, 5) y = trial.suggest_float("y2", 0, 3) v0 = 2 * x**2 + 2 * y**2 v1 = (x - 2)**2 + (y - 3)**2 return v0, v1
def objective_prune_without_report(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -15, 30) y = trial.suggest_float("y", -15, 30) v = x**2 + y**2 if v > 100: raise optuna.TrialPruned() return v
def objective(self, trial: Trial) -> float: """ Optuna optimization method. Parameters ---------- trial : optuna.Trial An optuna trial object. Returns ------- float The performance evaluation metric value for a single trial. """ suggest: Dict[str, float] = { "vect__max_df": trial.suggest_uniform( name="vect__max_df", low=self.hparams["vectorizer_hparams"]["max_df"][0], high=self.hparams["vectorizer_hparams"]["max_df"][1], ), "decomp__doc_topic_prior": trial.suggest_loguniform( name="decomp__doc_topic_prior", low=self.hparams["lda_hparams"]["alpha"][0], high=self.hparams["lda_hparams"]["alpha"][1], ), "decomp__topic_word_prior": trial.suggest_loguniform( name="decomp__topic_word_prior", low=self.hparams["lda_hparams"]["beta"][0], high=self.hparams["lda_hparams"]["beta"][1], ), "decomp__n_components": trial.suggest_int( name="decomp__n_components", low=self.hparams["lda_hparams"]["num_topics"][0], high=self.hparams["lda_hparams"]["num_topics"][1], ), "decomp__max_iter": trial.suggest_int( name="decomp__max_iter", low=self.hparams["lda_hparams"]["iterations"][0], high=self.hparams["lda_hparams"]["iterations"][1], ), "decomp__learning_decay": trial.suggest_uniform( name="decomp__learning_decay", low=self.hparams["lda_hparams"]["decay"][0], high=self.hparams["lda_hparams"]["decay"][1], ), "decomp__learning_offset": trial.suggest_float( name="decomp__learning_offset", low=self.hparams["lda_hparams"]["offset"][0], high=self.hparams["lda_hparams"]["offset"][1], ), } est: Pipeline = self.pipeline.set_params(**suggest).fit(self.X) return coherence(pipeline=est, X=self.X)
def objective(trial: Trial, train_X, train_y, test_X, test_y) -> float: params = { "n_estimators": trial.suggest_int('n_estimators', 0, 1000), 'max_depth': trial.suggest_int('max_depth', 2, 25), 'reg_alpha': trial.suggest_int('reg_alpha', 0, 10), 'reg_lambda': trial.suggest_int('reg_lambda', 0, 10), 'min_child_weight': trial.suggest_int('min_child_weight', 0, 20), 'gamma': trial.suggest_int('gamma', 0, 5), 'learning_rate': trial.suggest_loguniform('learning_rate', 0.0001, 0.5), 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.1, 1, 0.01), 'nthread': -1, 'scale_pos_weight': trial.suggest_int('scale_pos_weight', 1, 10), 'random_state': trial.suggest_int('random_state', 1, 30), 'subsample': trial.suggest_float('subsample', 0.5, 0.9) } model = XGBClassifier(**params) model.fit(train_X, train_y) return cross_val_score(model, test_X, test_y).mean()
def objective(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -10, 10) y = trial.suggest_float("y", -10, 10) trial.report(x, 0) trial.report(y, 1) trial.set_user_attr("x", x) trial.set_system_attr("y", y) return f(x, y)
def evaluate_trial(self, trial: optuna.Trial) -> float: self.__actualizar_prob_corte(trial.suggest_float('prob_corte', self.prob_corte_min, self.prob_corte_max)) variable_params = { 'eta': trial.suggest_float('eta', 0.01, 0.2), 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 0.7), 'reg_alpha': trial.suggest_float('reg_alpha', 0, 5), 'reg_lambda': trial.suggest_float('reg_lambda', 0, 100), 'gamma': trial.suggest_float('gamma', 0, 5), 'min_child_weight': trial.suggest_float('min_child_weight', 0, 5), 'max_leaves': trial.suggest_int('max_leaves', 32, 512), 'max_depth': trial.suggest_int('max_depth', 8, 50), 'max_bin': trial.suggest_int('max_bin', 20, 40) } params = {**self.fixed_params, **variable_params} booster = xgb.train(params, xgb.DMatrix(self.X, label=self.y, weight=self.weights), num_boost_round=5000, feval=self._evaluate, maximize=True, early_stopping_rounds=100, evals=[(xgb.DMatrix(self.X_val, label=self.y_val, weight=self.weights_val), 'validation')], verbose_eval=False) self.models.append(XGBoostModel(booster)) return booster.best_score
def objective(trial: opt.Trial): # only test dropping sozio economic facotrs drop_sozioeco = trial.suggest_categorical("drop_eco", [True, False]) # rest of preprocessing keeps default values # categrorial encoding, try identical encoders for all columns (for now) enc_name = trial.suggest_categorical("encoder", ["one-hot", "woe", "binary"]) enc = encoders[enc_name] x_tr = enc.fit_transform(x, y) param = { "verbosity": 0, "obective": "binary:logistic", "eval_metric": ["aucpr"], "max_depth": trial.suggest_int("max_depth", 4, 8), "booster": "gbtree", "lambda": trial.suggest_float("lambda", 1e-7, 0.5, log=True), "alpha": trial.suggest_float("alpha", 1e-8, 0.5, log=True), "subsample": trial.suggest_uniform("subsample", 0.5, 1.0), "eta": trial.suggest_loguniform("lr", 1e-5, 0.2), "gamma": trial.suggest_loguniform("gamma", 1e-8, 1.0), "grow_policy": trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"]) } dtrain = xgb.DMatrix(x_tr, label=y) cb = optuna.integration.XGBoostPruningCallback( trial, observation_key='test-aucpr') scores = xgb.cv(param, dtrain, nfold=5, stratified=True) test_aucpr = score['test-aucpr-mean'].values[-1] return test_aucpr
def sample_a2c_params(trial: optuna.Trial): """Sampler for A2C hyperparameters.""" gamma = 1.0 - trial.suggest_float("gamma", 0.0001, 0.1, log=True) max_grad_norm = trial.suggest_float("max_grad_norm", 0.3, 5.0, log=True) gae_lambda = 1.0 - trial.suggest_float("gae_lambda", 0.001, 0.2, log=True) n_steps = 2**trial.suggest_int("exponent_n_steps", 3, 10) learning_rate = trial.suggest_float("lr", 1e-5, 1, log=True) ent_coef = trial.suggest_float("ent_coef", 0.00000001, 0.1, log=True) ortho_init = trial.suggest_categorical("ortho_init", [False, True]) net_arch = trial.suggest_categorical("net_arch", ["tiny", "small"]) activation_fn = trial.suggest_categorical("activation_fn", ["tanh", "relu"]) # Display true values trial.set_user_attr("gamma_", gamma) trial.set_user_attr("gae_lambda_", gae_lambda) trial.set_user_attr("n_steps", n_steps) net_arch = [{ "pi": [64], "vf": [64] } if net_arch == "tiny" else { "pi": [64, 64], "vf": [64, 64] }] activation_fn = { "tanh": nn.Tanh, "relu": nn.ReLU, }[activation_fn] return { "n_steps": n_steps, "gamma": gamma, "gae_lambda": gae_lambda, "learning_rate": learning_rate, "ent_coef": ent_coef, "max_grad_norm": max_grad_norm, "policy_kwargs": { "net_arch": net_arch, "activation_fn": activation_fn, "ortho_init": ortho_init, }, }
def hp_search_optuna(trial: optuna.Trial): if torch.cuda.is_available(): logger.info("%s", torch.cuda.get_device_name(0)) global gopt opt = gopt # set config config = load_config(opt) config['opt'] = opt logger.info("%s", config) # set path set_path(config) # set search spaces lr = trial.suggest_float('lr', 1e-5, 1e-3, log=True) bsz = trial.suggest_categorical('batch_size', [32, 64, 128]) seed = trial.suggest_int('seed', 17, 42) epochs = trial.suggest_int('epochs', 1, opt.epoch) # prepare train, valid dataset train_loader, valid_loader = prepare_datasets(config, hp_search_bsz=bsz) with temp_seed(seed): # prepare model model = prepare_model(config) # create optimizer, scheduler, summary writer, scaler optimizer, scheduler, writer, scaler = prepare_osws( config, model, train_loader, hp_search_optuna_lr=lr) config['optimizer'] = optimizer config['scheduler'] = scheduler config['writer'] = writer config['scaler'] = scaler early_stopping = EarlyStopping(logger, patience=opt.patience, measure='f1', verbose=1) best_eval_f1 = -float('inf') for epoch in range(epochs): eval_loss, eval_f1, best_eval_f1 = train_epoch( model, config, train_loader, valid_loader, epoch, best_eval_f1) # early stopping if early_stopping.validate(eval_f1, measure='f1'): break if eval_f1 == best_eval_f1: early_stopping.reset(best_eval_f1) early_stopping.status() trial.report(eval_f1, epoch) if trial.should_prune(): raise optuna.TrialPruned() return eval_f1
def objective_fn( trial: Trial, device: int, direction: str, target_metric: str, base_serialization_dir: str, ): embedding_dim = trial.suggest_int("embedding_dim", 128, 256) max_filter_size = trial.suggest_int("max_filter_size", 3, 6) num_filters = trial.suggest_int("num_filters", 128, 256) output_dim = trial.suggest_int("output_dim", 128, 512) dropout = trial.suggest_float("dropout", 0, 1.0, log=False) lr = trial.suggest_float("lr", 1e-4, 1e-1, log=True) train_dataset, valid_dataset, vocab = prepare_data() model = create_model(vocab, embedding_dim, max_filter_size, num_filters, output_dim, dropout) if device > -1: model.to(torch.device("cuda:{}".format(device))) optimizer = SGD(model.parameters(), lr=lr) data_loader = DataLoader(train_dataset, batch_size=10, collate_fn=allennlp_collate) validation_data_loader = DataLoader(valid_dataset, batch_size=64, collate_fn=allennlp_collate) serialization_dir = os.path.join(base_serialization_dir, "trial_{}".format(trial.number)) trainer = GradientDescentTrainer( model=model, optimizer=optimizer, data_loader=data_loader, validation_data_loader=validation_data_loader, validation_metric=("+" if direction == "MAXIMIZE" else "-") + target_metric, patience=None, # `patience=None` since it could conflict with AllenNLPPruningCallback num_epochs=50, cuda_device=device, serialization_dir=serialization_dir, epoch_callbacks=[AllenNLPPruningCallback(trial, f"validation_{target_metric}")], ) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) return trainer.train()[f"best_validation_{target_metric}"]
def mlp(trial: optuna.Trial): hidden_layer_size = trial.suggest_int("hidden_layer_size", 10, 1000, 10) hidden_layer_count = trial.suggest_int("hidden_layer_count", 1, 10) hidden_layer = (hidden_layer_size, ) * hidden_layer_count activation_fun = trial.suggest_categorical("activation_fun", ['identity', 'tanh', 'relu']) alpha = trial.suggest_float("alpha", 0.000001, 0.0001) learning_rate = trial.suggest_categorical( 'learning_rate', ['constant', 'invscaling', 'adaptive']) classifier = MLPClassifier(hidden_layer_sizes=hidden_layer, activation=activation_fun, alpha=alpha, learning_rate=learning_rate) return evaluate_classifier(classifier)
def hp_search(trial: optuna.Trial, model_name: str, dataset, label_nbr, metric_name, reference_class, device): """ objective function for optuna.study optimizes for epoch number, lr and batch_size :param trial: optuna.Trial, trial of optuna, which will optimize for hyperparameters :param model_name: name of the pretrained model :param dataset: huggingface/nlp dataset object :param label_nbr: number of label for the model to output :param metric_name: name of the metric to maximize :param reference_class: reference class to calculate metrics for :param device: device where the training will occur, cuda recommended :return: metric after training """ lr = trial.suggest_float("lr", 1e-7, 1e-4, log=True) batch_size = trial.suggest_categorical("batch_size", [2, 4, 6]) epochs = trial.suggest_int("epochs", 1, 5) model = MultilabeledSequenceModel(pretrained_model_name=model_name, label_nbr=label_nbr).to(device) optimizer = AdamW(params=model.parameters(), lr=lr) for epoch in range(epochs): train_epoch(model, optimizer, dataset, batch_size, device) labels, preds = evaluate(model, dataset, batch_size, device) metric = calculate_metric(metric_name, labels, preds, reference_class) trial.report(metric, epoch) if trial.should_prune(): raise optuna.TrialPruned() return metric
def _construct_trial_grid(trial: optuna.Trial, param_space: Dict): param_grid = {} for name, params in param_space.items(): param_type = params[0] if param_type == "categorical": choices = params[1] param_grid[name] = trial.suggest_categorical(name, choices) elif param_type == "discrete_uniform": low, high, q = params[1], params[2], params[3] param_grid[name] = trial.suggest_discrete_uniform( name, low, high, q) elif param_type == "loguniform": low, high = params[1], params[2] param_grid[name] = trial.suggest_loguniform(name, low, high) elif param_type == "uniform": low, high = params[1], params[2] param_grid[name] = trial.suggest_uniform(name, low, high) elif param_type == "float": low, high = params[1], params[2] step, log = None, False if len(params) > 3: step = params[3] if len(params) > 4: log = params[4] param_grid[name] = trial.suggest_float(name, low, high, step=step, log=log) elif param_type == "int": low, high = params[1], params[2] step, log = 1, False if len(params) > 3: step = params[3] if len(params) > 4: log = params[4] param_grid[name] = trial.suggest_int(name, low, high, step=step, log=log) else: raise ValueError( f"Undefined sampling method given for trial object: {name}: {params}" ) return param_grid
def objective(self, trial: optuna.Trial) -> float: trial.suggest_float('lr', 1e-6, 1e-4) trial.suggest_categorical('batch_size', [4, 8, 14]) trial.suggest_float('weight_decay', 0.0, 0.1) trial.suggest_float('dropout', 0.0, 0.8) executor = optuna.integration.allennlp.AllenNLPExecutor( trial=trial, config_file=self.config_file, serialization_dir=self.MODEL_PATH + f'/trial_{trial.number}', metrics='best_validation_f1-measure-overall') return executor.run()
def objective(trial: Trial): delta = trial.suggest_float("delta", min_thresh, max_thresh) pipeline = create_doc_clustering_pipeline(delta, signal) pipeline.fit(train_X) gold_clusters, system_clusters = predict_baseline( logger, train_X, train_y, which=LEMMA_DELTA_BASELINE, doc_clustering_pipeline=pipeline) metrics = run_conll_evaluation(gold_clusters, system_clusters, single_meta_document=True, metrics="lea") lea_f1 = metrics.loc["lea", "f1"] return lea_f1
def __init__(self, trial: optuna.Trial) -> None: super().__init__() # We optimize the number of layers, hidden units in each layer and dropouts. layers = [] n_layers = trial.suggest_int("n_layers", 1, 3) dropout = trial.suggest_float("dropout", 0.2, 0.5) input_dim = 28 * 28 for i in range(n_layers): output_dim = trial.suggest_int("n_units_l{}".format(i), 4, 128, log=True) layers.append(nn.Linear(input_dim, output_dim)) layers.append(nn.Dropout(dropout)) layers.append(nn.ReLU()) input_dim = output_dim layers.append(nn.Linear(input_dim, 10)) self.model = nn.Sequential(*layers)
def objective(trial: Trial) -> float: params = { "epochs": trial.suggest_categorical("epochs", [50, 100, 200, 300, 400, 500]), "batch_size": 64, "num_hidden_layers": trial.suggest_int("num_hidden_layers", 0, 5), "learning_rate": trial.suggest_float("learning_rate", 1e-3, 0.1), "changepoints_range": trial.suggest_discrete_uniform("changepoints_range", 0.8, 0.95, 0.001), "n_changepoints": trial.suggest_int("n_changepoints", 20, 35), "seasonality_mode": "additive", "yearly_seasonality": False, "weekly_seasonality": True, "daily_seasonality": True, "loss_func": "MSE", } # fit_model m = NeuralProphet(**params) m.fit(train, freq="1D") future = m.make_future_dataframe(train, periods=len(valid), n_historic_predictions=True) forecast = m.predict(future) valid_forecast = forecast[forecast.y.isna()] val_rmse = mean_squared_error(valid_forecast.yhat1, valid, squared=False) return val_rmse
def get_params(trial: Trial, tunable_params: Dict, default_params: Dict): defaults = default_params.copy() for key in tunable_params: args = tunable_params[key] defaults[key] = trial.suggest_float(name=key, **args) return defaults
def objective(trial: optuna.Trial) -> float: trial.suggest_float("DROPOUT", dropout, dropout) executor = optuna.integration.AllenNLPExecutor( trial, input_config_file, tmp_dir) return executor.run()
def fixed_param(trial: op.Trial, name: str, value: float = 0.) -> float: return trial.suggest_float(name, value, value)