def _xgbclassifier_default(trial: optuna.trial.Trial): param = { 'silent': 1, 'objective': 'binary:logistic', 'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']), 'lambda': trial.suggest_loguniform('lambda', 1e-8, 1.0), 'alpha': trial.suggest_loguniform('alpha', 1e-8, 1.0) } if param['booster'] == 'gbtree' or param['booster'] == 'dart': param['max_depth'] = trial.suggest_int('max_depth', 1, 9) param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0) param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0) param['grow_policy'] = trial.suggest_categorical( 'grow_policy', ['depthwise', 'lossguide']) if param['booster'] == 'dart': param['sample_type'] = trial.suggest_categorical( 'sample_type', ['uniform', 'weighted']) param['normalize_type'] = trial.suggest_categorical( 'normalize_type', ['tree', 'forest']) param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0) param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0) return param
def __call__(self, trial: optuna.trial.Trial) -> float: data = TreeDataModule( self._filename, batch_size=trial.suggest_int("batch_size", 32, 160, 32), ) kwargs = { "lstm_size": trial.suggest_categorical("lstm_size", [512, 1024, 2048]), "dropout_prob": trial.suggest_float("dropout", 0.1, 0.5, step=0.1), "learning_rate": trial.suggest_float("lr", 1e-3, 1e-1, log=True), "weight_decay": trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True), } model = RouteDistanceModel(**kwargs) gpus = int(torch.cuda.is_available()) pruning_callback = PyTorchLightningPruningCallback( trial, monitor="val_monitor") trainer = Trainer( gpus=gpus, logger=True, # become a tensorboard logger checkpoint_callback=False, callbacks=[pruning_callback], # type: ignore max_epochs=EPOCHS, ) trainer.fit(model, datamodule=data) return trainer.callback_metrics["val_monitor"].item()
def mo_objective_test_upgrade(trial: optuna.trial.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", -5, 5) y = trial.suggest_int("y", 0, 10) z = cast(float, trial.suggest_categorical("z", [-5, 0, 5])) trial.set_system_attr("a", 0) trial.set_user_attr("b", 1) return x, x ** 2 + y ** 2 + z ** 2
def feedforward(config: BaseConfig, trial: optuna.trial.Trial) -> pl.LightningModule: """Returns a tunable PyTorch lightning feedforward module. Args: config (BaseConfig): the hard-coded configuration. trial (optuna.Trial): optuna trial. Returns: pl.LightningModule: a lightning module. """ model = FeedForward(num_inputs=config.NUM_INPUTS, num_outputs=config.NUM_OUTPUTS, num_hidden=trial.suggest_int('num_hidden', 1, 4), num_layers=trial.suggest_int('num_layers', 1, 2), dropout=trial.suggest_float('dropout', 0.0, 0.5), activation=trial.suggest_categorical( 'activation', ['relu', 'none'])) training_config = get_training_config( lr=trial.suggest_loguniform('lr', 1e-3, 1e-0), weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1), max_epochs=config.MAX_EPOCHS) pl_model = TemporalConvNet(training_config=training_config, lr=trial.suggest_loguniform('lr', 1e-3, 1e-0), weight_decay=trial.suggest_loguniform( 'weight_decay', 1e-5, 1e-1), max_epochs=config.MAX_EPOCHS) return pl_model
def modelExtraTreesClassifier(self, trial: optuna.trial.Trial): opt_params = dict(n_estimators=trial.suggest_int("n_estimators", 2, 2**10, log=True), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), max_depth=trial.suggest_int("max_depth", 2, 2**4), criterion=trial.suggest_categorical( "criterion", ["gini", "entropy"])) clf = ExtraTreesClassifier(n_estimators=100, criterion="gini", max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0., max_features="auto", max_leaf_nodes=None, min_impurity_decrease=0., min_impurity_split=None, bootstrap=False, oob_score=False, n_jobs=None, random_state=None, verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, max_samples=None) clf.set_params(**{**opt_params, **self.params}) return clf
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float: x = trial.suggest_uniform("x", -x_max, x_max) y = trial.suggest_loguniform("y", 20, 30) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) return (x - 2)**2 + (y - 25)**2 + z
def f(trial: optuna.trial.Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) trial.set_user_attr("train_loss", 3) raise ValueError() return x + y # 3.5
def objective(trial: optuna.trial.Trial) -> float: num_units = trial.suggest_int("NUM_UNITS", 16, 32) dropout_rate = trial.suggest_float("DROPOUT_RATE", 0.1, 0.2) optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"]) accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore return accuracy
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20.0, 30.0) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective_test_upgrade(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float. y = trial.suggest_int("y", 0, 10) z = cast(float, trial.suggest_categorical("z", [-5, 0, 5])) trial.set_system_attr("a", 0) trial.set_user_attr("b", 1) trial.report(0.5, step=0) return x ** 2 + y ** 2 + z ** 2
def _objective_func(trial: optuna.trial.Trial) -> float: x = trial.suggest_float("x", -1.0, 1.0) y = trial.suggest_float("y", 20, 30, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") mlflow.log_metric(metric_name, metric) return (x - 2)**2 + (y - 25)**2 + z
def objective0(trial: optuna.trial.Trial) -> float: p0 = trial.suggest_float("p0", 0, 10) p1 = trial.suggest_float("p1", 1, 10, log=True) p2 = trial.suggest_int("p2", 0, 10) p3 = trial.suggest_float("p3", 0, 9, step=3) p4 = trial.suggest_categorical("p4", ["10", "20", "30"]) assert isinstance(p4, str) return p0 + p1 + p2 + p3 + int(p4)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float: x = trial.suggest_float("x", -1.0, 1.0) y = trial.suggest_float("y", 20, 30, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) long_str = str(list(range(5000))) trial.set_user_attr("my_user_attr", long_str) return (x - 2)**2 + (y - 25)**2 + z
def objective_test_upgrade_distributions(trial: optuna.trial.Trial) -> float: x1 = trial.suggest_float("x1", -5, 5) x2 = trial.suggest_float("x2", 1e-5, 1e-3, log=True) x3 = trial.suggest_float("x3", -6, 6, step=2) y1 = trial.suggest_int("y1", 0, 10) y2 = trial.suggest_int("y2", 1, 20, log=True) y3 = trial.suggest_int("y3", 5, 15, step=3) z = cast(float, trial.suggest_categorical("z", [-5, 0, 5])) return x1**2 + x2**2 + x3**2 + y1**2 + y2**2 + y3**2 + z**2
def _objective_func(trial: optuna.trial.Trial) -> float: u = trial.suggest_int("u", 0, 10, step=2) v = trial.suggest_int("v", 1, 10, log=True) w = trial.suggest_float("w", -1.0, 1.0, step=0.1) x = trial.suggest_uniform("x", -1.0, 1.0) y = trial.suggest_loguniform("y", 20.0, 30.0) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) trial.set_user_attr("my_user_attr", "my_user_attr_value") return u + v + w + (x - 2)**2 + (y - 25)**2 + z
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]: x = trial.suggest_float("x", low=-10, high=10) y = trial.suggest_float("y", low=1, high=10, log=True) z = trial.suggest_categorical("z", (-1.0, 1.0)) assert isinstance(z, float) first_objective = (x - 2)**2 + (y - 25)**2 + z second_objective = (x - 2)**3 + (y - 25)**3 - z return first_objective, second_objective
def objective(trial: optuna.trial.Trial): encoder_num_layers = trial.suggest_int("encoder.num_layer", 1, 4) decoder_num_layers = trial.suggest_int("decoder.num_layer", 1, 4) transformer_d_model = trial.suggest_categorical( 'transformer.d_model', [64, 128, 256]) hparams["basic"]["transformer"]["d_model"] = transformer_d_model hparams["decoder"]["num_layer"] = decoder_num_layers hparams["encoder"]["num_layer"] = encoder_num_layers device = torch.device("cuda" if torch.cuda.is_available() else "cpu") task = Task(hparams, device) return task.train(trial)
def __init__(self, trial: optuna.trial.Trial) -> None: super(Net, self).__init__() self.activation = getattr( F, trial.suggest_categorical('activation', ['relu', 'elu'])) self.conv1 = nn.Conv2d(1, 32, 3, 1) self.conv2 = nn.Conv2d(32, 64, 3, 1) self.dropout1 = nn.Dropout2d( p=trial.suggest_uniform("dropout_prob1", 0.1, 0.9)) self.dropout2 = nn.Dropout2d( p=trial.suggest_uniform("dropout_prob2", 0.1, 0.9)) self.fc1 = nn.Linear(9216, 128) self.fc2 = nn.Linear(128, 10)
def search_hyperparam(trial: optuna.trial.Trial) -> Dict[str, Any]: """Search hyperparam from user-specified search space.""" epochs = trial.suggest_int("epochs", low=50, high=200, step=50) img_size = trial.suggest_categorical("img_size", [96, 112, 168, 224]) n_select = trial.suggest_int("n_select", low=0, high=6, step=2) batch_size = trial.suggest_int("batch_size", low=16, high=64, step=16) return { "EPOCHS": epochs, "IMG_SIZE": img_size, "n_select": n_select, "BATCH_SIZE": batch_size }
def _catboostclassifier_default(trial: optuna.trial.Trial): params = { 'iterations': trial.suggest_int('iterations', 50, 300), 'depth': trial.suggest_int('depth', 4, 10), 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.3), 'random_strength': trial.suggest_int('random_strength', 0, 100), 'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 0.01, 100.00), 'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']), 'od_wait': trial.suggest_int('od_wait', 10, 50) } return params
def f(trial: optuna.trial.Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) assert isinstance(y, float) trial.set_user_attr("train_loss", 3) trial.set_system_attr("foo", "bar") value = x + y # 3.5 # Test reported intermediate values, although it in practice is not "intermediate". trial.report(value, step=0) return value
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]: p0 = trial.suggest_float("p0", -10, 10) p1 = trial.suggest_uniform("p1", 3, 5) p2 = trial.suggest_loguniform("p2", 0.00001, 0.1) p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5) p4 = trial.suggest_int("p4", -20, -15) p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100])) p6 = trial.suggest_float("p6", -10, 10, step=1.0) p7 = trial.suggest_int("p7", 1, 7, log=True) return ( p0 + p1 + p2, p3 + p4 + p5 + p6 + p7, )
def _convert(trial: optuna.trial.Trial, template: TuningParametersTemplate) -> Dict[str, Any]: result: Dict[str, Any] = {} for k, v in template.params_dict.items(): if isinstance(v, RandInt): if v.log and v.q is not None: value = trial.suggest_float(name=k, low=0, high=1.0) result[k] = uniform_to_integers( value, low=v.low, high=v.high, q=v.q, # type: ignore log=True, include_high=v.include_high, ) else: _high: Any = v.high if v.include_high else v.high - 1 result[k] = trial.suggest_int(name=k, low=v.low, high=_high, step=v.q, log=v.log) elif isinstance(v, Rand): if v.log and v.q is not None: value = trial.suggest_float(name=k, low=0, high=1.0) result[k] = uniform_to_discrete( value, low=v.low, high=v.high, q=v.q, log=True, include_high=v.include_high, ) else: _high = v.high if v.q is not None and not v.include_high: _high -= _IGNORABLE_ERROR result[k] = trial.suggest_float(name=k, low=v.low, high=_high, step=v.q, log=v.log) elif isinstance(v, TransitionChoice): result[k] = v.values[trial.suggest_int(name=k, low=0, high=len(v.values) - 1)] elif isinstance(v, Choice): result[k] = trial.suggest_categorical(name=k, choices=v.values) else: # pragma: no cover raise NotImplementedError return result
def __call__(self, trial: optuna.trial.Trial): classifier_name = trial.suggest_categorical("classifier", ["SVC", "RandomForest"]) if classifier_name == "SVC": svc_c = trial.suggest_float("svc_c", 1e-10, 1e10, log=True) classifier_obj = sklearn.svm.SVC(C=svc_c, gamma="auto") else: rf_max_depth = trial.suggest_int("rf_max_depth", 2, 32, log=True) classifier_obj = ensemble.RandomForestClassifier( max_depth=rf_max_depth, n_estimators=10 ) score = cross_val_score(classifier_obj, self.X, self.y, n_jobs=1, cv=self.cv) accuracy = score.mean() return accuracy
def _objective(trial: optuna.trial.Trial) -> float: p0 = trial.suggest_float("p0", -3.3, 5.2) p1 = trial.suggest_float("p1", 2.0, 2.0) p2 = trial.suggest_float("p2", 0.0001, 0.3, log=True) p3 = trial.suggest_float("p3", 1.1, 1.1, log=True) p4 = trial.suggest_int("p4", -100, 8) p5 = trial.suggest_int("p5", -20, -20) p6 = trial.suggest_float("p6", 10, 20, step=2) p7 = trial.suggest_float("p7", 0.1, 1.0, step=0.1) p8 = trial.suggest_float("p8", 2.2, 2.2, step=0.5) p9 = trial.suggest_categorical("p9", ["9", "3", "0", "8"]) assert isinstance(p9, str) return p0 + p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + int(p9)
def _objective(self, trial: optuna.trial.Trial): cv = trial.suggest_int('cv', 2, 2**4) opt_params = dict( objective=trial.suggest_categorical("objective", ["Logloss", "CrossEntropy"]), boosting_type=trial.suggest_categorical("boosting_type", ["Ordered", "Plain"]), bootstrap_type=trial.suggest_categorical( "bootstrap_type", ["Bayesian", "Bernoulli", "MVS"]), # used_ram_limit="3gb", max_depth=trial.suggest_int("max_depth", 2, 2**4), learning_rate=trial.suggest_discrete_uniform( 'learning_rate', 0.001, 1, 0.001), # n_estimators=trial.suggest_int("n_estimators", 2, 2 ** 10, log=True), colsample_bylevel=trial.suggest_float("colsample_bylevel", 0.01, 0.1), reg_lambda=trial.suggest_float("reg_lambda", 1e-8, 100, log=True)) if opt_params["bootstrap_type"] == "Bayesian": opt_params["bagging_temperature"] = trial.suggest_float( "bagging_temperature", 0, 10) elif opt_params["bootstrap_type"] == "Bernoulli": opt_params["subsample"] = trial.suggest_float("subsample", 0.1, 1) if self.params is not None: opt_params.update(self.params) clf_oof = CatBoostClassifierOOF(self.X, self.y, params=opt_params, cv=cv, feval=self.feval) clf_oof.run() return clf_oof.oof_score # todo: f1
def _lgbmclassifier_default(trial: optuna.trial.Trial): # TODO: using LightGBMTuner params = { 'boosting_type': trial.suggest_categorical('boosting', ['gbdt', 'dart', 'goss']), 'objective': 'binary', 'metric': ['binary', 'binary_error', 'auc'], 'num_leaves': trial.suggest_int("num_leaves", 10, 500), 'learning_rate': trial.suggest_loguniform("learning_rate", 1e-5, 1), 'feature_fraction': trial.suggest_uniform("feature_fraction", 0.0, 1.0), } if params['boosting_type'] == 'dart': params['drop_rate'] = trial.suggest_loguniform('drop_rate', 1e-8, 1.0) params['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0) if params['boosting_type'] == 'goss': params['top_rate'] = trial.suggest_uniform('top_rate', 0.0, 1.0) params['other_rate'] = trial.suggest_uniform('other_rate', 0.0, 1.0 - params['top_rate']) return params
def get_optimizer(trial: optuna.trial.Trial, model: nn.Module) -> optim.Optimizer: def adam(model: nn.Module, lr: float, weight_decay: float) -> optim.Optimizer: return optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) def momentum(model: nn.Module, lr: float, weight_decay: float) -> optim.Optimizer: return optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay) optimizer_name = trial.suggest_categorical('optimizer', ['adam', 'momentum']) optimizer: Callable[[nn.Module, float, float], optim.Optimizer] = locals()[optimizer_name] lr = trial.suggest_loguniform('lr', 1e-5, 1e-1) weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3) return optimizer(model, lr, weight_decay)
def create_model(trail: optuna.trial.Trial): tf.keras.backend.clear_session() time.sleep(3) kwargs['name'] = f"{trail.number}-Gavin-Optuna" model_options = ["Transformer", "Performer"] model_option = trail.suggest_categorical("model_option", model_options) if model_option == "Transformer": model_type = TransformerIntegration elif model_option == "Performer": model_type = PerformerIntegration elif model_option == "FNet": model_type = FNetIntegration else: raise ValueError(f"Unknown Model Option: {model_option}") max_length = trail.suggest_int("MAX_LENGTH", 30, 40, step=10) num_layers = trail.suggest_int("NUM_LAYERS", 2, 6, step=2) d_model = trail.suggest_int("D_MODEL", 128, 1024, step=64) num_heads = trail.suggest_int("NUM_HEADS", 2, 16, step=2) units = trail.suggest_int("UNITS", 512, 4096, step=512) dropout = trail.suggest_float("DROPOUT", 0.01, 0.1, step=0.02) kwargs['max_len'] = max_length kwargs['num_layers'] = num_layers kwargs['d_model'] = d_model kwargs['num_heads'] = num_heads kwargs['units'] = units kwargs['dropout'] = dropout if model_option == "Performer": num_features = trail.suggest_int("num_features", d_model // 2, d_model, step=d_model // 4) kwargs['num_features'] = num_features print(f"Model Type: {model_option}") options_string = "" for k, v in kwargs.items(): options_string += f"{k}={v}\n" print(f"Model Options: \n\n{options_string}") return model_type(**kwargs)
def objective(trial: optuna.trial.Trial): df = pd.read_csv(META_PATH / "train_folds.csv") val_df = df[df["fold"] == 0].copy().reset_index(drop=True) txt_model_args = { "analyzer": trial.suggest_categorical("analyzer", ["word", "char", "char_wb"]), "ngram_range": (trial.suggest_int("ngram_range_low", low=1, high=3), 3), "max_features": trial.suggest_int("max_features", low=1024, high=1024 * 20, step=1024), "max_df": trial.suggest_float("max_df", low=0.9, high=1.0, step=0.01), "stop_words": "english", "binary": True, } model = TfidfVectorizer(**txt_model_args) text_embeds = model.fit_transform(val_df["title"]).toarray() text_embeds = torch.from_numpy(text_embeds) score, _ = validate_score(val_df, text_embeds, th=None) return score