def _xgbclassifier_default(trial: optuna.trial.Trial):
param = {
'silent':
1,
'objective':
'binary:logistic',
'booster':
trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),
'lambda':
trial.suggest_loguniform('lambda', 1e-8, 1.0),
'alpha':
trial.suggest_loguniform('alpha', 1e-8, 1.0)
}
if param['booster'] == 'gbtree' or param['booster'] == 'dart':
param['max_depth'] = trial.suggest_int('max_depth', 1, 9)
param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0)
param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0)
param['grow_policy'] = trial.suggest_categorical(
'grow_policy', ['depthwise', 'lossguide'])
if param['booster'] == 'dart':
param['sample_type'] = trial.suggest_categorical(
'sample_type', ['uniform', 'weighted'])
param['normalize_type'] = trial.suggest_categorical(
'normalize_type', ['tree', 'forest'])
param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0)
param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)
return param
def __call__(self, trial: optuna.trial.Trial) -> float:
data = TreeDataModule(
self._filename,
batch_size=trial.suggest_int("batch_size", 32, 160, 32),
)
kwargs = {
"lstm_size":
trial.suggest_categorical("lstm_size", [512, 1024, 2048]),
"dropout_prob":
trial.suggest_float("dropout", 0.1, 0.5, step=0.1),
"learning_rate":
trial.suggest_float("lr", 1e-3, 1e-1, log=True),
"weight_decay":
trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True),
}
model = RouteDistanceModel(**kwargs)
gpus = int(torch.cuda.is_available())
pruning_callback = PyTorchLightningPruningCallback(
trial, monitor="val_monitor")
trainer = Trainer(
gpus=gpus,
logger=True, # become a tensorboard logger
checkpoint_callback=False,
callbacks=[pruning_callback], # type: ignore
max_epochs=EPOCHS,
)
trainer.fit(model, datamodule=data)
return trainer.callback_metrics["val_monitor"].item()
def mo_objective_test_upgrade(trial: optuna.trial.Trial) -> Tuple[float, float]:
x = trial.suggest_float("x", -5, 5)
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
return x, x ** 2 + y ** 2 + z ** 2
def feedforward(config: BaseConfig,
trial: optuna.trial.Trial) -> pl.LightningModule:
"""Returns a tunable PyTorch lightning feedforward module.
Args:
config (BaseConfig): the hard-coded configuration.
trial (optuna.Trial): optuna trial.
Returns:
pl.LightningModule: a lightning module.
"""
model = FeedForward(num_inputs=config.NUM_INPUTS,
num_outputs=config.NUM_OUTPUTS,
num_hidden=trial.suggest_int('num_hidden', 1, 4),
num_layers=trial.suggest_int('num_layers', 1, 2),
dropout=trial.suggest_float('dropout', 0.0, 0.5),
activation=trial.suggest_categorical(
'activation', ['relu', 'none']))
training_config = get_training_config(
lr=trial.suggest_loguniform('lr', 1e-3, 1e-0),
weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1),
max_epochs=config.MAX_EPOCHS)
pl_model = TemporalConvNet(training_config=training_config,
lr=trial.suggest_loguniform('lr', 1e-3, 1e-0),
weight_decay=trial.suggest_loguniform(
'weight_decay', 1e-5, 1e-1),
max_epochs=config.MAX_EPOCHS)
return pl_model
def modelExtraTreesClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(n_estimators=trial.suggest_int("n_estimators",
2,
2**10,
log=True),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
max_depth=trial.suggest_int("max_depth", 2, 2**4),
criterion=trial.suggest_categorical(
"criterion", ["gini", "entropy"]))
clf = ExtraTreesClassifier(n_estimators=100,
criterion="gini",
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.,
max_features="auto",
max_leaf_nodes=None,
min_impurity_decrease=0.,
min_impurity_split=None,
bootstrap=False,
oob_score=False,
n_jobs=None,
random_state=None,
verbose=0,
warm_start=False,
class_weight=None,
ccp_alpha=0.0,
max_samples=None)
clf.set_params(**{**opt_params, **self.params})
return clf
def func(trial: optuna.trial.Trial, x_max: float = 1.0) -> float:
x = trial.suggest_uniform("x", -x_max, x_max)
y = trial.suggest_loguniform("y", 20, 30)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
return (x - 2)**2 + (y - 25)**2 + z
def f(trial: optuna.trial.Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
trial.set_user_attr("train_loss", 3)
raise ValueError()
return x + y # 3.5
def objective(trial: optuna.trial.Trial) -> float:
num_units = trial.suggest_int("NUM_UNITS", 16, 32)
dropout_rate = trial.suggest_float("DROPOUT_RATE", 0.1, 0.2)
optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"])
accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore
return accuracy
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return (x - 2) ** 2 + (y - 25) ** 2 + z
def objective_test_upgrade(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float.
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
trial.report(0.5, step=0)
return x ** 2 + y ** 2 + z ** 2
def _objective_func(trial: optuna.trial.Trial) -> float:
x = trial.suggest_float("x", -1.0, 1.0)
y = trial.suggest_float("y", 20, 30, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
mlflow.log_metric(metric_name, metric)
return (x - 2)**2 + (y - 25)**2 + z
def objective0(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_float("p0", 0, 10)
p1 = trial.suggest_float("p1", 1, 10, log=True)
p2 = trial.suggest_int("p2", 0, 10)
p3 = trial.suggest_float("p3", 0, 9, step=3)
p4 = trial.suggest_categorical("p4", ["10", "20", "30"])
assert isinstance(p4, str)
return p0 + p1 + p2 + p3 + int(p4)
def _objective_func_long_user_attr(trial: optuna.trial.Trial) -> float:
x = trial.suggest_float("x", -1.0, 1.0)
y = trial.suggest_float("y", 20, 30, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
long_str = str(list(range(5000)))
trial.set_user_attr("my_user_attr", long_str)
return (x - 2)**2 + (y - 25)**2 + z
def objective_test_upgrade_distributions(trial: optuna.trial.Trial) -> float:
x1 = trial.suggest_float("x1", -5, 5)
x2 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x3 = trial.suggest_float("x3", -6, 6, step=2)
y1 = trial.suggest_int("y1", 0, 10)
y2 = trial.suggest_int("y2", 1, 20, log=True)
y3 = trial.suggest_int("y3", 5, 15, step=3)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
return x1**2 + x2**2 + x3**2 + y1**2 + y2**2 + y3**2 + z**2
def _objective_func(trial: optuna.trial.Trial) -> float:
u = trial.suggest_int("u", 0, 10, step=2)
v = trial.suggest_int("v", 1, 10, log=True)
w = trial.suggest_float("w", -1.0, 1.0, step=0.1)
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return u + v + w + (x - 2)**2 + (y - 25)**2 + z
def _multiobjective_func(trial: optuna.trial.Trial) -> Tuple[float, float]:
x = trial.suggest_float("x", low=-10, high=10)
y = trial.suggest_float("y", low=1, high=10, log=True)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
first_objective = (x - 2)**2 + (y - 25)**2 + z
second_objective = (x - 2)**3 + (y - 25)**3 - z
return first_objective, second_objective
def objective(trial: optuna.trial.Trial):
encoder_num_layers = trial.suggest_int("encoder.num_layer", 1, 4)
decoder_num_layers = trial.suggest_int("decoder.num_layer", 1, 4)
transformer_d_model = trial.suggest_categorical(
'transformer.d_model', [64, 128, 256])
hparams["basic"]["transformer"]["d_model"] = transformer_d_model
hparams["decoder"]["num_layer"] = decoder_num_layers
hparams["encoder"]["num_layer"] = encoder_num_layers
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
task = Task(hparams, device)
return task.train(trial)
def __init__(self, trial: optuna.trial.Trial) -> None:
super(Net, self).__init__()
self.activation = getattr(
F, trial.suggest_categorical('activation', ['relu', 'elu']))
self.conv1 = nn.Conv2d(1, 32, 3, 1)
self.conv2 = nn.Conv2d(32, 64, 3, 1)
self.dropout1 = nn.Dropout2d(
p=trial.suggest_uniform("dropout_prob1", 0.1, 0.9))
self.dropout2 = nn.Dropout2d(
p=trial.suggest_uniform("dropout_prob2", 0.1, 0.9))
self.fc1 = nn.Linear(9216, 128)
self.fc2 = nn.Linear(128, 10)
def search_hyperparam(trial: optuna.trial.Trial) -> Dict[str, Any]:
"""Search hyperparam from user-specified search space."""
epochs = trial.suggest_int("epochs", low=50, high=200, step=50)
img_size = trial.suggest_categorical("img_size", [96, 112, 168, 224])
n_select = trial.suggest_int("n_select", low=0, high=6, step=2)
batch_size = trial.suggest_int("batch_size", low=16, high=64, step=16)
return {
"EPOCHS": epochs,
"IMG_SIZE": img_size,
"n_select": n_select,
"BATCH_SIZE": batch_size
}
def _catboostclassifier_default(trial: optuna.trial.Trial):
params = {
'iterations': trial.suggest_int('iterations', 50, 300),
'depth': trial.suggest_int('depth', 4, 10),
'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.3),
'random_strength': trial.suggest_int('random_strength', 0, 100),
'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 0.01, 100.00),
'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),
'od_wait': trial.suggest_int('od_wait', 10, 50)
}
return params
def f(trial: optuna.trial.Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
assert isinstance(y, float)
trial.set_user_attr("train_loss", 3)
trial.set_system_attr("foo", "bar")
value = x + y # 3.5
# Test reported intermediate values, although it in practice is not "intermediate".
trial.report(value, step=0)
return value
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
p0 = trial.suggest_float("p0", -10, 10)
p1 = trial.suggest_uniform("p1", 3, 5)
p2 = trial.suggest_loguniform("p2", 0.00001, 0.1)
p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5)
p4 = trial.suggest_int("p4", -20, -15)
p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100]))
p6 = trial.suggest_float("p6", -10, 10, step=1.0)
p7 = trial.suggest_int("p7", 1, 7, log=True)
return (
p0 + p1 + p2,
p3 + p4 + p5 + p6 + p7,
)
def _convert(trial: optuna.trial.Trial,
template: TuningParametersTemplate) -> Dict[str, Any]:
result: Dict[str, Any] = {}
for k, v in template.params_dict.items():
if isinstance(v, RandInt):
if v.log and v.q is not None:
value = trial.suggest_float(name=k, low=0, high=1.0)
result[k] = uniform_to_integers(
value,
low=v.low,
high=v.high,
q=v.q, # type: ignore
log=True,
include_high=v.include_high,
)
else:
_high: Any = v.high if v.include_high else v.high - 1
result[k] = trial.suggest_int(name=k,
low=v.low,
high=_high,
step=v.q,
log=v.log)
elif isinstance(v, Rand):
if v.log and v.q is not None:
value = trial.suggest_float(name=k, low=0, high=1.0)
result[k] = uniform_to_discrete(
value,
low=v.low,
high=v.high,
q=v.q,
log=True,
include_high=v.include_high,
)
else:
_high = v.high
if v.q is not None and not v.include_high:
_high -= _IGNORABLE_ERROR
result[k] = trial.suggest_float(name=k,
low=v.low,
high=_high,
step=v.q,
log=v.log)
elif isinstance(v, TransitionChoice):
result[k] = v.values[trial.suggest_int(name=k,
low=0,
high=len(v.values) - 1)]
elif isinstance(v, Choice):
result[k] = trial.suggest_categorical(name=k, choices=v.values)
else: # pragma: no cover
raise NotImplementedError
return result
def __call__(self, trial: optuna.trial.Trial):
classifier_name = trial.suggest_categorical("classifier", ["SVC", "RandomForest"])
if classifier_name == "SVC":
svc_c = trial.suggest_float("svc_c", 1e-10, 1e10, log=True)
classifier_obj = sklearn.svm.SVC(C=svc_c, gamma="auto")
else:
rf_max_depth = trial.suggest_int("rf_max_depth", 2, 32, log=True)
classifier_obj = ensemble.RandomForestClassifier(
max_depth=rf_max_depth, n_estimators=10
)
score = cross_val_score(classifier_obj, self.X, self.y, n_jobs=1, cv=self.cv)
accuracy = score.mean()
return accuracy
def _objective(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_float("p0", -3.3, 5.2)
p1 = trial.suggest_float("p1", 2.0, 2.0)
p2 = trial.suggest_float("p2", 0.0001, 0.3, log=True)
p3 = trial.suggest_float("p3", 1.1, 1.1, log=True)
p4 = trial.suggest_int("p4", -100, 8)
p5 = trial.suggest_int("p5", -20, -20)
p6 = trial.suggest_float("p6", 10, 20, step=2)
p7 = trial.suggest_float("p7", 0.1, 1.0, step=0.1)
p8 = trial.suggest_float("p8", 2.2, 2.2, step=0.5)
p9 = trial.suggest_categorical("p9", ["9", "3", "0", "8"])
assert isinstance(p9, str)
return p0 + p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + int(p9)
def _objective(self, trial: optuna.trial.Trial):
cv = trial.suggest_int('cv', 2, 2**4)
opt_params = dict(
objective=trial.suggest_categorical("objective",
["Logloss", "CrossEntropy"]),
boosting_type=trial.suggest_categorical("boosting_type",
["Ordered", "Plain"]),
bootstrap_type=trial.suggest_categorical(
"bootstrap_type", ["Bayesian", "Bernoulli", "MVS"]),
# used_ram_limit="3gb",
max_depth=trial.suggest_int("max_depth", 2, 2**4),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
# n_estimators=trial.suggest_int("n_estimators", 2, 2 ** 10, log=True),
colsample_bylevel=trial.suggest_float("colsample_bylevel", 0.01,
0.1),
reg_lambda=trial.suggest_float("reg_lambda", 1e-8, 100, log=True))
if opt_params["bootstrap_type"] == "Bayesian":
opt_params["bagging_temperature"] = trial.suggest_float(
"bagging_temperature", 0, 10)
elif opt_params["bootstrap_type"] == "Bernoulli":
opt_params["subsample"] = trial.suggest_float("subsample", 0.1, 1)
if self.params is not None:
opt_params.update(self.params)
clf_oof = CatBoostClassifierOOF(self.X,
self.y,
params=opt_params,
cv=cv,
feval=self.feval)
clf_oof.run()
return clf_oof.oof_score # todo: f1
def _lgbmclassifier_default(trial: optuna.trial.Trial):
# TODO: using LightGBMTuner
params = {
'boosting_type': trial.suggest_categorical('boosting', ['gbdt', 'dart', 'goss']),
'objective': 'binary',
'metric': ['binary', 'binary_error', 'auc'],
'num_leaves': trial.suggest_int("num_leaves", 10, 500),
'learning_rate': trial.suggest_loguniform("learning_rate", 1e-5, 1),
'feature_fraction': trial.suggest_uniform("feature_fraction", 0.0, 1.0),
}
if params['boosting_type'] == 'dart':
params['drop_rate'] = trial.suggest_loguniform('drop_rate', 1e-8, 1.0)
params['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)
if params['boosting_type'] == 'goss':
params['top_rate'] = trial.suggest_uniform('top_rate', 0.0, 1.0)
params['other_rate'] = trial.suggest_uniform('other_rate', 0.0, 1.0 - params['top_rate'])
return params
def get_optimizer(trial: optuna.trial.Trial,
model: nn.Module) -> optim.Optimizer:
def adam(model: nn.Module, lr: float,
weight_decay: float) -> optim.Optimizer:
return optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
def momentum(model: nn.Module, lr: float,
weight_decay: float) -> optim.Optimizer:
return optim.SGD(model.parameters(),
lr=lr,
momentum=0.9,
weight_decay=weight_decay)
optimizer_name = trial.suggest_categorical('optimizer',
['adam', 'momentum'])
optimizer: Callable[[nn.Module, float, float],
optim.Optimizer] = locals()[optimizer_name]
lr = trial.suggest_loguniform('lr', 1e-5, 1e-1)
weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
return optimizer(model, lr, weight_decay)
def create_model(trail: optuna.trial.Trial):
tf.keras.backend.clear_session()
time.sleep(3)
kwargs['name'] = f"{trail.number}-Gavin-Optuna"
model_options = ["Transformer", "Performer"]
model_option = trail.suggest_categorical("model_option", model_options)
if model_option == "Transformer":
model_type = TransformerIntegration
elif model_option == "Performer":
model_type = PerformerIntegration
elif model_option == "FNet":
model_type = FNetIntegration
else:
raise ValueError(f"Unknown Model Option: {model_option}")
max_length = trail.suggest_int("MAX_LENGTH", 30, 40, step=10)
num_layers = trail.suggest_int("NUM_LAYERS", 2, 6, step=2)
d_model = trail.suggest_int("D_MODEL", 128, 1024, step=64)
num_heads = trail.suggest_int("NUM_HEADS", 2, 16, step=2)
units = trail.suggest_int("UNITS", 512, 4096, step=512)
dropout = trail.suggest_float("DROPOUT", 0.01, 0.1, step=0.02)
kwargs['max_len'] = max_length
kwargs['num_layers'] = num_layers
kwargs['d_model'] = d_model
kwargs['num_heads'] = num_heads
kwargs['units'] = units
kwargs['dropout'] = dropout
if model_option == "Performer":
num_features = trail.suggest_int("num_features",
d_model // 2,
d_model,
step=d_model // 4)
kwargs['num_features'] = num_features
print(f"Model Type: {model_option}")
options_string = ""
for k, v in kwargs.items():
options_string += f"{k}={v}\n"
print(f"Model Options: \n\n{options_string}")
return model_type(**kwargs)
def objective(trial: optuna.trial.Trial):
df = pd.read_csv(META_PATH / "train_folds.csv")
val_df = df[df["fold"] == 0].copy().reset_index(drop=True)
txt_model_args = {
"analyzer":
trial.suggest_categorical("analyzer", ["word", "char", "char_wb"]),
"ngram_range": (trial.suggest_int("ngram_range_low", low=1,
high=3), 3),
"max_features":
trial.suggest_int("max_features", low=1024, high=1024 * 20, step=1024),
"max_df":
trial.suggest_float("max_df", low=0.9, high=1.0, step=0.01),
"stop_words":
"english",
"binary":
True,
}
model = TfidfVectorizer(**txt_model_args)
text_embeds = model.fit_transform(val_df["title"]).toarray()
text_embeds = torch.from_numpy(text_embeds)
score, _ = validate_score(val_df, text_embeds, th=None)
return score
