def objective(trial: optuna.trial.Trial) -> float:
# We optimize the number of layers, hidden units in each layer and dropouts.
n_layers = trial.suggest_int("n_layers", 1, 3)
dropout = trial.suggest_float("dropout", 0.2, 0.5)
output_dims = [
trial.suggest_int("n_units_l{}".format(i), 4, 128, log=True)
for i in range(n_layers)
]
model = LightningNet(dropout, output_dims)
datamodule = MNISTDataModule(data_dir=DIR, batch_size=BATCHSIZE)
trainer = pl.Trainer(
logger=True,
limit_val_batches=PERCENT_VALID_EXAMPLES,
checkpoint_callback=False,
max_epochs=EPOCHS,
gpus=-1 if torch.cuda.is_available() else None,
callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_acc")],
)
hyperparameters = dict(n_layers=n_layers,
dropout=dropout,
output_dims=output_dims)
trainer.logger.log_hyperparams(hyperparameters)
trainer.fit(model, datamodule=datamodule)
return trainer.callback_metrics["val_acc"].item()
def _objective(self, trial: optuna.trial.Trial):
cv = trial.suggest_int('cv', 2, 2**4)
opt_params = dict(
max_depth=trial.suggest_int("max_depth", 2, 2**4),
learning_rate=trial.suggest_float('learning_rate',
0.001,
1,
step=0.001),
# n_estimators=trial.suggest_int("n_estimators", 2, 2 ** 10, log=True),
gamma=trial.suggest_float('gamma', 1e-8, 1, log=True),
min_child_weight=trial.suggest_float('min_child_weight',
1e-8,
2**10,
log=True),
subsample=trial.suggest_float('subsample', 0.1, 1),
colsample_bytree=trial.suggest_float('colsample_bytree', 0.1, 1),
colsample_bylevel=trial.suggest_float('colsample_bylevel', 0.1, 1),
reg_alpha=trial.suggest_float('reg_alpha', 1e-8, 10, log=True),
reg_lambda=trial.suggest_float('reg_lambda', 1e-8, 10, log=True),
)
if self.params is not None:
opt_params.update(self.params)
clf_oof = XGBClassifierOOF(self.X,
self.y,
params=opt_params,
cv=cv,
feval=self.feval)
clf_oof.run()
return clf_oof.oof_score # todo: f1
def tcn(config: BaseConfig, trial: optuna.trial.Trial) -> pl.LightningModule:
"""Returns a tunable PyTorch lightning tcn module.
Args:
config (BaseConfig): the hard-coded configuration.
trial (optuna.Trial): optuna trial.
Returns:
pl.LightningModule: a lightning module.
"""
training_config = get_training_config(
lr=trial.suggest_loguniform('lr', 1e-3, 1e-0),
weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1),
max_epochs=config.MAX_EPOCHS)
tcn = TemporalConvNet(training_config=training_config,
num_inputs=config.NUM_INPUTS,
num_outputs=config.NUM_OUTPUTS,
num_hidden=trial.suggest_int('num_hidden', 1, 4),
kernel_size=trial.suggest_int('kernel_size', 2, 4),
num_layers=trial.suggest_int('num_layers', 1, 2),
dropout=trial.suggest_float('dropout', 0.1, 0.3))
return tcn
def modelExtraTreesClassifier(self, trial: optuna.trial.Trial):
opt_params = dict(n_estimators=trial.suggest_int("n_estimators",
2,
2**10,
log=True),
learning_rate=trial.suggest_discrete_uniform(
'learning_rate', 0.001, 1, 0.001),
max_depth=trial.suggest_int("max_depth", 2, 2**4),
criterion=trial.suggest_categorical(
"criterion", ["gini", "entropy"]))
clf = ExtraTreesClassifier(n_estimators=100,
criterion="gini",
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.,
max_features="auto",
max_leaf_nodes=None,
min_impurity_decrease=0.,
min_impurity_split=None,
bootstrap=False,
oob_score=False,
n_jobs=None,
random_state=None,
verbose=0,
warm_start=False,
class_weight=None,
ccp_alpha=0.0,
max_samples=None)
clf.set_params(**{**opt_params, **self.params})
return clf
def _preprocess(self, trial: optuna.trial.Trial) -> None:
if self.pbar is not None:
self.pbar.set_description(self.pbar_fmt.format(self.step_name, self.best_score))
if "lambda_l1" in self.target_param_names:
self.lgbm_params["lambda_l1"] = trial.suggest_loguniform("lambda_l1", 1e-8, 10.0)
if "lambda_l2" in self.target_param_names:
self.lgbm_params["lambda_l2"] = trial.suggest_loguniform("lambda_l2", 1e-8, 10.0)
if "num_leaves" in self.target_param_names:
tree_depth = self.lgbm_params.get("max_depth", _DEFAULT_TUNER_TREE_DEPTH)
max_num_leaves = 2 ** tree_depth if tree_depth > 0 else 2 ** _DEFAULT_TUNER_TREE_DEPTH
self.lgbm_params["num_leaves"] = trial.suggest_int("num_leaves", 2, max_num_leaves)
if "feature_fraction" in self.target_param_names:
# `GridSampler` is used for sampling feature_fraction value.
# The value 1.0 for the hyperparameter is always sampled.
param_value = min(trial.suggest_uniform("feature_fraction", 0.4, 1.0 + _EPS), 1.0)
self.lgbm_params["feature_fraction"] = param_value
if "bagging_fraction" in self.target_param_names:
# `TPESampler` is used for sampling bagging_fraction value.
# The value 1.0 for the hyperparameter might by sampled.
param_value = min(trial.suggest_uniform("bagging_fraction", 0.4, 1.0 + _EPS), 1.0)
self.lgbm_params["bagging_fraction"] = param_value
if "bagging_freq" in self.target_param_names:
self.lgbm_params["bagging_freq"] = trial.suggest_int("bagging_freq", 1, 7)
if "min_child_samples" in self.target_param_names:
# `GridSampler` is used for sampling min_child_samples value.
# The value 1.0 for the hyperparameter is always sampled.
param_value = int(trial.suggest_uniform("min_child_samples", 5, 100 + _EPS))
self.lgbm_params["min_child_samples"] = param_value
def objective(trial: optuna.trial.Trial):
settings = Settings(
learning_rate=trial.suggest_loguniform('learning_rate', 1e-5, 1e-2),
hidden1=trial.suggest_int('hidden1', 50, 200),
hidden2=trial.suggest_int('hidden2', 10, 50),
)
val_err = run_training(settings)
return val_err
def objective_test_upgrade_distributions(trial: optuna.trial.Trial) -> float:
x1 = trial.suggest_float("x1", -5, 5)
x2 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x3 = trial.suggest_float("x3", -6, 6, step=2)
y1 = trial.suggest_int("y1", 0, 10)
y2 = trial.suggest_int("y2", 1, 20, log=True)
y3 = trial.suggest_int("y3", 5, 15, step=3)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
return x1**2 + x2**2 + x3**2 + y1**2 + y2**2 + y3**2 + z**2
def _objective_func(trial: optuna.trial.Trial) -> float:
u = trial.suggest_int("u", 0, 10, step=2)
v = trial.suggest_int("v", 1, 10, log=True)
w = trial.suggest_float("w", -1.0, 1.0, step=0.1)
x = trial.suggest_uniform("x", -1.0, 1.0)
y = trial.suggest_loguniform("y", 20.0, 30.0)
z = trial.suggest_categorical("z", (-1.0, 1.0))
assert isinstance(z, float)
trial.set_user_attr("my_user_attr", "my_user_attr_value")
return u + v + w + (x - 2)**2 + (y - 25)**2 + z
def _objective(self, trial: optuna.trial.Trial):
cv = trial.suggest_int('cv', 2, 2**4)
opt_params = dict(
num_leaves=trial.suggest_int("num_leaves", 2, 2**8),
learning_rate=trial.suggest_float('learning_rate',
0.001,
1,
step=0.001),
min_child_samples=trial.suggest_int('min_child_samples', 2, 2**8),
min_child_weight=trial.suggest_float('min_child_weight',
1e-8,
1,
log=True),
min_split_gain=trial.suggest_float('min_split_gain',
1e-8,
1,
log=True),
bagging_fraction=trial.suggest_float('bagging_fraction', 0.4, 1),
bagging_freq=trial.suggest_int("bagging_freq", 0, 2**4),
feature_fraction=trial.suggest_float('feature_fraction', 0.4, 1),
lambda_l1=trial.suggest_float('lambda_l1', 1e-8, 10, log=True),
lambda_l2=trial.suggest_float('lambda_l2', 1e-8, 10, log=True),
)
if self.params is not None:
opt_params.update(self.params)
cv_result = lgb.cv(opt_params,
self.dtrain,
num_boost_round=10000,
nfold=cv,
stratified='reg' not in opt_params.get(
'application',
opt_params.get('objective', 'reg')),
feval=None,
early_stopping_rounds=100,
verbose_eval=100,
show_stdv=False,
seed=0,
eval_train_metric=False)
score = -1
self.best_num_boost_round = 0
for key in cv_result:
if 'mean' in key:
_ = cv_result[key]
score = _[-1]
self.best_num_boost_round = len(_)
print(
f'CV Score: {score if score != -1 else "cv_result donot contain mean-metric"}'
)
return score
def _catboostclassifier_default(trial: optuna.trial.Trial):
params = {
'iterations': trial.suggest_int('iterations', 50, 300),
'depth': trial.suggest_int('depth', 4, 10),
'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.3),
'random_strength': trial.suggest_int('random_strength', 0, 100),
'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 0.01, 100.00),
'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),
'od_wait': trial.suggest_int('od_wait', 10, 50)
}
return params
def search_hyperparam(trial: optuna.trial.Trial) -> Dict[str, Any]:
"""Search hyperparam from user-specified search space."""
epochs = trial.suggest_int("epochs", low=50, high=200, step=50)
img_size = trial.suggest_categorical("img_size", [96, 112, 168, 224])
n_select = trial.suggest_int("n_select", low=0, high=6, step=2)
batch_size = trial.suggest_int("batch_size", low=16, high=64, step=16)
return {
"EPOCHS": epochs,
"IMG_SIZE": img_size,
"n_select": n_select,
"BATCH_SIZE": batch_size
}
def optimize(trial: optuna.trial.Trial, data_dict: dict):
p = {
'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1),
'max_depth': trial.suggest_int('max_depth', 5, 30),
'max_leaves': trial.suggest_int('max_leaves', 5, 50),
'subsample': trial.suggest_uniform('subsample', 0.3, 1.0),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3,
1.0),
'min_child_weight': trial.suggest_int('min_child_weight', 5, 100),
'lambda': trial.suggest_uniform('lambda', 0.05, 0.2),
'alpha': trial.suggest_uniform('alpha', 0.05, 0.2),
'objective': 'binary:logistic',
'booster': 'gbtree',
'tree_method': 'gpu_hist',
'verbosity': 1,
'n_jobs': 10,
'eval_metric': 'auc'
}
print('Choosing parameters:', p)
scores = []
sizes = []
# gts = GroupTimeSeriesSplit()
data = data_dict['data']
target = data_dict['target']
date = data_dict['date']
gts = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10)
for i, (tr_idx, val_idx) in enumerate(gts.split(data, groups=date)):
sizes.append(len(tr_idx))
x_tr, x_val = copy.deepcopy(data.iloc[tr_idx]), copy.deepcopy(
data.iloc[val_idx])
y_tr, y_val = copy.deepcopy(target[tr_idx]), copy.deepcopy(
target[val_idx])
x_tr, x_val = calc_data_mean(x_tr, cache_dir='cache/', fold=i, train=True), \
calc_data_mean(x_val, cache_dir='cache/', fold=i, train=False)
d_tr = xgb.DMatrix(x_tr, label=y_tr)
d_val = xgb.DMatrix(x_val, label=y_val)
clf = xgb.train(p,
d_tr,
500, [(d_val, 'eval')],
early_stopping_rounds=50,
verbose_eval=True)
val_pred = clf.predict(d_val)
score = roc_auc_score(y_val, val_pred)
print(f'Fold {i} ROC AUC:\t', score)
scores.append(score)
del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score
rubbish = gc.collect()
print(scores)
avg_score = weighted_mean(scores, sizes)
print('Avg Score:', avg_score)
return avg_score
def objective(trial: optuna.trial.Trial) -> Tuple[float, float]:
p0 = trial.suggest_float("p0", -10, 10)
p1 = trial.suggest_uniform("p1", 3, 5)
p2 = trial.suggest_loguniform("p2", 0.00001, 0.1)
p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5)
p4 = trial.suggest_int("p4", -20, -15)
p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100]))
p6 = trial.suggest_float("p6", -10, 10, step=1.0)
p7 = trial.suggest_int("p7", 1, 7, log=True)
return (
p0 + p1 + p2,
p3 + p4 + p5 + p6 + p7,
)
def _convert(trial: optuna.trial.Trial,
template: TuningParametersTemplate) -> Dict[str, Any]:
result: Dict[str, Any] = {}
for k, v in template.params_dict.items():
if isinstance(v, RandInt):
if v.log and v.q is not None:
value = trial.suggest_float(name=k, low=0, high=1.0)
result[k] = uniform_to_integers(
value,
low=v.low,
high=v.high,
q=v.q, # type: ignore
log=True,
include_high=v.include_high,
)
else:
_high: Any = v.high if v.include_high else v.high - 1
result[k] = trial.suggest_int(name=k,
low=v.low,
high=_high,
step=v.q,
log=v.log)
elif isinstance(v, Rand):
if v.log and v.q is not None:
value = trial.suggest_float(name=k, low=0, high=1.0)
result[k] = uniform_to_discrete(
value,
low=v.low,
high=v.high,
q=v.q,
log=True,
include_high=v.include_high,
)
else:
_high = v.high
if v.q is not None and not v.include_high:
_high -= _IGNORABLE_ERROR
result[k] = trial.suggest_float(name=k,
low=v.low,
high=_high,
step=v.q,
log=v.log)
elif isinstance(v, TransitionChoice):
result[k] = v.values[trial.suggest_int(name=k,
low=0,
high=len(v.values) - 1)]
elif isinstance(v, Choice):
result[k] = trial.suggest_categorical(name=k, choices=v.values)
else: # pragma: no cover
raise NotImplementedError
return result
def evaluate_params(
trial: optuna.trial.Trial,
train_data: lgb.Dataset,
validation_data: lgb.Dataset,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["num_iterations"] = trial.suggest_int(
"num_iterations", 8, 128)
params["learning_rate"] = trial.suggest_uniform(
"learning_rate", 2**-5, 0.5)
params["num_leaves"] = trial.suggest_int("num_leaves", 8, 256)
params["max_depth"] = trial.suggest_int("max_depth", 4, 32)
params["min_data_in_leaf"] = trial.suggest_int(
"min_data_in_leaf", 4, 512)
params["min_sum_hessian_in_leaf"] = trial.suggest_uniform(
"min_sum_hessian_in_leaf", 2**-5, 0.25)
params["bagging_freq"] = trial.suggest_int("bagging_freq", 0, 1)
params["bagging_fraction"] = trial.suggest_uniform(
"bagging_fraction", 0.5, 1)
params["feature_fraction"] = trial.suggest_uniform(
"feature_fraction", 0.5, 1)
params["lambda_l1"] = trial.suggest_uniform("lambda_l1", 0, 64)
params["lambda_l2"] = trial.suggest_uniform("lambda_l2", 0, 64)
params["min_gain_to_split"] = trial.suggest_uniform(
"min_gain_to_split", 0, 0.25)
params["min_data_per_group"] = trial.suggest_int(
"min_data_per_group", 1, 512)
params["max_cat_threshold"] = trial.suggest_int(
"max_cat_threshold", 1, 512)
params["cat_l2"] = trial.suggest_uniform("cat_l2", 0, 64)
params["cat_smooth"] = trial.suggest_uniform("cat_smooth", 0, 2048)
params["max_cat_to_onehot"] = trial.suggest_int(
"max_cat_to_onehot", 1, 64)
params["max_bin"] = trial.suggest_int("max_bin", 32, 1024)
params["min_data_in_bin"] = trial.suggest_int(
"min_data_in_bin", 1, 64)
params["objective"] = self.objective
params["num_class"] = self.num_class
params["verbosity"] = -1
booster = lgb.Booster(params=params, train_set=train_data)
booster.add_valid(validation_data, "validation_set")
for step in range(params["num_iterations"]):
booster.update()
validation_loss = booster.eval_valid()[0][2]
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
def _objective(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_float("p0", -3.3, 5.2)
p1 = trial.suggest_float("p1", 2.0, 2.0)
p2 = trial.suggest_float("p2", 0.0001, 0.3, log=True)
p3 = trial.suggest_float("p3", 1.1, 1.1, log=True)
p4 = trial.suggest_int("p4", -100, 8)
p5 = trial.suggest_int("p5", -20, -20)
p6 = trial.suggest_float("p6", 10, 20, step=2)
p7 = trial.suggest_float("p7", 0.1, 1.0, step=0.1)
p8 = trial.suggest_float("p8", 2.2, 2.2, step=0.5)
p9 = trial.suggest_categorical("p9", ["9", "3", "0", "8"])
assert isinstance(p9, str)
return p0 + p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + int(p9)
def _suggest(self, trial: optuna.trial.Trial) -> Suggestion:
suggestions: Suggestion = dict()
for name, config in self.api_config.items():
low, high = config["range"]
log = config["space"] == "log"
if config["space"] == "logit":
assert 0 < low <= high < 1
low = np.log(low / (1 - low))
high = np.log(high / (1 - high))
if config["type"] == "real":
param = trial.suggest_float(name, low, high, log=log)
elif config["type"] == "int":
param = trial.suggest_int(name, low, high, log=log)
else:
# TODO(xadrianzetx) Support `suggest_categorical` if benchmark is extended.
raise RuntimeError("CategoricalDistribution is not supported in bayesmark.")
suggestions[name] = param if config["space"] != "logit" else 1 / (1 + np.exp(-param))
return suggestions
def mo_objective_test_upgrade(trial: optuna.trial.Trial) -> Tuple[float, float]:
x = trial.suggest_float("x", -5, 5)
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
return x, x ** 2 + y ** 2 + z ** 2
def f(trial: optuna.trial.Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
trial.set_user_attr("train_loss", 3)
raise ValueError()
return x + y # 3.5
def _xgbclassifier_default(trial: optuna.trial.Trial):
param = {
'silent':
1,
'objective':
'binary:logistic',
'booster':
trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),
'lambda':
trial.suggest_loguniform('lambda', 1e-8, 1.0),
'alpha':
trial.suggest_loguniform('alpha', 1e-8, 1.0)
}
if param['booster'] == 'gbtree' or param['booster'] == 'dart':
param['max_depth'] = trial.suggest_int('max_depth', 1, 9)
param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0)
param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0)
param['grow_policy'] = trial.suggest_categorical(
'grow_policy', ['depthwise', 'lossguide'])
if param['booster'] == 'dart':
param['sample_type'] = trial.suggest_categorical(
'sample_type', ['uniform', 'weighted'])
param['normalize_type'] = trial.suggest_categorical(
'normalize_type', ['tree', 'forest'])
param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0)
param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)
return param
def feedforward(config: BaseConfig,
trial: optuna.trial.Trial) -> pl.LightningModule:
"""Returns a tunable PyTorch lightning feedforward module.
Args:
config (BaseConfig): the hard-coded configuration.
trial (optuna.Trial): optuna trial.
Returns:
pl.LightningModule: a lightning module.
"""
model = FeedForward(num_inputs=config.NUM_INPUTS,
num_outputs=config.NUM_OUTPUTS,
num_hidden=trial.suggest_int('num_hidden', 1, 4),
num_layers=trial.suggest_int('num_layers', 1, 2),
dropout=trial.suggest_float('dropout', 0.0, 0.5),
activation=trial.suggest_categorical(
'activation', ['relu', 'none']))
training_config = get_training_config(
lr=trial.suggest_loguniform('lr', 1e-3, 1e-0),
weight_decay=trial.suggest_loguniform('weight_decay', 1e-5, 1e-1),
max_epochs=config.MAX_EPOCHS)
pl_model = TemporalConvNet(training_config=training_config,
lr=trial.suggest_loguniform('lr', 1e-3, 1e-0),
weight_decay=trial.suggest_loguniform(
'weight_decay', 1e-5, 1e-1),
max_epochs=config.MAX_EPOCHS)
return pl_model
def __call__(self, trial: optuna.trial.Trial) -> float:
data = TreeDataModule(
self._filename,
batch_size=trial.suggest_int("batch_size", 32, 160, 32),
)
kwargs = {
"lstm_size":
trial.suggest_categorical("lstm_size", [512, 1024, 2048]),
"dropout_prob":
trial.suggest_float("dropout", 0.1, 0.5, step=0.1),
"learning_rate":
trial.suggest_float("lr", 1e-3, 1e-1, log=True),
"weight_decay":
trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True),
}
model = RouteDistanceModel(**kwargs)
gpus = int(torch.cuda.is_available())
pruning_callback = PyTorchLightningPruningCallback(
trial, monitor="val_monitor")
trainer = Trainer(
gpus=gpus,
logger=True, # become a tensorboard logger
checkpoint_callback=False,
callbacks=[pruning_callback], # type: ignore
max_epochs=EPOCHS,
)
trainer.fit(model, datamodule=data)
return trainer.callback_metrics["val_monitor"].item()
def objective(trial: optuna.trial.Trial) -> float:
for i in range(N_REPORTS):
trial.report(i, step=i)
x = trial.suggest_float("x", -100, 100)
y = trial.suggest_int("y", -100, 100)
return x ** 2 + y ** 2
def objective(trial: optuna.trial.Trial) -> float:
num_units = trial.suggest_int("NUM_UNITS", 16, 32)
dropout_rate = trial.suggest_float("DROPOUT_RATE", 0.1, 0.2)
optimizer = trial.suggest_categorical("OPTIMIZER", ["sgd", "adam"])
accuracy = train_test_model(num_units, dropout_rate, optimizer) # type: ignore
return accuracy
def objective_for_binary_unet(args, trial: optuna.trial.Trial):
args.lr = trial.suggest_loguniform("lr", low=1e-5, high=1e-2)
args.edge_weight = trial.suggest_uniform("edge_weight", low=1, high=5)
args.wf = trial.suggest_int("wf", low=2, high=4)
args.depth = trial.suggest_int("depth", low=4, high=6)
pl_pruning_callback = PyTorchLightningPruningCallback(
trial, "val/f1_score")
ckpt_callback = train_binary_unet_model(args,
callbacks=[pl_pruning_callback])
best_f1_score = ckpt_callback.best_model_score.detach().cpu().numpy().item(
)
trial.set_user_attr("best_val_f1", best_f1_score)
trial.set_user_attr("best_model_path", ckpt_callback.best_model_path)
return best_f1_score
def objective_test_upgrade(trial: optuna.trial.Trial) -> float:
x = trial.suggest_uniform("x", -5, 5) # optuna==0.9.0 does not have suggest_float.
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
trial.report(0.5, step=0)
return x ** 2 + y ** 2 + z ** 2
def optimize(trial: optuna.trial.Trial, data_dict: dict):
p = {
'learning_rate': trial.suggest_uniform('learning_rate', 1e-4, 1e-1),
'max_depth': trial.suggest_int('max_depth', 5, 30),
'max_leaves': trial.suggest_int('max_leaves', 5, 50),
'subsample': trial.suggest_uniform('subsample', 0.3, 1.0),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.3,
1.0),
'min_child_weight': trial.suggest_int('min_child_weight', 5, 100),
'lambda': trial.suggest_uniform('lambda', 0.05, 0.2),
'alpha': trial.suggest_uniform('alpha', 0.05, 0.2),
'objective': 'reg:squarederror',
'booster': 'gbtree',
'tree_method': 'gpu_hist',
'verbosity': 1,
'n_jobs': 10,
'eval_metric': 'rmse'
}
print('Choosing parameters:', p)
scores = []
sizes = []
# gts = GroupTimeSeriesSplit()']
gts = pgs.PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=10)
for i, (tr_idx, val_idx) in enumerate(
gts.split(data_dict['data'], groups=data_dict['era'])):
x_tr, x_val = data_dict['data'][tr_idx], data_dict['data'][val_idx]
y_tr, y_val = data_dict['target'][tr_idx], data_dict['target'][val_idx]
d_tr = xgb.DMatrix(x_tr, label=y_tr)
d_val = xgb.DMatrix(x_val, label=y_val)
clf = xgb.train(p,
d_tr,
500, [(d_val, 'eval')],
early_stopping_rounds=50,
verbose_eval=True)
val_pred = clf.predict(d_val)
score = mean_squared_error(y_val, val_pred)
scores.append(score)
sizes.append(len(tr_idx) + len(val_idx))
del clf, val_pred, d_tr, d_val, x_tr, x_val, y_tr, y_val, score
rubbish = gc.collect()
print(scores)
avg_score = utils.weighted_mean(scores, sizes)
print('Avg Score:', avg_score)
return avg_score
def define_model(trial: optuna.trial.Trial) -> nn.Sequential:
n_layers = trial.suggest_int("n_layers", 1, 3)
dropout = trial.suggest_float("dropout", 0.2, 0.5)
input_dim = 28 * 28
layers = [nn.Flatten()]
for i in range(n_layers):
output_dim = trial.suggest_int("n_units_l{}".format(i),
4,
128,
log=True)
layers.append(nn.Linear(input_dim, output_dim))
layers.append(nn.ReLU())
layers.append(nn.Dropout(dropout))
input_dim = output_dim
layers.append(nn.Linear(input_dim, CLASSES))
return nn.Sequential(*layers)
def objective0(trial: optuna.trial.Trial) -> float:
p0 = trial.suggest_float("p0", 0, 10)
p1 = trial.suggest_float("p1", 1, 10, log=True)
p2 = trial.suggest_int("p2", 0, 10)
p3 = trial.suggest_float("p3", 0, 9, step=3)
p4 = trial.suggest_categorical("p4", ["10", "20", "30"])
assert isinstance(p4, str)
return p0 + p1 + p2 + p3 + int(p4)
def objective(trial: optuna.trial.Trial) -> float:
dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0000..0062}.tar").shuffle(20000).decode()
dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0000..0010}.tar").shuffle(20000).decode()
test_dataset_2 = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-gamma-10-{0063..0072}.tar").decode()
test_dataset = wds.WebDataset("/run/media/jacob/data/FACT_Dataset/fact-proton-10-{0011..0013}.tar").decode()
dataset = SampleEqually([dataset, dataset_2])
test_dataset = SampleEqually([test_dataset_2, test_dataset])
train_loader = DataLoader(dataset, num_workers=16, batch_size=4, pin_memory=True)
test_loader = DataLoader(test_dataset, num_workers=4, batch_size=1, pin_memory=True)
# We optimize the number of layers, hidden units in each layer and dropouts.
config = {
"sample_ratio_one": trial.suggest_uniform("sample_ratio_one", 0.1, 0.9),
"sample_radius_one": trial.suggest_uniform("sample_radius_one", 0.1, 0.9),
"sample_max_neighbor": trial.suggest_int("sample_max_neighbor", 8, 72),
"sample_ratio_two": trial.suggest_uniform("sample_ratio_two", 0.1, 0.9),
"sample_radius_two": trial.suggest_uniform("sample_radius_two", 0.1, 0.9),
"fc_1": trial.suggest_int("fc_1", 128, 256),
"fc_1_out": trial.suggest_int("fc_1_out", 32, 128),
"fc_2_out": trial.suggest_int("fc_2_out", 16, 96),
"dropout": trial.suggest_uniform("dropout", 0.1, 0.9),
}
num_classes = 2
import pytorch_lightning as pl
model = LitPointNet2(num_classes, lr=0.0001, config=config)
trainer = pl.Trainer(
logger=True,
limit_val_batches=10000,
limit_train_batches=10000,
checkpoint_callback=False,
auto_lr_find=True,
max_epochs=20,
gpus=1,
callbacks=[PyTorchLightningPruningCallback(trial, monitor="val/loss")],
)
trainer.logger.log_hyperparams(config)
trainer.tune(model=model, train_dataloader=train_loader, val_dataloaders=test_loader)
trainer.fit(model=model, train_dataloader=train_loader, val_dataloaders=test_loader)
return trainer.callback_metrics["val/loss"].item()
