def test_study(self):
def objective(trial):
x = trial.suggest_uniform('x', -1., 1.)
return x ** 2
n_trials = 20
study = optuna.create_study()
study.optimize(objective, n_trials=n_trials)
self.assertEqual(len(study.trials), n_trials)
def test_ask() -> None:
study = create_study()
trial = study.ask()
assert isinstance(trial, Trial)
def test_callbacks(n_jobs: int) -> None:
lock = threading.Lock()
def with_lock(f: CallbackFuncType) -> CallbackFuncType:
def callback(study: Study, trial: FrozenTrial) -> None:
with lock:
f(study, trial)
return callback
study = create_study()
def objective(trial: Trial) -> float:
return trial.suggest_int("x", 1, 1)
# Empty callback list.
study.optimize(objective, callbacks=[], n_trials=10, n_jobs=n_jobs)
# A callback.
values = []
callbacks = [with_lock(lambda study, trial: values.append(trial.value))]
study.optimize(objective, callbacks=callbacks, n_trials=10, n_jobs=n_jobs)
assert values == [1] * 10
# Two callbacks.
values = []
params = []
callbacks = [
with_lock(lambda study, trial: values.append(trial.value)),
with_lock(lambda study, trial: params.append(trial.params)),
]
study.optimize(objective, callbacks=callbacks, n_trials=10, n_jobs=n_jobs)
assert values == [1] * 10
assert params == [{"x": 1}] * 10
# If a trial is failed with an exception and the exception is caught by the study,
# callbacks are invoked.
states = []
callbacks = [with_lock(lambda study, trial: states.append(trial.state))]
study.optimize(
lambda t: 1 / 0,
callbacks=callbacks,
n_trials=10,
n_jobs=n_jobs,
catch=(ZeroDivisionError, ),
)
assert states == [TrialState.FAIL] * 10
# If a trial is failed with an exception and the exception isn't caught by the study,
# callbacks aren't invoked.
states = []
callbacks = [with_lock(lambda study, trial: states.append(trial.state))]
with pytest.raises(ZeroDivisionError):
study.optimize(lambda t: 1 / 0,
callbacks=callbacks,
n_trials=10,
n_jobs=n_jobs,
catch=())
assert states == []
def test_optimize_trivial_rdb_resume_study() -> None:
study = create_study("sqlite:///:memory:")
study.optimize(func, n_trials=10)
check_study(study)
def test_optimize_n_jobs_warning() -> None:
study = create_study()
with pytest.warns(FutureWarning):
study.optimize(func, n_trials=1, n_jobs=2)
def hyperparameters_optimization(self) -> None:
if self.verbose > 0:
print("Optimizing hyperparameters")
if self.storage is not None and self.study_name is None:
warnings.warn(
f"You passed a remote storage: {self.storage} but no `--study-name`."
"The study name will be generated by Optuna, make sure to re-use the same study name "
"when you want to do distributed hyperparameter optimization.")
if self.tensorboard_log is not None:
warnings.warn(
"Tensorboard log is deactivated when running hyperparameter optimization"
)
self.tensorboard_log = None
# TODO: eval each hyperparams several times to account for noisy evaluation
sampler = self._create_sampler(self.sampler)
pruner = self._create_pruner(self.pruner)
if self.verbose > 0:
print(f"Sampler: {self.sampler} - Pruner: {self.pruner}")
study = optuna.create_study(
sampler=sampler,
pruner=pruner,
storage=self.storage,
study_name=self.study_name,
load_if_exists=True,
direction="maximize",
)
try:
study.optimize(self.objective,
n_trials=self.n_trials,
n_jobs=self.n_jobs)
except KeyboardInterrupt:
pass
print("Number of finished trials: ", len(study.trials))
print("Best trial:")
trial = study.best_trial
print("Value: ", trial.value)
print("Params: ")
for key, value in trial.params.items():
print(f" {key}: {value}")
report_name = (
f"report_{self.env_id}_{self.n_trials}-trials-{self.n_timesteps}"
f"-{self.sampler}-{self.pruner}_{int(time.time())}.csv")
log_path = os.path.join(self.log_folder, self.algo, report_name)
if self.verbose:
print(f"Writing report to {log_path}")
# Write report
os.makedirs(os.path.dirname(log_path), exist_ok=True)
study.trials_dataframe().to_csv(log_path)
def test_group() -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = TPESampler(multivariate=True, group=True)
study = optuna.create_study(sampler=sampler)
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=2)
assert mock.call_count == 1
assert study.trials[-1].distributions == {
"x": distributions.IntUniformDistribution(low=0, high=10)
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_float("z", -3, 3), n_trials=1
)
assert mock.call_count == 1
assert study.trials[-1].distributions == {
"y": distributions.IntUniformDistribution(low=0, high=10),
"z": distributions.UniformDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10)
+ t.suggest_float("z", -3, 3)
+ t.suggest_float("u", 1e-2, 1e2, log=True)
+ bool(t.suggest_categorical("v", ["A", "B", "C"])),
n_trials=1,
)
assert mock.call_count == 2
assert study.trials[-1].distributions == {
"u": distributions.LogUniformDistribution(low=1e-2, high=1e2),
"v": distributions.CategoricalDistribution(choices=["A", "B", "C"]),
"y": distributions.IntUniformDistribution(low=0, high=10),
"z": distributions.UniformDistribution(low=-3, high=3),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_float("u", 1e-2, 1e2, log=True), n_trials=1)
assert mock.call_count == 3
assert study.trials[-1].distributions == {
"u": distributions.LogUniformDistribution(low=1e-2, high=1e2)
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(
lambda t: t.suggest_int("y", 0, 10) + t.suggest_int("w", 2, 8, log=True), n_trials=1
)
assert mock.call_count == 4
assert study.trials[-1].distributions == {
"y": distributions.IntUniformDistribution(low=0, high=10),
"w": distributions.IntLogUniformDistribution(low=2, high=8),
}
with patch.object(sampler, "_sample_relative", wraps=sampler._sample_relative) as mock:
study.optimize(lambda t: t.suggest_int("x", 0, 10), n_trials=1)
assert mock.call_count == 6
assert study.trials[-1].distributions == {
"x": distributions.IntUniformDistribution(low=0, high=10)
}
###################################################################################################
# Adding User Attributes to Studies
# ---------------------------------
#
# A :class:`~optuna.study.Study` object provides :func:`~optuna.study.Study.set_user_attr` method
# to register a pair of key and value as an user-defined attribute.
# A key is supposed to be a ``str``, and a value be any object serializable with ``json.dumps``.
import sklearn.datasets
import sklearn.model_selection
import sklearn.svm
import optuna
study = optuna.create_study(storage="sqlite:///example.db")
study.set_user_attr("contributors", ["Akiba", "Sano"])
study.set_user_attr("dataset", "MNIST")
###################################################################################################
# We can access annotated attributes with :attr:`~optuna.study.Study.user_attr` property.
study.user_attrs # {'contributors': ['Akiba', 'Sano'], 'dataset': 'MNIST'}
###################################################################################################
# :class:`~optuna.struct.StudySummary` object, which can be retrieved by
# :func:`~optuna.study.get_all_study_summaries`, also contains user-defined attributes.
study_summaries = optuna.get_all_study_summaries("sqlite:///example.db")
study_summaries[
0].user_attrs # {"contributors": ["Akiba", "Sano"], "dataset": "MNIST"}
w += beta * d * sa
action = action2
observation = observation2
if done:
# print("Episode finished after {} timesteps".format(t + 1))
# print(i_episode)
# tL.append(t)
M += t
break
env.close()
return M / 2000
import optuna
def objective(trial):
seed = trial.suggest_int('seed', 1, 100)
alpha = trial.suggest_loguniform('alpha', 1e-5, 1e-1)
beta = trial.suggest_loguniform('beta', 1e-5, 1e-1)
gamma = trial.suggest_discrete_uniform('gamma', 0.5, 1.0, 0.1)
return main(alpha, beta, gamma, seed)
study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=10000)
print(study.best_params)
print(study.best_value)
#'alpha': 0.00015057393338490765, 'seed': 46, 'gamma': 0.6, 'beta': 0.08719986440460901
batch_size=16,
verbose=0)
estimated_y_in_cv = model_selection.cross_val_predict(estimator,
train_x,
train_y,
cv=fold_number)
r2 = metrics.r2_score(train_y, estimated_y_in_cv)
return 1.0 - r2
#estimator.fit(train_x, train_y)
#return 1 - estimator.history['val_acc'][-1]
#history = model.fit(train_x, train_y, verbose = 0, epochs = 10, batch_size = 128, validation_split = 0.2, callbacks = [early_stop])
#return 1 - history.history['val_acc'][-1]
study = optuna.create_study()
study.optimize(objective, n_trials=trials)
print(study.best_params)
print(study.best_value)
best_optimizer = study.best_params['optimizer']
print(best_optimizer)
early_stop = keras.callbacks.EarlyStopping(monitor='accuracy', patience=10)
estimator = KerasRegressor(build_fn=create_model(**study.best_params),
epochs=5,
batch_size=16,
verbose=0)
estimator_history = estimator.fit(train_x, train_y)
results = model_selection.cross_val_score(estimator, train_x, train_y, cv=5)
def test_trials_dataframe(storage_mode: str, attrs: Tuple[str, ...],
multi_index: bool) -> None:
def f(trial: Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
assert isinstance(y, float)
trial.set_user_attr("train_loss", 3)
trial.set_system_attr("foo", "bar")
value = x + y # 3.5
# Test reported intermediate values, although it in practice is not "intermediate".
trial.report(value, step=0)
return value
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
study.optimize(f, n_trials=3)
df = study.trials_dataframe(attrs=attrs, multi_index=multi_index)
# Change index to access rows via trial number.
if multi_index:
df.set_index(("number", ""), inplace=True, drop=False)
else:
df.set_index("number", inplace=True, drop=False)
assert len(df) == 3
# Number columns are as follows (total of 13):
# non-nested: 6 (number, value, state, datetime_start, datetime_complete, duration)
# params: 2
# distributions: 2
# user_attrs: 1
# system_attrs: 1
# intermediate_values: 1
expected_n_columns = len(attrs)
if "params" in attrs:
expected_n_columns += 1
if "distributions" in attrs:
expected_n_columns += 1
assert len(df.columns) == expected_n_columns
for i in range(3):
assert df.number[i] == i
assert df.state[i] == "COMPLETE"
assert df.value[i] == 3.5
assert isinstance(df.datetime_start[i], pd.Timestamp)
assert isinstance(df.datetime_complete[i], pd.Timestamp)
if multi_index:
if "distributions" in attrs:
assert ("distributions", "x") in df.columns
assert ("distributions", "y") in df.columns
if "_trial_id" in attrs:
assert (
"trial_id",
"") in df.columns # trial_id depends on other tests.
if "duration" in attrs:
assert ("duration", "") in df.columns
assert df.params.x[i] == 1
assert df.params.y[i] == 2.5
assert df.user_attrs.train_loss[i] == 3
assert df.system_attrs.foo[i] == "bar"
else:
if "distributions" in attrs:
assert "distributions_x" in df.columns
assert "distributions_y" in df.columns
if "_trial_id" in attrs:
assert "trial_id" in df.columns # trial_id depends on other tests.
if "duration" in attrs:
assert "duration" in df.columns
assert df.params_x[i] == 1
assert df.params_y[i] == 2.5
assert df.user_attrs_train_loss[i] == 3
assert df.system_attrs_foo[i] == "bar"
def test_study_trials_dataframe_with_no_trials() -> None:
study_with_no_trials = create_study()
trials_df = study_with_no_trials.trials_dataframe()
assert trials_df.empty
def main():
_basic_columns = (
"environment",
"normalized_score_A",
"normalized_score_R",
"POIC",
"optimality_marginal",
"optimality_conditional",
"PIC",
"reward_marginal",
"reward_conditional",
"variance",
"temperatures"
"r_max",
"r_min",
"r_mean",
)
parser = argparse.ArgumentParser()
parser.add_argument("--n_trials",
type=int,
default=200,
help="n_trials for optuna")
parser.add_argument("--n_bins",
type=int,
default=100000,
help="number of bins")
parser.add_argument("--algo_max",
action="store_true",
help="max(r^algo, r^rand)")
parser.add_argument("--clip_persent",
type=float,
default=0.0,
help="top/bottom x percent clipping")
parser.add_argument("--sourse_path", type=str, default='./CartPole-v0.npy')
parser.add_argument("--root_dir", type=str, default='./results/')
parser.add_argument("--env", type=str, default='CartPole-v0')
args = parser.parse_args()
# save dir
output_dir = os.path.join(
args.root_dir,
'n_trials{}_clip_persent{}'.format(args.n_trials, args.clip_persent),
)
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, "{}_metrics.txt".format(args.env)),
"w") as f:
print("\t".join(_basic_columns), file=f)
with open(os.path.join(output_dir, "{}_tables.txt".format(args.env)),
"w") as f:
print(" & ".join(_basic_columns), file=f)
all_scores_per_param = np.load(args.sourse_path)
all_mean_scores = all_scores_per_param.mean(axis=1)
if args.clip_persent > 0:
upper = np.percentile(all_mean_scores, 100 - args.clip_persent)
lower = np.percentile(all_mean_scores, args.clip_persent)
all_scores_per_param = np.clip(all_scores_per_param, lower, upper)
all_scores = all_scores_per_param.flatten()
r_max = all_scores.max()
r_min = all_mean_scores.min()
r_mean = all_scores.mean()
variance = 0 if (r_max -
r_min) == 0 else all_scores.var() / (r_max - r_min)
if args.algo_max:
r_max = max(ALGORITHM_MAX[args.env], r_max)
def objective(trial):
temperature = trial.suggest_loguniform('temperature', 1e-4, 2e4)
p_o1 = np.exp((all_scores - r_max) / temperature).mean()
p_o1_ts = np.exp(
(all_scores_per_param - r_max) / temperature).mean(axis=1)
marginal = -p_o1 * np.log(p_o1 + 1e-12) - (
1 - p_o1) * np.log(1 - p_o1 + 1e-12)
conditional = np.mean(-p_o1_ts * np.log(p_o1_ts + 1e-12) -
(1 - p_o1_ts) * np.log(1 - p_o1_ts + 1e-12))
mutual_information = marginal - conditional
return mutual_information
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=args.n_trials)
# POIC
trial = study.best_trial
mi_o = trial.value
temperature = trial.params['temperature']
p_o1 = np.exp((all_scores - r_max) / temperature).mean()
p_o1_ts = np.exp((all_scores_per_param - r_max) / temperature).mean(axis=1)
h_o = -p_o1 * np.log(p_o1) - (1 - p_o1) * np.log(1 - p_o1)
h_o_t = np.mean(-p_o1_ts * np.log(p_o1_ts + 1e-12) -
(1 - p_o1_ts) * np.log(1 - p_o1_ts + 1e-12))
# PIC
bins = args.n_bins
hist = np.histogram(all_scores, bins=args.n_bins)
discretization_all = hist[0] / len(all_scores)
entropy_all = -np.sum(
discretization_all * np.log(discretization_all + 1e-12))
discretization_r_theta = [
np.histogram(x, bins=hist[1])[0] / len(x) for x in all_scores_per_param
]
entropy_r_theta = -np.mean([
np.sum(p_r_theta * np.log(p_r_theta + 1e-12))
for p_r_theta in discretization_r_theta
])
mi_r = entropy_all - entropy_r_theta
normalized_score_A = (ALGORITHM_AVG[args.env] - r_min) / (
max(ALGORITHM_MAX[args.env], r_max) - r_min)
normalized_score_R = (r_mean - r_min) / (
max(ALGORITHM_MAX[args.env], r_max) - r_min)
# save in scores.txt
values = (
args.env,
normalized_score_A,
normalized_score_R,
mi_o,
h_o,
h_o_t,
mi_r,
entropy_all,
entropy_r_theta,
variance,
temperature,
r_max,
r_min,
r_mean,
)
with open(os.path.join(output_dir, "{}_metrics.txt".format(args.env)),
"a+") as f:
print("\t".join(str(x) for x in values), file=f)
with open(os.path.join(output_dir, "{}_tables.txt".format(args.env)),
"a+") as f:
print(" & ".join(str(x) for x in values), file=f)
def test_optimize_trivial_in_memory_new() -> None:
study = optuna.create_study()
study.optimize(func, n_trials=10)
check_study(study)
model.add_seasonality(
name="quaterly",
period=365.25 / 4,
fourier_order=params["quaterly_fourier"],
prior_scale=params["quaterly_prior"],
)
train["cap"] = cap
train["floor"] = floor
model.fit(train)
future = model.make_future_dataframe(periods=144, freq="d")
future["cap"] = cap
future["floor"] = floor
forecast = model.predict(future)
valid_forecast = forecast.tail(7)
rmse = mean_squared_error(valid.y, valid_forecast.yhat, squared=False)
return rmse
if __name__ == "__main__":
study = optuna.create_study(
study_name="ontune hyperparameter",
direction="minimize",
sampler=TPESampler(seed=42),
)
study.optimize(objective, n_trials=args.trials)
prophet_params = study.best_params
joblib.dump(prophet_params, "../../parameters/" + args.params)
y = trial.suggest_int("y", 0, 10)
z = cast(float, trial.suggest_categorical("z", [-5, 0, 5]))
trial.set_system_attr("a", 0)
trial.set_user_attr("b", 1)
return x, x ** 2 + y ** 2 + z ** 2
if __name__ == "__main__":
parser = ArgumentParser(description="Create SQLite database for schema upgrade tests.")
parser.add_argument(
"--storage-url", default=f"sqlite:///test_upgrade_assets/{optuna.__version__}.db"
)
args = parser.parse_args()
# Create an empty study.
optuna.create_study(storage=args.storage_url, study_name="single_empty")
# Create a study for single-objective optimization.
study = optuna.create_study(storage=args.storage_url, study_name="single")
study.set_system_attr("c", 2)
study.set_user_attr("d", 3)
study.optimize(objective_test_upgrade, n_trials=1)
# Create a study for multi-objective optimization.
try:
optuna.create_study(
storage=args.storage_url, study_name="multi_empty", directions=["minimize", "minimize"]
)
study = optuna.create_study(
storage=args.storage_url, study_name="multi", directions=["minimize", "minimize"]
)
shuffle=True,
)
model = define_model(trial).to(DEVICE)
optimizer = torch.optim.Adam(
model.parameters(), trial.suggest_float("lr", 1e-5, 1e-1, log=True))
for epoch in range(10):
train_model(model, optimizer, train_loader)
flops, accuracy = eval_model(model, val_loader)
return flops, accuracy
###################################################################################################
# Run multi-objective optimization
# --------------------------------
#
# If your optimization problem is multi-objective,
# Optuna assumes that you will specify the optimization direction for each objective.
# Specifically, in this example, we want to minimize the FLOPS (we want a faster model)
# and maximize the accuracy. So we set ``directions`` to ``["minimize", "maximize"]``.
study = optuna.create_study(directions=["minimize", "maximize"])
study.optimize(objective, n_trials=30, timeout=300)
print("Number of finished trials: ", len(study.trials))
###################################################################################################
# Check trials on pareto front visually
optuna.visualization.plot_pareto_front(study,
target_names=["FLOPS", "accuracy"])
def test_get_observation_pairs() -> None:
def objective(trial: Trial) -> float:
x = trial.suggest_int("x", 5, 5)
if trial.number == 0:
return x
elif trial.number == 1:
trial.report(1, 4)
trial.report(2, 7)
raise TrialPruned()
elif trial.number == 2:
trial.report(float("nan"), 3)
raise TrialPruned()
elif trial.number == 3:
raise TrialPruned()
else:
raise RuntimeError()
# Test direction=minimize.
study = optuna.create_study(direction="minimize")
study.optimize(objective, n_trials=5, catch=(RuntimeError,))
scores = [
(-float("inf"), [5.0]), # COMPLETE
(-7, [2]), # PRUNED (with intermediate values)
(-3, [float("inf")]), # PRUNED (with a NaN intermediate value; it's treated as infinity)
(float("inf"), [0.0]), # PRUNED (without intermediate values)
]
assert _tpe.sampler._get_observation_pairs(study, ["x"], False) == (
{"x": [5.0, 5.0, 5.0, 5.0]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], False) == (
{"y": [None, None, None, None]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], True) == (
{"x": [5.0, 5.0, 5.0, 5.0]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], True) == ({"y": []}, [])
# Test direction=maximize.
study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=4)
study._storage.create_new_trial(study._study_id) # Create a running trial.
scores = [
(-float("inf"), [-5.0]), # COMPLETE
(-7, [-2]), # PRUNED (with intermediate values)
(-3, [float("inf")]), # PRUNED (with a NaN intermediate value; it's treated as infinity)
(float("inf"), [0.0]), # PRUNED (without intermediate values)
]
assert _tpe.sampler._get_observation_pairs(study, ["x"], False) == (
{"x": [5.0, 5.0, 5.0, 5.0]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], False) == (
{"y": [None, None, None, None]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["x"], True) == (
{"x": [5.0, 5.0, 5.0, 5.0]},
scores,
)
assert _tpe.sampler._get_observation_pairs(study, ["y"], True) == ({"y": []}, [])
def objective2(trial: Trial) -> float:
x = trial.suggest_int("x", 5, 5)
y = trial.suggest_int("y", 6, 6)
if trial.number == 0:
return x + y
elif trial.number == 1:
trial.report(1, 4)
trial.report(2, 7)
raise TrialPruned()
elif trial.number == 2:
trial.report(float("nan"), 3)
raise TrialPruned()
elif trial.number == 3:
raise TrialPruned()
else:
raise RuntimeError()
# Test direction=minimize.
study = optuna.create_study(direction="minimize")
study.optimize(objective2, n_trials=5, catch=(RuntimeError,))
assert _tpe.sampler._get_observation_pairs(study, ["x", "y"], True) == (
{"x": [5.0, 5.0, 5.0, 5.0], "y": [6.0, 6.0, 6.0, 6.0]},
[
(-float("inf"), [11.0]), # COMPLETE
(-7, [2]), # PRUNED (with intermediate values)
(
-3,
[float("inf")],
), # PRUNED (with a NaN intermediate value; it's treated as infinity)
(float("inf"), [0.0]), # PRUNED (without intermediate values)
],
)
# Test direction=maximize.
study = optuna.create_study(direction="maximize")
study.optimize(objective2, n_trials=4)
study._storage.create_new_trial(study._study_id) # Create a running trial.
assert _tpe.sampler._get_observation_pairs(study, ["x", "y"], True) == (
{"x": [5.0, 5.0, 5.0, 5.0], "y": [6.0, 6.0, 6.0, 6.0]},
[
(-float("inf"), [-11.0]), # COMPLETE
(-7, [-2]), # PRUNED (with intermediate values)
(
-3,
[float("inf")],
), # PRUNED (with a NaN intermediate value; it's treated as infinity)
(float("inf"), [0.0]), # PRUNED (without intermediate values)
],
)
# Just for logging purposes
log.debug(f" Chunked {d} data: {X[d].shape}")
log.debug(f" Chunked {d} data 2D: {X_2d[d].shape}")
log.debug(f" Chunked {d} target: {Y[d].shape}")
# Define dataloaders for PyTorch
loaders = torch_dataloaders(args, dsets, X, Y)
# ===== Optional: Hyperparameter tuning =====
# If you are doing hyperparameter optimization using optuna
if args.optuna_do:
study_name = args.exp + "_" + trial_ID
# For non-neural networks
if args.decode_type in args.input_2d_decoders:
study = optuna.create_study(
study_name=study_name,
pruner=optuna.pruners.SuccessiveHalvingPruner())
study.optimize(lambda trial: objective_2d(
trial, args, X_2d, Y, DIR_CHECK),
n_trials=args.optuna_trials)
# For neural networks
elif args.decode_type in args.input_3d_decoders:
study = optuna.create_study(
study_name=study_name,
pruner=optuna.pruners.SuccessiveHalvingPruner())
if args.fix_do:
study.optimize(lambda trial: objective_fix(
trial, args, loaders, DIR_CHECK),
n_trials=args.optuna_trials)
else:
study.optimize(lambda trial: objective(
best_ads = step2(N, XYR, best_ads, n=25000, a_th=a_th)
best_total = get_total_score(N, XYR, best_ads)
best_ads = step3(N, XYR, best_ads)
best_total = get_total_score(N, XYR, best_ads)
for i in range(N):
print(*best_ads[i])
logger.info("best_total: {:.06f}".format(best_total))
return best_total
def objective(trial):
a_th = trial.suggest_uniform("a_th", 0, 1)
inputs = read_from_file(Path("../tools/in/0000.txt"))
v1 = solve(*inputs, a_th=a_th)
inputs = read_from_file(Path("../tools/in/0001.txt"))
v2 = solve(*inputs, a_th=a_th)
inputs = read_from_file(Path("../tools/in/0002.txt"))
v3 = solve(*inputs, a_th=a_th)
return (v1 + v2 + v3) / 3
if __name__ == '__main__':
# random.seed(2021)
# inputs = read()
# solve(*inputs)
study = optuna.create_study(direction="maximize", study_name="strategy_5", storage="sqlite:///strategy_5.db")
study.optimize(objective, n_trials=100)
logger.info("[optuna] best a_th: {:.4f}".format(study.best_params["a_th"]))
logger.info("[optuna] best score: {:.4f}".format(study.best_value))
def hyperoptimize(
self,
n_trials: int = 64,
subset: Union[None, pd.core.series.Series] = None,
max_epochs: int = 128,
) -> dict:
"""Search for hyperparameters with greater out-of-sample performance.
Args:
n_trials: The number of hyperparameter sets to evaluate for each
time horizon. Return None if non-positive.
subset: A Boolean Series that is True for observations on which
to train and validate. If None, default to all observations not
flagged by self.test_col or self.predict_col.
Returns:
A dictionary containing the best-performing parameters.
"""
def evaluate_params(
trial: optuna.trial.Trial,
x_train: List[np.array],
y_train: np.array,
x_valid: List[np.array],
y_valid: np.array,
max_epochs: int,
) -> Union[None, dict]:
"""Compute out-of-sample performance for a parameter set."""
params = {}
params["BATCH_SIZE"] = trial.suggest_int(
"BATCH_SIZE", min(32, x_train[0].shape[0]),
x_train[0].shape[0])
params["DENSE_LAYERS"] = trial.suggest_int("DENSE_LAYERS", 0, 3)
params["DROPOUT_SHARE"] = trial.suggest_uniform(
"DROPOUT_SHARE", 0, 0.5)
params["EMBED_EXPONENT"] = trial.suggest_uniform(
"EMBED_EXPONENT", 0, 0.25)
params["EMBED_L2_REG"] = trial.suggest_uniform(
"EMBED_L2_REG", 0, 16.0)
if self.categorical_features:
params["POST_FREEZE_EPOCHS"] = trial.suggest_int(
"POST_FREEZE_EPOCHS", 4, max_epochs)
else:
params["POST_FREEZE_EPOCHS"] = 0
max_pre_freeze_epochs = max_epochs
params["PRE_FREEZE_EPOCHS"] = trial.suggest_int(
"PRE_FREEZE_EPOCHS", 4, max_epochs)
params["NODES_PER_DENSE_LAYER"] = trial.suggest_int(
"NODES_PER_DENSE_LAYER", 16, 1024)
construction_args = getfullargspec(
self.construct_embedding_network).args
self.model = self.construct_embedding_network(**{
k.lower(): v
for k, v in params.items() if k in construction_args
})
self.data[self.numeric_features] = self.data[
self.numeric_features].fillna(
self.config.get("NON_CAT_MISSING_VALUE", -1))
model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in self.config.items() if k in construction_args
})
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["PRE_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
model = freeze_embedding_layers(model)
model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
for step in range(params["POST_FREEZE_EPOCHS"]):
model.fit(x_train,
y_train,
batch_size=params["BATCH_SIZE"],
epochs=1)
validation_loss = model.evaluate(x_valid, y_valid)
trial.report(validation_loss, step + max_pre_freeze_epochs)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return validation_loss
default_params = {"BATCH_SIZE": 512, "PRE_FREEZE_EPOCHS": 16}
if n_trials <= 0:
return {
time_horizon: default_params
for time_horizon in range(self.n_intervals)
}
params = {}
if subset is None:
subset = ~self.data[self.test_col] & ~self.data[self.predict_col]
x_train = self.format_input_data(subset=subset
& ~self.data[self.validation_col])
x_valid = self.format_input_data(subset=subset
& self.data[self.validation_col])
y_surv = make_surv_array(
self.data[[self.duration_col, self.max_lead_col]].min(axis=1),
self.data[self.event_col],
np.arange(self.n_intervals + 1),
)
y_train = y_surv[[subset & ~self.data[self.validation_col]]]
y_valid = y_surv[[subset & self.data[self.validation_col]]]
study = optuna.create_study(
pruner=optuna.pruners.MedianPruner(),
sampler=optuna.samplers.TPESampler(
seed=self.config.get("SEED", 9999)),
)
study.optimize(
lambda trial: evaluate_params(trial, x_train, y_train, x_valid,
y_valid, max_epochs),
n_trials=n_trials,
)
params = study.best_params
if self.categorical_features:
default_params["POST_FREEZE_EPOCHS"] = 16
else:
default_params["POST_FREEZE_EPOCHS"] = 0
construction_args = getfullargspec(
self.construct_embedding_network).args
default_model = self.construct_embedding_network(
**{
k.lower(): v
for k, v in default_params.items() if k in construction_args
})
default_model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
default_model.fit(
x_train,
y_train,
batch_size=default_params["BATCH_SIZE"],
epochs=default_params["PRE_FREEZE_EPOCHS"],
)
default_model = freeze_embedding_layers(default_model)
default_model.compile(loss=surv_likelihood(self.n_intervals),
optimizer=Adam(amsgrad=True))
default_model.fit(
x_train,
y_train,
batch_size=default_params["BATCH_SIZE"],
epochs=default_params["POST_FREEZE_EPOCHS"],
)
default_validation_loss = default_model.evaluate(x_valid, y_valid)
if default_validation_loss <= study.best_value:
params = default_params
return params
# Eval dataset
root = dataset_root
if 'OTB' in args.dataset:
dataset_eval = OTBDataset(args.dataset, root)
elif 'LaSOT' == args.dataset:
dataset_eval = LaSOTDataset(args.dataset, root)
elif 'UAV' in args.dataset:
dataset_eval = UAVDataset(args.dataset, root)
elif 'NFS' in args.dataset:
dataset_eval = NFSDataset(args.dataset, root)
if args.dataset in ['VOT2016', 'VOT2017', 'VOT2018', 'VOT2019']:
dataset_eval = VOTDataset(args.dataset, root)
elif 'VOT2018-LT' == args.dataset:
dataset_eval = VOTLTDataset(args.dataset, root)
tune_result = os.path.join('tune_results', args.dataset)
if not os.path.isdir(tune_result):
os.makedirs(tune_result)
log_path = os.path.join(
tune_result, (args.snapshot).split('/')[-1].split('.')[0] + '.log')
logging.getLogger().setLevel(logging.INFO)
logging.getLogger().addHandler(logging.FileHandler(log_path))
optuna.logging.enable_propagation()
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=10000)
print('Best value: {} (params: {})\n'.format(study.best_value,
study.best_params))
lgb_eval = lgb.Dataset(x_valid, y_valid, reference=lgb_train)
model = lgb.train(
params,
lgb_train,
valid_sets=lgb_eval,
num_boost_round=100,
early_stopping_rounds=20,
verbose_eval=10,
)
y_pred = model.predict(x_valid, num_iteration=model.best_iteration)
score = np.sqrt(mean_squared_error(y_valid, y_pred))
return score
study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0))
study.optimize(objective, n_trials=50)
print(study.best_params)
"""
best_params：
{'num_leaves': 18,
'max_bin': 193,
'bagging_fraction': 0.6373487511442064,
'bagging_freq': 8,
'feature_fraction': 0.5690038074194459,
'min_data_in_leaf': 7,
'min_sum_hessian_in_leaf': 6}
"""
def test_optimize_trivial_in_memory_resume() -> None:
study = create_study()
study.optimize(func, n_trials=10)
study.optimize(func, n_trials=10)
check_study(study)
return metrics["best_validation_" + TARGET_METRIC]
if __name__ == "__main__":
if version.parse(allennlp.__version__) < version.parse("2.0.0"):
raise RuntimeError(
"`allennlp>=2.0.0` is required for this example."
" If you want to use `allennlp<2.0.0`, please install `optuna==2.5.0`"
" and refer to the following example:"
" https://github.com/optuna/optuna/blob/v2.5.0/examples/allennlp/allennlp_simple.py"
)
random.seed(41)
torch.manual_seed(41)
numpy.random.seed(41)
pruner = optuna.pruners.HyperbandPruner()
study = optuna.create_study(direction="maximize", pruner=pruner)
study.optimize(objective, n_trials=50, timeout=600)
print("Number of finished trials: ", len(study.trials))
print("Best trial:")
trial = study.best_trial
print(" Value: ", trial.value)
print(" Params: ")
for key, value in trial.params.items():
print(" {}: {}".format(key, value))
shutil.rmtree(MODEL_DIR)
def test_optimize_without_gc(collect_mock: Mock) -> None:
study = create_study()
study.optimize(func, n_trials=10, gc_after_trial=False)
check_study(study)
assert collect_mock.call_count == 0
worker_id, time_str))
mean_reward, std_reward = evaluate_model(agent.ppo_net,
env,
n_eval_episodes=10)
print("{}, {}".format(mean_reward, std_reward))
writer.close()
env.close()
return mean_reward
if __name__ == '__main__':
worker_id = 0
if len(sys.argv) > 1:
worker_id = int(sys.argv[1])
# run_exp()
name = 'crawler-JR'
db = 'sqlite:///example.db'
# cli command:
# optuna dashboard --study-name "crawler-JR" --storage "sqlite:///example.db"
# study = optuna.load_study(study_name='crawler-JR', storage='sqlite:///example.db')
try:
study = optuna.load_study(study_name=name, storage=db)
print("------- load study successful")
except:
optuna.create_study(storage=db, study_name=name)
study = optuna.load_study(study_name=name, storage=db)
print("******* create and load study successful")
study.optimize(objective, n_trials=1000)
def test_best_trials() -> None:
study = create_study(directions=["minimize", "maximize"])
study.optimize(lambda t: [2, 2], n_trials=1)
study.optimize(lambda t: [1, 1], n_trials=1)
study.optimize(lambda t: [3, 1], n_trials=1)
assert {tuple(t.values) for t in study.best_trials} == {(1, 1), (2, 2)}
def test_sample_independent_misc_arguments() -> None:
study = optuna.create_study(directions=["minimize", "maximize"])
dist = optuna.distributions.UniformDistribution(1.0, 100.0)
random.seed(128)
past_trials = [frozen_trial_factory(i, [random.random(), random.random()]) for i in range(32)]
# Prepare a trial and a sample for later checks.
trial = frozen_trial_factory(16, [0, 0])
sampler = MOTPESampler(seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
suggestion = sampler.sample_independent(study, trial, "param-a", dist)
# Test misc. parameters.
sampler = MOTPESampler(n_ehvi_candidates=13, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
sampler = MOTPESampler(gamma=lambda _: 5, seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
sampler = MOTPESampler(weights_above=lambda n: np.zeros(n), seed=0)
attrs = MockSystemAttr()
with patch.object(study._storage, "get_all_trials", return_value=past_trials), patch.object(
study._storage, "set_trial_system_attr", side_effect=attrs.set_trial_system_attr
), patch.object(study._storage, "get_trial", return_value=trial), patch(
"optuna.trial.Trial.system_attrs", new_callable=PropertyMock
) as mock1, patch(
"optuna.trial.FrozenTrial.system_attrs",
new_callable=PropertyMock,
) as mock2:
mock1.return_value = attrs.value
mock2.return_value = attrs.value
assert sampler.sample_independent(study, trial, "param-a", dist) != suggestion
trial.set_user_attr(key="best_booster", value=xgb_regressor
) # NOTE update the best model in the optuna's table.
return np.mean(rmse)
# SOURCE retrieve the best model - https://stackoverflow.com/questions/62144904/python-how-to-retrive-the-best-model-from-optuna-lightgbm-study#answer-63365355
def save_best(study, trial):
if study.best_trial.number == trial.number:
# Set the best booster as a trial attribute; accessible via study.trials_dataframe.
study.set_user_attr(key="best_booster",
value=trial.user_attrs["best_booster"])
# SOURCE retrieve the best number of estimators https://github.com/optuna/optuna/issues/1169
study = optuna.create_study(
direction="minimize",
sampler=optuna.samplers.TPESampler(seed=2021, multivariate=True),
pruner=optuna.pruners.MedianPruner(n_warmup_steps=3))
study.optimize(
lambda trial: objective(trial, x_train_full, y_train_full, params),
n_trials=N_TRIALS,
timeout=TIMEOUT,
n_jobs=1,
callbacks=[save_best])
# display params
hp = study.best_params
for key, value in hp.items():
print(f"{key:>20s} : {value}")
print(f"{'best objective value':>20s} : {study.best_value}")
# %%
# optuna.visualization.plot_optimization_history(study)
clf.partial_fit(x_train, y_train, classes=classes)
value = clf.score(x_valid, y_valid)
# Report intermediate objective value.
trial.report(value, step)
# Handle pruning based on the intermediate value.
if trial.should_prune():
raise optuna.TrialPruned()
return value
if __name__ == "__main__":
study = optuna.create_study(direction="maximize",
pruner=optuna.pruners.MedianPruner())
study.optimize(objective, n_trials=100, timeout=600)
# Visualize the optimization history.
plot_optimization_history(study).show()
# Visualize the learning curves of the trials.
plot_intermediate_values(study).show()
# Visualize high-dimensional parameter relationships.
plot_parallel_coordinate(study).show()
# Select parameters to visualize.
plot_parallel_coordinate(study, params=["lr_init", "n_units_l0"]).show()
# Visualize hyperparameter relationships.
