def objective(trial: Trial) -> Tuple[float, float]:
p0 = trial.suggest_float("p0", -10, 10)
p1 = trial.suggest_uniform("p1", 3, 5)
p2 = trial.suggest_loguniform("p2", 0.00001, 0.1)
p3 = trial.suggest_discrete_uniform("p3", 100, 200, q=5)
p4 = trial.suggest_int("p4", -20, -15)
p5 = cast(int, trial.suggest_categorical("p5", [7, 1, 100]))
p6 = trial.suggest_float("p6", -10, 10, step=1.0)
p7 = trial.suggest_int("p7", 1, 7, log=True)
return (
p0 + p1 + p2,
p3 + p4 + p5 + p6 + p7,
)
def test_check_distribution_suggest_discrete_uniform(
storage_mode: str) -> None:
sampler = samplers.RandomSampler()
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns() as record:
trial.suggest_discrete_uniform("x", 10, 20, 2)
trial.suggest_discrete_uniform("x", 10, 20, 2)
trial.suggest_discrete_uniform("x", 10, 22, 2)
# we expect exactly one warning (not counting ones caused by deprecation)
assert len([r for r in record if r.category != FutureWarning]) == 1
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20, 2)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.raises(ValueError):
trial.suggest_int("x", 10, 20, 2)
def test_suggest_low_equals_high(storage_init_func):
# type: (typing.Callable[[], storages.BaseStorage]) -> None
study = create_study(storage_init_func(), sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
# Parameter values are determined without suggestion when low == high.
with patch.object(trial, "_suggest", wraps=trial._suggest) as mock_object:
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting a param.
assert trial.suggest_uniform("a", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting a param.
assert trial.suggest_loguniform("b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert (
trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 0
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 0
def _get_params(self, trial: trial_module.Trial) -> Dict[str, Any]:
params = self.params.copy() # type: Dict[str, Any]
if self.param_distributions is None:
params["colsample_bylevel"] = trial.suggest_discrete_uniform(
"colsample_bylevel", 0.1, 1.0, 0.05
)
params["max_depth"] = trial.suggest_int("max_depth", 1, 7)
# https://catboost.ai/docs/concepts/parameter-tuning.html#tree-growing-policy
# params["num_leaves"] = trial.suggest_int(
# "num_leaves", 2, 2 ** params["max_depth"]
# )
# See https://github.com/Microsoft/LightGBM/issues/907
params["num_leaves"] = 31
params["min_data_in_leaf"] = trial.suggest_int(
"min_data_in_leaf",
1,
max(1, int(self.n_samples / params["num_leaves"])),
)
params["l2_leaf_reg"] = trial.suggest_loguniform("lambda_l2", 1e-09, 10.0)
if params["bootstrap_type"] == "Bayesian":
params["bagging_temperature"] = trial.suggest_discrete_uniform(
"bagging_temperature", 0.5, 0.95, 0.05
)
elif (
params["bootstrap_type"] == "Bernoulli"
or params["bootstrap_type"] == "Poisson"
):
params["subsample"] = trial.suggest_uniform("subsample", 0.1, 1)
return params
for name, distribution in self.param_distributions.items():
params[name] = trial._suggest(name, distribution)
return params
def test_suggest_discrete_uniform_range(
storage_mode: str, range_config: Dict[str, float]) -> None:
sampler = samplers.RandomSampler()
# Check upper endpoints.
mock = Mock()
mock.side_effect = lambda study, trial, param_name, distribution: distribution.high
with patch.object(
sampler, "sample_independent",
mock) as mock_object, StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns(UserWarning):
x = trial.suggest_discrete_uniform("x", range_config["low"],
range_config["high"],
range_config["q"])
assert x == range_config["mod_high"]
assert mock_object.call_count == 1
# Check lower endpoints.
mock = Mock()
mock.side_effect = lambda study, trial, param_name, distribution: distribution.low
with patch.object(
sampler, "sample_independent",
mock) as mock_object, StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage, sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with pytest.warns(UserWarning):
x = trial.suggest_discrete_uniform("x", range_config["low"],
range_config["high"],
range_config["q"])
assert x == range_config["low"]
assert mock_object.call_count == 1
def objective(trial: Trial) -> float:
x1 = trial.suggest_uniform("x1", 0.1, 3)
x2 = trial.suggest_loguniform("x2", 0.1, 3)
x3 = trial.suggest_discrete_uniform("x3", 0, 3, 1)
x4 = trial.suggest_int("x4", -3, 3)
x5 = trial.suggest_categorical("x5", [1.0, 1.1, 1.2])
if trial.number % 2 == 0:
# Conditional parameters are ignored unless `params` is specified and is not `None`.
x6 = trial.suggest_uniform("x6", 0.1, 3)
assert isinstance(x5, float)
value = x1**4 + x2 + x3 - x4**2 - x5
if trial.number % 2 == 0:
value += x6
return value
def test_check_distribution_suggest_float(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
sampler = samplers.RandomSampler()
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
x1 = trial.suggest_float("x1", 10, 20)
x2 = trial.suggest_uniform("x1", 10, 20)
assert x1 == x2
x3 = trial.suggest_float("x2", 1e-5, 1e-3, log=True)
x4 = trial.suggest_loguniform("x2", 1e-5, 1e-3)
assert x3 == x4
x5 = trial.suggest_float("x3", 10, 20, step=1.0)
x6 = trial.suggest_discrete_uniform("x3", 10, 20, 1.0)
assert x5 == x6
with pytest.raises(NotImplementedError):
trial.suggest_float("x4", 1e-5, 1e-2, log=True, step=1e-5)
def __call__(self, trial: Trial) -> float:
"""Calculate an objective value."""
train, val, test, num_users, num_items =\
preprocess_datasets(data=self.data, seed=12345)
# sample a set of hyperparameters.
config = yaml.safe_load(open('../config.yaml', 'r'))
eta = config['eta']
max_iters = config['max_iters']
batch_size = config['batch_size']
pre_iters = config['pre_iters']
post_iters = config['post_iters']
post_steps = config['post_steps']
dim = trial.suggest_discrete_uniform('dim', 5, 50, 5)
lam = trial.suggest_loguniform('lam', 1e-6, 1)
if '-at' in self.model_name:
epsilon = trial.suggest_loguniform('epsilon', 1e-3, 1)
ops.reset_default_graph()
tf.set_random_seed(12345)
sess = tf.Session()
if '-at' not in self.model_name:
model = MFIPS(num_users=num_users, num_items=num_items, dim=dim, lam=lam, eta=eta)
score, _, _, _, _, _ = train_mfips(
sess, model=model, data=self.data, train=train, val=val, test=test,
max_iters=max_iters, batch_size=batch_size, model_name=self.model_name)
else:
model = MFIPS(num_users=num_users, num_items=num_items, dim=dim, lam=lam, eta=eta, num=0)
model1 = MFIPS(num_users=num_users, num_items=num_items, dim=dim, lam=lam, eta=eta, num=1)
model2 = MFIPS(num_users=num_users, num_items=num_items, dim=dim, lam=lam, eta=eta, num=2)
score, _, _, _, _, _ = train_mfips_with_at(
sess, model=model, mfips1=model1, mfips2=model2, data=self.data,
train=train, val=val, test=test, epsilon=epsilon,
pre_iters=pre_iters, post_iters=post_iters, post_steps=post_steps,
batch_size=batch_size, model_name=self.model_name)
return score
def test_suggest_low_equals_high(storage_mode: str) -> None:
with patch.object(
distributions,
"_get_single_value",
wraps=distributions._get_single_value
) as mock_object, StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage,
sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 1
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 1
assert trial.suggest_loguniform("b", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 2
assert trial.suggest_loguniform(
"b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 2
assert trial.suggest_discrete_uniform(
"c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 3
assert (trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 3
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert mock_object.call_count == 4
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 4
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 5
assert trial.suggest_float("e", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 5
assert trial.suggest_float("f", 0.5, 0.5,
log=True) == 0.5 # Suggesting a param.
assert mock_object.call_count == 6
assert trial.suggest_float(
"f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 6
assert trial.suggest_float("g", 0.5, 0.5,
log=False) == 0.5 # Suggesting a param.
assert mock_object.call_count == 7
assert trial.suggest_float(
"g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 7
assert trial.suggest_float("h", 0.5, 0.5,
step=1.0) == 0.5 # Suggesting a param.
assert mock_object.call_count == 8
assert trial.suggest_float(
"h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 8
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting a param.
assert mock_object.call_count == 9
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting the same param.
assert mock_object.call_count == 9
def __call__(self, trial: Trial) -> float:
"""Calculate an objective value."""
logger = logging.getLogger(__name__) # Create a custom logger
# Create logging handlers
c_handler = logging.StreamHandler()
f_handler = logging.FileHandler(
Path(f'../logs/{self.data}/{self.model_name}/simulations.log'),
mode='w')
# Create logging formatters and add them to handlers
c_format = logging.Formatter('%(message)s')
f_format = logging.Formatter('%(message)s')
c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format)
logger.addHandler(c_handler)
logger.addHandler(f_handler)
train, val, test, num_users, num_items =\
preprocess_datasets(data=self.data, seed=rand_seed_val)
# sample a set of hyperparameters.
config = yaml.safe_load(open('../config.yaml', 'r'))
eta = config['eta']
max_iters = config['max_iters']
batch_size = config['batch_size']
pre_iters = config['pre_iters']
post_iters = config['post_iters']
post_steps = config['post_steps']
dim = trial.suggest_discrete_uniform('dim', 5, 50, 5)
lam = trial.suggest_loguniform('lam', 1e-6, 1)
if '-at' in self.model_name:
epsilon = trial.suggest_loguniform('epsilon', 1e-3, 1)
ops.reset_default_graph()
tf.set_random_seed(rand_seed_val)
sess = tf.Session()
if '-without_ipw' in self.model_name:
logger.debug('*** Without IPW ***')
model = MFMODEL(num_users=num_users,
num_items=num_items,
dim=dim,
lam=lam,
eta=eta)
score, _, _, _, _, _, _, _ = train_mfmodel_without_ipw(
sess,
model=model,
data=self.data,
train=train,
val=val,
test=test,
max_iters=max_iters,
batch_size=batch_size,
model_name=self.model_name)
elif '-at' not in self.model_name:
logger.debug(
'*** With IPW and without Asymmetric Tri-training ***')
model = MFMODEL(num_users=num_users,
num_items=num_items,
dim=dim,
lam=lam,
eta=eta)
score, _, _, _, _, _, _, _ = train_mfmodel(
sess,
model=model,
data=self.data,
train=train,
val=val,
test=test,
max_iters=max_iters,
batch_size=batch_size,
model_name=self.model_name)
else:
logger.debug('*** With IPW and Asymmetric Tri-training ***')
model = MFMODEL(num_users=num_users,
num_items=num_items,
dim=dim,
lam=lam,
eta=eta,
num=0)
model1 = MFMODEL(num_users=num_users,
num_items=num_items,
dim=dim,
lam=lam,
eta=eta,
num=1)
model2 = MFMODEL(num_users=num_users,
num_items=num_items,
dim=dim,
lam=lam,
eta=eta,
num=2)
score, _, _, _, _, _, _, _ = train_mfmodel_with_at(
sess,
model=model,
mfmodel1=model1,
mfmodel2=model2,
data=self.data,
train=train,
val=val,
test=test,
epsilon=epsilon,
pre_iters=pre_iters,
post_iters=post_iters,
post_steps=post_steps,
batch_size=batch_size,
model_name=self.model_name)
return score
def test_suggest_low_equals_high(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
study = create_study(storage_init_func(),
sampler=samplers.TPESampler(n_startup_trials=0))
trial = Trial(study, study._storage.create_new_trial(study._study_id))
with patch.object(
optuna.distributions,
"_get_single_value",
wraps=optuna.distributions._get_single_value) as mock_object:
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 1
assert trial.suggest_uniform("a", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 1
assert trial.suggest_loguniform("b", 1.0,
1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 2
assert trial.suggest_loguniform(
"b", 1.0, 1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 2
assert trial.suggest_discrete_uniform(
"c", 1.0, 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 3
assert (trial.suggest_discrete_uniform("c", 1.0, 1.0, 1.0) == 1.0
) # Suggesting the same param.
assert mock_object.call_count == 3
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting a param.
assert mock_object.call_count == 4
assert trial.suggest_int("d", 1, 1) == 1 # Suggesting the same param.
assert mock_object.call_count == 4
assert trial.suggest_float("e", 1.0, 1.0) == 1.0 # Suggesting a param.
assert mock_object.call_count == 5
assert trial.suggest_float("e", 1.0,
1.0) == 1.0 # Suggesting the same param.
assert mock_object.call_count == 5
assert trial.suggest_float("f", 0.5, 0.5,
log=True) == 0.5 # Suggesting a param.
assert mock_object.call_count == 6
assert trial.suggest_float(
"f", 0.5, 0.5, log=True) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 6
assert trial.suggest_float("g", 0.5, 0.5,
log=False) == 0.5 # Suggesting a param.
assert mock_object.call_count == 7
assert trial.suggest_float(
"g", 0.5, 0.5, log=False) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 7
assert trial.suggest_float("h", 0.5, 0.5,
step=1.0) == 0.5 # Suggesting a param.
assert mock_object.call_count == 8
assert trial.suggest_float(
"h", 0.5, 0.5, step=1.0) == 0.5 # Suggesting the same param.
assert mock_object.call_count == 8
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting a param.
assert mock_object.call_count == 9
assert trial.suggest_int("i", 1, 1,
log=True) == 1 # Suggesting the same param.
assert mock_object.call_count == 9
def __call__(self, name: str, trial: Trial):
return trial.suggest_discrete_uniform(name, self.low, self.high,
self.q)
