def test_search_space_transform_encoding() -> None:
trans = _SearchSpaceTransform({"x0": IntUniformDistribution(0, 3)})
assert len(trans.column_to_encoded_columns) == 1
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0]))
trans = _SearchSpaceTransform({"x0": CategoricalDistribution(["foo", "bar", "baz"])})
assert len(trans.column_to_encoded_columns) == 1
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0, 1, 2]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 0, 0]))
trans = _SearchSpaceTransform(
{
"x0": UniformDistribution(0, 3),
"x1": CategoricalDistribution(["foo", "bar", "baz"]),
"x3": DiscreteUniformDistribution(0, 1, q=0.2),
}
)
assert len(trans.column_to_encoded_columns) == 3
numpy.testing.assert_equal(trans.column_to_encoded_columns[0], numpy.array([0]))
numpy.testing.assert_equal(trans.column_to_encoded_columns[1], numpy.array([1, 2, 3]))
numpy.testing.assert_equal(trans.column_to_encoded_columns[2], numpy.array([4]))
numpy.testing.assert_equal(trans.encoded_column_to_column, numpy.array([0, 1, 1, 1, 2]))
def suggest_discrete_uniform(self, name: str, low: float, high: float, q: float) -> float:
"""Suggest a value for the discrete parameter.
The value is sampled from the range :math:`[\\mathsf{low}, \\mathsf{high}]`,
and the step of discretization is :math:`q`. More specifically,
this method returns one of the values in the sequence
:math:`\\mathsf{low}, \\mathsf{low} + q, \\mathsf{low} + 2 q, \\dots,
\\mathsf{low} + k q \\le \\mathsf{high}`,
where :math:`k` denotes an integer. Note that :math:`high` may be changed due to round-off
errors if :math:`q` is not an integer. Please check warning messages to find the changed
values.
Example:
Suggest a fraction of samples used for fitting the individual learners of
`GradientBoostingClassifier <https://scikit-learn.org/stable/modules/generated/
sklearn.ensemble.GradientBoostingClassifier.html>`_.
.. testcode::
import numpy as np
from sklearn.datasets import load_iris
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
import optuna
X, y = load_iris(return_X_y=True)
X_train, X_valid, y_train, y_valid = train_test_split(X, y)
def objective(trial):
subsample = trial.suggest_discrete_uniform("subsample", 0.1, 1.0, 0.1)
clf = GradientBoostingClassifier(subsample=subsample, random_state=0)
clf.fit(X_train, y_train)
return clf.score(X_valid, y_valid)
study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=3)
Args:
name:
A parameter name.
low:
Lower endpoint of the range of suggested values. ``low`` is included in the range.
high:
Upper endpoint of the range of suggested values. ``high`` is included in the range.
q:
A step of discretization.
Returns:
A suggested float value.
"""
distribution = DiscreteUniformDistribution(low=low, high=high, q=q)
self._check_distribution(name, distribution)
return self._suggest(name, distribution)
def test_search_space_transform_untransform_params() -> None:
search_space = {
"x0": DiscreteUniformDistribution(0, 1, q=0.2),
"x1": CategoricalDistribution(["foo", "bar", "baz", "qux"]),
"x2": IntLogUniformDistribution(1, 10),
"x3": CategoricalDistribution(["quux", "quuz"]),
"x4": UniformDistribution(2, 3),
"x5": LogUniformDistribution(1, 10),
"x6": IntUniformDistribution(2, 4),
"x7": CategoricalDistribution(["corge"]),
}
params = {
"x0": 0.2,
"x1": "qux",
"x2": 1,
"x3": "quux",
"x4": 2.0,
"x5": 1.0,
"x6": 2,
"x7": "corge",
}
trans = _SearchSpaceTransform(search_space)
trans_params = trans.transform(params)
untrans_params = trans.untransform(trans_params)
for name in params.keys():
assert untrans_params[name] == params[name]
def test_distributions(storage_init_func):
# type: (Callable[[], storages.BaseStorage]) -> None
def objective(trial):
# type: (Trial) -> float
trial.suggest_uniform("a", 0, 10)
trial.suggest_loguniform("b", 0.1, 10)
trial.suggest_discrete_uniform("c", 0, 10, 1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
study = create_study(storage_init_func())
study.optimize(objective, n_trials=1)
assert study.best_trial.distributions == {
"a": UniformDistribution(low=0, high=10),
"b": LogUniformDistribution(low=0.1, high=10),
"c": DiscreteUniformDistribution(low=0, high=10, q=1),
"d": IntUniformDistribution(low=0, high=10),
"e": CategoricalDistribution(choices=("foo", "bar", "baz")),
"f": IntLogUniformDistribution(low=1, high=10),
}
def test_sample_single_distribution(
sampler_class: Callable[[], BaseSampler]) -> None:
relative_search_space = {
"a": UniformDistribution(low=1.0, high=1.0),
"b": LogUniformDistribution(low=1.0, high=1.0),
"c": DiscreteUniformDistribution(low=1.0, high=1.0, q=1.0),
"d": IntUniformDistribution(low=1, high=1),
"e": IntLogUniformDistribution(low=1, high=1),
"f": CategoricalDistribution([1]),
"g": FloatDistribution(low=1.0, high=1.0),
"h": FloatDistribution(low=1.0, high=1.0, log=True),
"i": FloatDistribution(low=1.0, high=1.0, step=1.0),
"j": IntDistribution(low=1, high=1),
"k": IntDistribution(low=1, high=1, log=True),
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", optuna.exceptions.ExperimentalWarning)
sampler = sampler_class()
study = optuna.study.create_study(sampler=sampler)
# We need to test the construction of the model, so we should set `n_trials >= 2`.
for _ in range(2):
trial = study.ask(fixed_distributions=relative_search_space)
study.tell(trial, 1.0)
for param_name in relative_search_space.keys():
assert trial.params[param_name] == 1
def suggest_float(
self,
name: str,
low: float,
high: float,
*,
step: Optional[float] = None,
log: bool = False,
) -> float:
if step is not None:
if log:
raise ValueError(
"The parameter `step` is not supported when `log` is True."
)
else:
return self._suggest(
name,
DiscreteUniformDistribution(low=low, high=high, q=step))
else:
if log:
return self._suggest(
name, LogUniformDistribution(low=low, high=high))
else:
return self._suggest(name,
UniformDistribution(low=low, high=high))
def create_optuna_distribution_from_config(
config: MutableMapping[str, Any]) -> BaseDistribution:
kwargs = dict(config)
if isinstance(config["type"], str):
kwargs["type"] = DistributionType[config["type"]]
param = DistributionConfig(**kwargs)
if param.type == DistributionType.categorical:
assert param.choices is not None
return CategoricalDistribution(param.choices)
if param.type == DistributionType.int:
assert param.low is not None
assert param.high is not None
if param.log:
return IntLogUniformDistribution(int(param.low), int(param.high))
step = int(param.step) if param.step is not None else 1
return IntUniformDistribution(int(param.low),
int(param.high),
step=step)
if param.type == DistributionType.float:
assert param.low is not None
assert param.high is not None
if param.log:
return LogUniformDistribution(param.low, param.high)
if param.step is not None:
return DiscreteUniformDistribution(param.low, param.high,
param.step)
return UniformDistribution(param.low, param.high)
raise NotImplementedError(
f"{param.type} is not supported by Optuna sweeper.")
def search_space() -> Dict[str, BaseDistribution]:
return {
"c": CategoricalDistribution(("a", "b")),
"d": DiscreteUniformDistribution(-1, 9, 2),
"i": IntUniformDistribution(-1, 1),
"ii": IntUniformDistribution(-1, 3, 2),
"l": LogUniformDistribution(0.001, 0.1),
"u": UniformDistribution(-2, 2),
}
def search_space():
# type: () -> Dict[str, BaseDistribution]
return {
'c': CategoricalDistribution(('a', 'b')),
'd': DiscreteUniformDistribution(-1, 9, 2),
'i': IntUniformDistribution(-1, 1),
'l': LogUniformDistribution(0.001, 0.1),
'u': UniformDistribution(-2, 2),
}
def test_not_contained_param() -> None:
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": UniformDistribution(1.0, 10.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": LogUniformDistribution(1.0, 10.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0, log=True) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": DiscreteUniformDistribution(1.0, 10.0, 1.0)},
)
with pytest.warns(UserWarning):
assert trial.suggest_float("x", 10.0, 100.0, step=1.0) == 1.0
trial = create_trial(
value=0.2,
params={"x": 1.0},
distributions={"x": IntUniformDistribution(1, 10)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100) == 1
trial = create_trial(
value=0.2,
params={"x": 1},
distributions={"x": IntUniformDistribution(1, 10, 1)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100, 1) == 1
trial = create_trial(
value=0.2,
params={"x": 1},
distributions={"x": IntLogUniformDistribution(1, 10)},
)
with pytest.warns(UserWarning):
assert trial.suggest_int("x", 10, 100, log=True) == 1
def test_suggest_discrete_uniform(storage_init_func: Callable[[], storages.BaseStorage]) -> None:
mock = Mock()
mock.side_effect = [1.0, 2.0]
sampler = samplers.RandomSampler()
with patch.object(sampler, "sample_independent", mock) as mock_object:
study = create_study(storage_init_func(), sampler=sampler)
trial = Trial(study, study._storage.create_new_trial(study._study_id))
distribution = DiscreteUniformDistribution(low=0.0, high=3.0, q=1.0)
assert trial._suggest("x", distribution) == 1.0 # Test suggesting a param.
assert trial._suggest("x", distribution) == 1.0 # Test suggesting the same param.
assert trial._suggest("y", distribution) == 2.0 # Test suggesting a different param.
assert trial.params == {"x": 1.0, "y": 2.0}
assert mock_object.call_count == 2
def restore_old_distribution(distribution_json: str) -> str:
distribution = json_to_distribution(distribution_json)
old_distribution: BaseDistribution
# Float distributions.
if isinstance(distribution, FloatDistribution):
if distribution.log:
old_distribution = LogUniformDistribution(
low=distribution.low,
high=distribution.high,
)
else:
if distribution.step is not None:
old_distribution = DiscreteUniformDistribution(
low=distribution.low,
high=distribution.high,
q=distribution.step,
)
else:
old_distribution = UniformDistribution(
low=distribution.low,
high=distribution.high,
)
# Integer distributions.
elif isinstance(distribution, IntDistribution):
if distribution.log:
old_distribution = IntLogUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
else:
old_distribution = IntUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
# Categorical distribution.
else:
old_distribution = distribution
return distribution_to_json(old_distribution)
def test_frozen_trial_suggest_discrete_uniform() -> None:
trial = FrozenTrial(
number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={"x": 0.9},
distributions={"x": DiscreteUniformDistribution(0.0, 1.0, q=0.1)},
user_attrs={},
system_attrs={},
intermediate_values={},
)
assert trial.suggest_discrete_uniform("x", 0.0, 1.0, 0.1) == 0.9
with pytest.raises(ValueError):
trial.suggest_discrete_uniform("y", 0.0, 1.0, 0.1)
def test_distributions(storage_mode: str) -> None:
def objective(trial: Trial) -> float:
trial.suggest_float("a", 0, 10)
trial.suggest_float("b", 0.1, 10, log=True)
trial.suggest_float("c", 0, 10, step=1)
trial.suggest_int("d", 0, 10)
trial.suggest_categorical("e", ["foo", "bar", "baz"])
trial.suggest_int("f", 1, 10, log=True)
return 1.0
with StorageSupplier(storage_mode) as storage:
study = create_study(storage=storage)
study.optimize(objective, n_trials=1)
assert study.best_trial.distributions == {
"a": UniformDistribution(low=0, high=10),
"b": LogUniformDistribution(low=0.1, high=10),
"c": DiscreteUniformDistribution(low=0, high=10, q=1),
"d": IntUniformDistribution(low=0, high=10),
"e": CategoricalDistribution(choices=("foo", "bar", "baz")),
"f": IntLogUniformDistribution(low=1, high=10),
}
def suggest_float(
self,
name: str,
low: float,
high: float,
*,
step: Optional[float] = None,
log: bool = False,
) -> float:
"""Suggest a value for the floating point parameter.
.. versionadded:: 1.3.0
Example:
Suggest a momentum, learning rate and scaling factor of learning rate
for neural network training.
.. testcode::
import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
import optuna
X, y = load_iris(return_X_y=True)
X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
def objective(trial):
momentum = trial.suggest_float("momentum", 0.0, 1.0)
learning_rate_init = trial.suggest_float(
"learning_rate_init", 1e-5, 1e-3, log=True
)
power_t = trial.suggest_float("power_t", 0.2, 0.8, step=0.1)
clf = MLPClassifier(
hidden_layer_sizes=(100, 50),
momentum=momentum,
learning_rate_init=learning_rate_init,
solver="sgd",
random_state=0,
power_t=power_t,
)
clf.fit(X_train, y_train)
return clf.score(X_valid, y_valid)
study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=3)
Args:
name:
A parameter name.
low:
Lower endpoint of the range of suggested values. ``low`` is included in the range.
high:
Upper endpoint of the range of suggested values. ``high`` is included in the range.
step:
A step of discretization.
.. note::
The ``step`` and ``log`` arguments cannot be used at the same time. To set
the ``step`` argument to a float number, set the ``log`` argument to
:obj:`False`.
log:
A flag to sample the value from the log domain or not.
If ``log`` is true, the value is sampled from the range in the log domain.
Otherwise, the value is sampled from the range in the linear domain.
.. note::
The ``step`` and ``log`` arguments cannot be used at the same time. To set
the ``log`` argument to :obj:`True`, set the ``step`` argument to :obj:`None`.
Raises:
:exc:`ValueError`:
If ``step is not None`` and ``log = True`` are specified.
Returns:
A suggested float value.
.. seealso::
:ref:`configurations` tutorial describes more details and flexible usages.
"""
if step is not None:
if log:
raise ValueError("The parameter `step` is not supported when `log` is True.")
else:
distribution: Union[
DiscreteUniformDistribution, LogUniformDistribution, UniformDistribution
] = DiscreteUniformDistribution(low=low, high=high, q=step)
else:
if log:
distribution = LogUniformDistribution(low=low, high=high)
else:
distribution = UniformDistribution(low=low, high=high)
self._check_distribution(name, distribution)
return self._suggest(name, distribution)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog='%s %s' % (__script_name__, __version__),
description='Optimize parameter values of a game agent using optuna framework.',
epilog='%(prog)s')
parser.add_argument('--engine', required=True,
help='Engine filename or engine path and filename.')
parser.add_argument('--resign-movecount', required=False,
help='Number of move counts before the game is adjudicated as a loss.\n'
'This should be used together with --resign-score option. Example:\n'
'--resign-movecount 10 --resign-score 700\n'
'Will terminate the game when there are 10 successive -700 or worse score.')
parser.add_argument('--resign-score', required=False,
help='Score is centipawn where the game is considered resignable.\n'
'This should be used together with --resign-movecount option.')
parser.add_argument('--trials', required=False, type=int,
help='Trials to try, default=1000.',
default=1000)
parser.add_argument('--concurrency', required=False, type=int,
help='Number of game matches to run concurrently, default=1.',
default=1)
parser.add_argument('--games-per-trial', required=False, type=int,
help='Number of games per trial, default=32.\n'
'This should be even number.', default=32)
parser.add_argument('--study-name', required=False, type=str,
default='default_study_name',
help='The name of study. This can be used to resume\n'
'study sessions, default=default_study_name.')
parser.add_argument('--base-time-sec', required=False, type=int,
help='Base time in sec for time control, default=5.',
default=5)
parser.add_argument('--inc-time-sec', required=False, type=float,
help='Increment time in sec for time control, default=0.05.',
default=0.05)
parser.add_argument('--depth', required=False, type=int,
help='The maximum search depth that the engine is'
' allowed, default=1000.\n'
'Example:\n'
'python tuner.py --depth 6 ...\n'
'If depth is high say 24 and you want this depth\n'
'to be always respected increase the base time'
' control.\n'
'tuner.py --depth 24 --base-time-sec 300 ...',
default=1000)
parser.add_argument('--nodes', required=False, type=int,
help='The maximum search nodes that the engine is'
' allowed.\n'
'Example:\n'
'python tuner.py --nodes 1000 ...\n'
'Time and depth control will not be followed.')
parser.add_argument('--opening-file', required=True, type=str,
help='Start opening filename in pgn, fen or epd format.\n'
'If match manager is cutechess, you can use pgn, fen\n'
'or epd format. The format is hard-coded currently.\n'
'You have to modify the code.')
parser.add_argument('--opening-format', required=False, type=str,
help='Can be pgn, or epd for cutechess match manager,'
'default is pgn, for duel.py no need as it will use epd or fen.',
default='pgn')
parser.add_argument('--variant', required=False, type=str,
help='Game variant, default=normal.', default='normal')
parser.add_argument('--pgn-output', required=False, type=str,
help='Output pgn filename, default=optuna_games.pgn.',
default='optuna_games.pgn')
parser.add_argument('--plot', action='store_true', help='A flag to output plots in png.')
parser.add_argument('--initial-best-value', required=False, type=float,
help='The initial best value for the initial best\n'
'parameter values, default=0.5.', default=0.5)
parser.add_argument('--save-plots-every-trial', required=False, type=int,
help='Save plots every n trials, default=10.',
default=10)
parser.add_argument('--fix-base-param', action='store_true',
help='A flag to fix the parameter of base engine.\n'
'It will use the init or default parameter values.')
parser.add_argument('--match-manager', required=False, type=str,
help='The application that handles the engine match,'
' default=cutechess.',
default='cutechess')
parser.add_argument('--match-manager-path', required=True,
help='Match manager path and/or filename. Example:\n'
'cutechess:\n'
'--match-manager-path c:/chess/tourney_manager/cutechess/cutechess-cli.exe\n'
'duel.py for xboard engines:\n'
'--match-manager-path python c:/chess/tourney_manager/duel/duel.py\n'
'or\n'
'--match-manager-path c:/python3/python c:/chess/tourney_manager/duel/duel.py')
parser.add_argument('--protocol', required=False, type=str,
help='The protocol that the engine supports, can be'
' uci or cecp, default=uci.',
default='uci')
parser.add_argument('--sampler', required=False, nargs='*', action='append',
metavar=('name=', 'option_name='),
help='The sampler to be used in the study, default name=tpe.\n'
'name can be tpe or cmaes or skopt, examples:\n'
'--sampler name=tpe ei_samples=50 ...\n'
' default ei_samples=24\n'
'--sampler name=tpe multivariate=true ...\n'
' default multivariate is false.\n'
'--sampler name=cmaes ...\n'
'--sampler name=skopt acquisition_function=LCB ...\n'
' default acquisition_function=gp_hedge\n'
' Can be LCB or EI or PI or gp_hedge\n'
' Example to explore, with LCB and kappa, high kappa would explore, low would exploit:\n'
' --sampler name=skopt acquisition_function=LCB kappa=10000\n'
' Example to exploit, with EI or PI and xi, high xi would explore, low would exploit:\n'
' --sampler name=skopt acquisition_function=EI xi=0.0001\n'
' Note: negative xi does not work with PI, but will work with EI.\n'
' Ref.: https://scikit-optimize.github.io/stable/auto_examples/exploration-vs-exploitation.html#sphx-glr-auto-examples-exploration-vs-exploitation-py\n'
' skopt has base_estimator options namely: GP, RF, ET and GBRT, default is GP.\n'
' GP=Gaussian Process, RF=Random Forest, ET=Extra Tree, GBRT=Gradient Boosted Regressor Tree.\n'
' Example:\n'
' --sampler name=skopt base_estimator=GBRT acquisition_function=EI ...\n')
parser.add_argument('--threshold-pruner', required=False, nargs='*', action='append',
metavar=('result=', 'games='),
help='A trial pruner used to prune or stop unpromising'
' trials.\n'
'Example:\n'
'tuner.py --threshold-pruner result=0.45 games=50 interval=1 ...\n'
'Assuming games per trial is 100, after 50 games, check\n'
'the score of the match, if this is below 0.45, then\n'
'prune the trial or stop the engine match. Get new param\n'
'from optimizer and start a new trial.\n'
'Default values:\n'
'result=0.25, games=games_per_trial/2, interval=1\n'
'Example:\n'
'tuner.py --threshold-pruner ...',
default=None)
parser.add_argument('--input-param', required=True, type=str,
help='The parameters that will be optimized.\n'
'Example 1 with 1 parameter:\n'
'--input-param \"{\'pawn\': {\'default\': 92,'
' \'min\': 90, \'max\': 120, \'step\': 2}}\"\n'
'Example 2 with 2 parameters:\n'
'--input-param \"{\'pawn\': {\'default\': 92,'
' \'min\': 90, \'max\': 120, \'step\': 2},'
' \'knight\': {\'default\': 300, \'min\': 250,'
' \'max\': 350, \'step\': 2}}\"\n'
'Example 3 with 1 parameter but float value:\n'
'--input-param \"{\'CPuct\': {\'default\': 0.5,'
' \'min\': 0.1, \'max\': 3.0, \'step\': 0.05, \'type\': \'float\'}}\"'
)
parser.add_argument('-v', '--version', action='version', version=f'{__version__}')
parser.add_argument('--common-param', required=False, type=str,
help='The parameters that will be sent to both test and base engines.\n'
'Make sure that this param is not included in the input-param.\n'
'Example:\n'
'--common-param \"{\'RookOpenFile\': 92, \'KnightOutpost\': 300}\"')
args = parser.parse_args()
trials = args.trials
init_value = args.initial_best_value
save_plots_every_trial = args.save_plots_every_trial
fix_base_param = args.fix_base_param
common_param = args.common_param
if common_param is not None:
common_param = ast.literal_eval(common_param)
# Number of games should be even for a fair engine match.
games_per_trial = args.games_per_trial
games_per_trial += 1 if (args.games_per_trial % 2) != 0 else 0
rounds = games_per_trial//2
good_result_cnt = 0
study_name = args.study_name
storage_file = f'{study_name}.db'
logger.info(f'{__script_name__} {__version__}')
logger.info(f'trials: {trials}, games_per_trial: {rounds * 2}, sampler: {args.sampler}\n')
# Convert the input param string to a dict of dict and sort by key.
input_param = ast.literal_eval(args.input_param)
input_param = OrderedDict(sorted(input_param.items()))
logger.info(f'input param: {input_param}\n')
init_param = Objective.set_param(input_param)
# Adjust save_plots_every_trial if trials is lower than it so
# that max_cycle is 1 or more and studies can continue. The plot
# will be generated after the study.
if trials < save_plots_every_trial:
save_plots_every_trial = trials
max_cycle = trials // save_plots_every_trial
n_trials = save_plots_every_trial
cycle = 0
# Define sampler to use, default is TPE.
sampler = Objective.get_sampler(args.sampler)
# ThresholdPruner as trial pruner, if result of a match is below result
# threshold after games threshold then prune the trial. Get new param
# from optimizer and continue with the next trial.
# --threshold-pruner result=0.45 games=50 --games-per-trial 100 ...
pruner, th_pruner = Objective.get_pruner(args.threshold_pruner, games_per_trial)
logger.info('Starting optimization ...')
while cycle < max_cycle:
cycle += 1
# Define study.
study = optuna.create_study(study_name=study_name,
direction='maximize',
storage=f'sqlite:///{storage_file}',
load_if_exists=True, sampler=sampler,
pruner=pruner)
# Get the best value from previous study session.
best_param, best_value, is_study = {}, 0.0, False
try:
best_value = study.best_value
is_study = True
except ValueError:
logger.warning('Warning, best value from previous trial is not found!')
except:
logger.exception('Unexpected error:', sys.exc_info()[0])
raise
logger.info(f'study best value: {best_value}')
# Get the best param values from previous study session.
try:
best_param = copy.deepcopy(study.best_params)
except ValueError:
logger.warning('Warning, best param from previous trial is not found!.')
except:
logger.exception('Unexpected error:', sys.exc_info()[0])
raise
logger.info(f'study best param: {best_param}')
old_trial_num = len(study.trials)
# Get the good result count before we resume the study.
if is_panda_ok and not fix_base_param and is_study:
df = study.trials_dataframe(attrs=('value', 'state'))
for index, row in df.iterrows():
if row['value'] > init_value and row['state'] == 'COMPLETE':
good_result_cnt += 1
# If there is no trial recorded yet we will initialize our study
# with default values from the engine.
if not is_study:
distri = {}
init_trial_value = init_value
for k, v in input_param.items():
if 'type' in v and v['type'] == 'float':
distri.update({k: DiscreteUniformDistribution(v['min'], v['max'], v['step'])})
else:
distri.update({k: IntUniformDistribution(v['min'], v['max'], v['step'])})
init_trial = optuna.trial.create_trial(
params=copy.deepcopy(init_param),
distributions=copy.deepcopy(distri),
value=init_trial_value,
)
study.add_trial(init_trial)
best_param = study.best_params
best_value = study.best_value
# Begin param optimization.
# https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize
study.optimize(Objective(args.engine, input_param, best_param,
best_value, init_param, init_value,
args.variant, args.opening_file,
args.opening_format, old_trial_num,
args.pgn_output, args.nodes,
args.base_time_sec, args.inc_time_sec,
rounds, args.concurrency,
args.protocol, fix_base_param,
args.match_manager, args.match_manager_path,
good_result_cnt,
args.depth, games_per_trial, th_pruner,
common_param, args.resign_movecount,
args.resign_score),
n_trials=n_trials)
# Create and save plots after this study session is completed.
save_plots(study, study_name, input_param, args.plot)
# Build pandas dataframe, print and save to csv file.
if is_panda_ok:
df = study.trials_dataframe(attrs=('number', 'value', 'params',
'state'))
logger.info(f'{df.to_string(index=False)}\n')
df.to_csv(f'{study_name}.csv', index=False)
# Show the best param, value and trial number.
logger.info(f'study best param: {study.best_params}')
logger.info(f'study best value: {study.best_value}')
logger.info(f'study best trial number: {study.best_trial.number}\n')
# Output for match manager.
option_output = ''
for k, v in study.best_params.items():
option_output += f'option.\'{k}\'={v} '
logger.info(f'{option_output}\n')
distribution) for _ in range(100)
])
assert np.all(points >= distribution.low)
assert np.all(points < distribution.high)
assert not isinstance(
study.sampler.sample_independent(study, _create_new_trial(study), "x",
distribution),
np.floating,
)
@parametrize_sampler
@pytest.mark.parametrize(
"distribution",
[
DiscreteUniformDistribution(-10, 10, 0.1),
DiscreteUniformDistribution(-10.2, 10.2, 0.1)
],
)
def test_discrete_uniform(sampler_class: Callable[[], BaseSampler],
distribution: DiscreteUniformDistribution) -> None:
study = optuna.study.create_study(sampler=sampler_class())
points = np.array([
study.sampler.sample_independent(study, _create_new_trial(study), "x",
distribution) for _ in range(100)
])
assert np.all(points >= distribution.low)
assert np.all(points <= distribution.high)
assert not isinstance(
study.sampler.sample_independent(study, _create_new_trial(study), "x",
def suggest_discrete_uniform(self, name: str, low: float, high: float,
q: float) -> float:
discrete = DiscreteUniformDistribution(low=low, high=high, q=q)
return self._suggest(name, discrete)
},
IntLogUniformDistribution(1, 100),
),
({
"type": "float",
"low": 0,
"high": 1
}, UniformDistribution(0, 1)),
(
{
"type": "float",
"low": 0,
"high": 10,
"step": 2
},
DiscreteUniformDistribution(0, 10, 2),
),
(
{
"type": "float",
"low": 1,
"high": 100,
"log": True
},
LogUniformDistribution(1, 100),
),
],
)
def test_create_optuna_distribution_from_config(input: Any,
expected: Any) -> None:
actual = _impl.create_optuna_distribution_from_config(input)
from optuna.distributions import DiscreteUniformDistribution
from optuna.distributions import IntLogUniformDistribution
from optuna.distributions import IntUniformDistribution
from optuna.distributions import LogUniformDistribution
from optuna.distributions import UniformDistribution
@pytest.mark.parametrize(
"param,distribution",
[
(0, IntUniformDistribution(0, 3)),
(1, IntLogUniformDistribution(1, 10)),
(2, IntUniformDistribution(0, 10, step=2)),
(0.0, UniformDistribution(0, 3)),
(1.0, LogUniformDistribution(1, 10)),
(0.2, DiscreteUniformDistribution(0, 1, q=0.2)),
("foo", CategoricalDistribution(["foo"])),
("bar", CategoricalDistribution(["foo", "bar", "baz"])),
],
)
def test_search_space_transform_shapes_dtypes(param: Any, distribution: BaseDistribution) -> None:
trans = _SearchSpaceTransform({"x0": distribution})
trans_params = trans.transform({"x0": param})
if isinstance(distribution, CategoricalDistribution):
expected_bounds_shape = (len(distribution.choices), 2)
expected_params_shape = (len(distribution.choices),)
else:
expected_bounds_shape = (1, 2)
expected_params_shape = (1,)
assert trans.bounds.shape == expected_bounds_shape
},
IntLogUniformDistribution(1, 100),
),
({
"type": "float",
"low": 0,
"high": 1
}, UniformDistribution(0, 1)),
(
{
"type": "float",
"low": 0,
"high": 10,
"step": 2
},
DiscreteUniformDistribution(0, 10, 2),
),
(
{
"type": "float",
"low": 1,
"high": 100,
"log": True
},
LogUniformDistribution(1, 100),
),
],
)
def test_create_optuna_distribution_from_config(input: Any,
expected: Any) -> None:
actual = _impl.create_optuna_distribution_from_config(input)
