def test_create_new_trial_with_template_trial(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
now = datetime.now()
template_trial = FrozenTrial(
state=TrialState.COMPLETE,
value=10000,
datetime_start=now,
datetime_complete=now,
params={'x': 0.5},
distributions={'x': UniformDistribution(0, 1)},
user_attrs={'foo': 'bar'},
system_attrs={
'baz': 123,
'_number': 55 # This entry is ignored.
},
intermediate_values={
1: 10,
2: 100,
3: 1000
},
number=-1, # dummy value (unused).
trial_id=-1, # dummy value (unused).
)
study_id = storage.create_new_study()
trial_id = storage.create_new_trial(study_id,
template_trial=template_trial)
trials = storage.get_all_trials(study_id)
assert len(trials) == 1
assert trials[0].trial_id == trial_id
assert trials[0].number == 0
assert trials[0].state == template_trial.state
assert trials[0].value == template_trial.value
assert trials[0].datetime_start == template_trial.datetime_start
assert trials[0].datetime_complete == template_trial.datetime_complete
assert trials[0].params == template_trial.params
assert trials[0].distributions == template_trial.distributions
assert trials[0].user_attrs == template_trial.user_attrs
assert trials[0].intermediate_values == template_trial.intermediate_values
# TODO(Yanase): Remove number from system_attrs after adding TrialModel.number.
template_trial.system_attrs['_number'] = 0
assert trials[0].system_attrs == template_trial.system_attrs
def _create_frozen_trial():
# type: () -> FrozenTrial
return FrozenTrial(number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={'x': 10},
distributions={'x': UniformDistribution(5, 12)},
user_attrs={},
system_attrs={},
intermediate_values={})
def test_frozen_trial_repr():
# type: () -> None
trial = FrozenTrial(number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={'x': 10},
distributions={'x': UniformDistribution(5, 12)},
user_attrs={},
system_attrs={},
intermediate_values={})
assert trial == eval(repr(trial))
def _create_frozen_trial(params, param_distributions):
# type: (Dict[str, Any], Dict[str, distributions.BaseDistribution]) -> FrozenTrial
return FrozenTrial(
number=0,
value=1.0,
state=optuna.structs.TrialState.COMPLETE,
user_attrs={},
system_attrs={},
params=params,
distributions=param_distributions,
intermediate_values={},
datetime_start=None,
datetime_complete=None,
trial_id=0,
)
def _create_frozen_trial(params, param_distributions, state=TrialState.COMPLETE, number=0):
# type: (Dict[str, Any], Dict[str, BaseDistribution], TrialState, int) -> FrozenTrial
return FrozenTrial(
number=number,
value=1.,
state=state,
user_attrs={},
system_attrs={},
params=params,
distributions=param_distributions,
intermediate_values={},
datetime_start=None,
datetime_complete=None,
trial_id=number,
)
def test_frozen_trial_deprecated() -> None:
with pytest.warns(FutureWarning):
FrozenTrial(
number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.datetime.now(),
datetime_complete=datetime.datetime.now(),
params={"x": 10},
distributions={"x": UniformDistribution(5, 12)},
user_attrs={},
system_attrs={},
intermediate_values={},
)
def test_create_new_trial_with_template_trial(storage_init_func):
# type: (Callable[[], BaseStorage]) -> None
storage = storage_init_func()
now = datetime.now()
template_trial = FrozenTrial(
state=TrialState.COMPLETE,
value=10000,
datetime_start=now,
datetime_complete=now,
params={"x": 0.5},
distributions={"x": UniformDistribution(0, 1)},
user_attrs={"foo": "bar"},
system_attrs={
"baz": 123,
},
intermediate_values={
1: 10,
2: 100,
3: 1000
},
number=55, # This entry is ignored.
trial_id=-1, # dummy value (unused).
)
study_id = storage.create_new_study()
trial_id = storage.create_new_trial(study_id,
template_trial=template_trial)
trials = storage.get_all_trials(study_id)
assert len(trials) == 1
assert trials[0].trial_id == trial_id
assert trials[0].number == 0
assert trials[0].state == template_trial.state
assert trials[0].value == template_trial.value
assert trials[0].datetime_start == template_trial.datetime_start
assert trials[0].datetime_complete == template_trial.datetime_complete
assert trials[0].params == template_trial.params
assert trials[0].distributions == template_trial.distributions
assert trials[0].user_attrs == template_trial.user_attrs
assert trials[0].intermediate_values == template_trial.intermediate_values
assert trials[0].system_attrs == template_trial.system_attrs
def _create_frozen_trial(params, param_distributions):
# type: (Dict[str, Any], Dict[str, distributions.BaseDistribution]) -> FrozenTrial
params_in_internal_repr = {}
for param_name, param_value in params.items():
params_in_internal_repr[param_name] = param_distributions[
param_name].to_internal_repr(param_value)
return FrozenTrial(
number=0,
value=1.,
state=optuna.structs.TrialState.COMPLETE,
user_attrs={},
system_attrs={},
params=params,
params_in_internal_repr=params_in_internal_repr,
distributions=param_distributions,
intermediate_values={},
datetime_start=None,
datetime_complete=None,
trial_id=0,
)
def test_frozen_trial_validate():
# type: () -> None
# Valid.
valid_trial = FrozenTrial(number=0,
trial_id=0,
state=TrialState.COMPLETE,
value=0.2,
datetime_start=datetime.now(),
datetime_complete=datetime.now(),
params={'x': 10},
distributions={'x': UniformDistribution(5, 12)},
user_attrs={},
system_attrs={},
intermediate_values={})
valid_trial._validate()
# Invalid: `datetime_start` is not set.
invalid_trial = valid_trial._replace(datetime_start=None)
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is `RUNNING` and `datetime_complete` is set.
invalid_trial = valid_trial._replace(state=TrialState.RUNNING)
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is not `RUNNING` and `datetime_complete` is not set.
for state in [TrialState.COMPLETE, TrialState.PRUNED, TrialState.FAIL]:
invalid_trial = valid_trial._replace(state=state,
datetime_complete=None)
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: `state` is `COMPLETE` and `value` is not set.
invalid_trial = valid_trial._replace(value=None)
with pytest.raises(ValueError):
invalid_trial._validate()
# Invalid: Inconsistent `params` and `distributions`
inconsistent_pairs = [
# `params` has an extra element.
({
'x': 0.1,
'y': 0.5
}, {
'x': UniformDistribution(0, 1)
}),
# `distributions` has an extra element.
({
'x': 0.1
}, {
'x': UniformDistribution(0, 1),
'y': LogUniformDistribution(0, 1)
}),
# The value of `x` isn't contained in the distribution.
({
'x': -0.5
}, {
'x': UniformDistribution(0, 1)
})
] # type: List[Tuple[Dict[str, Any], Dict[str, BaseDistribution]]]
for params, distributions in inconsistent_pairs:
invalid_trial = valid_trial._replace(params=params,
distributions=distributions)
with pytest.raises(ValueError):
invalid_trial._validate()
'x': UniformDistribution(low=1., high=2.),
'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza'))
} # type: Dict[str, BaseDistribution]
# TODO(Yanase): Remove _number from system_attrs after adding TrialModel.number.
EXAMPLE_TRIALS = [
FrozenTrial(
number=0, # dummy
value=1.,
state=TrialState.COMPLETE,
user_attrs={},
system_attrs={'_number': 0},
params={
'x': 0.5,
'y': 'Ginza'
},
distributions=EXAMPLE_DISTRIBUTIONS,
intermediate_values={
0: 2.,
1: 3.
},
datetime_start=None, # dummy
datetime_complete=None, # dummy
trial_id=-1, # dummy id
),
FrozenTrial(
number=0, # dummy
value=2.,
state=TrialState.RUNNING,
user_attrs={
'tags': ['video', 'classification'],
EXAMPLE_DISTRIBUTIONS = {
"x": UniformDistribution(low=1.0, high=2.0),
"y": CategoricalDistribution(choices=("Otemachi", "Tokyo", "Ginza")),
} # type: Dict[str, BaseDistribution]
EXAMPLE_TRIALS = [
FrozenTrial(
number=0, # dummy
value=1.0,
state=TrialState.COMPLETE,
user_attrs={},
system_attrs={},
params={
"x": 0.5,
"y": "Ginza"
},
distributions=EXAMPLE_DISTRIBUTIONS,
intermediate_values={
0: 2.0,
1: 3.0
},
datetime_start=None, # dummy
datetime_complete=None, # dummy
trial_id=-1, # dummy id
),
FrozenTrial(
number=0, # dummy
value=2.0,
state=TrialState.RUNNING,
user_attrs={
"tags": ["video", "classification"],
'x': UniformDistribution(low=1., high=2.),
'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza'))
} # type: Dict[str, BaseDistribution]
EXAMPLE_TRIALS = [
FrozenTrial(
trial_id=-1, # dummy id
value=1.,
state=TrialState.COMPLETE,
user_attrs={},
system_attrs={},
params={
'x': 0.5,
'y': 'Ginza'
},
intermediate_values={
0: 2.,
1: 3.
},
params_in_internal_repr={
'x': .5,
'y': 2.
},
datetime_start=None, # dummy
datetime_complete=None # dummy
),
FrozenTrial(
trial_id=-1, # dummy id
value=2.,
state=TrialState.RUNNING,
user_attrs={