def _build_frozen_trial_from_trial_model(
trial: models.TrialModel) -> FrozenTrial:
return FrozenTrial(
number=trial.number,
state=trial.state,
value=trial.value,
datetime_start=trial.datetime_start,
datetime_complete=trial.datetime_complete,
params={
p.param_name: distributions.json_to_distribution(
p.distribution_json).to_external_repr(p.param_value)
for p in trial.params
},
distributions={
p.param_name:
distributions.json_to_distribution(p.distribution_json)
for p in trial.params
},
user_attrs={
attr.key: json.loads(attr.value_json)
for attr in trial.user_attributes
},
system_attrs={
attr.key: json.loads(attr.value_json)
for attr in trial.system_attributes
},
intermediate_values={
value.step: value.value
for value in trial.values
},
trial_id=trial.trial_id,
)
def _build_frozen_trial_from_trial_model(trial: models.TrialModel) -> FrozenTrial:
values: Optional[List[float]]
if trial.values:
values = [0 for _ in trial.values]
for value_model in trial.values:
values[value_model.objective] = value_model.value
else:
values = None
return FrozenTrial(
number=trial.number,
state=trial.state,
value=None,
values=values,
datetime_start=trial.datetime_start,
datetime_complete=trial.datetime_complete,
params={
p.param_name: distributions.json_to_distribution(
p.distribution_json
).to_external_repr(p.param_value)
for p in trial.params
},
distributions={
p.param_name: distributions.json_to_distribution(p.distribution_json)
for p in trial.params
},
user_attrs={attr.key: json.loads(attr.value_json) for attr in trial.user_attributes},
system_attrs={
attr.key: json.loads(attr.value_json) for attr in trial.system_attributes
},
intermediate_values={v.step: v.intermediate_value for v in trial.intermediate_values},
trial_id=trial.trial_id,
)
def test_set_trial_param_to_check_distribution_json():
# type: () -> None
example_distributions = {
'x': UniformDistribution(low=1., high=2.),
'y': CategoricalDistribution(choices=('Otemachi', 'Tokyo', 'Ginza'))
} # type: Dict[str, BaseDistribution]
storage = create_test_storage()
session = storage.scoped_session()
study_id = storage.create_new_study_id()
trial_id = storage.create_new_trial_id(study_id)
storage.set_trial_param(trial_id, 'x', 1.5, example_distributions['x'])
storage.set_trial_param(trial_id, 'y', 2, example_distributions['y'])
# test setting new name
result_1 = session.query(TrialParamModel). \
filter(TrialParamModel.param_name == 'x').one()
assert json_to_distribution(
result_1.distribution_json) == example_distributions['x']
result_2 = session.query(TrialParamModel). \
filter(TrialParamModel.param_name == 'y').one()
assert json_to_distribution(
result_2.distribution_json) == example_distributions['y']
def test_abbreviated_json_to_distribution() -> None:
for key in EXAMPLE_ABBREVIATED_JSONS:
distribution_actual = distributions.json_to_distribution(
EXAMPLE_ABBREVIATED_JSONS[key])
assert distribution_actual == EXAMPLE_DISTRIBUTIONS[key]
unknown_json = '{"type": "unknown", "low": 1.0, "high": 2.0}'
pytest.raises(ValueError,
lambda: distributions.json_to_distribution(unknown_json))
invalid_distribution = (
'{"type": "float", "low": 0.0, "high": -100.0}',
'{"type": "float", "low": 7.3, "high": 7.2, "log": true}',
'{"type": "float", "low": -30.0, "high": -40.0, "step": 3.0}',
'{"type": "float", "low": 1.0, "high": 100.0, "step": 0.0}',
'{"type": "float", "low": 1.0, "high": 100.0, "step": -1.0}',
'{"type": "int", "low": 123, "high": 100}',
'{"type": "int", "low": 123, "high": 100, "step": 2}',
'{"type": "int", "low": 123, "high": 100, "log": true}',
'{"type": "int", "low": 1, "high": 100, "step": 0}',
'{"type": "int", "low": 1, "high": 100, "step": -1}',
'{"type": "categorical", "choices": []}',
)
for distribution in invalid_distribution:
pytest.raises(ValueError,
lambda: distributions.json_to_distribution(distribution))
def test_set_trial_param_to_check_distribution_json():
# type: () -> None
example_distributions = {
"x": UniformDistribution(low=1.0, high=2.0),
"y": CategoricalDistribution(choices=("Otemachi", "Tokyo", "Ginza")),
} # type: Dict[str, BaseDistribution]
storage = create_test_storage()
session = storage.scoped_session()
study_id = storage.create_new_study()
trial_id = storage.create_new_trial(study_id)
storage.set_trial_param(trial_id, "x", 1.5, example_distributions["x"])
storage.set_trial_param(trial_id, "y", 2, example_distributions["y"])
# test setting new name
result_1 = session.query(TrialParamModel).filter(
TrialParamModel.param_name == "x").one()
assert json_to_distribution(
result_1.distribution_json) == example_distributions["x"]
result_2 = session.query(TrialParamModel).filter(
TrialParamModel.param_name == "y").one()
assert json_to_distribution(
result_2.distribution_json) == example_distributions["y"]
def test_json_to_distribution() -> None:
for key in EXAMPLE_JSONS:
distribution_actual = distributions.json_to_distribution(EXAMPLE_JSONS[key])
assert distribution_actual == EXAMPLE_DISTRIBUTIONS[key]
unknown_json = '{"name": "UnknownDistribution", "attributes": {"low": 1.0, "high": 2.0}}'
pytest.raises(ValueError, lambda: distributions.json_to_distribution(unknown_json))
def _check_compatibility_with_previous_trial_param_distributions(self, session):
# type: (orm.Session) -> None
trial = TrialModel.find_or_raise_by_id(self.trial_id, session)
previous_record = session.query(TrialParamModel).join(TrialModel). \
filter(TrialModel.study_id == trial.study_id). \
filter(TrialParamModel.param_name == self.param_name).first()
if previous_record is not None:
distributions.check_distribution_compatibility(
distributions.json_to_distribution(previous_record.distribution_json),
distributions.json_to_distribution(self.distribution_json))
def _set_trial_param(self, data: Dict[str, Any], worker_id: str) -> None:
number = _id.get_trial_number(data["trial_id"])
name = data["name"]
self.trials[number].params[name] = data["value"]
self.trials[number].distributions[
name] = distributions.json_to_distribution(data["distribution"])
def migrate_new_distribution(distribution_json: str) -> str:
distribution = json_to_distribution(distribution_json)
new_distribution = _convert_old_distribution_to_new_distribution(
distribution,
suppress_warning=True,
)
return distribution_to_json(new_distribution)
def test_backward_compatibility_int_uniform_distribution():
# type: () -> None
json_str = '{"name": "IntUniformDistribution", "attributes": {"low": 1, "high": 10}}'
actual = distributions.json_to_distribution(json_str)
expected = distributions.IntUniformDistribution(low=1, high=10)
assert actual == expected
def __to_frozen_trial(self, trial):
number = trial["number"]
state = trial["state"]
datetime_start = trial["datetime_start"]
datetime_complete = trial["datetime_complete"]
params = trial["params"]
distributions = {}
for p in params:
distributions[p] = json_to_distribution(trial["distributions"][p])
user_attrs = trial["user_attrs"]
system_attrs = trial["system_attrs"]
trial["intermediate_values"]
intkey_val = {}
for k in trial["intermediate_values"]:
intk = int(k)
intkey_val[k] = trial["intermediate_values"][k]
intermediate_values = intkey_val
trial_id = trial["trial_id"]
value = trial["value"]
ret = FrozenTrial(number=number,
state=TrialState(state),
datetime_start=datetime_start,
datetime_complete=datetime_complete,
params=params,
distributions=distributions,
user_attrs=user_attrs,
system_attrs=system_attrs,
intermediate_values=intermediate_values,
trial_id=trial_id,
value=value)
return ret
def _set_trial_param_without_commit(
self,
session: orm.Session,
trial_id: int,
param_name: str,
param_value_internal: float,
distribution: distributions.BaseDistribution,
) -> None:
trial = models.TrialModel.find_or_raise_by_id(trial_id, session)
self.check_trial_is_updatable(trial_id, trial.state)
trial_param = models.TrialParamModel.find_by_trial_and_param_name(
trial, param_name, session)
if trial_param is not None:
# Raise error in case distribution is incompatible.
distributions.check_distribution_compatibility(
distributions.json_to_distribution(
trial_param.distribution_json), distribution)
trial_param.param_value = param_value_internal
trial_param.distribution_json = distributions.distribution_to_json(
distribution)
else:
trial_param = models.TrialParamModel(
trial_id=trial_id,
param_name=param_name,
param_value=param_value_internal,
distribution_json=distributions.distribution_to_json(
distribution),
)
trial_param.check_and_add(session)
def set_trial_param(self, trial_id, param_name, param_value_internal,
distribution):
# type: (int, str, float, distributions.BaseDistribution) -> bool
session = self.scoped_session()
trial = models.TrialModel.find_or_raise_by_id(trial_id, session)
trial_param = \
models.TrialParamModel.find_by_trial_and_param_name(trial, param_name, session)
if trial_param is not None:
# Raise error in case distribution is incompatible.
distributions.check_distribution_compatibility(
distributions.json_to_distribution(
trial_param.distribution_json), distribution)
# Return False when distribution is compatible but parameter has already been set.
return False
param = models.TrialParamModel(
trial_id=trial_id,
param_name=param_name,
param_value=param_value_internal,
distribution_json=distributions.distribution_to_json(distribution))
param.check_and_add(session)
commit_success = self._commit_with_integrity_check(session)
return commit_success
def _set_trial_param_without_commit(self, session, trial_id, param_name,
param_value_internal, distribution):
# type: (orm.Session, int, str, float, distributions.BaseDistribution) -> bool
trial = models.TrialModel.find_or_raise_by_id(trial_id, session)
self.check_trial_is_updatable(trial_id, trial.state)
trial_param = \
models.TrialParamModel.find_by_trial_and_param_name(trial, param_name, session)
if trial_param is not None:
# Raise error in case distribution is incompatible.
distributions.check_distribution_compatibility(
distributions.json_to_distribution(
trial_param.distribution_json), distribution)
# Terminate transaction explicitly to avoid connection timeout during transaction.
self._commit(session)
# Return False when distribution is compatible but parameter has already been set.
return False
param = models.TrialParamModel(
trial_id=trial_id,
param_name=param_name,
param_value=param_value_internal,
distribution_json=distributions.distribution_to_json(distribution))
param.check_and_add(session)
return True
def deserialize_frozentrial(data):
data["state"] = getattr(TrialState, data["state"])
data["distributions"] = {
k: json_to_distribution(v) for k, v in data["distributions"].items()
}
if data["datetime_start"] is not None:
data["datetime_start"] = deserialize_datetime(data["datetime_start"])
if data["datetime_complete"] is not None:
data["datetime_complete"] = deserialize_datetime(data["datetime_complete"])
trail = FrozenTrial(**data)
return trail
def test_categorical_internal_representation() -> None:
c = EXAMPLE_DISTRIBUTIONS["c1"]
for choice in c.choices:
if isinstance(choice, float) and np.isnan(choice):
assert np.isnan(c.to_external_repr(c.to_internal_repr(choice)))
else:
assert c.to_external_repr(c.to_internal_repr(choice)) == choice
# We need to create new objects to compare NaNs.
# See https://github.com/optuna/optuna/pull/3567#pullrequestreview-974939837.
c_ = distributions.json_to_distribution(EXAMPLE_JSONS["c1"])
for choice in cast(distributions.CategoricalDistribution, c_).choices:
if isinstance(choice, float) and np.isnan(choice):
assert np.isnan(c.to_external_repr(c.to_internal_repr(choice)))
else:
assert c.to_external_repr(c.to_internal_repr(choice)) == choice
def set_trial_param(
self,
comm,
trial_id: int,
param_name: str,
param_value_internal: float,
distribution: BaseDistribution,
storage_name: str = None,
) -> None:
distribution = json_to_distribution(distribution)
return self.get_storage(storage_name).set_trial_param(
trial_id=trial_id,
param_name=param_name,
param_value_internal=param_value_internal,
distribution=distribution,
)
def handle_message(self, message):
kind, v = next(iter(message.items()))
if kind == 'SetStudyDirection':
d = v['direction']
if d == 'NOT_SET':
self.direction = structs.StudyDirection.NOT_SET
elif d == 'MINIMIZE':
self.direction = structs.StudyDirection.MINIMIZE
else:
self.direction = structs.StudyDirection.MAXIMIZE
elif kind == 'SetStudyUserAttr':
self.user_attrs[v['key']] = v['value']
elif kind == 'SetStudySystemAttr':
self.system_attrs[v['key']] = v['value']
elif kind == 'CreateTrial':
t = self._trial(v['trial_id'])
self.trials[t.trial_id] = t._replace(
datetime_start=datetime.fromtimestamp(
v['timestamp']['secs'])) # TODO: nanos
elif kind == 'SetTrialState':
t = self._trial(v['trial_id'])
t = t._replace(state=str_to_trial_state(v['state']))
if v['state'] != 'RUNNING':
t = t._replace(datetime_complete=datetime.fromtimestamp(
v['timestamp']['secs'])) # TODO: nanos
self.trials[t.trial_id] = t
elif kind == 'SetTrialParam':
t = self._trial(v['trial_id'])
distribution = distributions.json_to_distribution(
v['value']['distribution'])
t.params[v['key']] = distribution.to_external_repr(
v['value']['value'])
t.params_in_internal_repr[v['key']] = v['value']['value']
elif kind == 'SetTrialValue':
t = self._trial(v['trial_id'])
self.trials[t.trial_id] = t._replace(value=v['value'])
elif kind == 'SetTrialIntermediateValue':
t = self._trial(v['trial_id'])
t.intermediate_values[v['step']] = v['value']
elif kind == 'SetTrialUserAttr':
t = self._trial(v['trial_id'])
t.user_attrs[v['key']] = v['value']
elif kind == 'SetTrialSystemAttr':
t = self._trial(v['trial_id'])
t.system_attrs[v['key']] = v['value']
else:
raise NotImplementedError(str(message))
def restore_old_distribution(distribution_json: str) -> str:
distribution = json_to_distribution(distribution_json)
old_distribution: BaseDistribution
# Float distributions.
if isinstance(distribution, FloatDistribution):
if distribution.log:
old_distribution = LogUniformDistribution(
low=distribution.low,
high=distribution.high,
)
else:
if distribution.step is not None:
old_distribution = DiscreteUniformDistribution(
low=distribution.low,
high=distribution.high,
q=distribution.step,
)
else:
old_distribution = UniformDistribution(
low=distribution.low,
high=distribution.high,
)
# Integer distributions.
elif isinstance(distribution, IntDistribution):
if distribution.log:
old_distribution = IntLogUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
else:
old_distribution = IntUniformDistribution(
low=distribution.low,
high=distribution.high,
step=distribution.step,
)
# Categorical distribution.
else:
old_distribution = distribution
return distribution_to_json(old_distribution)
def _check_and_set_param_distribution(
self,
study_id: int,
trial_id: int,
param_name: str,
param_value_internal: float,
distribution: distributions.BaseDistribution,
) -> None:
session = self.scoped_session()
# Acquire lock.
#
# Assume that study exists.
models.StudyModel.find_by_id(study_id, session, for_update=True)
models.TrialModel.find_or_raise_by_id(trial_id, session)
previous_record = (
session.query(models.TrialParamModel)
.join(models.TrialModel)
.filter(models.TrialModel.study_id == study_id)
.filter(models.TrialParamModel.param_name == param_name)
.first()
)
if previous_record is not None:
distributions.check_distribution_compatibility(
distributions.json_to_distribution(previous_record.distribution_json),
distribution,
)
session.add(
models.TrialParamModel(
trial_id=trial_id,
param_name=param_name,
param_value=param_value_internal,
distribution_json=distributions.distribution_to_json(distribution),
)
)
# Release lock.
session.commit()
def _check_or_set_param_distribution(
self,
trial_id: int,
param_name: str,
param_value_internal: float,
distribution: distributions.BaseDistribution,
) -> None:
session = self.scoped_session()
# Acquire a lock of this trial.
trial = models.TrialModel.find_by_id(trial_id,
session,
for_update=True)
if trial is None:
raise KeyError(models.NOT_FOUND_MSG)
previous_record = (session.query(models.TrialParamModel).join(
models.TrialModel).filter(
models.TrialModel.study_id == trial.study_id).filter(
models.TrialParamModel.param_name == param_name).first())
if previous_record is not None:
distributions.check_distribution_compatibility(
distributions.json_to_distribution(
previous_record.distribution_json),
distribution,
)
else:
session.add(
models.TrialParamModel(
trial_id=trial_id,
param_name=param_name,
param_value=param_value_internal,
distribution_json=distributions.distribution_to_json(
distribution),
))
# Release lock.
session.commit()
def dict_to_trial(d):
params = {}
params_in_internal_repr = {}
for k, v in d['params'].items():
distribution = distributions.json_to_distribution(v['distribution'])
params[k] = distribution.to_external_repr(v['value'])
params_in_internal_repr[k] = v['value']
return structs.FrozenTrial(
trial_id=d['trial_id'],
state=str_to_trial_state(d['state']),
params=params,
user_attrs=d['user_attrs'],
system_attrs=d['system_attrs'],
value=d['value'],
intermediate_values=dict(
(int(k), v) for k, v in d['intermediate_values'].items()),
params_in_internal_repr=params_in_internal_repr,
datetime_start=datetime.fromtimestamp(d['datetime_start']),
datetime_complete=datetime.fromtimestamp(d['datetime_end'])
if d['datetime_end'] else None,
)
def get_all_study_summaries(self) -> List[StudySummary]:
with _create_scoped_session(self.scoped_session) as session:
summarized_trial = (
session.query(
models.TrialModel.study_id,
functions.min(models.TrialModel.datetime_start).label("datetime_start"),
functions.count(models.TrialModel.trial_id).label("n_trial"),
)
.group_by(models.TrialModel.study_id)
.with_labels()
.subquery()
)
study_summary_stmt = session.query(
models.StudyModel.study_id,
models.StudyModel.study_name,
summarized_trial.c.datetime_start,
functions.coalesce(summarized_trial.c.n_trial, 0).label("n_trial"),
).select_from(orm.outerjoin(models.StudyModel, summarized_trial))
study_summary = study_summary_stmt.all()
_directions = defaultdict(list)
for d in session.query(models.StudyDirectionModel).all():
_directions[d.study_id].append(d.direction)
_user_attrs = defaultdict(list)
for a in session.query(models.StudyUserAttributeModel).all():
_user_attrs[d.study_id].append(a)
_system_attrs = defaultdict(list)
for a in session.query(models.StudySystemAttributeModel).all():
_system_attrs[d.study_id].append(a)
study_summaries = []
for study in study_summary:
directions = _directions[study.study_id]
best_trial: Optional[models.TrialModel] = None
try:
if len(directions) > 1:
raise ValueError
elif directions[0] == StudyDirection.MAXIMIZE:
best_trial = models.TrialModel.find_max_value_trial(
study.study_id, 0, session
)
else:
best_trial = models.TrialModel.find_min_value_trial(
study.study_id, 0, session
)
except ValueError:
best_trial_frozen: Optional[FrozenTrial] = None
if best_trial:
value = models.TrialValueModel.find_by_trial_and_objective(
best_trial, 0, session
)
assert value
params = (
session.query(
models.TrialParamModel.param_name,
models.TrialParamModel.param_value,
models.TrialParamModel.distribution_json,
)
.filter(models.TrialParamModel.trial_id == best_trial.trial_id)
.all()
)
param_dict = {}
param_distributions = {}
for param in params:
distribution = distributions.json_to_distribution(param.distribution_json)
param_dict[param.param_name] = distribution.to_external_repr(
param.param_value
)
param_distributions[param.param_name] = distribution
user_attrs = models.TrialUserAttributeModel.where_trial_id(
best_trial.trial_id, session
)
system_attrs = models.TrialSystemAttributeModel.where_trial_id(
best_trial.trial_id, session
)
intermediate = models.TrialIntermediateValueModel.where_trial_id(
best_trial.trial_id, session
)
best_trial_frozen = FrozenTrial(
best_trial.number,
TrialState.COMPLETE,
value.value,
best_trial.datetime_start,
best_trial.datetime_complete,
param_dict,
param_distributions,
{i.key: json.loads(i.value_json) for i in user_attrs},
{i.key: json.loads(i.value_json) for i in system_attrs},
{value.step: value.intermediate_value for value in intermediate},
best_trial.trial_id,
)
user_attrs = _user_attrs.get(study.study_id, [])
system_attrs = _system_attrs.get(study.study_id, [])
study_summaries.append(
StudySummary(
study_name=study.study_name,
direction=None,
directions=directions,
best_trial=best_trial_frozen,
user_attrs={i.key: json.loads(i.value_json) for i in user_attrs},
system_attrs={i.key: json.loads(i.value_json) for i in system_attrs},
n_trials=study.n_trial,
datetime_start=study.datetime_start,
study_id=study.study_id,
)
)
return study_summaries
def get_all_study_summaries(self) -> List[StudySummary]:
session = self.scoped_session()
summarized_trial = (session.query(
models.TrialModel.study_id,
functions.min(
models.TrialModel.datetime_start).label("datetime_start"),
functions.count(models.TrialModel.trial_id).label("n_trial"),
).group_by(models.TrialModel.study_id).with_labels().subquery())
study_summary_stmt = session.query(
models.StudyModel.study_id,
models.StudyModel.study_name,
models.StudyModel.direction,
summarized_trial.c.datetime_start,
functions.coalesce(summarized_trial.c.n_trial, 0).label("n_trial"),
).select_from(orm.outerjoin(models.StudyModel, summarized_trial))
study_summary = study_summary_stmt.all()
study_summaries = []
for study in study_summary:
best_trial: Optional[models.TrialModel] = None
try:
if study.direction == StudyDirection.MAXIMIZE:
best_trial = models.TrialModel.find_max_value_trial(
study.study_id, session)
else:
best_trial = models.TrialModel.find_min_value_trial(
study.study_id, session)
except ValueError:
best_trial_frozen: Optional[FrozenTrial] = None
if best_trial:
params = (session.query(
models.TrialParamModel.param_name,
models.TrialParamModel.param_value,
models.TrialParamModel.distribution_json,
).filter(models.TrialParamModel.trial_id ==
best_trial.trial_id).all())
param_dict = {}
param_distributions = {}
for param in params:
distribution = distributions.json_to_distribution(
param.distribution_json)
param_dict[
param.param_name] = distribution.to_external_repr(
param.param_value)
param_distributions[param.param_name] = distribution
user_attrs = session.query(
models.TrialUserAttributeModel).filter(
models.TrialUserAttributeModel.trial_id ==
best_trial.trial_id)
system_attrs = session.query(
models.TrialSystemAttributeModel).filter(
models.TrialSystemAttributeModel.trial_id ==
best_trial.trial_id)
intermediate = session.query(models.TrialValueModel).filter(
models.TrialValueModel.trial_id == best_trial.trial_id)
best_trial_frozen = FrozenTrial(
best_trial.number,
TrialState.COMPLETE,
best_trial.value,
best_trial.datetime_start,
best_trial.datetime_complete,
param_dict,
param_distributions,
{i.key: json.loads(i.value_json)
for i in user_attrs},
{i.key: json.loads(i.value_json)
for i in system_attrs},
{value.step: value.value
for value in intermediate},
best_trial.trial_id,
)
user_attrs = session.query(models.StudyUserAttributeModel).filter(
models.StudyUserAttributeModel.study_id == study.study_id)
system_attrs = session.query(
models.StudySystemAttributeModel).filter(
models.StudySystemAttributeModel.study_id ==
study.study_id)
study_summaries.append(
StudySummary(
study_name=study.study_name,
direction=study.direction,
best_trial=best_trial_frozen,
user_attrs={
i.key: json.loads(i.value_json)
for i in user_attrs
},
system_attrs={
i.key: json.loads(i.value_json)
for i in system_attrs
},
n_trials=study.n_trial,
datetime_start=study.datetime_start,
study_id=study.study_id,
))
# Terminate transaction explicitly to avoid connection timeout during transaction.
self._commit(session)
return study_summaries
def _merge_trials_orm(
self,
trials, # type: List[models.TrialModel]
trial_params, # type: List[models.TrialParamModel]
trial_intermediate_values, # type: List[models.TrialValueModel]
trial_user_attrs, # type: List[models.TrialUserAttributeModel]
trial_system_attrs # type: List[models.TrialSystemAttributeModel]
):
# type: (...) -> List[structs.FrozenTrial]
id_to_trial = {}
for trial in trials:
id_to_trial[trial.trial_id] = trial
id_to_params = defaultdict(
list) # type: Dict[int, List[models.TrialParamModel]]
for param in trial_params:
id_to_params[param.trial_id].append(param)
id_to_values = defaultdict(
list) # type: Dict[int, List[models.TrialValueModel]]
for value in trial_intermediate_values:
id_to_values[value.trial_id].append(value)
id_to_user_attrs = \
defaultdict(list) # type: Dict[int, List[models.TrialUserAttributeModel]]
for user_attr in trial_user_attrs:
id_to_user_attrs[user_attr.trial_id].append(user_attr)
id_to_system_attrs = \
defaultdict(list) # type: Dict[int, List[models.TrialSystemAttributeModel]]
for system_attr in trial_system_attrs:
id_to_system_attrs[system_attr.trial_id].append(system_attr)
result = []
for trial_id, trial in id_to_trial.items():
params = {}
param_distributions = {}
for param in id_to_params[trial_id]:
distribution = distributions.json_to_distribution(
param.distribution_json)
params[param.param_name] = distribution.to_external_repr(
param.param_value)
param_distributions[param.param_name] = distribution
intermediate_values = {}
for value in id_to_values[trial_id]:
intermediate_values[value.step] = value.value
user_attrs = {}
for user_attr in id_to_user_attrs[trial_id]:
user_attrs[user_attr.key] = json.loads(user_attr.value_json)
system_attrs = {}
for system_attr in id_to_system_attrs[trial_id]:
system_attrs[system_attr.key] = json.loads(
system_attr.value_json)
result.append(
structs.FrozenTrial(number=trial.number,
state=trial.state,
params=params,
distributions=param_distributions,
user_attrs=user_attrs,
system_attrs=system_attrs,
value=trial.value,
intermediate_values=intermediate_values,
datetime_start=trial.datetime_start,
datetime_complete=trial.datetime_complete,
trial_id=trial_id))
return result
def test_check_distribution_compatibility() -> None:
# test the same distribution
for key in EXAMPLE_JSONS:
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS[key], EXAMPLE_DISTRIBUTIONS[key])
# We need to create new objects to compare NaNs.
# See https://github.com/optuna/optuna/pull/3567#pullrequestreview-974939837.
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS[key],
distributions.json_to_distribution(EXAMPLE_JSONS[key]))
# test different distribution classes
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["fl"]),
)
# test compatibility between IntDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["id"],
EXAMPLE_DISTRIBUTIONS["i"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"], EXAMPLE_DISTRIBUTIONS["il"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"], EXAMPLE_DISTRIBUTIONS["id"])
# test compatibility between FloatDistributions.
distributions.check_distribution_compatibility(EXAMPLE_DISTRIBUTIONS["fd"],
EXAMPLE_DISTRIBUTIONS["f"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"], EXAMPLE_DISTRIBUTIONS["fl"])
with pytest.raises(ValueError):
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"], EXAMPLE_DISTRIBUTIONS["fd"])
# test dynamic value range (CategoricalDistribution)
pytest.raises(
ValueError,
lambda: distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["c2"],
distributions.CategoricalDistribution(choices=("Roppongi",
"Akasaka")),
),
)
# test dynamic value range (except CategoricalDistribution)
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["i"],
distributions.IntDistribution(low=-3, high=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["il"],
distributions.IntDistribution(low=1, high=13, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["id"],
distributions.IntDistribution(low=-3, high=2, step=2))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["f"],
distributions.FloatDistribution(low=-3.0, high=-2.0))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fl"],
distributions.FloatDistribution(low=0.1, high=1.0, log=True))
distributions.check_distribution_compatibility(
EXAMPLE_DISTRIBUTIONS["fd"],
distributions.FloatDistribution(low=-1.0, high=11.0, step=0.5))
def _merge_trials_orm(
self,
trials, # type: List[models.TrialModel]
trial_params, # type: List[models.TrialParamModel]
trial_intermediate_values, # type: List[models.TrialValueModel]
trial_user_attrs, # type: List[models.TrialUserAttributeModel]
trial_system_attrs # type: List[models.TrialSystemAttributeModel]
):
# type: (...) -> List[structs.FrozenTrial]
id_to_trial = {}
for trial in trials:
id_to_trial[trial.trial_id] = trial
id_to_params = defaultdict(
list) # type: Dict[int, List[models.TrialParamModel]]
for param in trial_params:
id_to_params[param.trial_id].append(param)
id_to_values = defaultdict(
list) # type: Dict[int, List[models.TrialValueModel]]
for value in trial_intermediate_values:
id_to_values[value.trial_id].append(value)
id_to_user_attrs = \
defaultdict(list) # type: Dict[int, List[models.TrialUserAttributeModel]]
for user_attr in trial_user_attrs:
id_to_user_attrs[user_attr.trial_id].append(user_attr)
id_to_system_attrs = \
defaultdict(list) # type: Dict[int, List[models.TrialSystemAttributeModel]]
for system_attr in trial_system_attrs:
id_to_system_attrs[system_attr.trial_id].append(system_attr)
temp_trials = []
for trial_id, trial in id_to_trial.items():
params = {}
param_distributions = {}
for param in id_to_params[trial_id]:
distribution = distributions.json_to_distribution(
param.distribution_json)
params[param.param_name] = distribution.to_external_repr(
param.param_value)
param_distributions[param.param_name] = distribution
intermediate_values = {}
for value in id_to_values[trial_id]:
intermediate_values[value.step] = value.value
user_attrs = {}
for user_attr in id_to_user_attrs[trial_id]:
user_attrs[user_attr.key] = json.loads(user_attr.value_json)
system_attrs = {}
for system_attr in id_to_system_attrs[trial_id]:
system_attrs[system_attr.key] = json.loads(
system_attr.value_json)
# `-1` is a dummy value.
# It will be replaced by a proper value before returned to the caller.
#
# TODO(ohta): Use trial.number after TrialModel.number is added.
trial_number = -1
temp_trials.append(
structs.FrozenTrial(number=trial_number,
state=trial.state,
params=params,
distributions=param_distributions,
user_attrs=user_attrs,
system_attrs=system_attrs,
value=trial.value,
intermediate_values=intermediate_values,
datetime_start=trial.datetime_start,
datetime_complete=trial.datetime_complete,
trial_id=trial_id))
result = []
for temp_trial in temp_trials:
# [NOTE]
# We set actual trial numbers here to avoid calling `self.get_trial_number_from_id()`
# within the above loop.
#
# This is because `self.get_trial_number_from_id()` may call `session.commit()`
# internally, which causes unintended changes of the states of `trials`.
# (see https://github.com/pfnet/optuna/pull/349#issuecomment-475086642 for details)
trial_number = self.get_trial_number_from_id(temp_trial.trial_id)
result.append(temp_trial._replace(number=trial_number))
return result
