def dummy_trial():
"""Return a dummy trial object"""
trial = Trial()
trial.params = [
Trial.Param(name='a', type='real', value=0.0),
Trial.Param(name='b', type='integer', value=1),
Trial.Param(name='c', type='categorical', value='Some')]
return trial
def test_value_equal(self, exp_config):
"""Compare Param objects using __eq__"""
trials = Trial.build(exp_config[1])
assert trials[0].params[0] == Trial.Param(
**exp_config[1][0]['params'][0])
assert trials[0].params[1] != Trial.Param(
**exp_config[1][0]['params'][0])
def dummy_trial():
"""Return a dummy trial object"""
trial = Trial()
trial._params = [
Trial.Param(name="a", type="real", value=0.0),
Trial.Param(name="b", type="integer", value=1),
Trial.Param(name="c", type="categorical", value="Some"),
]
return trial
def test_composite_adapter_backward(self, dummy_param, trials):
"""Test :meth:`orion.core.evc.adapters.CompositeAdapter.backward` with two adapters"""
new_param = Trial.Param(name="second_normal_prior",
type="integer",
value=1)
dimension_addition_adapter = DimensionAddition(new_param)
dimension_deletion_adapter = DimensionDeletion(new_param)
composite_adapter = CompositeAdapter(dimension_deletion_adapter)
adapted_trials = composite_adapter.backward(trials)
assert adapted_trials[0]._params[-1] == new_param
assert adapted_trials[4]._params[-1] == new_param
assert adapted_trials[-1]._params[-1] == new_param
composite_adapter = CompositeAdapter(dimension_addition_adapter,
dimension_deletion_adapter)
adapted_trials = composite_adapter.backward(trials)
assert len(adapted_trials) == len(trials)
assert new_param not in (adapted_trials[0]._params)
assert new_param not in (adapted_trials[4]._params)
assert new_param not in (adapted_trials[-1]._params)
def test_value_not_allowed_type(self):
"""Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`.
Same for `Trial.Param`.
"""
with pytest.raises(ValueError):
v = Trial.Result(name='asfda', type='hoho')
v = Trial.Result()
with pytest.raises(ValueError):
v.type = 'asfda'
with pytest.raises(ValueError):
v = Trial.Param(name='asfda', type='hoho')
v = Trial.Param()
with pytest.raises(ValueError):
v.type = 'asfda'
def test_dimension_deletion_backward_already_existing(self, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionDeletion.backward`
with valid param and invalid trials because param already exist
"""
new_param = Trial.Param(name="normal_prior", type="integer", value=1)
dimension_deletion_adapter = DimensionDeletion(new_param)
with pytest.raises(RuntimeError) as exc_info:
dimension_deletion_adapter.backward(trials)
assert "Provided trial does not have a compatible configuration" in str(
exc_info.value)
def test_dimension_deletion_forward(self, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionDeletion.forward`
with valid param and valid trials
"""
new_param = Trial.Param(name="second_normal_prior",
type="integer",
value=1)
dimension_deletion_adapter = DimensionDeletion(new_param)
sampler = DimensionBuilder().build("random",
"uniform(10, 100, discrete=True)")
for trial in trials:
random_param = new_param.to_dict()
random_param["value"] = sampler.sample()
trial._params.append(Trial.Param(**random_param))
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 0
trials[0]._params[-1].value = 1
assert trials[0]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 1
trials[4]._params[-1].value = 1
assert trials[4]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 2
trials[-1]._params[-1].value = 1
assert trials[-1]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 3
assert new_param not in (adapted_trials[0]._params)
assert new_param not in (adapted_trials[1]._params)
assert new_param not in (adapted_trials[2]._params)
def test_dimension_addition_backward(self, dummy_param, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionAddition.backward`
with valid param and valid trials
"""
new_param = Trial.Param(name='second_normal_prior',
type='integer',
value=1)
dimension_addition_adapter = DimensionAddition(new_param)
sampler = DimensionBuilder().build('random',
'uniform(10, 100, discrete=True)')
for trial in trials:
random_param = new_param.to_dict()
random_param['value'] = sampler.sample()
trial.params.append(Trial.Param(**random_param))
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 0
trials[0].params[-1].value = 1
assert trials[0].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 1
trials[4].params[-1].value = 1
assert trials[4].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 2
trials[-1].params[-1].value = 1
assert trials[-1].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 3
assert new_param not in (adapted_trials[0].params)
assert new_param not in (adapted_trials[1].params)
assert new_param not in (adapted_trials[2].params)
def _params(self):
"""See `~orion.core.worker.trial.Trial`"""
if self.memory is not None:
return self.memory._params
types = self.storage.metadata["params_types"]
params = flatten(self.storage.parameters)
return [
OrionTrial.Param(name=add_leading_slash(name),
value=params.get(name),
type=vtype) for name, vtype in types.items()
]
def test_dimension_addition_forward(self, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionAddition.forward`
with valid param and trials
"""
new_param = Trial.Param(name='second_normal_prior',
type='integer',
value=1)
dimension_addition_adapter = DimensionAddition(new_param)
adapted_trials = dimension_addition_adapter.forward(trials)
assert adapted_trials[0].params[-1] == new_param
assert adapted_trials[4].params[-1] == new_param
assert adapted_trials[-1].params[-1] == new_param
def test_dimension_deletion_backward(self, dummy_param, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionDeletion.backward`
with valid param and valid trials
"""
new_param = Trial.Param(name="second_normal_prior",
type="integer",
value=1)
dimension_deletion_adapter = DimensionDeletion(new_param)
adapted_trials = dimension_deletion_adapter.backward(trials)
assert adapted_trials[0]._params[-1] == new_param
assert adapted_trials[4]._params[-1] == new_param
assert adapted_trials[-1]._params[-1] == new_param
def generate_trials(priors, n_trials):
"""Generate trials by sampling from priors. Each trial has one param per prior."""
trials = []
for _ in range(n_trials):
params = []
for name, prior in priors.items():
dimension = DimensionBuilder().build(name, prior)
value = dimension.sample()[0]
params.append(
Trial.Param(name=name, type=dimension.type,
value=value).to_dict())
trials.append(Trial(params=params))
return trials
def test_working_dir(self, tmp_path, params):
trial = Trial(experiment=0,
exp_working_dir=tmp_path,
params=params,
parent=1)
assert trial.working_dir == os.path.join(tmp_path, trial.id)
assert trial.get_working_dir() == os.path.join(tmp_path, trial.id)
trial._params.append(
Trial.Param(name="/epoch", type="fidelity", value=1))
assert trial.id != trial.hash_params
assert trial.get_working_dir(ignore_fidelity=True,
ignore_lie=True,
ignore_parent=True) == os.path.join(
tmp_path, trial.hash_params)
assert trial.get_working_dir(ignore_parent=True) != trial.working_dir
def trials(small_prior, large_prior, normal_prior, disjoint_prior,
integer_prior, categorical_prior, multidim_prior):
"""Trials with dimensions for all priors defined as fixtures"""
N_TRIALS = 10
priors = dict((name, prior) for (name, prior) in locals().items()
if isinstance(name, str) and name.endswith("_prior"))
trials = []
for _ in range(N_TRIALS):
params = []
for name, prior in priors.items():
dimension = DimensionBuilder().build(name, prior)
value = dimension.sample()[0]
params.append(
Trial.Param(name=name, type=dimension.type,
value=value).to_dict())
trials.append(Trial(params=params))
return trials
def __init__(self, param):
"""Initialize and instantiate the param if necessary
Parameters
----------
param: instance of `orion.core.worker.trial.Trial.Param` or `dict`
A parameter object which defines the name and default value of the name dimension.
It can be either a dictionary definition or an instance of Param. If the former, then
a parameter is instantiated from the dictionary.
"""
if isinstance(param, dict):
param = Trial.Param(**param)
elif not isinstance(param, Trial.Param):
raise TypeError(
"Invalid param argument type ('%s'). "
"Param argument must be a Param object or a dictionnary "
"as defined by Trial.Param.to_dict()." % str(type(param)))
self.param = param
def test_composite_configuration(dummy_param):
"""Test :meth:`orion.core.evc.adapters.CompositeAdapter.configuration`"""
new_param = Trial.Param(name="second_normal_prior",
type="integer",
value=1)
dimension_addition_adapter = DimensionAddition(dummy_param)
dimension_deletion_adapter = DimensionDeletion(new_param)
composite_adapter = CompositeAdapter(dimension_addition_adapter,
dimension_deletion_adapter)
configuration = composite_adapter.configuration
assert configuration[0] == dimension_addition_adapter.configuration[0]
assert configuration[1] == dimension_deletion_adapter.configuration[0]
assert (BaseAdapter.build(configuration).adapters[0].configuration[0] ==
configuration[0])
assert (BaseAdapter.build(configuration).adapters[1].configuration[0] ==
configuration[1])
def dummy_param():
"""Give dummy param integer param with value 1"""
return Trial.Param(name='dummy', type='integer', value=1)
def test_param_bad_init(self):
"""Other than `Trial.Param.__slots__` are not allowed in __init__ too."""
with pytest.raises(AttributeError):
Trial.Param(ispii='iela')
def dummy_param():
"""Give dummy param integer param with value 1"""
return Trial.Param(name="dummy", type="integer", value=1)
