def __init__(self):
if import_error:
raise import_error
self.builder = DimensionBuilder()
self.dim_leaves = {
'uniform': self.uniform,
'normal': self.normal,
'categorical': self.choices,
'ordinal': self.ordinal
}
def generate_trials(priors, n_trials):
"""Generate trials by sampling from priors. Each trial has one param per prior."""
trials = []
for _ in range(n_trials):
params = []
for name, prior in priors.items():
dimension = DimensionBuilder().build(name, prior)
value = dimension.sample()[0]
params.append(
Trial.Param(name=name, type=dimension.type,
value=value).to_dict())
trials.append(Trial(params=params))
return trials
def new_dimension_with_default_conflict(old_config, new_config):
"""Generate a new dimension conflict with default value for new experiment configuration"""
name = "new"
prior = "normal(0, 2, default_value=0.001)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.NewDimensionConflict(old_config, new_config, dimension,
prior)
def new_dimension_same_prior_conflict(old_config, new_config):
"""Generate a new dimension conflict with different prior for renaming tests"""
name = "new"
prior = "uniform(-10, 10)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.NewDimensionConflict(old_config, new_config, dimension,
prior)
def new_cat_dimension_conflict(old_config, new_config):
"""Generate a new dimension conflict with categorical prior for new experiment configuration"""
name = 'new-cat'
prior = 'choices(["hello", 2])'
dimension = DimensionBuilder().build(name, prior)
return evc.conflicts.NewDimensionConflict(old_config, new_config,
dimension, prior)
def missing_dimension_conflict(old_config, new_config):
"""Generate a missing dimension conflict"""
name = "missing"
prior = "uniform(-10, 10)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.MissingDimensionConflict(old_config, new_config,
dimension, prior)
def new_dimension_conflict(old_config, new_config):
"""Generate a new dimension conflict for new experiment configuration"""
name = "new"
prior = "normal(0, 2)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.NewDimensionConflict(old_config, new_config, dimension,
prior)
def missing_dimension_with_default_conflict(old_config, new_config):
"""Generate a missing dimension conflict with a default value"""
name = "missing"
prior = "uniform(-10, 10, default_value=0.0)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.MissingDimensionConflict(old_config, new_config,
dimension, prior)
def missing_cat_dimension_conflict(old_config, new_config):
"""Generate a missing dimension conflict with categorical prior for new experiment
configuration
"""
name = 'missing-cat'
prior = 'choices(["goodbye", 5])'
dimension = DimensionBuilder().build(name, prior)
return evc.conflicts.MissingDimensionConflict(old_config, new_config, dimension, prior)
def add_default_config_for_missing(conflict, value):
"""Create a missing dimension conflict with a default value for the prior"""
conflict_with_default = copy.deepcopy(conflict)
conflict_with_default.prior = (conflict_with_default.prior[:-1] +
f", default_value={value})")
conflict_with_default.dimension = DimensionBuilder().build(
conflict_with_default.dimension.name, conflict_with_default.prior)
return conflict_with_default
def missing_conflict_with_identical_prior(old_config, new_config,
new_dimension_conflict):
"""Generate a missing dimension conflict which have the same prior as the new dim conflict"""
name = 'missing-idem'
prior = new_dimension_conflict.prior
dimension = DimensionBuilder().build(name, prior)
return evc.conflicts.MissingDimensionConflict(old_config, new_config,
dimension, prior)
def changed_dimension_conflict(old_config, new_config):
"""Generate a changed dimension conflict"""
name = "changed"
old_prior = "uniform(-10, 10)"
new_prior = "normal(0, 2)"
dimension = DimensionBuilder().build(name, old_prior)
return conflicts.ChangedDimensionConflict(old_config, new_config,
dimension, old_prior, new_prior)
def missing_dimension_from_config_conflict(old_config_with_script_conf,
new_config_with_script_conf):
"""Generate a missing dimension conflict in the config file"""
name = "dropped"
prior = "uniform(-1, 5)"
dimension = DimensionBuilder().build(name, prior)
return conflicts.MissingDimensionConflict(old_config_with_script_conf,
new_config_with_script_conf,
dimension, prior)
def build_space(model, optimizer, **space_config):
space = Space()
dimension_builder = DimensionBuilder()
full_space_config = {'optimizer': OPTIMIZER_SPACES[optimizer]}
full_space_config['optimizer'].update(space_config.get('optimizer', {}))
for name, prior in flatten(full_space_config).items():
if not prior:
continue
try:
space[name] = dimension_builder.build(name, prior)
except TypeError as e:
print(str(e))
print('Ignoring key {} with prior {}'.format(name, prior))
return space
def trials(small_prior, large_prior, normal_prior, disjoint_prior,
integer_prior, categorical_prior, multidim_prior):
"""Trials with dimensions for all priors defined as fixtures"""
N_TRIALS = 10
priors = dict((name, prior) for (name, prior) in locals().items()
if isinstance(name, str) and name.endswith("_prior"))
trials = []
for _ in range(N_TRIALS):
params = []
for name, prior in priors.items():
dimension = DimensionBuilder().build(name, prior)
value = dimension.sample()[0]
params.append(
Trial.Param(name=name, type=dimension.type,
value=value).to_dict())
trials.append(Trial(params=params))
return trials
def test_dimension_addition_backward(self, dummy_param, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionAddition.backward`
with valid param and valid trials
"""
new_param = Trial.Param(name='second_normal_prior',
type='integer',
value=1)
dimension_addition_adapter = DimensionAddition(new_param)
sampler = DimensionBuilder().build('random',
'uniform(10, 100, discrete=True)')
for trial in trials:
random_param = new_param.to_dict()
random_param['value'] = sampler.sample()
trial.params.append(Trial.Param(**random_param))
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 0
trials[0].params[-1].value = 1
assert trials[0].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 1
trials[4].params[-1].value = 1
assert trials[4].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 2
trials[-1].params[-1].value = 1
assert trials[-1].params[-1] == new_param
adapted_trials = dimension_addition_adapter.backward(trials)
assert len(adapted_trials) == 3
assert new_param not in (adapted_trials[0].params)
assert new_param not in (adapted_trials[1].params)
assert new_param not in (adapted_trials[2].params)
def test_dimension_deletion_forward(self, trials):
"""Test :meth:`orion.core.evc.adapters.DimensionDeletion.forward`
with valid param and valid trials
"""
new_param = Trial.Param(name="second_normal_prior",
type="integer",
value=1)
dimension_deletion_adapter = DimensionDeletion(new_param)
sampler = DimensionBuilder().build("random",
"uniform(10, 100, discrete=True)")
for trial in trials:
random_param = new_param.to_dict()
random_param["value"] = sampler.sample()
trial._params.append(Trial.Param(**random_param))
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 0
trials[0]._params[-1].value = 1
assert trials[0]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 1
trials[4]._params[-1].value = 1
assert trials[4]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 2
trials[-1]._params[-1].value = 1
assert trials[-1]._params[-1] == new_param
adapted_trials = dimension_deletion_adapter.forward(trials)
assert len(adapted_trials) == 3
assert new_param not in (adapted_trials[0]._params)
assert new_param not in (adapted_trials[1]._params)
assert new_param not in (adapted_trials[2]._params)
def __init__(self, name, old_prior, new_prior):
"""Initialize and instantiate dimensions
Parameters
----------
name: `str`
Name of the dimension.
old_prior: `str`
string definition as parsable by `orion.core.io.space_builder`.
new_prior: `str`
string definition as parsable by `orion.core.io.space_builder`.
"""
self.name = name
self.old_prior = old_prior
self.new_prior = new_prior
self.old_dimension = DimensionBuilder().build("old", old_prior)
self.new_dimension = DimensionBuilder().build("new", new_prior)
if self.old_dimension.shape != self.new_dimension.shape:
log.warning(
"Oríon does not support yet adaptations on prior shape changes. All trials "
"of different shapes will be ignored.")
def __init__(self, name, old_prior, new_prior):
"""Initialize and instantiate dimensions
Parameters
----------
name: `str`
Name of the dimension.
old_prior: `str`
string definition as parsable by `orion.core.io.space_builder`.
new_prior: `str`
string definition as parsable by `orion.core.io.space_builder`.
"""
self.name = name
self.old_prior = old_prior
self.new_prior = new_prior
self.old_dimension = DimensionBuilder().build('old', old_prior)
self.new_dimension = DimensionBuilder().build('new', new_prior)
if self.old_dimension.shape != self.new_dimension.shape:
raise NotImplementedError(
"Oríon does not support yet adaptations on prior "
"shape changes.")
class _OrionSpaceBuilder:
def __init__(self):
if import_error:
raise import_error
self.builder = DimensionBuilder()
self.dim_leaves = {
'uniform': self.uniform,
'normal': self.normal,
'categorical': self.choices,
'ordinal': self.ordinal
}
def uniform(self, name, lower, upper, log, discrete, quantization=None):
self.builder.name = name
if quantization is not None:
print('Orion does not support quantization')
if log:
return self.builder.loguniform(lower, upper, discrete=discrete)
return self.builder.uniform(lower, upper, discrete=discrete)
def normal(self, name, loc, scale, discrete, log, quantization=None):
print(name, loc, scale, discrete, log, quantization)
self.builder.name = name
if quantization is not None:
print('Orion does not support quantization')
if log is True:
raise NotImplementedError('Orion does not provide LogNormal')
return self.builder.normal(loc, scale, discrete=discrete)
def choices(self, name, options):
self.builder.name = name
return self.builder.choices(options)
def ordinal(self, name, *args, **kwargs):
raise NotImplementedError('Orion does not provide Ordinal dim')
def cond_leaf(self, mode, leaf, hyper_parameter, ctx=None):
raise NotImplementedError('Orion does not support conditionals')
def cond_node(self, mode, node, hyper_parameter, ctx=None):
raise NotImplementedError('Orion does not support conditionals')
def dim_leaf(self, fun_name, **kwargs):
fun = self.dim_leaves.get(fun_name)
if fun is None:
raise NotImplementedError(f'{fun_name} is missing')
return fun(**kwargs)
def dim_node(self, node, **kwargs):
space = OrionSpace.Space()
for k, dim_expr in node.space_tree.items():
dim = dim_expr.visit(self)
if isinstance(dim, OrionSpace.Space):
for sub_k, sub_dim in dim.items():
sub_dim.name = f'{k}.{sub_k}'
space.register(sub_dim)
else:
space.register(dim)
if dim_expr.condition is not None:
print('Orion does not support conditionals')
if dim_expr.forbidden is not None:
print('Orion does not support conditionals')
return space
@staticmethod
def _to_dictionary(handle, val):
return sort_dict({k: v for k, v in zip(handle.keys(), val)})
@staticmethod
def sample(handle, n_samples, seed):
return [
_OrionSpaceBuilder._to_dictionary(handle, p)
for p in handle.sample(n_samples, seed)
]
def dimbuilder():
"""Return a `DimensionBuilder` instance."""
return DimensionBuilder()
def build_space():
space = Space()
space['a'] = DimensionBuilder().build('a', 'uniform(0.1, 2.0)')
space['b'] = DimensionBuilder().build('b', 'uniform(0, 10.0)')
return space