def dummy_trial(): """Return a dummy trial object""" trial = Trial() trial.params = [ Trial.Param(name='a', type='real', value=0.0), Trial.Param(name='b', type='integer', value=1), Trial.Param(name='c', type='categorical', value='Some')] return trial
def test_value_equal(self, exp_config): """Compare Param objects using __eq__""" trials = Trial.build(exp_config[1]) assert trials[0].params[0] == Trial.Param( **exp_config[1][0]['params'][0]) assert trials[0].params[1] != Trial.Param( **exp_config[1][0]['params'][0])
def dummy_trial(): """Return a dummy trial object""" trial = Trial() trial._params = [ Trial.Param(name="a", type="real", value=0.0), Trial.Param(name="b", type="integer", value=1), Trial.Param(name="c", type="categorical", value="Some"), ] return trial
def test_composite_adapter_backward(self, dummy_param, trials): """Test :meth:`orion.core.evc.adapters.CompositeAdapter.backward` with two adapters""" new_param = Trial.Param(name="second_normal_prior", type="integer", value=1) dimension_addition_adapter = DimensionAddition(new_param) dimension_deletion_adapter = DimensionDeletion(new_param) composite_adapter = CompositeAdapter(dimension_deletion_adapter) adapted_trials = composite_adapter.backward(trials) assert adapted_trials[0]._params[-1] == new_param assert adapted_trials[4]._params[-1] == new_param assert adapted_trials[-1]._params[-1] == new_param composite_adapter = CompositeAdapter(dimension_addition_adapter, dimension_deletion_adapter) adapted_trials = composite_adapter.backward(trials) assert len(adapted_trials) == len(trials) assert new_param not in (adapted_trials[0]._params) assert new_param not in (adapted_trials[4]._params) assert new_param not in (adapted_trials[-1]._params)
def test_value_not_allowed_type(self): """Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`. Same for `Trial.Param`. """ with pytest.raises(ValueError): v = Trial.Result(name='asfda', type='hoho') v = Trial.Result() with pytest.raises(ValueError): v.type = 'asfda' with pytest.raises(ValueError): v = Trial.Param(name='asfda', type='hoho') v = Trial.Param() with pytest.raises(ValueError): v.type = 'asfda'
def test_dimension_deletion_backward_already_existing(self, trials): """Test :meth:`orion.core.evc.adapters.DimensionDeletion.backward` with valid param and invalid trials because param already exist """ new_param = Trial.Param(name="normal_prior", type="integer", value=1) dimension_deletion_adapter = DimensionDeletion(new_param) with pytest.raises(RuntimeError) as exc_info: dimension_deletion_adapter.backward(trials) assert "Provided trial does not have a compatible configuration" in str( exc_info.value)
def test_dimension_deletion_forward(self, trials): """Test :meth:`orion.core.evc.adapters.DimensionDeletion.forward` with valid param and valid trials """ new_param = Trial.Param(name="second_normal_prior", type="integer", value=1) dimension_deletion_adapter = DimensionDeletion(new_param) sampler = DimensionBuilder().build("random", "uniform(10, 100, discrete=True)") for trial in trials: random_param = new_param.to_dict() random_param["value"] = sampler.sample() trial._params.append(Trial.Param(**random_param)) adapted_trials = dimension_deletion_adapter.forward(trials) assert len(adapted_trials) == 0 trials[0]._params[-1].value = 1 assert trials[0]._params[-1] == new_param adapted_trials = dimension_deletion_adapter.forward(trials) assert len(adapted_trials) == 1 trials[4]._params[-1].value = 1 assert trials[4]._params[-1] == new_param adapted_trials = dimension_deletion_adapter.forward(trials) assert len(adapted_trials) == 2 trials[-1]._params[-1].value = 1 assert trials[-1]._params[-1] == new_param adapted_trials = dimension_deletion_adapter.forward(trials) assert len(adapted_trials) == 3 assert new_param not in (adapted_trials[0]._params) assert new_param not in (adapted_trials[1]._params) assert new_param not in (adapted_trials[2]._params)
def test_dimension_addition_backward(self, dummy_param, trials): """Test :meth:`orion.core.evc.adapters.DimensionAddition.backward` with valid param and valid trials """ new_param = Trial.Param(name='second_normal_prior', type='integer', value=1) dimension_addition_adapter = DimensionAddition(new_param) sampler = DimensionBuilder().build('random', 'uniform(10, 100, discrete=True)') for trial in trials: random_param = new_param.to_dict() random_param['value'] = sampler.sample() trial.params.append(Trial.Param(**random_param)) adapted_trials = dimension_addition_adapter.backward(trials) assert len(adapted_trials) == 0 trials[0].params[-1].value = 1 assert trials[0].params[-1] == new_param adapted_trials = dimension_addition_adapter.backward(trials) assert len(adapted_trials) == 1 trials[4].params[-1].value = 1 assert trials[4].params[-1] == new_param adapted_trials = dimension_addition_adapter.backward(trials) assert len(adapted_trials) == 2 trials[-1].params[-1].value = 1 assert trials[-1].params[-1] == new_param adapted_trials = dimension_addition_adapter.backward(trials) assert len(adapted_trials) == 3 assert new_param not in (adapted_trials[0].params) assert new_param not in (adapted_trials[1].params) assert new_param not in (adapted_trials[2].params)
def _params(self): """See `~orion.core.worker.trial.Trial`""" if self.memory is not None: return self.memory._params types = self.storage.metadata["params_types"] params = flatten(self.storage.parameters) return [ OrionTrial.Param(name=add_leading_slash(name), value=params.get(name), type=vtype) for name, vtype in types.items() ]
def test_dimension_addition_forward(self, trials): """Test :meth:`orion.core.evc.adapters.DimensionAddition.forward` with valid param and trials """ new_param = Trial.Param(name='second_normal_prior', type='integer', value=1) dimension_addition_adapter = DimensionAddition(new_param) adapted_trials = dimension_addition_adapter.forward(trials) assert adapted_trials[0].params[-1] == new_param assert adapted_trials[4].params[-1] == new_param assert adapted_trials[-1].params[-1] == new_param
def test_dimension_deletion_backward(self, dummy_param, trials): """Test :meth:`orion.core.evc.adapters.DimensionDeletion.backward` with valid param and valid trials """ new_param = Trial.Param(name="second_normal_prior", type="integer", value=1) dimension_deletion_adapter = DimensionDeletion(new_param) adapted_trials = dimension_deletion_adapter.backward(trials) assert adapted_trials[0]._params[-1] == new_param assert adapted_trials[4]._params[-1] == new_param assert adapted_trials[-1]._params[-1] == new_param
def generate_trials(priors, n_trials): """Generate trials by sampling from priors. Each trial has one param per prior.""" trials = [] for _ in range(n_trials): params = [] for name, prior in priors.items(): dimension = DimensionBuilder().build(name, prior) value = dimension.sample()[0] params.append( Trial.Param(name=name, type=dimension.type, value=value).to_dict()) trials.append(Trial(params=params)) return trials
def test_working_dir(self, tmp_path, params): trial = Trial(experiment=0, exp_working_dir=tmp_path, params=params, parent=1) assert trial.working_dir == os.path.join(tmp_path, trial.id) assert trial.get_working_dir() == os.path.join(tmp_path, trial.id) trial._params.append( Trial.Param(name="/epoch", type="fidelity", value=1)) assert trial.id != trial.hash_params assert trial.get_working_dir(ignore_fidelity=True, ignore_lie=True, ignore_parent=True) == os.path.join( tmp_path, trial.hash_params) assert trial.get_working_dir(ignore_parent=True) != trial.working_dir
def trials(small_prior, large_prior, normal_prior, disjoint_prior, integer_prior, categorical_prior, multidim_prior): """Trials with dimensions for all priors defined as fixtures""" N_TRIALS = 10 priors = dict((name, prior) for (name, prior) in locals().items() if isinstance(name, str) and name.endswith("_prior")) trials = [] for _ in range(N_TRIALS): params = [] for name, prior in priors.items(): dimension = DimensionBuilder().build(name, prior) value = dimension.sample()[0] params.append( Trial.Param(name=name, type=dimension.type, value=value).to_dict()) trials.append(Trial(params=params)) return trials
def __init__(self, param): """Initialize and instantiate the param if necessary Parameters ---------- param: instance of `orion.core.worker.trial.Trial.Param` or `dict` A parameter object which defines the name and default value of the name dimension. It can be either a dictionary definition or an instance of Param. If the former, then a parameter is instantiated from the dictionary. """ if isinstance(param, dict): param = Trial.Param(**param) elif not isinstance(param, Trial.Param): raise TypeError( "Invalid param argument type ('%s'). " "Param argument must be a Param object or a dictionnary " "as defined by Trial.Param.to_dict()." % str(type(param))) self.param = param
def test_composite_configuration(dummy_param): """Test :meth:`orion.core.evc.adapters.CompositeAdapter.configuration`""" new_param = Trial.Param(name="second_normal_prior", type="integer", value=1) dimension_addition_adapter = DimensionAddition(dummy_param) dimension_deletion_adapter = DimensionDeletion(new_param) composite_adapter = CompositeAdapter(dimension_addition_adapter, dimension_deletion_adapter) configuration = composite_adapter.configuration assert configuration[0] == dimension_addition_adapter.configuration[0] assert configuration[1] == dimension_deletion_adapter.configuration[0] assert (BaseAdapter.build(configuration).adapters[0].configuration[0] == configuration[0]) assert (BaseAdapter.build(configuration).adapters[1].configuration[0] == configuration[1])
def dummy_param(): """Give dummy param integer param with value 1""" return Trial.Param(name='dummy', type='integer', value=1)
def test_param_bad_init(self): """Other than `Trial.Param.__slots__` are not allowed in __init__ too.""" with pytest.raises(AttributeError): Trial.Param(ispii='iela')
def dummy_param(): """Give dummy param integer param with value 1""" return Trial.Param(name="dummy", type="integer", value=1)