def test_conditions(condition): space = Space('ConfigSpace') a = space.normal('a', 1, 2, quantization=0.01) b = space.normal('b', 1, 2, quantization=0.01) b.enable_if(condition(a, 1.5)) print(space.sample())
def test_conditions_and(): space = Space('ConfigSpace') a = space.normal('a', 1, 2, quantization=0.01) b = space.normal('b', 1, 2, quantization=0.01) b.enable_if(both(gt(a, 1), lt(a, 2))) print(space.sample())
def test_conditions_in(): space = Space('ConfigSpace') a = space.normal('a', 1, 2, quantization=0.01) b = space.normal('b', 1, 2, quantization=0.01) b.enable_if(contains(a, [1, 1.5, 2])) print(space.sample())
def test_forbid_and(): space = Space('ConfigSpace') a = space.uniform('a', 1, 2, quantization=0.01) a.forbid_equal(1) a.forbid_in([1, 2]) print(space.sample())
def test_conditions_or(): space = Space('ConfigSpace') a = space.normal('a', 1, 2, quantization=0.01) b = space.normal('b', 1, 2, quantization=0.01) b.enable_if(either(eq(a, 1), ne(a, 2))) print(space.sample())
def make_big_space(backend='ConfigSpace'): def make_subspace(sspace): sspace.uniform('uniform_float', 0, 1) sspace.loguniform('loguniform_float', 2, 3) sspace.normal('normal_float', 0, 1) sspace.lognormal('lognormal_float', 2, 3) sspace.uniform('uniform_float_q', 0, 1, quantization=0.01) sspace.loguniform('loguniform_float_q', 2, 3, quantization=0.01) sspace.normal('normal_float_q', 0, 1, quantization=0.01) sspace.lognormal('lognormal_float_q', 2, 1, quantization=0.01) sspace.uniform('uniform_int', 0, 1, discrete=True) sspace.loguniform('loguniform_int', 2, 3, discrete=True) sspace.normal('normal_int', 0, 1, discrete=True) sspace.lognormal('lognormal_int', 2, 3, discrete=True) sspace.choices('choices', ['a', 'b', 'c', 'd']) sspace.ordinal('ordinal', ['a', 'b', 'c', 'd']) sspace.variable('epoch') sspace.identity('uid') return sspace space = Space(backend=backend) make_subspace(space) make_subspace(space.subspace('sub')) return space
def make_space(backend='ConfigSapce'): space = Space(backend=backend) optim = space.categorical('optimizer', ['sgd', 'adam']) sgd_lr = space.loguniform('optimizer.lr', 1, 2, quantization=0.01) sgd_lr.enable_if(either(eq(optim, 'adam'), eq(optim, 'sgd'))) sgd_lr.forbid_equal(1) return space
def test_categorical(backend): space = Space(backend=backend) space.categorical('cat', ['a', 'b', 'c']) space.categorical('caw', a=0.2, b=0.1, c=0.7) space.categorical('cad', dict(a=0.2, b=0.1, c=0.7)) print(space.sample())
def test_uniform(backend): space = Space(backend=backend) for discrete in [True, False]: for log in [True, False]: for q in [None, 0.01, 1]: space.uniform(f'a_{discrete}_{log}_{q}', 1, 2, discrete=discrete, log=log, quantization=q) print(space.sample())
def test_hyperband_api(): import random params = Space.from_dict({'a': 'uniform(0, 1)'}) hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params) assert not hpo.is_done() for rung in range(3): params_set = hpo.suggest() for i, params in enumerate(params_set): print(i, params) try: hpo.suggest() raise RuntimeError() except WaitingForTrials: pass except OptimizationIsDone: pass for i, params in enumerate(params_set): v = random.uniform(0, 1) if i == len(params_set) - 1: v = 1e-10 hpo.observe(params, v) print('-------') assert hpo.is_done() print(hpo.result()) print(hpo.info())
def __init__(self, fidelity: Fidelity, space: Union[Space, Dict], seed=0, **kwargs): self.identity = 'uid' for k, v in kwargs.items(): debug(f'used parameter ({k}: {v})') if isinstance(space, dict): space = Space.from_dict(space) if space is not None: space.identity(self.identity) if isinstance(fidelity, dict): fidelity = Fidelity.from_dict(fidelity) self.fidelity = fidelity self.space = space self.seed = seed self.seed_time = 0 self.manual_insert = 0 self.manual_samples = [] self.manual_fidelity = [] self.trials = OrderedDict()
def test_normal(backend): for discrete in [True, False]: for log in [True, False]: for q in [None, 0.01, 1]: space = Space(backend=backend) space.normal(f'a_{discrete}_{log}_{q}', loc=1, scale=2, discrete=discrete, log=log, quantization=q) try: print(space.sample()) except NotImplementedError: assert backend == 'Orion' and log is True
def test_build_bounds(): space = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'uniform(1, 15)' }) lower, upper = build_bounds(space.instantiate('Orion')) assert lower.tolist() == [-5, 1] assert upper.tolist() == [10, 15] space = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(1, 15)' }) lower, upper = build_bounds(space.instantiate('Orion')) assert lower.tolist() == [-5, numpy.log(1)] assert upper.tolist() == [10, numpy.log(15)]
def test_serialization_is_same(backend): import copy space = make_big_space(backend) serialized = space.serialize() new_space = Space.from_dict(copy.deepcopy(serialized)) new_serialized = new_space.serialize() assert serialized == new_serialized new_space.sample(**{'sub.epoch': 1, 'epoch': 2})
def test_sample_in_log_space(robo): params = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(12, 15)' }) robo = RoBO(FIDELITY, params, count=15, n_init=10) while not robo.is_done(): samples = robo.suggest() for sample in samples: robo.observe(sample['uid'], branin(**sample)) assert 12 <= sample['y'] <= 15
def test_serialization_sample_big(): import copy space = make_big_space() serialized = space.serialize() print(json.dumps(serialized, indent=2)) new_space = Space.from_dict(copy.deepcopy(serialized)) new_serialized = new_space.serialize() assert serialized == new_serialized print(json.dumps(new_space.sample(2, **{'sub.epoch': 1, 'epoch': 2}), indent=2))
def test_hyperband_simple_sequential(): import random params = Space.from_dict({'a': 'uniform(0, 1)'}) hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params) for params in hpo: hpo.observe(params, result=random.uniform(0, 1)) assert hpo.is_done() print(hpo.result()) print(hpo.info())
def load_state_dict(self, state): state = decompress_dict(state) self.space = Space.from_dict(state['space']) self.seed = state['seed'] self.manual_samples = state['manual_samples'] self.manual_fidelity = state['manual_fidelity'] self.manual_insert = state['manual_insert'] self.seed_time = state['seed_time'] self.fidelity = Fidelity.from_dict(state['fidelity']) self.trials = OrderedDict( (k, Trial.from_dict(t)) for k, t in state['trials']) return self
def test_subspace(backend): space = Space(backend=backend) space.normal('a', 1, 2, quantization=0.01) subspace = space.subspace('b') subspace.normal('a', 1, 2, quantization=0.01) print(space.sample())
def test_convert_samples_to_x(): space = { 'a': 'uniform(lower=-1, upper=1)', 'b': 'uniform(lower=-1, upper=1)', 'c': 'loguniform(lower=1, upper=10)' } samples = Space.from_dict(space).sample(10, seed=1) X = convert_samples_to_x(samples, space) assert X.shape == (10, 3) assert X[0, 0] == samples[0]['a'] assert X[0, 1] == samples[0]['b'] assert X[0, 2] == numpy.log(samples[0]['c'])
def simulate_hpo(model, space, budget, seed): # NOTE: We hardcode for random search for now. # Sample points samples = Space.from_dict(space).sample(budget, seed=seed) X = convert_samples_to_x(samples, space) # Score points mean, std = model.predict(X) scores = mean # Select best min_index = numpy.argmin(scores) best_point = X[min_index] return convert_x_to_samples(X[min_index].reshape(1, -1), space)[0]
def test_ordinal(backend): space = Space(backend=backend) try: space.ordinal('ord', ['a', 'b', 'c']) print(space.sample()) print(space.sample()) print(space.sample()) except NotImplementedError: assert backend == 'Orion'
def test_X_with_log_dims(robo): params = Space.from_dict({ 'x': 'uniform(-5, 10)', 'y': 'loguniform(1, 15)' }) robo = RoBO(FIDELITY, params, count=5, n_init=2) while not robo.is_done(): samples = robo.suggest() for sample in samples: robo.observe(sample['uid'], branin(**sample)) assert robo.X.shape == (5, 2) assert robo.y.shape == (5,) assert robo.X.min(0)[0] >= -5 assert robo.X.min(0)[1] >= numpy.log(1) assert robo.X.max(0)[0] <= 10 assert robo.X.max(0)[1] <= numpy.log(15)
def fit_model(X, y, space, seed=1): n_samples = 50 # kernel = GPy.kern.Matern52(all_x.shape[1], variance=0.01, ARD=True) # model = GPModel_MCMC(n_samples=n_samples, kernel=kernel) # with all_logging_disabled(logging.ERROR): # model.updateModel(all_x, all_y, None, None) orion_space = Space.from_dict(space).instantiate('Orion') lower, upper = build_bounds(orion_space) # TODO set the seeds model = build_model(lower, upper, model_type='gp_mcmc', model_seed=1, prior_seed=1) model.train(X, y) return model
def build_data(budget, variables, defaults, space): epochs = 5 # defaults = {'a': 0, 'b': 1, 'c': 2, 'd': 3} # params = {'c': 2, 'd': 3, 'epoch': epochs} n_vars = len(variables) objectives = numpy.arange(budget * (epochs + 1)) numpy.random.RandomState(0).shuffle(objectives) objectives = objectives.reshape((epochs + 1, budget, 1)) params = Space.from_dict(space).sample(budget, seed=1) trials = OrderedDict() for trial_params in params: config = copy.deepcopy( dict(list(variables.items()) + list(defaults.items()))) config.update(trial_params) config['uid'] = compute_identity(config, IDENTITY_SIZE) # NOTE: We don't need objectives trials[config['uid']] = Trial(config) metrics = dict() for trial_i, trial_uid in enumerate(trials.keys()): metrics[trial_uid] = [{ 'epoch': i, 'obj': objectives[i, trial_i, 0] } for i in range(epochs + 1)] data = [] param_names = list(sorted(space.keys())) return create_valid_curves_xarray(trials, metrics, sorted(variables.keys()), epochs, param_names, seed=1)
def test_deserialize_orion(): cs = Space.from_dict(orion_space) print(cs) print(cs.sample(fid=1)) print(json.dumps(cs.serialize(), indent=2))
from sspace import Space space = Space() space.uniform('lr', 0, 1) space.ordinal('epoch', [1, 2, 3]) s1 = space.sample(seed=0) s2 = space.sample(seed=1) s1p = space.sample(seed=0) print(s1) print(s2) print(s1p) # [OrderedDict([('epoch', 1), ('optimizer', 'adam')])] # [OrderedDict([('epoch', 2), ('optimizer', 'adam')])] # [OrderedDict([('epoch', 1), ('optimizer', 'adam')])]
from sspace import Space, either, eq import json if __name__ == '__main__': space = Space(backend='ConfigSpace') optim = space.categorical('optimizer', ['sgd', 'adam']) sgd_lr = space.loguniform('optimizer.lr', 1, 2, quantization=0.01) sgd_lr.enable_if(either(eq(optim, 'adam'), eq(optim, 'sgd'))) sgd_lr.forbid_equal(1) for sample in space.sample(2): print(sample) print(json.dumps(space.serialize(), indent=2))