def test_conditions(condition):
space = Space('ConfigSpace')
a = space.normal('a', 1, 2, quantization=0.01)
b = space.normal('b', 1, 2, quantization=0.01)
b.enable_if(condition(a, 1.5))
print(space.sample())
def test_conditions_and():
space = Space('ConfigSpace')
a = space.normal('a', 1, 2, quantization=0.01)
b = space.normal('b', 1, 2, quantization=0.01)
b.enable_if(both(gt(a, 1), lt(a, 2)))
print(space.sample())
def test_conditions_in():
space = Space('ConfigSpace')
a = space.normal('a', 1, 2, quantization=0.01)
b = space.normal('b', 1, 2, quantization=0.01)
b.enable_if(contains(a, [1, 1.5, 2]))
print(space.sample())
def test_forbid_and():
space = Space('ConfigSpace')
a = space.uniform('a', 1, 2, quantization=0.01)
a.forbid_equal(1)
a.forbid_in([1, 2])
print(space.sample())
def test_conditions_or():
space = Space('ConfigSpace')
a = space.normal('a', 1, 2, quantization=0.01)
b = space.normal('b', 1, 2, quantization=0.01)
b.enable_if(either(eq(a, 1), ne(a, 2)))
print(space.sample())
def make_big_space(backend='ConfigSpace'):
def make_subspace(sspace):
sspace.uniform('uniform_float', 0, 1)
sspace.loguniform('loguniform_float', 2, 3)
sspace.normal('normal_float', 0, 1)
sspace.lognormal('lognormal_float', 2, 3)
sspace.uniform('uniform_float_q', 0, 1, quantization=0.01)
sspace.loguniform('loguniform_float_q', 2, 3, quantization=0.01)
sspace.normal('normal_float_q', 0, 1, quantization=0.01)
sspace.lognormal('lognormal_float_q', 2, 1, quantization=0.01)
sspace.uniform('uniform_int', 0, 1, discrete=True)
sspace.loguniform('loguniform_int', 2, 3, discrete=True)
sspace.normal('normal_int', 0, 1, discrete=True)
sspace.lognormal('lognormal_int', 2, 3, discrete=True)
sspace.choices('choices', ['a', 'b', 'c', 'd'])
sspace.ordinal('ordinal', ['a', 'b', 'c', 'd'])
sspace.variable('epoch')
sspace.identity('uid')
return sspace
space = Space(backend=backend)
make_subspace(space)
make_subspace(space.subspace('sub'))
return space
def make_space(backend='ConfigSapce'):
space = Space(backend=backend)
optim = space.categorical('optimizer', ['sgd', 'adam'])
sgd_lr = space.loguniform('optimizer.lr', 1, 2, quantization=0.01)
sgd_lr.enable_if(either(eq(optim, 'adam'), eq(optim, 'sgd')))
sgd_lr.forbid_equal(1)
return space
def test_categorical(backend):
space = Space(backend=backend)
space.categorical('cat', ['a', 'b', 'c'])
space.categorical('caw', a=0.2, b=0.1, c=0.7)
space.categorical('cad', dict(a=0.2, b=0.1, c=0.7))
print(space.sample())
def test_uniform(backend):
space = Space(backend=backend)
for discrete in [True, False]:
for log in [True, False]:
for q in [None, 0.01, 1]:
space.uniform(f'a_{discrete}_{log}_{q}', 1, 2,
discrete=discrete,
log=log,
quantization=q)
print(space.sample())
def test_hyperband_api():
import random
params = Space.from_dict({'a': 'uniform(0, 1)'})
hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params)
assert not hpo.is_done()
for rung in range(3):
params_set = hpo.suggest()
for i, params in enumerate(params_set):
print(i, params)
try:
hpo.suggest()
raise RuntimeError()
except WaitingForTrials:
pass
except OptimizationIsDone:
pass
for i, params in enumerate(params_set):
v = random.uniform(0, 1)
if i == len(params_set) - 1:
v = 1e-10
hpo.observe(params, v)
print('-------')
assert hpo.is_done()
print(hpo.result())
print(hpo.info())
def __init__(self,
fidelity: Fidelity,
space: Union[Space, Dict],
seed=0,
**kwargs):
self.identity = 'uid'
for k, v in kwargs.items():
debug(f'used parameter ({k}: {v})')
if isinstance(space, dict):
space = Space.from_dict(space)
if space is not None:
space.identity(self.identity)
if isinstance(fidelity, dict):
fidelity = Fidelity.from_dict(fidelity)
self.fidelity = fidelity
self.space = space
self.seed = seed
self.seed_time = 0
self.manual_insert = 0
self.manual_samples = []
self.manual_fidelity = []
self.trials = OrderedDict()
def test_normal(backend):
for discrete in [True, False]:
for log in [True, False]:
for q in [None, 0.01, 1]:
space = Space(backend=backend)
space.normal(f'a_{discrete}_{log}_{q}',
loc=1, scale=2,
discrete=discrete,
log=log,
quantization=q)
try:
print(space.sample())
except NotImplementedError:
assert backend == 'Orion' and log is True
def test_build_bounds():
space = Space.from_dict({
'x': 'uniform(-5, 10)',
'y': 'uniform(1, 15)'
})
lower, upper = build_bounds(space.instantiate('Orion'))
assert lower.tolist() == [-5, 1]
assert upper.tolist() == [10, 15]
space = Space.from_dict({
'x': 'uniform(-5, 10)',
'y': 'loguniform(1, 15)'
})
lower, upper = build_bounds(space.instantiate('Orion'))
assert lower.tolist() == [-5, numpy.log(1)]
assert upper.tolist() == [10, numpy.log(15)]
def test_serialization_is_same(backend):
import copy
space = make_big_space(backend)
serialized = space.serialize()
new_space = Space.from_dict(copy.deepcopy(serialized))
new_serialized = new_space.serialize()
assert serialized == new_serialized
new_space.sample(**{'sub.epoch': 1, 'epoch': 2})
def test_sample_in_log_space(robo):
params = Space.from_dict({
'x': 'uniform(-5, 10)',
'y': 'loguniform(12, 15)'
})
robo = RoBO(FIDELITY, params, count=15, n_init=10)
while not robo.is_done():
samples = robo.suggest()
for sample in samples:
robo.observe(sample['uid'], branin(**sample))
assert 12 <= sample['y'] <= 15
def test_serialization_sample_big():
import copy
space = make_big_space()
serialized = space.serialize()
print(json.dumps(serialized, indent=2))
new_space = Space.from_dict(copy.deepcopy(serialized))
new_serialized = new_space.serialize()
assert serialized == new_serialized
print(json.dumps(new_space.sample(2, **{'sub.epoch': 1, 'epoch': 2}), indent=2))
def test_hyperband_simple_sequential():
import random
params = Space.from_dict({'a': 'uniform(0, 1)'})
hpo = Hyperband(Fidelity(0, 100, 10, 'epochs'), params)
for params in hpo:
hpo.observe(params, result=random.uniform(0, 1))
assert hpo.is_done()
print(hpo.result())
print(hpo.info())
def load_state_dict(self, state):
state = decompress_dict(state)
self.space = Space.from_dict(state['space'])
self.seed = state['seed']
self.manual_samples = state['manual_samples']
self.manual_fidelity = state['manual_fidelity']
self.manual_insert = state['manual_insert']
self.seed_time = state['seed_time']
self.fidelity = Fidelity.from_dict(state['fidelity'])
self.trials = OrderedDict(
(k, Trial.from_dict(t)) for k, t in state['trials'])
return self
def test_subspace(backend):
space = Space(backend=backend)
space.normal('a', 1, 2, quantization=0.01)
subspace = space.subspace('b')
subspace.normal('a', 1, 2, quantization=0.01)
print(space.sample())
def test_convert_samples_to_x():
space = {
'a': 'uniform(lower=-1, upper=1)',
'b': 'uniform(lower=-1, upper=1)',
'c': 'loguniform(lower=1, upper=10)'
}
samples = Space.from_dict(space).sample(10, seed=1)
X = convert_samples_to_x(samples, space)
assert X.shape == (10, 3)
assert X[0, 0] == samples[0]['a']
assert X[0, 1] == samples[0]['b']
assert X[0, 2] == numpy.log(samples[0]['c'])
def simulate_hpo(model, space, budget, seed):
# NOTE: We hardcode for random search for now.
# Sample points
samples = Space.from_dict(space).sample(budget, seed=seed)
X = convert_samples_to_x(samples, space)
# Score points
mean, std = model.predict(X)
scores = mean
# Select best
min_index = numpy.argmin(scores)
best_point = X[min_index]
return convert_x_to_samples(X[min_index].reshape(1, -1), space)[0]
def test_ordinal(backend):
space = Space(backend=backend)
try:
space.ordinal('ord', ['a', 'b', 'c'])
print(space.sample())
print(space.sample())
print(space.sample())
except NotImplementedError:
assert backend == 'Orion'
def test_X_with_log_dims(robo):
params = Space.from_dict({
'x': 'uniform(-5, 10)',
'y': 'loguniform(1, 15)'
})
robo = RoBO(FIDELITY, params, count=5, n_init=2)
while not robo.is_done():
samples = robo.suggest()
for sample in samples:
robo.observe(sample['uid'], branin(**sample))
assert robo.X.shape == (5, 2)
assert robo.y.shape == (5,)
assert robo.X.min(0)[0] >= -5
assert robo.X.min(0)[1] >= numpy.log(1)
assert robo.X.max(0)[0] <= 10
assert robo.X.max(0)[1] <= numpy.log(15)
def fit_model(X, y, space, seed=1):
n_samples = 50
# kernel = GPy.kern.Matern52(all_x.shape[1], variance=0.01, ARD=True)
# model = GPModel_MCMC(n_samples=n_samples, kernel=kernel)
# with all_logging_disabled(logging.ERROR):
# model.updateModel(all_x, all_y, None, None)
orion_space = Space.from_dict(space).instantiate('Orion')
lower, upper = build_bounds(orion_space)
# TODO set the seeds
model = build_model(lower,
upper,
model_type='gp_mcmc',
model_seed=1,
prior_seed=1)
model.train(X, y)
return model
def build_data(budget, variables, defaults, space):
epochs = 5
# defaults = {'a': 0, 'b': 1, 'c': 2, 'd': 3}
# params = {'c': 2, 'd': 3, 'epoch': epochs}
n_vars = len(variables)
objectives = numpy.arange(budget * (epochs + 1))
numpy.random.RandomState(0).shuffle(objectives)
objectives = objectives.reshape((epochs + 1, budget, 1))
params = Space.from_dict(space).sample(budget, seed=1)
trials = OrderedDict()
for trial_params in params:
config = copy.deepcopy(
dict(list(variables.items()) + list(defaults.items())))
config.update(trial_params)
config['uid'] = compute_identity(config, IDENTITY_SIZE)
# NOTE: We don't need objectives
trials[config['uid']] = Trial(config)
metrics = dict()
for trial_i, trial_uid in enumerate(trials.keys()):
metrics[trial_uid] = [{
'epoch': i,
'obj': objectives[i, trial_i, 0]
} for i in range(epochs + 1)]
data = []
param_names = list(sorted(space.keys()))
return create_valid_curves_xarray(trials,
metrics,
sorted(variables.keys()),
epochs,
param_names,
seed=1)
def test_deserialize_orion():
cs = Space.from_dict(orion_space)
print(cs)
print(cs.sample(fid=1))
print(json.dumps(cs.serialize(), indent=2))
from sspace import Space
space = Space()
space.uniform('lr', 0, 1)
space.ordinal('epoch', [1, 2, 3])
s1 = space.sample(seed=0)
s2 = space.sample(seed=1)
s1p = space.sample(seed=0)
print(s1)
print(s2)
print(s1p)
# [OrderedDict([('epoch', 1), ('optimizer', 'adam')])]
# [OrderedDict([('epoch', 2), ('optimizer', 'adam')])]
# [OrderedDict([('epoch', 1), ('optimizer', 'adam')])]
from sspace import Space, either, eq
import json
if __name__ == '__main__':
space = Space(backend='ConfigSpace')
optim = space.categorical('optimizer', ['sgd', 'adam'])
sgd_lr = space.loguniform('optimizer.lr', 1, 2, quantization=0.01)
sgd_lr.enable_if(either(eq(optim, 'adam'), eq(optim, 'sgd')))
sgd_lr.forbid_equal(1)
for sample in space.sample(2):
print(sample)
print(json.dumps(space.serialize(), indent=2))
