def test_opt():
space = DesignSpace().parse([{
'name': 'x1',
'type': 'num',
'lb': -3.0,
'ub': 3.0
}])
acq = ToyExample()
opt = EvolutionOpt(space, acq, pop=10)
rec = opt.optimize(initial_suggest=space.sample(3))
x, xe = space.transform(rec)
assert (approx(1.0, 1e-3) == acq(x, xe)[:, 0].squeeze().item())
def test_opt_int():
space = DesignSpace().parse([{
'name': 'x1',
'type': 'num',
'lb': -3.0,
'ub': 3.0
}, {
'name': 'x2',
'type': 'int',
'lb': -3.0,
'ub': 3.0
}])
acq = ToyExample()
opt = EvolutionOpt(space, acq, pop=10)
rec = opt.optimize()
assert (approx(1.0,
1e-3) == acq(*space.transform(rec))[:, 0].squeeze().item())
def test_contextual_opt(opt_cls):
space = DesignSpace().parse([
{'name' : 'x0', 'type' : 'int', 'lb' : 0, 'ub' : 7},
{'name' : 'x1', 'type' : 'int', 'lb' : 0, 'ub' : 7},
])
opt = opt_cls(space, rand_sample = 2, model_name = 'rf')
for i in range(2):
n_suggestions = 8 if opt.support_parallel_opt else 1
rec = opt.suggest(n_suggestions = n_suggestions, fix_input = {'x0' : 3})
y = (rec[['x0', 'x1']].values ** 2).sum(axis = 1, keepdims =True)
assert((rec['x0'] == 3).all())
opt.observe(rec, y)
def parse_space_from_bayesmark(api_config) -> DesignSpace:
"""
Parse design space of bayesmark (https://github.com/uber/bayesmark)
"""
space = DesignSpace()
params = []
for param_name in api_config:
param_conf = api_config[param_name]
param_type = param_conf['type']
param_space = param_conf.get('space', None)
param_range = param_conf.get("range", None)
param_values = param_conf.get("values", None)
bo_param_conf = {'name' : param_name}
if param_type == 'int': # ignore 'log' space # TODO: support log-scale int
bo_param_conf['type'] = 'int'
bo_param_conf['lb'] = param_range[0]
bo_param_conf['ub'] = param_range[1]
elif param_type == 'bool':
bo_param_conf['type'] = 'bool'
elif param_type in ('cat', 'ordinal'):
bo_param_conf['type'] = 'cat'
bo_param_conf['categories'] = list(set(param_values))
elif param_type == 'real':
if param_space in ('log', 'logit'):
bo_param_conf['type'] = 'pow'
bo_param_conf['base'] = 10
bo_param_conf['lb'] = param_range[0]
bo_param_conf['ub'] = param_range[1]
else:
bo_param_conf['type'] = 'num'
bo_param_conf['lb'] = param_range[0]
bo_param_conf['ub'] = param_range[1]
else:
assert False, "type %s not handled in API" % param_type
params.append(bo_param_conf)
space.parse(params)
return space
def test_opt(model_name, opt_cls):
space = DesignSpace().parse([
{'name' : 'x0', 'type' : 'num', 'lb' : -3, 'ub' : 7},
{'name' : 'x1', 'type' : 'cat', 'categories' : ['a', 'b', 'c']}
])
opt = opt_cls(space, rand_sample = 10, model_name = model_name)
for i in range(11):
num_suggest = 8 if opt.support_parallel_opt else 1
rec = opt.suggest(n_suggestions = num_suggest)
y = obj(rec)
if y.shape[0] > 1 and i > 0:
y[np.argmax(y.reshape(-1))] = np.inf
opt.observe(rec, y)
if opt.y.shape[0] > 11:
break
def test_mo():
space = DesignSpace().parse([{
'name': 'x1',
'type': 'num',
'lb': -3.0,
'ub': 3.0
}, {
'name': 'x2',
'type': 'int',
'lb': -3.0,
'ub': 3.0
}])
acq = ToyExampleMO()
opt = EvolutionOpt(space, acq, pop=10)
rec = opt.optimize()
assert (rec.shape[0] == 10)
def test_opt_fix():
space = DesignSpace().parse([{
'name': 'x1',
'type': 'num',
'lb': -3.0,
'ub': 3.0
}, {
'name': 'x2',
'type': 'num',
'lb': -3.0,
'ub': 3.0
}])
acq = ToyExample()
opt = EvolutionOpt(space, acq, pop=10)
rec = opt.optimize(fix_input={'x1': 1.0})
print(rec)
assert (rec['x1'].values == approx(1.0, 1e-3))
def test_design_space():
space = DesignSpace().parse([{
'name': 'x0',
'type': 'num',
'lb': 0,
'ub': 7
}, {
'name': 'x1',
'type': 'int',
'lb': 0,
'ub': 7
}, {
'name': 'x2',
'type': 'pow',
'lb': 1e-4,
'ub': 1e-2,
'base': 10
}, {
'name': 'x3',
'type': 'cat',
'categories': ['a', 'b', 'c']
}, {
'name': 'x4',
'type': 'bool'
}])
assert space.numeric_names == ['x0', 'x1', 'x2', 'x4']
assert space.enum_names == ['x3']
assert space.num_paras == 5
assert space.num_numeric == 4
assert space.num_categorical == 1
samp = space.sample(10)
x, xe = space.transform(samp)
x_, xe_ = space.transform(space.inverse_transform(x, xe))
assert (x - x_).abs().max() < 1e-4
assert (space.opt_lb <= space.opt_ub).all()
assert not space.paras['x0'].is_discrete
assert space.paras['x1'].is_discrete
assert not space.paras['x2'].is_discrete
assert space.paras['x3'].is_discrete
assert space.paras['x4'].is_discrete
dir_list.sort()
# [t_lateral_f, left_right_rim_res, left_rim_f, four_side_rim_res, four_side_rim_latf, angvel] [5,]
param_list = [{
'name': 'res',
'type': 'num',
'lb': .5,
'ub': .95
}, {
'name': 'f',
'type': 'num',
'lb': 0.,
'ub': .8
}]
np.random.seed(99)
space = DesignSpace().parse(param_list)
opt = HEBO(space)
# init value
rec = pd.DataFrame({
'res': [.9, .7, .95, .59, .67, .75, .62, .8],
'f': [.1, .3, .45, .6, .01, .4, .6, .7]
})
data = []
objs = []
params = []
n_sugg = 15
fileHeader = ["iters", "res", "f", "objs"]
with open(
def test_cond(output_noise, rand_prior, num_processes):
space = DesignSpace().parse([
{
'name': 'param_A',
'type': 'bool'
},
{
'name': 'param_B',
'type': 'num',
'lb': 0,
'ub': 1
},
{
'name': 'param_C',
'type': 'num',
'lb': 0,
'ub': 1
},
{
'name': 'param_D',
'type': 'num',
'lb': 0,
'ub': 1
},
{
'name': 'param_A2',
'type': 'cat',
'categories': ['a', 'b', 'c']
},
{
'name': 'param_B2',
'type': 'num',
'lb': 0,
'ub': 1
},
])
X = space.sample(50)
y = torch.randn(50, 1)
cond = {
'param_A': None,
'param_B': ('param_A', True),
'param_C': ('param_A', True),
'param_D':
('param_A',
[True]), # XXX: enable value could be scaler value or list of values
'param_A2': None,
'param_B2': ('param_A2', ['a'])
}
model = ConditionalDeepEnsemble(4,
0,
1,
space,
cond,
num_epoch=1,
rand_prior=rand_prior,
num_processes=num_processes,
output_noise=output_noise)
model.fit(*space.transform(X), y)
model.fit(*space.transform(X), y) # test warm-starting
with torch.no_grad():
py, ps2 = model.predict(*space.transform(X))
disabled = ((X.param_A == False) & (X.param_A2 != 'a')).values
assert (py[disabled].var() == approx(0., abs=1e-4))
def sklearn_tuner(model_class,
space_config: [dict],
X: np.ndarray,
y: np.ndarray,
metric: Callable,
greater_is_better: bool = True,
n_splits=5,
max_iter=16,
report=False) -> (dict, pd.DataFrame):
"""Tuning sklearn estimator
Parameters:
-------------------
model_class: class of sklearn estimator
space_config: list of dict, specifying search space
X, y: data used to for cross-valiation
metrics: metric function in sklearn.metrics
greater_is_better: whether a larger metric value is better
n_splits: split data into `n_splits` parts for cross validation
max_iter: number of trials
Returns:
-------------------
Best hyper-parameters and all visited data
Example:
-------------------
from sklearn.datasets import load_boston
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score, mean_squared_error
from bo.sklearn_tuner import sklearn_tuner
space_cfg = [
{'name' : 'max_depth', 'type' : 'int', 'lb' : 1, 'ub' : 20},
{'name' : 'min_samples_leaf', 'type' : 'num', 'lb' : 1e-4, 'ub' : 0.5},
{'name' : 'max_features', 'type' : 'cat', 'categories' : ['auto', 'sqrt', 'log2']},
{'name' : 'bootstrap', 'type' : 'bool'},
{'name' : 'min_impurity_decrease', 'type' : 'pow', 'lb' : 1e-4, 'ub' : 1.0},
]
X, y = load_boston(return_X_y = True)
result = sklearn_tuner(RandomForestRegressor, space_cfg, X, y, metric = r2_score, max_iter = 16)
"""
space = DesignSpace().parse(space_config)
opt = HEBO(space)
for i in range(max_iter):
rec = opt.suggest()
model = model_class(**rec.iloc[0].to_dict())
pred = cross_val_predict(model,
X,
y,
cv=KFold(n_splits=n_splits, shuffle=True))
score_v = metric(y, pred)
sign = -1. if greater_is_better else 1.0
opt.observe(rec, np.array([sign * score_v]))
print('Iter %d, best metric: %g' % (i, sign * opt.y.min()))
best_id = np.argmin(opt.y.reshape(-1))
best_hyp = opt.X.iloc[best_id]
df_report = opt.X.copy()
df_report['metric'] = sign * opt.y
if report:
return best_hyp.to_dict(), df_report
else:
return best_hyp.to_dict()