def test_hp_to_skopt_space(self):
"""tests that if we give space as hp.space, then can we get .x_iters and .best_paras
successfully.
"""
ai4water_args = {
'model': 'XGBoostRegressor',
'inputs':
['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10'],
'outputs': ['target']
}
opt = HyperOpt("tpe",
param_space=[
Integer(low=1000,
high=2000,
name='n_estimators',
num_samples=5),
Integer(low=3,
high=6,
name='max_depth',
num_samples=5),
Categorical(['gbtree', 'gblinear', 'dart'],
name='booster')
],
ai4water_args=ai4water_args,
data=data,
backend='hyperopt',
num_iterations=5)
opt.fit()
check_attrs(opt, 3, ai4water_args)
assert isinstance(list(opt.best_paras().values())[-1], str)
def test_named_custom_bayes(self):
dims = [
Integer(low=1000, high=2000, name='n_estimators'),
Integer(low=3, high=6, name='max_depth'),
Real(low=1e-5, high=0.1, name='learning_rate'),
Categorical(categories=["gbtree", "dart"], name="booster")
]
def f(**kwargs):
kwargs['objective'] = 'reg:squarederror'
kwargs = Jsonize(kwargs)()
model = Model(inputs=inputs,
outputs=outputs,
lookback=1,
batches="2d",
val_data="same",
test_fraction=0.3,
model={"xgboostregressor": kwargs},
transformation=None,
data=data,
prefix='testing',
verbosity=0)
model.fit(indices="random")
t, p = model.predict(indices=model.test_indices, prefix='test')
mse = RegressionMetrics(t, p).mse()
print(f"Validation mse {mse}")
return mse
opt = HyperOpt(
"bayes",
objective_fn=f,
param_space=dims,
acq_func='EI', # Expected Improvement.
n_calls=12,
# acq_optimizer='auto',
x0=[1000, 3, 0.01, "gbtree"],
n_random_starts=3, # the number of random initialization points
random_state=2)
opt.fit()
check_attrs(opt, 4)
return
def test_bayes(self):
# testing for sklearn-based model with gp_min
# https://scikit-optimize.github.io/stable/modules/generated/skopt.BayesSearchCV.html
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
if int(''.join(sklearn.__version__.split('.')[1])) > 22:
# https://github.com/scikit-optimize/scikit-optimize/issues/978
pass
else:
opt = HyperOpt("bayes",
objective_fn=SVC(),
param_space={
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly',
'rbf']),
},
n_iter=32,
random_state=0)
# executes bayesian optimization
_ = opt.fit(X_train, y_train)
# model can be saved, used for predictions or scoring
np.testing.assert_almost_equal(0.9736842105263158,
opt.score(X_test, y_test), 5)
print("BayesSearchCV test passed")
return
def run_unified_interface(algorithm,
backend,
num_iterations,
num_samples=None):
def fn(**suggestion):
model = Model(inputs=inputs,
outputs=outputs,
model={"xgboostregressor": suggestion},
data=data,
prefix=f'test_{algorithm}_xgboost_{backend}',
verbosity=0)
model.fit(indices="random")
t, p = model.predict(indices=model.test_indices, prefix='test')
mse = RegressionMetrics(t, p).mse()
return mse
search_space = [
Categorical(['gbtree', 'dart'], name='booster'),
Integer(low=1000,
high=2000,
name='n_estimators',
num_samples=num_samples),
Real(low=1.0e-5,
high=0.1,
name='learning_rate',
num_samples=num_samples)
]
optimizer = HyperOpt(algorithm,
objective_fn=fn,
param_space=search_space,
backend=backend,
num_iterations=num_iterations,
opt_path=os.path.join(
os.getcwd(),
f'results\\test_{algorithm}_xgboost_{backend}'))
optimizer.fit()
check_attrs(optimizer, 3)
for f in [
"fanova_importance.html", "convergence.png", "iterations.json",
"iterations_sorted.json"
]:
fpath = os.path.join(optimizer.opt_path, f)
assert os.path.exists(fpath)
return optimizer
def test_ai4water_bayes(self):
dims = [
Integer(low=1000, high=2000, name='n_estimators'),
Integer(low=3, high=6, name='max_depth'),
Real(low=1e-5, high=0.1, name='learning_rate'),
Categorical(categories=["gbtree", "dart"], name="booster")
]
ai4water_args = {
"inputs": inputs,
"outputs": outputs,
"lookback": 1,
"batches": "2d",
"val_data": "same",
"test_fraction": 0.3,
"model": {
"xgboostregressor": {}
},
#"ml_model_args": {'objective': 'reg:squarederror'}, TODO
"transformation": None
}
opt = HyperOpt(
"bayes",
param_space=dims,
ai4water_args=ai4water_args,
data=data,
acq_func='EI', # Expected Improvement.
n_calls=12,
# acq_optimizer='auto',
x0=[1000, 3, 0.01, "gbtree"],
n_random_starts=3, # the number of random initialization points
random_state=2)
opt.fit()
check_attrs(opt, 4, ai4water_args)
return
lookback=int(suggestion['lookback']),
lr=float(suggestion['lr']),
batch_size=int(suggestion['batch_size']),
data=data['224206'],
verbosity=0,
epochs=500,
prefix=_suffix)
h = model.fit()
return np.min(h.history['val_loss'])
num_samples = 4
d = [
Categorical(categories=['relu', 'sigmoid', 'tanh', 'linear'],
name='activation'),
Integer(low=3, high=15, name='lookback', num_samples=num_samples),
Categorical(categories=[16, 32, 64, 128], name='batch_size'),
Real(low=1e-5, high=0.001, name='lr', num_samples=num_samples)
]
x0 = ['relu', 5, 32, 0.0001]
optimizer = HyperOpt(m,
objective_fn=objective_fn,
param_space=d,
num_iterations=50,
x0=x0,
use_named_args=True,
opt_path=os.path.join(os.getcwd(),
f'results{SEP}{_suffix}'))
r = optimizer.fit()
def test_integer_grid(self):
grit = [1, 2, 3, 4, 5]
r = Integer(grid=grit)
np.testing.assert_array_equal(grit, r.grid)
def test_integer_steps(self):
r = Integer(low=10, high=100, step=20)
grit = r.grid
assert len(grit) == 5
def test_integer_num_samples(self):
r = Integer(low=10, high=100, num_samples=20)
grit = r.grid
assert len(grit) == 20