def suggest(self, n_suggestions = 1, fix_input = None):
if self.X.shape[0] < self.rand_sample:
sample = self.space.sample(n_suggestions)
if fix_input is not None:
for k, v in fix_input.items():
sample[k] = v
return sample
else:
X, Xe = self.space.transform(self.X)
y = torch.FloatTensor(self.y)
num_uniqs = None if Xe.shape[1] == 0 else [len(self.space.paras[name].categories) for name in self.space.enum_names]
self.model = get_model(self.model_name, X.shape[1], Xe.shape[1], y.shape[1], num_uniqs = num_uniqs, **self.model_conf)
self.model.fit(X, Xe, y)
acq = GeneralAcq(
self.model,
self.num_obj,
self.num_constr,
kappa = self.kappa,
c_kappa = self.c_kappa,
use_noise = self.use_noise)
opt = EvolutionOpt(self.space, acq, pop = 100, iters = 200)
suggest = opt.optimize()
with torch.no_grad():
py, ps2 = self.model.predict(*self.space.transform(suggest))
largest_uncert_id = np.argmax(np.log(ps2).sum(axis = 1))
if suggest.shape[0] >= n_suggestions:
selected_id = np.random.choice(suggest.shape[0], n_suggestions).tolist()
if largest_uncert_id not in selected_id:
selected_id[0] = largest_uncert_id
return suggest.iloc[selected_id]
else:
rand_samp = self.space.sample(n_suggestions - suggest.shape[0])
suggest = pd.concat([suggest, rand_samp], axis = 0, ignore_index = True)
return suggest
def test_thompson_sampling(model_name):
model = get_model(model_name, 1, 0, 1)
Xc = torch.randn(10, 1)
Xe = None
y = Xc**2
model = get_model(model_name, 1, 0, 1, num_epochs=30)
model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape))
with torch.no_grad():
if model.support_ts:
f = model.sample_f()
py = f(Xc, Xe)
assert (r2_score(y.numpy(), py.numpy()) > 0.5)
else:
with pytest.raises(NotImplementedError):
f = model.sample_f()
def test_thompson_sampling(model_name):
model = get_model(model_name, 1, 0, 1)
if model.support_ts:
assert False, "Test cases for TS have not been concstructed"
else:
with pytest.raises(RuntimeError):
f = model.sample_f()
def test_fit_with_cont_only(model_name):
Xc = torch.randn(50, 1)
Xe = None
y = Xc
model = get_model(model_name, 1, 0, 1, num_epochs = 30)
model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape))
with torch.no_grad():
py, ps2 = model.predict(Xc, Xe)
check_overfitted(y, py, ps2)
def test_fit_with_enum_only(model_name):
Xc = None
Xe = torch.randint(2, (50, 1))
y = Xe.float()
model = get_model(model_name, 0, 1, 1, num_uniqs = [2], num_epochs = 30)
model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape))
with torch.no_grad():
py, ps2 = model.predict(Xc, Xe)
check_overfitted(y, py, ps2)
def gather_mo_models():
model_list = []
for name in model_dict:
try:
model = get_model(name, 2, 0, 2)
model_list.append(name)
except AssertionError as e:
assert str(e) == "Model only support single-output"
return model_list
def test_noise(model_name):
Xc = torch.randn(50, 1)
Xe = None
y = Xc + 0.01 * torch.randn(50, 1)
model = get_model(model_name, 1, 0, 1, num_epochs = 1)
model.fit(Xc, Xe, y)
with torch.no_grad():
assert(model.noise.shape == torch.Size([model.num_out]))
assert((model.noise >= 0).all())
def test_fit_with_nan(model_name):
x = torch.randn(10, 1)
y = x + 1e-4 * torch.randn(10, 1)
y[0] = np.nan
model = get_model(model_name, 1, 0, 1, num_epochs=5)
try:
model.fit(x, None, y)
except:
assert False, "Model won't fit with NaN"
with torch.no_grad():
py, ps2 = model.predict(x, None)
assert (torch.isfinite(py).all())
assert (torch.isfinite(ps2).all())
def test_fit(model_name, num_out):
"""
Model should be able to overfit simple linear model with training R2 > 0.8
"""
model = get_model(model_name, 2, 0, num_out, num_epochs=20)
X, y = rand_dataset(num_out)
model.fit(X, None, y)
with torch.no_grad():
py, ps2 = model.predict(X, None)
ps = ps2.sqrt()
assert (r2_score(y.numpy(), py.numpy()) > 0.5)
assert (py + 3 * ps > y).sum() > 0.9 * y.numel()
assert (py - 3 * ps < y).sum() > 0.9 * y.numel()
def test_grad(model_name):
Xc = torch.randn(50, 1)
Xe = None
y = Xc + 0.01 * torch.randn(50, 1)
model = get_model(model_name, 1, 0, 1, num_epochs = 30)
model.fit(Xc, Xe, y)
if model.support_grad:
X_tst = torch.randn(50, 1)
X_tst.requires_grad = True
py, _ = model.predict(X_tst, None)
obj = py.sum()
obj.backward()
assert(X_tst.grad is not None)
def test_fit_with_cont_enum(model_name):
Xc = torch.randn(50, 1)
Xe = torch.randint(2, (50, 1))
y = torch.zeros(50, 1)
y[Xe.squeeze() == 1] = Xc[Xe.squeeze() == 1]
y[Xe.squeeze() == 0] = -1 * Xc[Xe.squeeze() == 0]
model = get_model(model_name, 1, 1, 1, num_uniqs = [2], num_epochs = 30)
model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape))
with torch.no_grad():
py, ps2 = model.predict(Xc, Xe)
check_overfitted(y, py, ps2)
samp = model.sample_y(Xc, Xe, 50).mean(axis = 0)
assert r2_score(y.numpy(), samp.numpy()) > 0.8
def test_grad(model_name, num_out):
model = get_model(model_name, 2, 0, num_out, num_epochs=2)
X, y = rand_dataset(num_out)
model.fit(X, None, y)
X.requires_grad = True
py, ps2 = model.predict(X, None)
ps = ps2.sqrt()
y = (py + ps).sum()
y.backward()
assert (X.grad.shape == X.shape)
delta = 0.001
Xdelta = X.detach().clone()
Xdelta[0, 0] += delta
py, ps2 = model.predict(Xdelta, None)
ydelta = (py + ps2.sqrt()).sum()
grad_delta = ((ydelta - y) / delta).detach()
assert (grad_delta.numpy() == approx(X.grad[0, 0].numpy(), abs=0.2))
def suggest(self, n_suggestions = 1, fix_input = None):
assert n_suggestions == 1
if self.X.shape[0] < self.rand_sample:
sample = self.space.sample(n_suggestions)
if fix_input is not None:
for k, v in fix_input.items():
sample[k] = v
return sample
else:
X, Xe = self.space.transform(self.X)
y = torch.FloatTensor(self.y)
num_uniqs = None if Xe.shape[1] == 0 else [len(self.space.paras[name].categories) for name in self.space.enum_names]
model = get_model(self.model_name, X.shape[1], Xe.shape[1], y.shape[1], num_uniqs = num_uniqs, warp = False)
model.fit(X, Xe, y)
acq = LCB(model, kappa = 2.0)
opt = EvolutionOpt(self.space, acq, pop = 100, iters = 100)
suggest = self.X.iloc[[np.argmin(self.y.reshape(-1))]]
return opt.optimize(initial_suggest = suggest, fix_input = fix_input)
def test_warm_start(model_name):
Xc = torch.randn(64, 1)
y = Xc + 0.01 * torch.randn(Xc.shape[0], 1)
model = get_model(model_name,
1,
0,
1,
num_epochs=5,
output_noise=False,
batch_size=64)
if model.support_warm_start:
model.fit(Xc, None, y)
with torch.no_grad():
py, ps2 = model.predict(Xc, None)
err1 = ((py - y)**2).mean()
model.fit(Xc, None, y)
with torch.no_grad():
py, ps2 = model.predict(Xc, None)
err2 = ((py - y)**2).mean()
assert (err2 < err1)
else:
pytest.skip('%s does not support warm start' % model_name)
def suggest(self, n_suggestions=1):
if self.X.shape[0] < 4 * n_suggestions:
df_suggest = self.quasi_sample(n_suggestions)
x_guess = []
for i, row in df_suggest.iterrows():
x_guess.append(row.to_dict())
else:
X, Xe = self.space.transform(self.X)
try:
if self.y.min() <= 0:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='yeo-johnson'))
else:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='box-cox'))
if y.std() < 0.5:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='yeo-johnson'))
if y.std() < 0.5:
raise RuntimeError('Power transformation failed')
model = get_model(self.model_name, self.space.num_numeric,
self.space.num_categorical, 1,
**self.model_config)
model.fit(X, Xe, y)
except:
print('Error fitting GP')
y = torch.FloatTensor(self.y).clone()
filt, q = self.filter(y)
print('Q = %g, kept = %d/%d' %
(q, y.shape[0], self.y.shape[0]))
X = X[filt]
Xe = Xe[filt]
y = y[filt]
model = get_model(self.model_name, self.space.num_numeric,
self.space.num_categorical, 1,
**self.model_config)
model.fit(X, Xe, y)
print('Noise level: %g' % model.noise, flush=True)
best_id = np.argmin(self.y.squeeze())
best_x = self.X.iloc[[best_id]]
best_y = y.min()
py_best, ps2_best = model.predict(*self.space.transform(best_x))
py_best = py_best.detach().numpy().squeeze()
ps_best = ps2_best.sqrt().detach().numpy().squeeze()
# XXX: minimize (mu, -1 * sigma)
# s.t. LCB < best_y
iter = max(1, self.X.shape[0] // n_suggestions)
upsi = 0.5
delta = 0.01
kappa = np.sqrt(
upsi * 2 *
np.log(iter**(2.0 + self.X.shape[1] / 2.0) * 3 * np.pi**2 /
(3 * delta)))
acq = MACE(model, py_best, kappa=kappa) # LCB < py_best
mu = Mean(model)
sig = Sigma(model, linear_a=-1.)
opt = EvolutionOpt(self.space,
acq,
pop=100,
iters=100,
verbose=True)
rec = opt.optimize(initial_suggest=best_x).drop_duplicates()
rec = rec[self.check_unique(rec)]
cnt = 0
while rec.shape[0] < n_suggestions:
rand_rec = self.quasi_sample(n_suggestions - rec.shape[0])
rand_rec = rand_rec[self.check_unique(rand_rec)]
rec = rec.append(rand_rec, ignore_index=True)
cnt += 1
if cnt > 3:
break
if rec.shape[0] < n_suggestions:
rand_rec = self.quasi_sample(n_suggestions - rec.shape[0])
rec = rec.append(rand_rec, ignore_index=True)
select_id = np.random.choice(rec.shape[0],
n_suggestions,
replace=False).tolist()
x_guess = []
with torch.no_grad():
py_all = mu(*self.space.transform(rec)).squeeze().numpy()
ps_all = -1 * sig(*self.space.transform(rec)).squeeze().numpy()
best_pred_id = np.argmin(py_all)
best_unce_id = np.argmax(ps_all)
if best_unce_id not in select_id and n_suggestions > 2:
select_id[0] = best_unce_id
if best_pred_id not in select_id and n_suggestions > 2:
select_id[1] = best_pred_id
rec_selected = rec.iloc[select_id].copy()
py, ps2 = model.predict(*self.space.transform(rec_selected))
rec_selected['py'] = py.squeeze().numpy()
rec_selected['ps'] = ps2.sqrt().squeeze().numpy()
print(rec_selected)
print('Best y is %g %g %g %g' %
(self.y.min(), best_y, py_best, ps_best),
flush=True)
for idx in select_id:
x_guess.append(rec.iloc[idx].to_dict())
for rec in x_guess:
for name in rec:
if self.api_config[name]['type'] == 'int':
rec[name] = int(rec[name])
return x_guess
def test_sample(model_name, num_out):
model = get_model(model_name, 2, 0, num_out, num_epochs=1)
X, y = rand_dataset(num_out)
model.fit(X, None, y)
assert (model.sample_y(X, None,
5).shape == torch.Size([5, X.shape[0], y.shape[1]]))
def test_noise(model_name, num_out):
model = get_model(model_name, 2, 0, num_out, num_epochs=5)
X, y = rand_dataset(num_out)
model.fit(X, None, y)
assert (model.noise.shape == torch.Size([model.num_out]))
assert ((model.noise >= 0).all())
def test_init(model_name, num_out):
model = get_model(model_name, 2, 0, num_out)
assert (model.num_out == num_out)
def test_model_can_be_initialized(model_name):
model = get_model(model_name, 1, 0, 1)
model_class = get_model_class(model_name)
model = model_class(1, 0, 1)
def suggest(self, n_suggestions=1, fix_input=None):
if self.X.shape[0] < self.rand_sample:
sample = self.quasi_sample(n_suggestions, fix_input)
return sample
else:
X, Xe = self.space.transform(self.X)
try:
if self.y.min() <= 0:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='yeo-johnson'))
else:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='box-cox'))
if y.std() < 0.5:
y = torch.FloatTensor(
power_transform(self.y / self.y.std(),
method='yeo-johnson'))
if y.std() < 0.5:
raise RuntimeError('Power transformation failed')
model = get_model(self.model_name, self.space.num_numeric,
self.space.num_categorical, 1,
**self.model_config)
model.fit(X, Xe, y)
except:
y = torch.FloatTensor(self.y).clone()
model = get_model(self.model_name, self.space.num_numeric,
self.space.num_categorical, 1,
**self.model_config)
model.fit(X, Xe, y)
best_id = np.argmin(self.y.squeeze())
best_x = self.X.iloc[[best_id]]
best_y = y.min()
py_best, ps2_best = model.predict(*self.space.transform(best_x))
py_best = py_best.detach().numpy().squeeze()
ps_best = ps2_best.sqrt().detach().numpy().squeeze()
iter = max(1, self.X.shape[0] // n_suggestions)
upsi = 0.5
delta = 0.01
# kappa = np.sqrt(upsi * 2 * np.log(iter ** (2.0 + self.X.shape[1] / 2.0) * 3 * np.pi**2 / (3 * delta)))
kappa = np.sqrt(upsi * 2 *
((2.0 + self.X.shape[1] / 2.0) * np.log(iter) +
np.log(3 * np.pi**2 / (3 * delta))))
acq = MACE(model, py_best, kappa=kappa) # LCB < py_best
mu = Mean(model)
sig = Sigma(model, linear_a=-1.)
opt = EvolutionOpt(self.space,
acq,
pop=100,
iters=100,
verbose=False,
es=self.es)
rec = opt.optimize(initial_suggest=best_x,
fix_input=fix_input).drop_duplicates()
rec = rec[self.check_unique(rec)]
cnt = 0
while rec.shape[0] < n_suggestions:
rand_rec = self.quasi_sample(n_suggestions - rec.shape[0],
fix_input)
rand_rec = rand_rec[self.check_unique(rand_rec)]
rec = rec.append(rand_rec, ignore_index=True)
cnt += 1
if cnt > 3:
# sometimes the design space is so small that duplicated sampling is unavoidable
break
if rec.shape[0] < n_suggestions:
rand_rec = self.quasi_sample(n_suggestions - rec.shape[0],
fix_input)
rec = rec.append(rand_rec, ignore_index=True)
select_id = np.random.choice(rec.shape[0],
n_suggestions,
replace=False).tolist()
x_guess = []
with torch.no_grad():
py_all = mu(*self.space.transform(rec)).squeeze().numpy()
ps_all = -1 * sig(*self.space.transform(rec)).squeeze().numpy()
best_pred_id = np.argmin(py_all)
best_unce_id = np.argmax(ps_all)
if best_unce_id not in select_id and n_suggestions > 2:
select_id[0] = best_unce_id
if best_pred_id not in select_id and n_suggestions > 2:
select_id[1] = best_pred_id
rec_selected = rec.iloc[select_id].copy()
return rec_selected
def test_model_can_be_initialized(model_name):
model = get_model(model_name, 1, 0, 1)
def test_is_basic_model(model_name):
assert(issubclass(type(get_model(model_name, 1, 0, 1)), BaseModel))
