def suggest(self, n_suggestions = 1, fix_input = None): if self.X.shape[0] < self.rand_sample: sample = self.space.sample(n_suggestions) if fix_input is not None: for k, v in fix_input.items(): sample[k] = v return sample else: X, Xe = self.space.transform(self.X) y = torch.FloatTensor(self.y) num_uniqs = None if Xe.shape[1] == 0 else [len(self.space.paras[name].categories) for name in self.space.enum_names] self.model = get_model(self.model_name, X.shape[1], Xe.shape[1], y.shape[1], num_uniqs = num_uniqs, **self.model_conf) self.model.fit(X, Xe, y) acq = GeneralAcq( self.model, self.num_obj, self.num_constr, kappa = self.kappa, c_kappa = self.c_kappa, use_noise = self.use_noise) opt = EvolutionOpt(self.space, acq, pop = 100, iters = 200) suggest = opt.optimize() with torch.no_grad(): py, ps2 = self.model.predict(*self.space.transform(suggest)) largest_uncert_id = np.argmax(np.log(ps2).sum(axis = 1)) if suggest.shape[0] >= n_suggestions: selected_id = np.random.choice(suggest.shape[0], n_suggestions).tolist() if largest_uncert_id not in selected_id: selected_id[0] = largest_uncert_id return suggest.iloc[selected_id] else: rand_samp = self.space.sample(n_suggestions - suggest.shape[0]) suggest = pd.concat([suggest, rand_samp], axis = 0, ignore_index = True) return suggest
def test_thompson_sampling(model_name): model = get_model(model_name, 1, 0, 1) Xc = torch.randn(10, 1) Xe = None y = Xc**2 model = get_model(model_name, 1, 0, 1, num_epochs=30) model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape)) with torch.no_grad(): if model.support_ts: f = model.sample_f() py = f(Xc, Xe) assert (r2_score(y.numpy(), py.numpy()) > 0.5) else: with pytest.raises(NotImplementedError): f = model.sample_f()
def test_thompson_sampling(model_name): model = get_model(model_name, 1, 0, 1) if model.support_ts: assert False, "Test cases for TS have not been concstructed" else: with pytest.raises(RuntimeError): f = model.sample_f()
def test_fit_with_cont_only(model_name): Xc = torch.randn(50, 1) Xe = None y = Xc model = get_model(model_name, 1, 0, 1, num_epochs = 30) model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape)) with torch.no_grad(): py, ps2 = model.predict(Xc, Xe) check_overfitted(y, py, ps2)
def test_fit_with_enum_only(model_name): Xc = None Xe = torch.randint(2, (50, 1)) y = Xe.float() model = get_model(model_name, 0, 1, 1, num_uniqs = [2], num_epochs = 30) model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape)) with torch.no_grad(): py, ps2 = model.predict(Xc, Xe) check_overfitted(y, py, ps2)
def gather_mo_models(): model_list = [] for name in model_dict: try: model = get_model(name, 2, 0, 2) model_list.append(name) except AssertionError as e: assert str(e) == "Model only support single-output" return model_list
def test_noise(model_name): Xc = torch.randn(50, 1) Xe = None y = Xc + 0.01 * torch.randn(50, 1) model = get_model(model_name, 1, 0, 1, num_epochs = 1) model.fit(Xc, Xe, y) with torch.no_grad(): assert(model.noise.shape == torch.Size([model.num_out])) assert((model.noise >= 0).all())
def test_fit_with_nan(model_name): x = torch.randn(10, 1) y = x + 1e-4 * torch.randn(10, 1) y[0] = np.nan model = get_model(model_name, 1, 0, 1, num_epochs=5) try: model.fit(x, None, y) except: assert False, "Model won't fit with NaN" with torch.no_grad(): py, ps2 = model.predict(x, None) assert (torch.isfinite(py).all()) assert (torch.isfinite(ps2).all())
def test_fit(model_name, num_out): """ Model should be able to overfit simple linear model with training R2 > 0.8 """ model = get_model(model_name, 2, 0, num_out, num_epochs=20) X, y = rand_dataset(num_out) model.fit(X, None, y) with torch.no_grad(): py, ps2 = model.predict(X, None) ps = ps2.sqrt() assert (r2_score(y.numpy(), py.numpy()) > 0.5) assert (py + 3 * ps > y).sum() > 0.9 * y.numel() assert (py - 3 * ps < y).sum() > 0.9 * y.numel()
def test_grad(model_name): Xc = torch.randn(50, 1) Xe = None y = Xc + 0.01 * torch.randn(50, 1) model = get_model(model_name, 1, 0, 1, num_epochs = 30) model.fit(Xc, Xe, y) if model.support_grad: X_tst = torch.randn(50, 1) X_tst.requires_grad = True py, _ = model.predict(X_tst, None) obj = py.sum() obj.backward() assert(X_tst.grad is not None)
def test_fit_with_cont_enum(model_name): Xc = torch.randn(50, 1) Xe = torch.randint(2, (50, 1)) y = torch.zeros(50, 1) y[Xe.squeeze() == 1] = Xc[Xe.squeeze() == 1] y[Xe.squeeze() == 0] = -1 * Xc[Xe.squeeze() == 0] model = get_model(model_name, 1, 1, 1, num_uniqs = [2], num_epochs = 30) model.fit(Xc, Xe, y + 1e-2 * torch.randn(y.shape)) with torch.no_grad(): py, ps2 = model.predict(Xc, Xe) check_overfitted(y, py, ps2) samp = model.sample_y(Xc, Xe, 50).mean(axis = 0) assert r2_score(y.numpy(), samp.numpy()) > 0.8
def test_grad(model_name, num_out): model = get_model(model_name, 2, 0, num_out, num_epochs=2) X, y = rand_dataset(num_out) model.fit(X, None, y) X.requires_grad = True py, ps2 = model.predict(X, None) ps = ps2.sqrt() y = (py + ps).sum() y.backward() assert (X.grad.shape == X.shape) delta = 0.001 Xdelta = X.detach().clone() Xdelta[0, 0] += delta py, ps2 = model.predict(Xdelta, None) ydelta = (py + ps2.sqrt()).sum() grad_delta = ((ydelta - y) / delta).detach() assert (grad_delta.numpy() == approx(X.grad[0, 0].numpy(), abs=0.2))
def suggest(self, n_suggestions = 1, fix_input = None): assert n_suggestions == 1 if self.X.shape[0] < self.rand_sample: sample = self.space.sample(n_suggestions) if fix_input is not None: for k, v in fix_input.items(): sample[k] = v return sample else: X, Xe = self.space.transform(self.X) y = torch.FloatTensor(self.y) num_uniqs = None if Xe.shape[1] == 0 else [len(self.space.paras[name].categories) for name in self.space.enum_names] model = get_model(self.model_name, X.shape[1], Xe.shape[1], y.shape[1], num_uniqs = num_uniqs, warp = False) model.fit(X, Xe, y) acq = LCB(model, kappa = 2.0) opt = EvolutionOpt(self.space, acq, pop = 100, iters = 100) suggest = self.X.iloc[[np.argmin(self.y.reshape(-1))]] return opt.optimize(initial_suggest = suggest, fix_input = fix_input)
def test_warm_start(model_name): Xc = torch.randn(64, 1) y = Xc + 0.01 * torch.randn(Xc.shape[0], 1) model = get_model(model_name, 1, 0, 1, num_epochs=5, output_noise=False, batch_size=64) if model.support_warm_start: model.fit(Xc, None, y) with torch.no_grad(): py, ps2 = model.predict(Xc, None) err1 = ((py - y)**2).mean() model.fit(Xc, None, y) with torch.no_grad(): py, ps2 = model.predict(Xc, None) err2 = ((py - y)**2).mean() assert (err2 < err1) else: pytest.skip('%s does not support warm start' % model_name)
def suggest(self, n_suggestions=1): if self.X.shape[0] < 4 * n_suggestions: df_suggest = self.quasi_sample(n_suggestions) x_guess = [] for i, row in df_suggest.iterrows(): x_guess.append(row.to_dict()) else: X, Xe = self.space.transform(self.X) try: if self.y.min() <= 0: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='yeo-johnson')) else: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='box-cox')) if y.std() < 0.5: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='yeo-johnson')) if y.std() < 0.5: raise RuntimeError('Power transformation failed') model = get_model(self.model_name, self.space.num_numeric, self.space.num_categorical, 1, **self.model_config) model.fit(X, Xe, y) except: print('Error fitting GP') y = torch.FloatTensor(self.y).clone() filt, q = self.filter(y) print('Q = %g, kept = %d/%d' % (q, y.shape[0], self.y.shape[0])) X = X[filt] Xe = Xe[filt] y = y[filt] model = get_model(self.model_name, self.space.num_numeric, self.space.num_categorical, 1, **self.model_config) model.fit(X, Xe, y) print('Noise level: %g' % model.noise, flush=True) best_id = np.argmin(self.y.squeeze()) best_x = self.X.iloc[[best_id]] best_y = y.min() py_best, ps2_best = model.predict(*self.space.transform(best_x)) py_best = py_best.detach().numpy().squeeze() ps_best = ps2_best.sqrt().detach().numpy().squeeze() # XXX: minimize (mu, -1 * sigma) # s.t. LCB < best_y iter = max(1, self.X.shape[0] // n_suggestions) upsi = 0.5 delta = 0.01 kappa = np.sqrt( upsi * 2 * np.log(iter**(2.0 + self.X.shape[1] / 2.0) * 3 * np.pi**2 / (3 * delta))) acq = MACE(model, py_best, kappa=kappa) # LCB < py_best mu = Mean(model) sig = Sigma(model, linear_a=-1.) opt = EvolutionOpt(self.space, acq, pop=100, iters=100, verbose=True) rec = opt.optimize(initial_suggest=best_x).drop_duplicates() rec = rec[self.check_unique(rec)] cnt = 0 while rec.shape[0] < n_suggestions: rand_rec = self.quasi_sample(n_suggestions - rec.shape[0]) rand_rec = rand_rec[self.check_unique(rand_rec)] rec = rec.append(rand_rec, ignore_index=True) cnt += 1 if cnt > 3: break if rec.shape[0] < n_suggestions: rand_rec = self.quasi_sample(n_suggestions - rec.shape[0]) rec = rec.append(rand_rec, ignore_index=True) select_id = np.random.choice(rec.shape[0], n_suggestions, replace=False).tolist() x_guess = [] with torch.no_grad(): py_all = mu(*self.space.transform(rec)).squeeze().numpy() ps_all = -1 * sig(*self.space.transform(rec)).squeeze().numpy() best_pred_id = np.argmin(py_all) best_unce_id = np.argmax(ps_all) if best_unce_id not in select_id and n_suggestions > 2: select_id[0] = best_unce_id if best_pred_id not in select_id and n_suggestions > 2: select_id[1] = best_pred_id rec_selected = rec.iloc[select_id].copy() py, ps2 = model.predict(*self.space.transform(rec_selected)) rec_selected['py'] = py.squeeze().numpy() rec_selected['ps'] = ps2.sqrt().squeeze().numpy() print(rec_selected) print('Best y is %g %g %g %g' % (self.y.min(), best_y, py_best, ps_best), flush=True) for idx in select_id: x_guess.append(rec.iloc[idx].to_dict()) for rec in x_guess: for name in rec: if self.api_config[name]['type'] == 'int': rec[name] = int(rec[name]) return x_guess
def test_sample(model_name, num_out): model = get_model(model_name, 2, 0, num_out, num_epochs=1) X, y = rand_dataset(num_out) model.fit(X, None, y) assert (model.sample_y(X, None, 5).shape == torch.Size([5, X.shape[0], y.shape[1]]))
def test_noise(model_name, num_out): model = get_model(model_name, 2, 0, num_out, num_epochs=5) X, y = rand_dataset(num_out) model.fit(X, None, y) assert (model.noise.shape == torch.Size([model.num_out])) assert ((model.noise >= 0).all())
def test_init(model_name, num_out): model = get_model(model_name, 2, 0, num_out) assert (model.num_out == num_out)
def test_model_can_be_initialized(model_name): model = get_model(model_name, 1, 0, 1) model_class = get_model_class(model_name) model = model_class(1, 0, 1)
def suggest(self, n_suggestions=1, fix_input=None): if self.X.shape[0] < self.rand_sample: sample = self.quasi_sample(n_suggestions, fix_input) return sample else: X, Xe = self.space.transform(self.X) try: if self.y.min() <= 0: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='yeo-johnson')) else: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='box-cox')) if y.std() < 0.5: y = torch.FloatTensor( power_transform(self.y / self.y.std(), method='yeo-johnson')) if y.std() < 0.5: raise RuntimeError('Power transformation failed') model = get_model(self.model_name, self.space.num_numeric, self.space.num_categorical, 1, **self.model_config) model.fit(X, Xe, y) except: y = torch.FloatTensor(self.y).clone() model = get_model(self.model_name, self.space.num_numeric, self.space.num_categorical, 1, **self.model_config) model.fit(X, Xe, y) best_id = np.argmin(self.y.squeeze()) best_x = self.X.iloc[[best_id]] best_y = y.min() py_best, ps2_best = model.predict(*self.space.transform(best_x)) py_best = py_best.detach().numpy().squeeze() ps_best = ps2_best.sqrt().detach().numpy().squeeze() iter = max(1, self.X.shape[0] // n_suggestions) upsi = 0.5 delta = 0.01 # kappa = np.sqrt(upsi * 2 * np.log(iter ** (2.0 + self.X.shape[1] / 2.0) * 3 * np.pi**2 / (3 * delta))) kappa = np.sqrt(upsi * 2 * ((2.0 + self.X.shape[1] / 2.0) * np.log(iter) + np.log(3 * np.pi**2 / (3 * delta)))) acq = MACE(model, py_best, kappa=kappa) # LCB < py_best mu = Mean(model) sig = Sigma(model, linear_a=-1.) opt = EvolutionOpt(self.space, acq, pop=100, iters=100, verbose=False, es=self.es) rec = opt.optimize(initial_suggest=best_x, fix_input=fix_input).drop_duplicates() rec = rec[self.check_unique(rec)] cnt = 0 while rec.shape[0] < n_suggestions: rand_rec = self.quasi_sample(n_suggestions - rec.shape[0], fix_input) rand_rec = rand_rec[self.check_unique(rand_rec)] rec = rec.append(rand_rec, ignore_index=True) cnt += 1 if cnt > 3: # sometimes the design space is so small that duplicated sampling is unavoidable break if rec.shape[0] < n_suggestions: rand_rec = self.quasi_sample(n_suggestions - rec.shape[0], fix_input) rec = rec.append(rand_rec, ignore_index=True) select_id = np.random.choice(rec.shape[0], n_suggestions, replace=False).tolist() x_guess = [] with torch.no_grad(): py_all = mu(*self.space.transform(rec)).squeeze().numpy() ps_all = -1 * sig(*self.space.transform(rec)).squeeze().numpy() best_pred_id = np.argmin(py_all) best_unce_id = np.argmax(ps_all) if best_unce_id not in select_id and n_suggestions > 2: select_id[0] = best_unce_id if best_pred_id not in select_id and n_suggestions > 2: select_id[1] = best_pred_id rec_selected = rec.iloc[select_id].copy() return rec_selected
def test_model_can_be_initialized(model_name): model = get_model(model_name, 1, 0, 1)
def test_is_basic_model(model_name): assert(issubclass(type(get_model(model_name, 1, 0, 1)), BaseModel))