def _initialize_bayesian_gp(xr, xg, y): """ Initialize a Bayesian GP where we use the elgacy data to form priors on the model's (constant) mean function * kernel parameters. Get Gaussians for priors based on empirical moments of models trained on legacy task. For now, weight each legacy task equally """ assert xg.shape[1] == 1 # Not sure how this might work otherwise # 1) Train models on legacy tasks to get data to form priors: legacy_parameters = { "scales": [], "variances": [], "jitters": [], "biases": [] } for xgi in np.unique(xg): if xgi == "0": continue i = np.where(xg == xgi)[0] xi, yi = xr[i], y[i] kern = Rbf(xi.shape[1], ARD=True) model = GPR(xi, yi, Rbf(xi.shape[1], ARD=True), mean_function=gptorch.mean_functions.Constant(1), likelihood=gptorch.likelihoods.Gaussian(variance=0.001)) # We don't need a good model--just a ballpark of reasonable # hyperparameters so we can get a prior. model.optimize(method="Adam", max_iter=200) # ASSUMPTION: gptorch using ExpTransform as BayesianGP will! # Valid for at least gptorch version <= 0.3. # Get raw, torch param values legacy_parameters["scales"].append(model.kernel.length_scales.data) legacy_parameters["variances"].append(model.kernel.variance.data) legacy_parameters["jitters"].append(model.likelihood.variance.data) legacy_parameters["biases"].append(model.mean_function.val.data) for key in legacy_parameters.keys(): legacy_parameters[key] = torch.stack(legacy_parameters[key]) # 2) Initialize the Bayesian GP: i = np.where(xg == "0")[0] model = BayesianGP(xr[i], y[i].flatten()) model.raw_scales_prior = Normal(legacy_parameters["scales"].mean(dim=0), legacy_parameters["scales"].std(dim=0)) model.raw_variance_prior = Normal( legacy_parameters["variances"].mean(dim=0), legacy_parameters["variances"].std(dim=0)) model.raw_jitter_prior = Normal(legacy_parameters["jitters"].mean(dim=0), legacy_parameters["jitters"].std(dim=0)) model.bias_prior = Normal(legacy_parameters["biases"].mean(dim=0), legacy_parameters["biases"].std(dim=0)) return model
def _predict_fy_samples(self, attr): """ attr="predict_f_samples" or "predict_y_samples" """ # TODO mock a GPModel? Using GPR for the moment. n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(dx, ARD=True) gp = GPR(x, y, kern) f = getattr(gp, attr) n_test = 5 x_test = np.random.randn(n_test, dx) samples = f(x_test) assert isinstance(samples, np.ndarray) assert samples.ndim == 3 # [sample x n_test x dy] assert samples.shape == (1, n_test, dy) n_samples = 3 samples_2 = f(x_test, n_samples=n_samples) assert isinstance(samples_2, np.ndarray) assert samples_2.ndim == 3 # [sample x n_test x dy] assert samples_2.shape == (n_samples, n_test, dy) x_test_torch = TensorType(x_test) samples_torch = f(x_test_torch) assert isinstance(samples_torch, TensorType) assert samples_torch.ndimension() == 3 # [1 x n_test x dy] assert samples_torch.shape == (1, n_test, dy)
def _predict_fy(self, attr): """ attr='predict_f' or 'predict_y' """ n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(dx, ARD=True) gp = GPR(x, y, kern) n_test = 5 x_test = np.random.randn(n_test, dx) f = getattr(gp, attr) mu, v = f(x_test) for result in [mu, v]: assert isinstance(result, np.ndarray) assert result.ndim == 2 # [n_test x dy] assert result.shape == (n_test, dy) x_test_torch = TensorType(x_test) mu_torch, v_torch = f(x_test_torch) for result in [mu_torch, v_torch]: assert isinstance(result, TensorType) assert result.ndimension() == 2 # [n_test x dy] assert result.shape == (n_test, dy)
def test_predict_cuda(self): n, n_test, dx, dy = 5, 7, 3, 2 x, y = torch.randn(n, dx), torch.randn(n, dy) kern = Rbf(x.shape[1], ARD=True) model = GPR(x, y, kern) model.cuda() x_test = torch.randn(n_test, dx, dtype=torch_dtype).cuda() for t in model._predict(x_test): # mean, variance assert t.is_cuda
def test_init(self): n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(x.shape[1], ARD=True) # init w/ numpy GPR(x, y, kern) # init w/ PyTorch tensors: GPR(TensorType(y), TensorType(x), kern) # init w/ a mean function: GPR(x, y, kern, mean_function=torch.nn.Linear(dx, dy))
def initialize_model(xr, xg, y, model_type): if model_type == "GP": assert xg.shape[1] == 1 i = np.where(xg.flatten() == "0")[0] xr, xg, y = xr[i], xg[i], y[i] return GPR( xr, y, Rbf(xr.shape[1], ARD=True), likelihood=gptorch.likelihoods.Gaussian(variance=0.001) ) else: return experiment_utils.initialize_model(xr, xg, y, model_type)
def test_predict(self): n, n_test, dx, dy = 5, 7, 3, 2 x, y = torch.randn(n, dx), torch.randn(n, dy) kern = Rbf(x.shape[1], ARD=True) model = GPR(x, y, kern) x_test = torch.randn(n_test, dx) mu_var, var = model._predict(x_test) assert all([e == a for e, a in zip(mu_var.shape, (n_test, dy))]) assert all([e == a for e, a in zip(var.shape, (n_test, dy))]) mu_cov, cov = model._predict(x_test, diag=False) assert all([e == a for e, a in zip(mu_cov.shape, (n_test, dy))]) assert all([e == a for e, a in zip(cov.shape, (n_test, n_test))])
def initialize_model(xr, xg, y, model_type): """ Initialize the metamodel to be used for BO. """ if model_type == "BEGP": return EGP(xr, xg, y) elif model_type == "EGP": return EGP(xr, xg, y, embedder_type=DeterministicEmbedder) elif model_type == "GP": x = TensorType(np.zeros((0, xr.shape[1]))) y = TensorType(np.zeros((0, 1))) return GPR(x, y, Rbf(x.shape[1], ARD=True), likelihood=gptorch.likelihoods.Gaussian(variance=0.001)) elif model_type == "BGP": return _initialize_bayesian_gp(xr, xg, y) raise ValueError("Unexpected model type %s" % model_type)
def test_predict_y_samples(self): # TODO mock a GPModel? Using GPR for the moment. n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(dx, ARD=True) gp = GPR(y, x, kern) n_test = 5 x_test = np.random.randn(n_test, dx) y_samples = gp.predict_y_samples(x_test) assert isinstance(y_samples, th.Tensor) assert y_samples.ndimension() == 3 # [sample x n_test x dy] assert y_samples.shape == (1, n_test, dy) n_samples = 3 y_samples_2 = gp.predict_y_samples(x_test, n_samples=n_samples) assert isinstance(y_samples_2, th.Tensor) assert y_samples_2.ndimension() == 3 # [sample x n_test x dy] assert y_samples_2.shape == (n_samples, n_test, dy)
def _predict_fy_samples_cuda(self, attr): n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(dx, ARD=True) gp = GPR(x, y, kern) f = getattr(gp, attr) n_test = 5 x_test = np.random.randn(n_test, dx) x_test_torch = TensorType(x_test) gp.cuda() # Numpy input: samples_cuda_np = f(x_test) assert isinstance(samples_cuda_np, np.ndarray) # PyTorch (cpu) input samples_cuda_torch = f(x_test_torch) assert samples_cuda_torch.device == x_test_torch.device # PyTorch (GPU) input samples_cuda_gpu = f(x_test_torch.to("cuda")) assert samples_cuda_gpu.is_cuda
def _get_model(): n, dx, dy = 5, 3, 2 x, y = np.random.randn(n, dx), np.random.randn(n, dy) kern = Rbf(dx, ARD=True) return GPR(x, y, kern)