def main(): nTrain = 200 nQuery = [150, 100] nDims = 2 # Make test dataset: X = np.random.uniform(0, 1.0, size=(nTrain, nDims)) X = (X + 0.2 * (X > 0.5))/1.2 X = X[np.argsort(X[:, 0])] noise = np.random.normal(loc=0.0, scale=0.1, size=(nTrain,)) Y = (np.cos(3*np.pi*X[:, 0]) + np.cos(3*np.pi*X[:, 1])) + noise data_mean = np.mean(Y, axis=0) ys = Y-data_mean # make a gridded query Xsx = np.linspace(0., 1., nQuery[0]) Xsy = np.linspace(0., 1., nQuery[1]) xv, yv = np.meshgrid(Xsx, Xsy) Xs = np.vstack((xv.ravel(), yv.ravel())).T # Compose isotropic kernel: def kerneldef(h, k): a = h(0.1, 5, 0.1) b = h(0.1, 5, 0.1) logsigma = h(-6, 1) return a*k(gp.kernels.gaussian, b) + k(gp.kernels.lognoise, logsigma) hyper_params = gp.learn(X, ys, kerneldef, verbose=True, ftol=1e-5, maxiter=2000) print(gp.describe(kerneldef, hyper_params)) regressor = gp.condition(X, ys, kerneldef, hyper_params) query = gp.query(regressor, Xs) post_mu = gp.mean(query) + data_mean post_var = gp.variance(query, noise=True) # Shift outputs back: ax = pl.subplot(131) pl.scatter(X[:, 0], X[:, 1], s=20, c=Y, linewidths=0) pl.axis('equal') pl.title('Training') pl.subplot(132, sharex=ax, sharey=ax) pl.scatter(Xs[:, 0], Xs[:, 1], s=20, c=post_mu, linewidths=0) pl.axis('equal') pl.title('Prediction') pl.subplot(133, sharex=ax, sharey=ax) pl.scatter(Xs[:, 0], Xs[:, 1], s=20, c=np.sqrt(post_var), linewidths=0) pl.axis('equal') pl.title('Stdev') pl.tight_layout() pl.show()
def main(): nTrain = 20 nQuery = 100 nDims = 1 noise_level = 0.05 # Test dataset---------------------------------------------- X = np.random.uniform(0, 1.0, size=(nTrain, nDims)) X = X[np.argsort(X[:, 0])] # n*d underlyingFunc = (lambda x: np.sin(2*np.pi*x) + 5 + np.random.normal(loc=0.0, scale=0.05, size=(x.shape[0], 1))) y = underlyingFunc(X) + noise_level * np.random.randn(nTrain, 1) y = y.ravel() Xs = np.linspace(0., 1., nQuery)[:, np.newaxis] data_mean = np.mean(y) ys = y - data_mean # ---------------------------------------------------------- # Define a pathological GP kernel: def kerneldef(h, k): a = h(0.1, 5, 0.1) b = h(0.1, 5, 0.1) logsigma = h(-6, 1) return (a*k(gp.kernels.gaussian, b) + .1*b*k(gp.kernels.matern3on2, a) + k(gp.kernels.lognoise, logsigma)) # Learning signal and noise hyperparameters hyper_params = gp.learn(X, ys, kerneldef, verbose=False, ftol=1e-15, maxiter=2000) # old_hyper_params = [1.48, .322, np.log(0.0486)] # print(gp.describe(kerneldef, old_hyper_params)) print(gp.describe(kerneldef, hyper_params)) regressor = gp.condition(X, ys, kerneldef, hyper_params) query = gp.query(regressor, Xs) post_mu = gp.mean(query) + data_mean # post_cov = gp.covariance(query) post_var = gp.variance(query, noise=True) # Plot fig = pl.figure() ax = fig.add_subplot(111) ax.plot(Xs, post_mu, 'k-') upper = post_mu + 2*np.sqrt(post_var) lower = post_mu - 2*np.sqrt(post_var) ax.fill_between(Xs.ravel(), upper, lower, facecolor=(0.9, 0.9, 0.9), edgecolor=(0.5, 0.5, 0.5)) ax.plot(regressor.X[:, 0], regressor.y+data_mean, 'r.') pl.show()
def predict(self, Xq, real=True): """ Predict the query mean and variance using the Gaussian process model. Infers the mean and variance of the Gaussian process at given locations using the data collected so far .. note :: [Properties Modified] (None) Parameters ---------- Xq : numpy.ndarray Query points real : bool, optional To use only the real observations or also the virtual observations Returns ------- numpy.ndarray Expectance of the prediction at the given locations numpy.ndarray Variance of the prediction at the given locations """ assert self.hyperparams, "Sampler is not trained yet. " \ "Possibly not enough observations provided." # To use only the real data, extract the real data and compute the # regressors using only the real data if real: X_real, y_real = self.get_real_data() regressors = [gp.condition(X_real, y_real[:, i_task] - self.y_mean[i_task], self.kerneldef, self.hyperparams[i_task]) for i_task in range(self.n_tasks)] # Otherwise, just use the regressors we already have else: regressors = self.regressors # Compute using the standard predictor sequence predictors = [gp.query(r, Xq) for r in regressors] yq_exp = [gp.mean(p) for p in predictors] yq_var = [gp.variance(p) for p in predictors] return np.asarray(yq_exp).T + self.y_mean, np.asarray(yq_var).T
def update_regressors(self): """ Update the regressors of the Gaussian process model. Only makes sense to do this after hyperparameters are learned .. note :: [Properties Modified] regressors .. note :: [Further Work] Use Cholesky Update here correctly to cache regressors and improve efficiency """ if self.hyperparams is None: return # raise ValueError('Hyperparameters are not learned yet.' + # 'Regressors cannot be updated.') # Create the regressors if it hasn't already been self.regressors = [] for i_task in range(self.n_tasks): self.regressors.append( gp.condition(self.X(), self.y()[:, i_task] - self.y_mean[i_task], self.kerneldef, self.hyperparams[i_task]))
def main(): # # Settings # # Algorithmic properties nbases = 50 lenscale = 1 # For all basis functions that take lengthscales lenscale2 = 0.5 # For the Combo basis noise = 1 order = 7 # For polynomial basis rate = 0.9 eta = 1e-5 passes = 1000 batchsize = 100 reg = 1 # np.random.seed(100) N = 500 Ns = 250 # Dataset selection # dataset = 'sinusoid' dataset = 'gp1D' # Dataset properties lenscale_true = 0.7 # For the gpdraw dataset noise_true = 0.1 basis = 'RKS' # basis = 'FF' # basis = 'RBF' # basis = 'Linear' # basis = 'Poly' # basis = 'Combo' # # Make Data # # Sinusoid if dataset == 'sinusoid': Xtrain = np.linspace(-2 * np.pi, 2 * np.pi, N)[:, np.newaxis] ytrain = np.sin(Xtrain).flatten() + np.random.randn(N) * noise Xtest = np.linspace(-2 * np.pi, 2 * np.pi, Ns)[:, np.newaxis] ftest = np.sin(Xtest).flatten() # Random RBF GP elif dataset == 'gp1D': Xtrain, ytrain, Xtest, ftest = \ gen_gausprocess_se(N, Ns, lenscale=lenscale_true, noise=noise_true) else: raise ValueError('Invalid dataset!') # # Make Bases # if basis == 'FF': base = basis_functions.FastFood(nbases, Xtrain.shape[1]) elif basis == 'RKS': base = basis_functions.RandomRBF(nbases, Xtrain.shape[1]) elif basis == 'RBF': base = basis_functions.RadialBasis(Xtrain) elif basis == 'Linear': base = basis_functions.LinearBasis(onescol=True) elif basis == 'Poly': base = basis_functions.PolynomialBasis(order) elif basis == 'Combo': base1 = basis_functions.RandomRBF(nbases, Xtrain.shape[1]) base2 = basis_functions.LinearBasis(onescol=True) base3 = basis_functions.FastFood(nbases, Xtrain.shape[1]) base = base1 + base2 + base3 else: raise ValueError('Invalid basis!') # Set up optimisation # learning_params = gp.OptConfig() # learning_params.sigma = gp.auto_range(kdef) # learning_params.noise = gp.Range([1e-5], [1e5], [1]) # learning_params.walltime = 60 # # Learn regression parameters and predict # if basis == 'Linear' or basis == 'Poly': hypers = [] elif basis == 'FF' or basis == 'RKS' or basis == 'RBF': hypers = [lenscale] elif basis == 'Combo': hypers = [lenscale, lenscale2] else: raise ValueError('Invalid basis!') params_elbo = regression.learn(Xtrain, ytrain, base, hypers, var=noise**2, regulariser=reg) Ey_e, Vf_e, Vy_e = regression.predict(Xtest, base, *params_elbo) Sy_e = np.sqrt(Vy_e) # # Nonparametric variational inference GLM # llhood = likelihoods.Gaussian() lparams = [noise**2] params_glm = glm.learn(Xtrain, ytrain, llhood, lparams, base, hypers, regulariser=reg, use_sgd=True, rate=rate, postcomp=10, eta=eta, batchsize=batchsize, maxit=passes) Ey_g, Vf_g, Eyn, Eyx = glm.predict_meanvar(Xtest, llhood, base, *params_glm) Vy_g = Vf_g + params_glm[2][0] Sy_g = np.sqrt(Vy_g) # # Learn GP and predict # def kdef(h, k): return (h(1e-5, 1., 0.5) * k(kern.gaussian, h(1e-5, 1e5, lenscale)) + k(kern.lognoise, h(-4, 1, -3))) hyper_params = gp.learn(Xtrain, ytrain, kdef, verbose=True, ftol=1e-15, maxiter=passes) regressor = gp.condition(Xtrain, ytrain, kdef, hyper_params) query = gp.query(regressor, Xtest) Ey_gp = gp.mean(query) Vf_gp = gp.variance(query) Vy_gp = gp.variance(query, noise=True) Sy_gp = np.sqrt(Vy_gp) # import ipdb; ipdb.set_trace() # # Evaluate LL and SMSE # LL_elbo = mll(ftest, Ey_e, Vf_e) LL_gp = mll(ftest, Ey_gp, Vf_gp) LL_g = mll(ftest, Ey_g, Vy_g) smse_elbo = smse(ftest, Ey_e) smse_gp = smse(ftest, Ey_gp) smse_glm = smse(ftest, Ey_g) log.info("A la Carte, LL: {}, smse = {}, noise: {}, hypers: {}" .format(LL_elbo, smse_elbo, np.sqrt(params_elbo[3]), params_elbo[2])) log.info("GP, LL: {}, smse = {}, noise: {}, hypers: {}" .format(LL_gp, smse_gp, hyper_params[1], hyper_params[0])) log.info("GLM, LL: {}, smse = {}, noise: {}, hypers: {}" .format(LL_g, smse_glm, np.sqrt(params_glm[2][0]), params_glm[3])) # # Plot # Xpl_t = Xtrain.flatten() Xpl_s = Xtest.flatten() # Training/Truth pl.plot(Xpl_t, ytrain, 'k.', label='Training') pl.plot(Xpl_s, ftest, 'k-', label='Truth') # ELBO Regressor pl.plot(Xpl_s, Ey_e, 'g-', label='Bayesian linear regression') pl.fill_between(Xpl_s, Ey_e - 2 * Sy_e, Ey_e + 2 * Sy_e, facecolor='none', edgecolor='g', linestyle='--', label=None) # GP # pl.plot(Xpl_s, Ey_gp, 'b-', label='GP') # pl.fill_between(Xpl_s, Ey_gp - 2 * Sy_gp, Ey_gp + 2 * Sy_gp, # facecolor='none', edgecolor='b', linestyle='--', # label=None) # GLM Regressor pl.plot(Xpl_s, Ey_g, 'm-', label='GLM') pl.fill_between(Xpl_s, Ey_g - 2 * Sy_g, Ey_g + 2 * Sy_g, facecolor='none', edgecolor='m', linestyle='--', label=None) pl.legend() pl.grid(True) pl.title('Regression demo') pl.ylabel('y') pl.xlabel('x') pl.show()
# # Predict Revrand # Ey, Vf, _, _ = glm.predict_meanvar(X_test, llhood, base, *params) Vy = Vf + params[2][0] Sy = np.sqrt(Vy) # # Predict GP # regressor = gp.condition(X_train_sub, y_train_sub, kdef, hyper_params) query = gp.query(regressor, X_test) Ey_gp = gp.mean(query) Vf_gp = gp.variance(query) Vy_gp = gp.variance(query, noise=True) Sy_gp = np.sqrt(Vy_gp) # # Validation # log.info("Subset GP smse = {}, msll = {},\n\thypers = {}, noise = {}." .format(smse(y_test, Ey_gp), msll(y_test, Ey_gp, Vy_gp, y_train), hyper_params[0], hyper_params[1])) log.info("Revrand smse = {}, msll = {},\n\thypers = {}, noise = {}."
ftol=1e-15, maxiter=1000) # # Predict Revrand # Ey, Vf, _, _ = glm.predict_meanvar(X_test, llhood, base, *params) Vy = Vf + params[2][0] Sy = np.sqrt(Vy) # # Predict GP # regressor = gp.condition(X_train_sub, y_train_sub, kdef, hyper_params) query = gp.query(regressor, X_test) Ey_gp = gp.mean(query) Vf_gp = gp.variance(query) Vy_gp = gp.variance(query, noise=True) Sy_gp = np.sqrt(Vy_gp) # # Validation # log.info("Subset GP smse = {}, msll = {},\n\thypers = {}, noise = {}.".format( smse(y_test, Ey_gp), msll(y_test, Ey_gp, Vy_gp, y_train), hyper_params[0], hyper_params[1])) log.info("Revrand smse = {}, msll = {},\n\thypers = {}, noise = {}.".format( smse(y_test, Ey), msll(y_test, Ey, Vy, y_train), params[2],