def generate_regression_task_dataset(n_tasks): kernel = RBF(length_scale=1., length_scale_bounds=(1e-5, 1e5)) gpr = GaussianProcessRegressor(kernel=kernel) X = np.linspace(0, 100, 1000) X = np.atleast_2d(X).T f_samples = [] for i in range(n_tasks): # sample a function from a zero mean GP f = gpr.sample_y(X, random_state=i) f_samples.append(f) return f_samples
def test_y_multioutput(): """ Test that GPR can deal with multi-dimensional target values""" y_2d = np.vstack((y, y*2)).T # Test for fixed kernel that first dimension of 2d GP equals the output # of 1d GP and that second dimension is twice as large kernel = RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None, normalize_y=False) gpr.fit(X, y) gpr_2d = GaussianProcessRegressor(kernel=kernel, optimizer=None, normalize_y=False) gpr_2d.fit(X, y_2d) y_pred_1d, y_std_1d = gpr.predict(X2, return_std=True) y_pred_2d, y_std_2d = gpr_2d.predict(X2, return_std=True) _, y_cov_1d = gpr.predict(X2, return_cov=True) _, y_cov_2d = gpr_2d.predict(X2, return_cov=True) assert_almost_equal(y_pred_1d, y_pred_2d[:, 0]) assert_almost_equal(y_pred_1d, y_pred_2d[:, 1] / 2) # Standard deviation and covariance do not depend on output assert_almost_equal(y_std_1d, y_std_2d) assert_almost_equal(y_cov_1d, y_cov_2d) y_sample_1d = gpr.sample_y(X2, n_samples=10) y_sample_2d = gpr_2d.sample_y(X2, n_samples=10) assert_almost_equal(y_sample_1d, y_sample_2d[:, 0]) # Test hyperparameter optimization for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True) gpr.fit(X, y) gpr_2d = GaussianProcessRegressor(kernel=kernel, normalize_y=True) gpr_2d.fit(X, np.vstack((y, y)).T) assert_almost_equal(gpr.kernel_.theta, gpr_2d.kernel_.theta, 4)
class Gaussian_data(): def __init__(self, l=1, e=.1): self.length_scale = l self.noise_level = e self.kernel = RBF(length_scale=l) + WhilteKernel(noise_level=e) self.gp = GaussianProcessRegressor(kernel=self.kernel) def gen(self, n=5, x_min=-2, x_max=2): x = np.random.rand(x_min, x_max, n).sort() y = self.gp.sample_y(x) return x, y
def test_sample_statistics(kernel): # Test that statistics of samples drawn from GP are correct. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_mean, y_cov = gpr.predict(X2, return_cov=True) samples = gpr.sample_y(X2, 300000) # More digits accuracy would require many more samples assert_almost_equal(y_mean, np.mean(samples, 1), 1) assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(), np.var(samples, 1) / np.diag(y_cov).max(), 1)
def test_sample_statistics(): """ Test that statistics of samples drawn from GP are correct.""" for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_mean, y_cov = gpr.predict(X2, return_cov=True) samples = gpr.sample_y(X2, 300000) # More digits accuracy would require many more samples assert_almost_equal(y_mean, np.mean(samples, 1), 2) assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(), np.var(samples, 1) / np.diag(y_cov).max(), 1)
def gaussian_process_reg(particles, weights): # GP regression kernel = 1.0 * RBF(1.0) gp = GaussianProcessRegressor(kernel=kernel, random_state=0) gp.fit(particles, weights) sampled_weights_array = gp.sample_y(particles) gp_weights = [] for item in sampled_weights_array: gp_weights.append(item[0]) weights[:] = gp_weights[:]
def _simulate_single_equation(X, scale): """X: [n, num of parents], x: [n]""" z = np.random.normal(scale=scale, size=n) pa_size = X.shape[1] if pa_size == 0: return z if sem_type == 'mlp': hidden = 100 W1 = np.random.uniform(low=0.5, high=2.0, size=[pa_size, hidden]) W1[np.random.rand(*W1.shape) < 0.5] *= -1 W2 = np.random.uniform(low=0.5, high=2.0, size=hidden) W2[np.random.rand(hidden) < 0.5] *= -1 x = sigmoid(X @ W1) @ W2 + z elif sem_type == 'mim': w1 = np.random.uniform(low=0.5, high=2.0, size=pa_size) w1[np.random.rand(pa_size) < 0.5] *= -1 w2 = np.random.uniform(low=0.5, high=2.0, size=pa_size) w2[np.random.rand(pa_size) < 0.5] *= -1 w3 = np.random.uniform(low=0.5, high=2.0, size=pa_size) w3[np.random.rand(pa_size) < 0.5] *= -1 x = np.tanh(X @ w1) + np.cos(X @ w2) + np.sin(X @ w3) + z elif sem_type == 'gp': from sklearn.gaussian_process import GaussianProcessRegressor gp = GaussianProcessRegressor() x = gp.sample_y(X, random_state=None).flatten() + z elif sem_type == 'gp-add': from sklearn.gaussian_process import GaussianProcessRegressor gp = GaussianProcessRegressor() x = sum([ gp.sample_y(X[:, i, None], random_state=None).flatten() for i in range(X.shape[1]) ]) + z else: raise ValueError('Unknown sem type. In a nonlinear model, \ the options are as follows: mlp, mim, \ gp, gp-add, or quadratic.') return x
class SmoothFunctionCreator(): def __init__(self, seed=42): self._gp = GaussianProcessRegressor() x_train = np.array([0.0, 2.0, 6.0, 10.0])[:, np.newaxis] source_train = np.array([0.0, 1.0, -1.0, 0.0]) self._gp.fit(x_train, source_train) self._random_state = np.random.RandomState(seed) def sample(self, n_samples): x = np.linspace(0.0, 10.0, 100)[:, np.newaxis] source = self._gp.sample_y(x, n_samples, random_state=self._random_state) target = gaussian_filter1d(source, 1, order=1, axis=0) target = np.tanh(10.0 * target) return source, target
def kernel_draw(N, Lim, kernel=TEST): #ASL=1 # Specify Gaussian Process gp = GaussianProcessRegressor(kernel=kernel) # Plot prior X_ = np.linspace(0, scale * Lim, N) y_samples = gp.sample_y(X_[:, np.newaxis], 1) print(y_samples.shape) return stretch * np.squeeze(y_samples.T)
def prior_and_posterior(kernel): gp = GaussianProcessRegressor(kernel=kernel) plt.figure(figsize=(8, 8)) # Plot prior plt.subplot(2, 1, 1) Xstar = np.linspace(0, 5, 100) mu, sigma = gp.predict(Xstar[:, np.newaxis], return_std=True) plt.plot(Xstar, mu, 'k', lw=3, zorder=9) plt.fill_between(Xstar, mu - sigma, mu + sigma, alpha=0.2, color='k') y_samples = gp.sample_y(Xstar[:, np.newaxis], 10) plt.plot(Xstar, y_samples, lw=1) plt.xlim(0, 5) plt.ylim(-3, 3) plt.title("Prior (kernel: %s)" % kernel, fontsize=12) # Generate data and fit GP rng = np.random.RandomState(4) X = rng.uniform(0, 5, 10)[:, np.newaxis] y = np.sin((X[:, 0] - 2.5)**2) gp.fit(X, y) # Plot posterior plt.subplot(2, 1, 2) mu, sigma = gp.predict(Xstar[:, np.newaxis], return_std=True) plt.plot(Xstar, mu, 'k', lw=3, zorder=9) plt.fill_between(Xstar, mu - sigma, mu + sigma, alpha=0.2, color='k') y_samples = gp.sample_y(Xstar[:, np.newaxis], 10) plt.plot(Xstar, y_samples, lw=1) plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0)) plt.xlim(0, 5) plt.ylim(-3, 3) plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)), fontsize=12) plt.show()
def gp_bootstrap_noise(X, Y, kernel, n_samples=50, alpha=20.0): # First, we fit a GP model to the data # (with noise added, represented by the parameter alpha): m = GaussianProcessRegressor(kernel=kernel, alpha=alpha) m.fit(X, Y) gp_samples = m.sample_y(X, n_samples=n_samples).T boot_samples = [] for si in range(n_samples): boot_samples.append((X, gp_samples[si][0])) return boot_samples
class GPDataGen(DataGenerator): def __init__(self, params=None, param_sampler=None, kernel=RBF(), mean_fn=None, xs_sampler=None): self.params = params self.param_sampler = None self.xs_sampler = xs_sampler self.mean_fn = mean_fn self.kernel = kernel self.gp = GaussianProcessRegressor(self.kernel) def generate_data(self, xs, params=None): xs = xs.reshape((-1, 1)) return self.gp.sample_y(xs.reshape((-1, 1))).flatten()
def pass_arg(Xx, nsim, tr_size): print("tr_Size:",tr_size) tr_size = int(tr_size) use_YPhy = 0 #List of lakes to choose from lake = ['mendota' , 'mille_lacs'] lake_num = 0 # 0 : mendota , 1 : mille_lacs lake_name = lake[lake_num] # Load features (Xc) and target values (Y) data_dir = '../../../../data/' filename = lake_name + '.mat' mat = spio.loadmat(data_dir + filename, squeeze_me=True, variable_names=['Y','Xc_doy','Modeled_temp']) Xc = mat['Xc_doy'] Y = mat['Y'] Xc = Xc[:,:-1] scaler = preprocessing.StandardScaler() # train and test data trainX, testX, trainY, testY = train_test_split(Xc, Y, train_size=tr_size/Xc.shape[0], test_size=tr_size/Xc.shape[0], random_state=42, shuffle=True) trainY=trainY[:,np.newaxis] #trainX = scaler.fit_transform(trainX) #trainY = scaler.fit_transform(trainY) kernel = C(5.0, (0.1, 1e2)) * RBF(length_scale = [1] * trainX.shape[1], length_scale_bounds=(1e-1, 1e15)) gp = GaussianProcessRegressor(kernel=kernel, alpha =.1, n_restarts_optimizer=10) gp.fit(trainX, trainY) #y_pred1, sigma1 = gp.predict(testX, return_std=True) # scale the uniform numbers to original space # max and min value in each column max_in_column_Xc = np.max(trainX,axis=0) min_in_column_Xc = np.min(trainX,axis=0) # Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc) Xc_org = Xx*(max_in_column_Xc-min_in_column_Xc) + min_in_column_Xc print(gp.kernel_) samples = gp.sample_y(Xc_org, n_samples=int(nsim)).T return np.squeeze(samples)
class DriftSimulator(): def __init__(self, n_samples, rt_range, n_gp_points, gp_params, data_params): self.n_samples = n_samples X = np.linspace(rt_range[0], rt_range[1], n_gp_points)[:, np.newaxis] y = np.random.normal(data_params[0], data_params[1], n_gp_points) kernel = gp_params[0] * RBF(length_scale=gp_params[1], length_scale_bounds=(1e-1, 10.0)) self.gp = GaussianProcessRegressor(kernel=kernel) self.gp.fit(X, y) def get_drift(self, model_idx, rt): samples = self.gp.sample_y(np.array([[rt]]), n_samples=self.n_samples) return samples[0][model_idx] def plot_drifts(self): NotImplementedError()
def test_sample_y_shapes(normalize_y, n_targets): """Check the shapes of y_samples in single-output (n_targets=0) and multi-output settings, including the edge case when n_targets=1, where the sklearn convention is to squeeze the predictions. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/22175 """ rng = np.random.RandomState(1234) n_features, n_samples_train = 6, 9 # Number of spatial locations to predict at n_samples_X_test = 7 # Number of sample predictions per test point n_samples_y_test = 5 y_train_shape = (n_samples_train, ) if n_targets is not None: y_train_shape = y_train_shape + (n_targets, ) # By convention single-output data is squeezed upon prediction if n_targets is not None and n_targets > 1: y_test_shape = (n_samples_X_test, n_targets, n_samples_y_test) else: y_test_shape = (n_samples_X_test, n_samples_y_test) X_train = rng.randn(n_samples_train, n_features) X_test = rng.randn(n_samples_X_test, n_features) y_train = rng.randn(*y_train_shape) model = GaussianProcessRegressor(normalize_y=normalize_y) # FIXME: before fitting, the estimator does not have information regarding # the number of targets and default to 1. This is inconsistent with the shape # provided after `fit`. This assert should be made once the following issue # is fixed: # https://github.com/scikit-learn/scikit-learn/issues/22430 # y_samples = model.sample_y(X_test, n_samples=n_samples_y_test) # assert y_samples.shape == y_test_shape model.fit(X_train, y_train) y_samples = model.sample_y(X_test, n_samples=n_samples_y_test) assert y_samples.shape == y_test_shape
def generate_slow_background(x, scale=500, npoints=1000): kernel = 1.0 * RBF(length_scale=scale) X_ = np.linspace(0, x[-1], npoints) #print(X_.shape) gp_bg = GaussianProcessRegressor(kernel=kernel) y_mean, y_std = gp_bg.predict(X_[:, np.newaxis], return_std=True) y_samples = gp_bg.sample_y(X_[:, np.newaxis], 1) #y_samples -= np.min(y_s) #th=0.5 #y_samples[y_samples <= -th] = -th #y_samples += th + 1e-7 #print(y_samples.shape) f2 = interp1d(X_, y_samples[::, 0], kind='cubic', bounds_error=False) res = f2(np.arange(len(x))) #res[res<=0] = 0 res -= np.min(res) return res
def get_gp_prediction(x: np.ndarray, y: np.ndarray, n_samples: int, n_points: int = 100): """Fit a GP to observed P( Y | X ) and sample some functions from it. Args: x: x-values (features) y: y-values (targets/labels) n_samples: The number of GP samples to use as basis functions. n_points: The number of points to subsample form x and y to fit each GP. Returns: a function that takes as input an array either of shape (n,) or (k, n) and outputs: if input is 1D -> output: (n, n_samples) if input is 2D -> output: (k, n, n_samples) """ kernel = PairwiseKernel(metric='poly') + RBF() gp = GaussianProcessRegressor(kernel=kernel, alpha=0.4, n_restarts_optimizer=0, normalize_y=True) xmin = np.min(x) xmax = np.max(x) xx = np.linspace(xmin, xmax, n_points) y_samples = [] rng = onp.random.RandomState(0) for i in range(n_samples): logging.info("Subsample 200 points and fit GP to P(Y|X)...") idx = rng.choice(len(x), 200, replace=False) gp.fit(x[idx, np.newaxis], y[idx]) logging.info(f"Get a sample functions from the GP") y_samples.append(gp.sample_y(xx[:, np.newaxis], 1)) y_samples = np.array(y_samples).squeeze() logging.info(f"Shape of samples: {y_samples.shape}") def predict(inputs: np.ndarray) -> np.ndarray: return interp1d(inputs, xmin, xmax, y_samples).T return jit(vmap(predict))
class GPRealization(Function): """ A callable class that is the realization of a gaussian process, f:R->R. """ def __init__(self, kernel=None, x0=0, x1=1, num_init=100): """ Parameters ---------- kernel : Kernel x0 : float left boundary x1 : float right boundary num_init : int the number of equidistant samples between x0 and x1 used to build the function """ super().__init__() # RBF(length_scale_bounds=(0.3, 10.0)) self.gp = GaussianProcessRegressor(kernel=kernel) x = np.linspace(x0, x1, num_init) x = x[:, np.newaxis] y = self.gp.sample_y(x, n_samples=1, random_state=np.random.RandomState()) y = y[:, 0] self.gp.fit(x, y) def reset(self, reset_params=True): """ parameters are chosen """ self.evals = 0 if reset_params: raise NotImplementedError( 'No reset of params implemented for GPRealization') def __call__(self, x): return self.gp.predict([[x]])[0] def integral(self, x_0=0, x_1=1): """ return the integral from x_0 to x_1 """ return quad(self, x_0, x_1)[0]
class GPTSLearner: def __init__(self, arms, sigma, restart_optimizer=0, cost_kernel=1, lenght_scale_kernel=1): self.name_learner = "GPTS learner" self.arms = arms self.means = np.ones(len(arms)) self.sigmas = np.ones(len(arms)) * sigma self.pulled_arms = list() self.collected_rewards = list() self.users_sampled = [0, 0, 0] kernel = C(1.0, (1e-1, 1e1)) * RBF(1.0, (1e-1, 1e1)) self.gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-10, normalize_y=True, n_restarts_optimizer=3, random_state=17) def update(self, idx_pulled_arm, reward): self.pulled_arms.append(self.arms[idx_pulled_arm]) self.collected_rewards.append(reward) # Split pulled_arms in elements with at least 2 arrays each one. x = np.atleast_2d(self.pulled_arms).T y = self.collected_rewards # X: Training Data # Y: Target Values self.gp = self.gp.fit(x, y) self.means, self.sigmas = self.gp.predict(np.atleast_2d(self.arms).T, return_std=True) self.sigmas = np.maximum(self.sigmas, 1e-2) self.means = np.maximum(self.means, 0) def sample_arms(self): values = np.round(self.gp.sample_y(np.atleast_2d(self.arms).T), 3) return np.where(values > 0, values, 0)
def sample_greedy(df, n_reps=5, n_gp_samples=30): print("== Sampling for coefficient estimation, greedily...") logger = CoeffSamplingLogger("greedy") for _ in range(n_reps): n_samples = min(len(df) - 1, MAX_N_SAMPLES) idxs = [np.random.randint(len(df))] for i in tqdm(range(1, n_samples)): df_sampled = df.iloc[df.index.isin(idxs), :] df_rest = df.iloc[~df.index.isin(idxs), :] logger.tick(df_sampled) gp = GaussianProcessRegressor(kernel=coeff_est_kernel, normalize_y=True) gp.fit( df_sampled.loc[:, ("lat", "lng", "pred", "electrification")], df_sampled["true"]) preds = gp.sample_y(df_rest.loc[:, ("lat", "lng", "pred", "electrification")], n_samples=n_gp_samples) idx = df_rest.index[np.argmax(np.std(preds, axis=1))] idxs.append(idx) logger.clear_run() return logger
def gp_fit(df): # Convert time to float in days all_times = df.index.values time_x = df.index.to_julian_date().values time_x -= time_x.min() kernel = 1**2 * RBF(length_scale=7, length_scale_bounds=(1, 20)) + WhiteKernel( noise_level=0.1**2, noise_level_bounds=(1e-7, 40)) gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-10, normalize_y=True, n_restarts_optimizer=10) gp.fit(time_x.reshape(-1, 1), df['weight'].values) y_pred, y_std = gp.predict(time_x.reshape(-1, 1), return_std=True) gp_samples = gp.sample_y(np.array([time_x[0], time_x[-1]]).reshape(-1, 1), n_samples=100000) return y_pred, y_std, gp_samples
def fitGaussianProc(patDXdTdata, patAvgXdata, params): ''' Fits a GP on the change data (x, dx/dt) Parameters ---------- patDXdTdata patAvgXdata estimNoise lengthScaleFactors plotTrajParams Returns ------- ''' # Mesh the input space for evaluations of the real function, the prediction and # its MSE assert(CTL == 1) nrBiomk = patDXdTdata.shape[0] #minX = np.amin(patAvgXdata, axis=0) #maxX = np.amax(patAvgXdata, axis=0) minX = np.array([np.nanmin(patAvgXdata[b], axis=0) for b in range(nrBiomk)]) maxX = np.array([np.nanmax(patAvgXdata[b], axis=0) for b in range(nrBiomk)]) assert not any(np.isnan(minX)) assert not any(np.isnan(maxX)) intervalSize = maxX-minX minX -= intervalSize/0.5 maxX += intervalSize/0.5 #print minX.shape, maxX.shape nrPointsToEval = 5000 x_pred = np.zeros((nrPointsToEval, nrBiomk),float) dXdT_pred = np.zeros((nrPointsToEval, nrBiomk),float) sigma_pred = np.zeros((nrPointsToEval, nrBiomk),float) nrSamples = 100 posteriorSamples = np.zeros((nrSamples, nrPointsToEval, nrBiomk),float) # print(avgXdata.shape, diag.shape) # print(avgXdata[diag == CTL,:].shape) # ctlXMean = np.nanmean(avgXdata[diag == CTL,:], axis = 0) # ctlXStd = np.nanstd(avgXdata[diag == CTL,:], axis = 0) # ctldXdTMean = np.nanmean(dXdTdata[diag == CTL,:], axis = 0) # ctldXdTStd = np.nanstd(dXdTdata[diag == CTL,:], axis = 0) # allXMean = np.nanmean(avgXdata, axis = 0) # allXStd = np.nanstd(avgXdata, axis = 0) # alldXdTMean = np.nanmean(dXdTdata, axis = 0) # alldXdTStd = np.nanstd(dXdTdata, axis = 0) patXMean = np.array([np.nanmean(patAvgXdata[b], axis=0) for b in range(nrBiomk)]) patXStd = np.array([np.nanstd(patAvgXdata[b], axis=0) for b in range(nrBiomk)]) patdXdTMean = np.array([np.nanmean(patDXdTdata[b], axis=0) for b in range(nrBiomk)]) patdXdTStd = np.array([np.nanstd(patDXdTdata[b], axis=0) for b in range(nrBiomk)]) gpList = [] for b in range(nrBiomk): points = np.linspace(minX[b], maxX[b], nrPointsToEval) #print points.shape X = patAvgXdata[b] Y = patDXdTdata[b] notNanInd = np.logical_not(np.isnan(X)) X = X[notNanInd] Y = Y[notNanInd] X = X.reshape(-1,1) Y = Y.reshape(-1,1) # X = (X - allXMean[b]) / allXStd[b] # standardizing the inputs and outputs # Y = (Y - alldXdTMean[b]) / alldXdTStd[b] # minX[b] = (minX[b] - allXMean[b]) / allXStd[b] # maxX[b] = (maxX[b] - allXMean[b]) / allXStd[b] X = (X - patXMean[b]) / patXStd[b] # standardizing the inputs and outputs # Y = (Y - patdXdTMean[b]) / patdXdTStd[b] Y = Y / patdXdTStd[b] minX[b] = (minX[b] - patXMean[b]) / patXStd[b] maxX[b] = (maxX[b] - patXMean[b]) / patXStd[b] #print 'Xshape, Yshape', X.shape, Y.shape lower, upper = np.abs(1/np.max(X)), np.abs(1/(np.min(X)+1e-6)) if lower > upper: lower, upper = upper, lower mid = 1/np.abs(np.mean(X)) # print("X", X[:20],'Y', Y[:20]) # print(minX, maxX) #lengthScale = (np.max(X)-np.min(X)) lengthScale = params['lengthScaleFactors'][b] * (np.max(X) - np.min(X))/2 estimNoise = np.var(Y)/2 # this should be variance, as it is placed as is on the diagonal of the kernel, which is a covariance matrix #estimAlpha = np.ravel((np.std(Y))**2) #estimAlpha = np.var(Y)/2 estimAlpha = np.std(Y)*2 boundsFactor = 2.0 #estimAlpha = 0 #need to specity bounds as the lengthScale is optimised in the fit rbfKernel = ConstantKernel(1.0, constant_value_bounds="fixed") * RBF(length_scale=lengthScale, length_scale_bounds=(float(lengthScale)/boundsFactor, 1*lengthScale)) whiteKernel = ConstantKernel(1.0, constant_value_bounds="fixed") * WhiteKernel(noise_level=estimNoise, noise_level_bounds=(float(estimNoise)/boundsFactor, boundsFactor*estimNoise)) #rbfKernel = 1 * RBF(length_scale=lengthScale) #whiteKernel = 1 * WhiteKernel(noise_level=estimNoise) kernel = rbfKernel + whiteKernel #kernel = 1.0 * RBF(length_scale=lengthScale) print('\nbiomk %d lengthScale %f noise %f alpha %f'% (b, lengthScale, estimNoise, estimAlpha)) #print estimAlpha.shape normalizeYflag = False #normalizeYflag = True gp = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer='fmin_l_bfgs_b', n_restarts_optimizer=100, normalize_y=normalizeYflag) #gp = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer=None, n_restarts_optimizer=100, normalize_y=True) assert not any(np.isnan(X)) assert not any(np.isnan(Y)) # Fit to data using Maximum Likelihood Estimation of the parameters gp.fit(X, Y) print("optimised kernel", gp.kernel_)#, " theta", gp.kernel_.theta, " bounds", gp.kernel_.bounds) #gpNonOpt = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer=None, normalize_y=False) #gpNonOpt.fit(X,Y) #print("non-optimised kernel", gpNonOpt.kernel_)#, " theta", gpNonOpt.kernel_.theta, " bounds", gpNonOpt.kernel_.bounds) #gp = gpNonOpt # Make the prediction on the meshed x-axis (ask for Cov matrix as well) x_pred[:,b] = np.linspace(minX[b], maxX[b], nrPointsToEval) assert not any(np.isnan(x_pred[:,b])) dXdT_predCurr, cov_matrix = gp.predict(x_pred[:,b].reshape(-1,1), return_cov=True) # make sure dXdT is not too low, otherwise truncate the [minX, maxX] interval dXdTthresh = 1e-10 tooLowMask = np.abs(np.ravel(dXdT_predCurr)) < dXdTthresh print(tooLowMask.shape) if np.sum(tooLowMask) > nrPointsToEval/10: print("Warning dXdT is too low, will restict the [minxX, maxX] interval") goodIndicesMask = np.logical_not(tooLowMask) #print(x_pred.shape, goodIndicesMask.shape) #print(x_pred[goodIndicesMask, b]) minX[b] = min(x_pred[goodIndicesMask,b]) maxX[b] = max(x_pred[goodIndicesMask,b]) x_pred[:, b] = np.linspace(minX[b], maxX[b], nrPointsToEval) dXdT_predCurr, cov_matrix = gp.predict(x_pred[:,b].reshape(-1,1), return_cov=True) MSE = np.diagonal(cov_matrix) dXdT_pred[:,b] = np.ravel(dXdT_predCurr) sigma_pred[:,b] = np.ravel(np.sqrt(MSE)) samples = gp.sample_y(x_pred[:,b].reshape(-1,1), n_samples=nrSamples, random_state=0) posteriorSamples[:,:,b] = np.squeeze(samples).T # renormalize the Xs and Ys # x_pred[:,b] = x_pred[:,b] * allXStd[b] + allXMean[b] # dXdT_pred[:,b] = dXdT_pred[:,b] * alldXdTStd[b] + alldXdTMean[b] # sigma_pred[:,b] = sigma_pred[:,b] * alldXdTStd[b] # posteriorSamples[:,:,b] = posteriorSamples[:,:,b]*alldXdTStd[b] + alldXdTMean[b] # renormalize the Xs and Ys # x_pred[:, b] = x_pred[:, b] * patXStd[b] + patXMean[b] # dXdT_pred[:, b] = dXdT_pred[:, b] * patdXdTStd[b] + patdXdTMean[b] # sigma_pred[:, b] = sigma_pred[:, b] * patdXdTStd[b] # posteriorSamples[:, :, b] = posteriorSamples[:, :, b] * patdXdTStd[b] + patdXdTMean[b] x_pred[:, b] = x_pred[:, b] * patXStd[b] + patXMean[b] dXdT_pred[:, b] = dXdT_pred[:, b] * patdXdTStd[b] sigma_pred[:, b] = sigma_pred[:, b] * patdXdTStd[b] posteriorSamples[:, :, b] = posteriorSamples[:, :, b] * patdXdTStd[b] # diagCol = plotTrajParams['diagColors'] # fig = pl.figure(1) # nrDiags = np.unique(diag).shape[0] # for diagNr in range(1, nrDiags + 1): # print(avgXdata.shape, diag.shape, dXdTdata.shape, diagCol, diagNr) # pl.scatter(avgXdata[diag == diagNr, b], dXdTdata[diag == diagNr, b], color = diagCol[diagNr - 1]) # # modelCol = 'r' # red # pl.plot(x_pred[:, b], dXdT_pred[:, b], '%s-' % modelCol, label = u'Prediction') # pl.fill(np.concatenate([x_pred[:, b], x_pred[::-1, b]]), np.concatenate( # [dXdT_pred[:, b] - 1.9600 * sigma_pred[:, b], (dXdT_pred[:, b] + 1.9600 * sigma_pred[:, b])[::-1]]), alpha = .5, # fc = modelCol, ec = 'None', label = '95% confidence interval') # for s in range(nrSamples): # pl.plot(x_pred[:, b], posteriorSamples[s, :, b]) # fig.show() gpParams = gp.get_params(deep=True) #print 'kernel', gp.kernel #print 'gpParams', gpParams gpList.append(gp) #print(adsa) return x_pred, dXdT_pred, sigma_pred, gpList, posteriorSamples
from sklearn.gaussian_process.kernels import (RBF, ExpSineSquared) kernel = RBF(length_scale=0.5, length_scale_bounds='fixed') #ExpSineSquared(length_scale=1.0, periodicity=1.0, length_scale_bounds='fixed', periodicity_bounds='fixed') gp = GaussianProcessRegressor(kernel=kernel) samples = 5 # Plot prior plt.figure(figsize=(10,10)) ax = plt.subplot(1, 1, 1) X_ = np.linspace(-5, 5, 500) y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) plt.plot(X_, y_mean, 'k--', lw=1, zorder=9) plt.fill_between(X_, y_mean - 2*y_std, y_mean + 2*y_std, alpha=0.1, color='k') y_samples = gp.sample_y(X_[:, np.newaxis], samples) plt.plot(X_, y_samples, lw=1) plt.xlim(-5, 5) plt.ylim(-3, 3) #plt.title("Prior (kernel: %s)" % kernel, fontsize=12) plt.xticks([5], [r"$x$"], rotation=0, fontsize=30) plt.yticks([3], [r"$f(x)$"], rotation=0, fontsize=30) ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) ax.tick_params(axis=u'both', which=u'both',length=0) # Generate data and fit GP rng = np.random.RandomState(11)
def _random_function(self, node: str, parent_values: Dict[str, Tensor], seed=None) -> Tensor: # Ordering parent_values = collections.OrderedDict([ (par_node, parent_values[par_node]) for par_node in self.model[node]['parents'] ]) parent_values = list(parent_values.values()) if len(parent_values) == 0: return torch.tensor(0.0) old_parent_values = self.model[node]['stacked_parent_values'] old_random_effects = self.model[node]['stacked_random_effects'] parent_values = torch.stack(parent_values).T gp = GaussianProcessRegressor(kernel=RBF(length_scale=self.bandwidth), optimizer=None) if old_parent_values is None and old_random_effects is None: # Sampling for the first time node_values = torch.tensor( gp.sample_y(parent_values, random_state=seed)).reshape(-1) self.model[node]['stacked_random_effects'] = node_values self.model[node]['stacked_parent_values'] = parent_values return node_values else: # Sampling, when random function was already evaluated if len(old_random_effects) >= self.interpolation_switch: if self.model[node]['interpolator'] is None: logger.warning( 'Using interpolation instead of Gaussian process!') if old_parent_values.shape[1] == 1: interpolator = interp1d(old_parent_values.squeeze(), old_random_effects, fill_value=0.0, bounds_error=False) else: interpolator = LinearNDInterpolator( old_parent_values.squeeze(), old_random_effects, fill_value=0.0) self.model[node]['interpolator'] = interpolator # else: # interpolator = LinearNDInterpolator(old_parent_values, old_random_effects, fill_value=0.0) return torch.tensor(self.model[node]['interpolator']( parent_values.squeeze())) else: gp.fit(old_parent_values, old_random_effects) node_values = torch.tensor( gp.sample_y(parent_values, random_state=seed)).reshape(-1) self.model[node]['stacked_random_effects'] = torch.cat( [old_random_effects, node_values]) self.model[node]['stacked_parent_values'] = torch.cat( [old_parent_values, parent_values], dim=0) return node_values
# Visualization of prior pylab.figure(0, figsize=(10, 8)) X_nn = gp.kernel.k2._project_manifold(X_[:, None]) pylab.subplot(3, 2, 1) for i in range(X_nn.shape[1]): pylab.plot(X_, X_nn[:, i], label="Manifold-dim %d" % i) pylab.legend(loc="best") pylab.xlim(-7.5, 7.5) pylab.title("Prior mapping to manifold") pylab.subplot(3, 2, 2) y_mean, y_std = gp.predict(X_[:, None], return_std=True) pylab.plot(X_, y_mean, 'k', lw=3, zorder=9, label="mean") pylab.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.5, color='k') y_samples = gp.sample_y(X_[:, None], 10) pylab.plot(X_, y_samples, color='b', lw=1) pylab.plot(X_, y_samples[:, 0], color='b', lw=1, label="samples") # just for the legend pylab.legend(loc="best") pylab.xlim(-7.5, 7.5) pylab.ylim(-4, 3) pylab.title("Prior samples") # Generate data and fit GP X = np.random.uniform(-5, 5, 40)[:, None] y = np.sin(X[:, 0]) + (X[:, 0] > 0) gp.fit(X, y) # Visualization of posterior X_nn = gp.kernel_.k2._project_manifold(X_[:, None])
class GaussianProcessFilter(FilteringModel): def __init__(self, kernel: Optional[Kernel] = None, **kwargs): """ GaussianProcessFilter model This model uses the GaussianProcessRegressor of scikit-learn to fit a Gaussian Process to the supplied TimeSeries. This can then be used to obtain samples or the mean values of the Gaussian Process at the times of the TimeSeries. Parameters ---------- kernel : sklearn.gaussian_process.kernels.Kernel, default: None The kernel specifying the covariance function of the Gaussian Process. If None is passed, the default in scikit-learn is used. Note that the kernel hyperparameters are optimized during fitting unless the bounds are marked as “fixed”. **kwargs Additional keyword arguments passed to `sklearn.gaussian_process.GaussianProcessRegressor`. """ super().__init__() self.model = GaussianProcessRegressor(kernel=kernel, **kwargs) def filter(self, series: TimeSeries, num_samples: int = 1) -> TimeSeries: """ Fits the Gaussian Process on the observations and returns samples from the Gaussian Process, or its mean values if `num_samples` is set to 1. Parameters ---------- series : TimeSeries The series of observations used to infer the values according to the specified Gaussian Process. This must be a deterministic series (containing one sample). num_samples: int, default: 1 Number of times a prediction is sampled from the Gaussian Process. If set to 1, instead the mean values will be returned. Returns ------- TimeSeries A stochastic `TimeSeries` sampled from the Gaussian Process, or its mean if `num_samples` is set to 1. """ raise_if_not(series.is_deterministic, 'The input series for the Gaussian Process filter must be ' 'deterministic (observations).') super().filter(series) values = series.values(copy=False) if series.has_datetime_index: times = np.arange(series.n_timesteps).reshape(-1, 1) else: times = series.time_index.values.reshape(-1, 1) not_nan_mask = np.all(~np.isnan(values), axis=1) self.model.fit(times[not_nan_mask, :], values[not_nan_mask, :]) if num_samples == 1: filtered_values = self.model.predict(times) else: filtered_values = self.model.sample_y(times, n_samples=num_samples) return TimeSeries.from_times_and_values(series.time_index, filtered_values)
class WrappedGaussianProcess(MultiOutputMixin, RegressorMixin, BaseEstimator): r"""Wrapped Gaussian Process. The implementation is based on the algorithm 4 of [1]. Parameters ---------- space : Manifold Manifold. metric : RiemannianMetric Riemannian metric. prior : function Associate to each input a manifold valued point. kernel : kernel instance, default=None The kernel specifying the covariance function of the GP. If None is passed, the kernel ``ConstantKernel(1.0, constant_value_bounds="fixed") * RBF(1.0, length_scale_bounds="fixed")`` is used as default. Note that the kernel hyperparameters are optimized during fitting unless the bounds are marked as "fixed". alpha : float or ndarray of shape (n_samples,), default=1e-10 Value added to the diagonal of the kernel matrix during fitting. This can prevent a potential numerical issue during fitting, by ensuring that the calculated values form a positive definite matrix. It can also be interpreted as the variance of additional Gaussian measurement noise on the training observations. Note that this is different from using a `WhiteKernel`. If an array is passed, it must have the same number of entries as the data used for fitting and is used as datapoint-dependent noise level. Allowing to specify the noise level directly as a parameter is mainly for convenience and for consistency with :class:`~sklearn.linear_model.Ridge`. optimizer : "fmin_l_bfgs_b" or callable, default="fmin_l_bfgs_b" Can either be one of the internally supported optimizers for optimizing the kernel's parameters, specified by a string, or an externally defined optimizer passed as a callable. If a callable is passed, it must have the signature:: def optimizer(obj_func, initial_theta, bounds): # * 'obj_func': the objective function to be minimized, which # takes the hyperparameters theta as a parameter and an # optional flag eval_gradient, which determines if the # gradient is returned additionally to the function value # * 'initial_theta': the initial value for theta, which can be # used by local optimizers # * 'bounds': the bounds on the values of theta .... # Returned are the best found hyperparameters theta and # the corresponding value of the target function. return theta_opt, func_min Per default, the L-BFGS-B algorithm from `scipy.optimize.minimize` is used. If None is passed, the kernel's parameters are kept fixed. Available internal optimizers are: `{'fmin_l_bfgs_b'}`. n_restarts_optimizer : int, default=0 The number of restarts of the optimizer for finding the kernel's parameters which maximize the log-marginal likelihood. The first run of the optimizer is performed from the kernel's initial parameters, the remaining ones (if any) from thetas sampled log-uniform randomly from the space of allowed theta-values. If greater than 0, all bounds must be finite. Note that `n_restarts_optimizer == 0` implies that one run is performed. copy_X_train : bool, default=True If True, a persistent copy of the training data is stored in the object. Otherwise, just a reference to the training data is stored, which might cause predictions to change if the data is modified externally. random_state : int, RandomState instance or None, default=None Determines random number generation used to initialize the centers. Pass an int for reproducible results across multiple function calls. [1] Mallasto, A. and Feragen, A. Wrapped gaussian process regression on riemannian manifolds. In 2018 IEEE/CVF Conference on Computer Vision and Pattern Recognition """ def __init__( self, space, metric, prior, kernel=None, *, alpha=1e-10, optimizer="fmin_l_bfgs_b", n_restarts_optimizer=0, copy_X_train=True, random_state=None, ): if metric is None: metric = space.metric self.metric = metric self.space = space self.prior = prior self.copy_X_train = copy_X_train self.y_train_ = None self.tangent_y_train_ = None self.y_train_shape_ = None self._euclidean_gpr = GaussianProcessRegressor( kernel=kernel, alpha=alpha, optimizer=optimizer, n_restarts_optimizer=n_restarts_optimizer, normalize_y=False, copy_X_train=copy_X_train, random_state=random_state, ) self.__dict__.update(self._euclidean_gpr.__dict__) self.log_marginal_likelihood = self._euclidean_gpr.log_marginal_likelihood def _get_tangent_targets(self, X, y): """Compute the tangent targets, using the provided prior. Parameters ---------- X : array-like of shape (n_samples, n_features) or list of object Feature vectors or other representations of training data. y : array-like of shape (n_samples,) or (n_samples, n_targets) or (n_samples, n1_targets, n2_targets) for matrix-valued targets. Target values. The target must belongs to the manifold space Returns ------- tangent_y : array-like of shape (n_samples,) or (n_samples, n_targets) or (n_samples, n1_targets, n2_targets) Target projected on the associated (by the prior) tangent space. """ base_points = self.prior(X) return self.metric.log(y, base_point=base_points) def fit(self, X, y): """Fit Wrapped Gaussian process regression model. The Wrapped Gaussian process is fit through the following steps: - Compute the tangent dataset using the prior - Fit a Gaussian process regression on the tangent dataset - Store the resulting euclidean Gaussian process Parameters ---------- X : array-like of shape (n_samples, n_features) or list of object Feature vectors or other representations of training data. y : array-like of shape (n_samples,) or (n_samples, n_targets) or (n_samples, n1_targets, n2_targets) Target values. The target must belongs to the manifold space Returns ------- self : object WrappedGaussianProcessRegressor class instance. """ if not gs.all(self.space.belongs(y)): raise AttributeError( "The target values must belongs to the given space") # compute the tangent dataset using the prior tangent_y = self._get_tangent_targets(X, y) self.y_train_shape_ = y.shape[ 1:] # this is really useful when the samples are matrices, or tensor of dim>1 tangent_y = gs.reshape(tangent_y, (y.shape[0], -1)) # flatten the samples. # fit a gpr on the tangent dataset self._euclidean_gpr.fit(X, tangent_y) # update the attributes of the wgpr using the new attributes of the gpr self.__dict__.update(self._euclidean_gpr.__dict__) self.y_train_ = y self.tangent_y_train_ = tangent_y # = self._euclidean_gpr.y_train_ return self def predict(self, X, return_tangent_std=False, return_tangent_cov=False): """Predict using the Gaussian process regression model. A fitted Wrapped Gaussian process can be use to predict values through the following steps: - Use the stored Gaussian process regression on the dataset to return tangent predictions - Compute the base-points using the prior - Map the tangent predictions on the manifold via the metric's exp with the base-points yielded by the prior We can also predict based on an unfitted model by using the GP prior. In addition to the mean of the predictive distribution, optionally also returns its standard deviation (`return_std=True`) or covariance (`return_cov=True`). Note that at most one of the two can be requested. Parameters ---------- X : array-like of shape (n_samples, n_features) or list of object Query points where the GP is evaluated. return_tangent_std : bool, default=False If True, the standard-deviation of the predictive distribution on at the query points in the tangent space is returned along with the mean. return_tangent_cov : bool, default=False If True, the covariance of the joint predictive distribution at the query points in the tangent space is returned along with the mean. Returns ------- y_mean : ndarray of shape (n_samples,) or (n_samples, n_targets) Mean of predictive distribution a query points. y_std : ndarray of shape (n_samples,) or (n_samples, n_targets), optional Standard deviation of predictive distribution at query points in the tangent space. Only returned when `return_std` is True. y_cov : ndarray of shape (n_samples, n_samples) or \ (n_samples, n_samples, n_targets), optional Covariance of joint predictive distribution a query points in the tangent space. Only returned when `return_cov` is True. In the case where the target is matrix valued, return the covariance of the vectorized prediction. """ euc_result = self._euclidean_gpr.predict(X, return_cov=return_tangent_cov, return_std=return_tangent_std) return_multiple = return_tangent_std or return_tangent_cov tangent_means = euc_result[0] if return_multiple else euc_result base_points = self.prior(X) tangent_means = gs.reshape( gs.cast(tangent_means, dtype=X.dtype), (X.shape[0], *self.y_train_shape_), ) y_mean = self.metric.exp(tangent_means, base_point=base_points) if return_multiple: tangent_std_cov = gs.cast(euc_result[1], dtype=X.dtype) return (y_mean, tangent_std_cov) return y_mean def sample_y(self, X, n_samples=1, random_state=0): """Draw samples from Wrapped Gaussian process and evaluate at X. A fitted Wrapped Gaussian process can be use to sample values through the following steps: - Use the stored Gaussian process regression on the dataset to sample tangent values - Compute the base-points using the prior - Flatten (and repeat if needed) both the base-points and the tangent samples to benefit from vectorized computation. - Map the tangent samples on the manifold via the metric's exp with the flattened and repeated base-points yielded by the prior Parameters ---------- X : array-like of shape (n_samples_X, n_features) or list of object Query points where the WGP is evaluated. n_samples : int, default=1 Number of samples drawn from the Wrapped Gaussian process per query point. random_state : int, RandomState instance or None, default=0 Determines random number generation to randomly draw samples. Pass an int for reproducible results across multiple function calls. Returns ------- y_samples : ndarray of shape (n_samples_X, n_samples), or \ (n_samples_X, *target_shape, n_samples) Values of n_samples samples drawn from wrapped Gaussian process and evaluated at query points. """ tangent_samples = self._euclidean_gpr.sample_y(X, n_samples, random_state) tangent_samples = gs.cast(tangent_samples, dtype=X.dtype) if gs.ndim(tangent_samples) > 2: tangent_samples = gs.moveaxis(tangent_samples, -2, -1) flat_tangent_samples = gs.reshape(tangent_samples, (-1, *self.y_train_shape_)) base_points = gs.repeat(self.prior(X), n_samples, axis=0) flat_y_samples = self.metric.exp(flat_tangent_samples, base_point=base_points) y_samples = gs.reshape(flat_y_samples, (X.shape[0], n_samples, *self.y_train_shape_)) if gs.ndim(tangent_samples) > 2: y_samples = gs.moveaxis(y_samples, 1, -1) return y_samples
#X_ = np.array([[0], [1.1], [1.3], [2.2], [2.8], [3.6], [3.7], [4.6], [4.7], [4.8]]) #Y_ = np.array([[-0.1], [0.9], [1], [0.1], [0], [0.7], [0.8], [-1.1], [-1], [-0.8]]) X = np.linspace(0, 5, 100)[:, None] # RBF kernel = C(1.0) * RBF(length_scale=1) gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-5, n_restarts_optimizer=10) pylab.figure(0, figsize=(14, 12)) pylab.subplot(3, 2, 1) ymean, y_std = gp.predict(X, return_std=True) pylab.plot(X, ymean, 'k', lw=3, zorder=9, label="mean") pylab.fill_between(X[:, 0], ymean - y_std, ymean + y_std, alpha=0.5, color='k') y_samples = gp.sample_y(X, 10) pylab.plot(X, y_samples, color='b', lw=2) pylab.plot(X, y_samples[:, 0], color='b', lw=2, label="sample") pylab.legend(loc="best") pylab.xlim(0, 5) pylab.ylim(-3, 3) pylab.title("Prior Samples") #Matern kernel = C(1.0) * Matern(length_scale=1, nu=1.5) gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-5, n_restarts_optimizer=10) pylab.subplot(3, 2, 2) ymean, y_std = gp.predict(X, return_std=True)
plt.plot(x, y_pred, 'b-', label=u'Prediction') #plt.fill(np.concatenate([x, x[::-1]]), # np.concatenate([y_pred - 1.9600 * sigma, # (y_pred + 1.9600 * sigma)[::-1]]), # alpha=.5, fc='b', ec='None', label='95% confidence interval') plt.fill(np.concatenate([x, x[::-1]]), np.concatenate([y_pred - 1.9600 * sigma, (y_pred + 1.9600 * sigma)[::-1]]), alpha=.5, fc='b', ec='None', label='95% confidence interval') plt.xlabel('$x$') plt.ylabel('$f(x)$') plt.ylim(-10, 20) plt.legend(loc='upper left') fig = plt.figure() y_samples = gp.sample_y(x, 10) plt.plot(x, y_samples, lw=1) plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$') plt.plot(X, y, 'r.', markersize=10, label=u'Observations') #plt.plot(x, y_pred, 'b-', label=u'Prediction') #plt.fill(np.concatenate([x, x[::-1]]), # np.concatenate([y_pred - 1.9600 * sigma, # (y_pred + 1.9600 * sigma)[::-1]]), # alpha=.5, fc='b', ec='None', label='95% confidence interval') plt.xlabel('$x$') plt.ylabel('$f(x)$') plt.ylim(-10, 20) plt.legend(loc='upper left') fig = plt.figure() #y_samples = gp.sample_y(x, 10)
alpha=0.01, # float or array-like of shape(n_sample), default=1e-10 optimizer= "fmin_l_bfgs_b", # "fmin_l_bfgs_b” or callable, default="fmin_l_bfgs_b" n_restarts_optimizer=0, # int, default=0 normalize_y=False, # boolean, optional (default: False) copy_X_train=True, # bool, default=True random_state=None, # int or RandomState, default=None ) model.fit(train_x, train_y) y_pred, y_std = model.predict(x.reshape(-1, 1), return_std=True) # y_pred, y_std = model.predict(x.reshape(-1, 1), return_std=True) log_marginal_likelihood = model.log_marginal_likelihood() # 対数周辺尤度 params = model.get_params() # 設定パラメータの取得(辞書) scores = model.score(train_x, train_y) # 決定係数R^2 # params = model.set_params() # 設定パラメータの設定(辞書) k_samples = model.sample_y(train_x, n_samples=5) # 事後分布のカーネル関数をランダムに5つサンプリング X_train = model.X_train_ y_train = model.y_train_ kernel = model.kernel_ # 予測に使用されたカーネル(最適化済みで最初に設定したパラメータとは異なる) L = model.L_ alpha = model.alpha_ log_marginal_likelihood_value = model.log_marginal_likelihood_value_ # 対数周辺尤度 # plot fig = plt.figure(figsize=(6, 4)) ax1 = fig.add_subplot(111) for i in range(k_samples.shape[1]): ax1.plot(train_x, k_samples[:, i]) ax1.scatter(train_x, train_y,
# Plot posterior """ @BUG line below is what we need, but it gives an error """ y_mean, y_std = gpr.predict(X_[:, np.newaxis], return_std=True) """ @TOTO be able to remove 2 following lines and have above @BUG solved """ # y_mean = gpr.predict(X_[:, np.newaxis], return_std=False) # y_std = np.zeros(len(y_mean)) y_mean = y_mean + continuous_from_array(X_, np_priors)[:, np.newaxis] plt.plot(X_, y_mean, 'r', zorder=9) plt.fill_between(X_, (y_mean - y_std[:, np.newaxis]).flatten(), (y_mean + y_std[:, np.newaxis]).flatten(), alpha=0.4, color='b') plt.fill_between(X_, (y_mean - 3 * y_std[:, np.newaxis]).flatten(), (y_mean + 3 * y_std[:, np.newaxis]).flatten(), alpha=0.1, color='b') y_samples = gpr.sample_y(X_[:, np.newaxis], 10) ''' plt.plot(X_, y_samples, lw=1) ''' plt.scatter(target[:, 0], target[:, 1], marker='+', c='m', s=50, zorder=10, edgecolors=(0, 0, 0)) plt.scatter(X[:, 0], target_y_train[:, np.newaxis], marker='+', c='g', s=50, zorder=10, edgecolors=(0, 0, 0)) plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gpr.kernel_, gpr.log_marginal_likelihood(gpr.kernel_.theta)), fontsize=12) plt.ylim(-1, 1.2 * np.max(target[:, 1])) time_stop_np_gp = time.clock() print(f'Execution numpy GP : {time_stop_np_gp - time_start_np_gp}')
import matplotlib.pyplot as plt from scipy.stats import norm, multivariate_normal x = np.linspace(0, 1, 100) x1 = np.linspace(-5, 0, 100) x2 = np.linspace(0, 5, 100) K = norm.pdf(10 * np.abs(np.subtract(*np.meshgrid(x, x)))) plt.figure() for _ in range(10): plt.plot(x, multivariate_normal.rvs(np.zeros(100, dtype=np.float), K)) #X, y = make_friedman2(n_samples=500, noise=0, random_state=0) kernel = RBF(0.1) #DotProduct() + WhiteKernel() gpr = GaussianProcessRegressor(kernel=kernel, random_state=0, optimizer=None) #.fit(X, y) plt.figure() for i in range(10): y1 = gpr.sample_y(x[:, None], random_state=i).squeeze() #gpr.fit(x1[:, None], y1) plt.plot(x, y1) #plt.plot(x2, gpr.sample_y(x2[:, None], random_state=i).squeeze()) plt.show() #print(gpr.score(X, y)) #print(gpr.predict(X[:2,:], return_std=True)) print(gpr.get_params())
class GaussianProcess(oneDcurve): def __init__(self, x, y, dy, mask=None, **args): '''Fit a GP (Gaussian Process) spline to the data. [args] can be any argument recognized by Matern kernel''' oneDcurve.__init__(self, x, y, dy, mask) self.pars = { 'diff_degree':None, 'scale':None, 'amp':None} for key in args: if key not in self.pars and key != "mean": raise TypeError("%s is an invalid keyword argument for this method" % key) if key != "mean": self.pars[key] = args[key] if 'mean' in args: self.mean = args['mean'] else: self.mean = lambda x: x*0 + num.median(self.y) # Make sure the data conform to the spine requirements self.median = num.median(self.y) self._setup() self.realization = None def __str__(sef): return "Gaussian Process" def help(self): print("scale: Scale over which the function varies") print("amp: Amplitude of typical function variations") print("diff_degree: Roughly, the degree of differentiability") def _setup(self): '''Given the current set of params, setup the interpolator.''' from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import Matern, ConstantKernel globals()['GaussianProcessRegressor'] = GaussianProcessRegressor globals()['Matern'] = Matern globals()['ConstantKernel'] = ConstantKernel x,y,dy = self._regularize() if self.diff_degree is None: self.diff_degree = 2 if self.amp is None: self.amp = num.std(y - self.mean(x)) if self.scale is None: #self.scale = (self.x.max() - self.x.min())/2 self.scale = 30 self.kernel = ConstantKernel(self.amp, constant_value_bounds='fixed')*\ Matern(length_scale=self.scale, nu=self.diff_degree+0.5, length_scale_bounds='fixed') Y = y - self.mean(x) X = num.array([x]).T self.gpr = GaussianProcessRegressor(kernel=self.kernel, alpha=dy*dy).fit(X,Y) self.setup = True self.realization = None def __call__(self, x): '''Interpolate at point [x]. Returns a 3-tuple: (y, mask) where [y] is the interpolated point, and [mask] is a boolean array with the same shape as [x] and is True where interpolated and False where extrapolated''' if not self.setup: self._setup() if len(num.shape(x)) < 1: scalar = True else: scalar = False x = num.atleast_1d(x) if self.realization is not None: #res = self.realization(x.reshape(-1,1),random_state=self._seed)[:,0] res = splev(x, self.realization) else: res = self.gpr.predict(x.reshape(-1,1)) res = res + self.mean(x) if scalar: return res[0],self.x.min() <= x[0] <= self.x.max() else: return res,num.greater_equal(x, self.x.min())*\ num.less_equal(x, self.x.max()) def domain(self): return (self.x.min(),self.x.max()) def error(self, x): '''Returns the error in the interpolator at points [x].''' if not self.setup: self._setup() if len(num.shape(x)) < 1: scalar = True else: scalar = False x = num.atleast_1d(x) res,sigma = self.gpr.predict(x.reshape(-1,1), return_std=True) if scalar: return sigma[0] else: return sigma def draw(self): '''Generate a random realization of the spline, based on the data.''' if not self.setup: self._setup() # scikit-learn seems to have a bug. We have to make a realization # and make a smoothing spline to approximate it. tmin,tmax = self.domain() t = num.arange(tmin, tmax+1, 1.0) seed = num.random.randint(2**32) y = self.gpr.sample_y(t.reshape(-1,1), random_state=seed)[:,0] self.realization = splrep(t, y, k=3, s=0) def reset_mean(self): self.realization = None def rchisquare(self): chisq = self.chisquare() if len(self.x) < 5: return -1 return chisq/(len(self.x) - 4) def deriv(self, x, n=1): '''Returns the nth derivative of the function at x.''' if len(num.shape(x)) < 1: scalar = True else: scalar = False xs = num.atleast_1d(x) f = lambda x: self.__call__(x)[0] res = deriv(f, xs, dx=self.scale/100., n=n) if scalar: return res[0] else: return res def find_extrema(self, xmin=None, xmax=None): '''Find the position and values of the maxima/minima. Returns a tuple: (roots,vals,ypps) where roots are the x-values where the extrema occur, vals are the y-values at these points, and ypps are the 2nd derivatives. Optionally, only search for roots between xmin and xmax''' #evaluate the 1st derivative at sacle/10 intervals (that should be # enough) if xmin is None: xmin = self.x.min() if xmax is None: xmax = self.x.max() dx = min(self.scale*1.0/20, (xmax-xmin)/5.0) xs = num.arange(xmin, xmax, dx) dys = self.deriv(xs, n=1) pids = num.greater(dys, 0) inds = num.nonzero(num.logical_xor(pids[1:],pids[:-1]))[0] if len(inds) == 0: return (num.array([]), num.array([]), num.array([])) ret = [] for i in range(len(inds)): try: res = brentq(self.deriv, xs[inds[i]-1], xs[inds[i]+1]) ret.append(res) except: continue if len(ret) == 0: return (num.array([]), num.array([]), num.array([])) ret = num.array(ret) vals = self.__call__(ret)[0] curvs = self.deriv(ret, n=2) curvs = num.where(curvs > 0, 1, curvs) curvs = num.where(curvs < 0, -1, curvs) return ret,vals,curvs def intercept(self, y): '''Find the value of x for which the interpolator goes through [y]''' xs = num.arange(self.x.min(), self.x.max(), self.scale/10) f = lambda x: self.__call__(x)[0] - y ys = f(xs) pids = num.greater(ys, 0) if num.alltrue(pids) or num.alltrue(-pids): return None ret = [] inds = num.nonzero(pids[1:] - pids[:-1])[0] for i in range(len(inds)): ret.append(brentq(f, xs[inds[i]], xs[inds[i]+1])) ret = num.array(ret) return ret
def pass_arg_upd(Xx, nsim, tr_size, pre_trained_hyperparamters): print("tr_Size:", tr_size) #pre_tr_size = 100 tr_size = int(tr_size) #List of lakes to choose from lake = ['mendota', 'mille_lacs'] lake_num = 0 # 0 : mendota , 1 : mille_lacs lake_name = lake[lake_num] # Load features (Xc) and target values (Y) data_dir = '../../../../data/' filename = lake_name + '.mat' mat = spio.loadmat(data_dir + filename, squeeze_me=True, variable_names=['Y', 'Xc_doy', 'Modeled_temp']) Xc = mat['Xc_doy'] Y = mat['Y'] Xc = Xc[:, :-1] # train and test data trainX, testX, trainY, testY = train_test_split( Xc, Y, train_size=tr_size / Xc.shape[0], test_size=tr_size / Xc.shape[0], random_state=42, shuffle=True) ## train and test data #trainX, trainY = Xc[:tr_size,:-1], Y[:tr_size] #testX, testY = Xc[-50:,:-1], Y[-50:] # # Loading unsupervised data # unsup_filename = lake_name + '_sampled.mat' # unsup_mat = spio.loadmat(data_dir+unsup_filename, squeeze_me=True, # variable_names=['Xc_doy1','Xc_doy2']) # uX1 = unsup_mat['Xc_doy1'] # Xc at depth i for every pair of consecutive depth values # uX2 = unsup_mat['Xc_doy2'] # Xc at depth i + 1 for every pair of consecutive depth values # uX1 = uX1[:pre_tr_size,:-1] # uX2 = uX2[:pre_tr_size,:-1] # uY1 = uX1[:pre_tr_size,-1:] # uY2 = uX2[:pre_tr_size,-1:] # kernel = C(5.0, (1e-2, 1e3)) * RBF(length_scale = [1] * trainX.shape[1], length_scale_bounds=(1e-3, 1e4)) # gp1 = GaussianProcessRegressor(kernel=kernel, alpha =1.2, n_restarts_optimizer=0) # gp1.fit(uX1, uY1) # # pre-trained model parameters # pre_trained_hyperparamters = gp1.kernel_ # Updated model gp2 = GaussianProcessRegressor(kernel=pre_trained_hyperparamters, alpha=1.5, n_restarts_optimizer=10) gp2.fit(trainX, trainY) # scale the uniform numbers to original space # max and min value in each column max_in_column_Xc = np.max(trainX, axis=0) min_in_column_Xc = np.min(trainX, axis=0) # Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc) Xc_org = Xx * (max_in_column_Xc - min_in_column_Xc) + min_in_column_Xc samples = gp2.sample_y(Xc_org, n_samples=int(nsim)).T return samples
1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5)] for fig_index, kernel in enumerate(kernels): # Specify Gaussian Process gp = GaussianProcessRegressor(kernel=kernel) # Plot prior plt.figure(fig_index, figsize=(8, 8)) plt.subplot(2, 1, 1) X_ = np.linspace(0, 5, 100) y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) plt.plot(X_, y_mean, 'k', lw=3, zorder=9) plt.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.2, color='k') y_samples = gp.sample_y(X_[:, np.newaxis], 10) plt.plot(X_, y_samples, lw=1) plt.xlim(0, 5) plt.ylim(-3, 3) plt.title("Prior (kernel: %s)" % kernel, fontsize=12) # Generate data and fit GP rng = np.random.RandomState(4) X = rng.uniform(0, 5, 10)[:, np.newaxis] y = np.sin((X[:, 0] - 2.5) ** 2) gp.fit(X, y) # Plot posterior plt.subplot(2, 1, 2) X_ = np.linspace(0, 5, 100) y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)