def generate_regression_task_dataset(n_tasks):
kernel = RBF(length_scale=1., length_scale_bounds=(1e-5, 1e5))
gpr = GaussianProcessRegressor(kernel=kernel)
X = np.linspace(0, 100, 1000)
X = np.atleast_2d(X).T
f_samples = []
for i in range(n_tasks):
# sample a function from a zero mean GP
f = gpr.sample_y(X, random_state=i)
f_samples.append(f)
return f_samples
def test_y_multioutput():
""" Test that GPR can deal with multi-dimensional target values"""
y_2d = np.vstack((y, y*2)).T
# Test for fixed kernel that first dimension of 2d GP equals the output
# of 1d GP and that second dimension is twice as large
kernel = RBF(length_scale=1.0)
gpr = GaussianProcessRegressor(kernel=kernel, optimizer=None,
normalize_y=False)
gpr.fit(X, y)
gpr_2d = GaussianProcessRegressor(kernel=kernel, optimizer=None,
normalize_y=False)
gpr_2d.fit(X, y_2d)
y_pred_1d, y_std_1d = gpr.predict(X2, return_std=True)
y_pred_2d, y_std_2d = gpr_2d.predict(X2, return_std=True)
_, y_cov_1d = gpr.predict(X2, return_cov=True)
_, y_cov_2d = gpr_2d.predict(X2, return_cov=True)
assert_almost_equal(y_pred_1d, y_pred_2d[:, 0])
assert_almost_equal(y_pred_1d, y_pred_2d[:, 1] / 2)
# Standard deviation and covariance do not depend on output
assert_almost_equal(y_std_1d, y_std_2d)
assert_almost_equal(y_cov_1d, y_cov_2d)
y_sample_1d = gpr.sample_y(X2, n_samples=10)
y_sample_2d = gpr_2d.sample_y(X2, n_samples=10)
assert_almost_equal(y_sample_1d, y_sample_2d[:, 0])
# Test hyperparameter optimization
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
gpr.fit(X, y)
gpr_2d = GaussianProcessRegressor(kernel=kernel, normalize_y=True)
gpr_2d.fit(X, np.vstack((y, y)).T)
assert_almost_equal(gpr.kernel_.theta, gpr_2d.kernel_.theta, 4)
class Gaussian_data():
def __init__(self, l=1, e=.1):
self.length_scale = l
self.noise_level = e
self.kernel = RBF(length_scale=l) + WhilteKernel(noise_level=e)
self.gp = GaussianProcessRegressor(kernel=self.kernel)
def gen(self, n=5, x_min=-2, x_max=2):
x = np.random.rand(x_min, x_max, n).sort()
y = self.gp.sample_y(x)
return x, y
def test_sample_statistics(kernel):
# Test that statistics of samples drawn from GP are correct.
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
y_mean, y_cov = gpr.predict(X2, return_cov=True)
samples = gpr.sample_y(X2, 300000)
# More digits accuracy would require many more samples
assert_almost_equal(y_mean, np.mean(samples, 1), 1)
assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(),
np.var(samples, 1) / np.diag(y_cov).max(), 1)
def test_sample_statistics():
""" Test that statistics of samples drawn from GP are correct."""
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
y_mean, y_cov = gpr.predict(X2, return_cov=True)
samples = gpr.sample_y(X2, 300000)
# More digits accuracy would require many more samples
assert_almost_equal(y_mean, np.mean(samples, 1), 2)
assert_almost_equal(np.diag(y_cov) / np.diag(y_cov).max(), np.var(samples, 1) / np.diag(y_cov).max(), 1)
def gaussian_process_reg(particles, weights):
# GP regression
kernel = 1.0 * RBF(1.0)
gp = GaussianProcessRegressor(kernel=kernel, random_state=0)
gp.fit(particles, weights)
sampled_weights_array = gp.sample_y(particles)
gp_weights = []
for item in sampled_weights_array:
gp_weights.append(item[0])
weights[:] = gp_weights[:]
def _simulate_single_equation(X, scale):
"""X: [n, num of parents], x: [n]"""
z = np.random.normal(scale=scale, size=n)
pa_size = X.shape[1]
if pa_size == 0:
return z
if sem_type == 'mlp':
hidden = 100
W1 = np.random.uniform(low=0.5,
high=2.0,
size=[pa_size, hidden])
W1[np.random.rand(*W1.shape) < 0.5] *= -1
W2 = np.random.uniform(low=0.5, high=2.0, size=hidden)
W2[np.random.rand(hidden) < 0.5] *= -1
x = sigmoid(X @ W1) @ W2 + z
elif sem_type == 'mim':
w1 = np.random.uniform(low=0.5, high=2.0, size=pa_size)
w1[np.random.rand(pa_size) < 0.5] *= -1
w2 = np.random.uniform(low=0.5, high=2.0, size=pa_size)
w2[np.random.rand(pa_size) < 0.5] *= -1
w3 = np.random.uniform(low=0.5, high=2.0, size=pa_size)
w3[np.random.rand(pa_size) < 0.5] *= -1
x = np.tanh(X @ w1) + np.cos(X @ w2) + np.sin(X @ w3) + z
elif sem_type == 'gp':
from sklearn.gaussian_process import GaussianProcessRegressor
gp = GaussianProcessRegressor()
x = gp.sample_y(X, random_state=None).flatten() + z
elif sem_type == 'gp-add':
from sklearn.gaussian_process import GaussianProcessRegressor
gp = GaussianProcessRegressor()
x = sum([
gp.sample_y(X[:, i, None], random_state=None).flatten()
for i in range(X.shape[1])
]) + z
else:
raise ValueError('Unknown sem type. In a nonlinear model, \
the options are as follows: mlp, mim, \
gp, gp-add, or quadratic.')
return x
class SmoothFunctionCreator():
def __init__(self, seed=42):
self._gp = GaussianProcessRegressor()
x_train = np.array([0.0, 2.0, 6.0, 10.0])[:, np.newaxis]
source_train = np.array([0.0, 1.0, -1.0, 0.0])
self._gp.fit(x_train, source_train)
self._random_state = np.random.RandomState(seed)
def sample(self, n_samples):
x = np.linspace(0.0, 10.0, 100)[:, np.newaxis]
source = self._gp.sample_y(x, n_samples, random_state=self._random_state)
target = gaussian_filter1d(source, 1, order=1, axis=0)
target = np.tanh(10.0 * target)
return source, target
def kernel_draw(N, Lim, kernel=TEST):
#ASL=1
# Specify Gaussian Process
gp = GaussianProcessRegressor(kernel=kernel)
# Plot prior
X_ = np.linspace(0, scale * Lim, N)
y_samples = gp.sample_y(X_[:, np.newaxis], 1)
print(y_samples.shape)
return stretch * np.squeeze(y_samples.T)
def prior_and_posterior(kernel):
gp = GaussianProcessRegressor(kernel=kernel)
plt.figure(figsize=(8, 8))
# Plot prior
plt.subplot(2, 1, 1)
Xstar = np.linspace(0, 5, 100)
mu, sigma = gp.predict(Xstar[:, np.newaxis], return_std=True)
plt.plot(Xstar, mu, 'k', lw=3, zorder=9)
plt.fill_between(Xstar, mu - sigma, mu + sigma, alpha=0.2, color='k')
y_samples = gp.sample_y(Xstar[:, np.newaxis], 10)
plt.plot(Xstar, y_samples, lw=1)
plt.xlim(0, 5)
plt.ylim(-3, 3)
plt.title("Prior (kernel: %s)" % kernel, fontsize=12)
# Generate data and fit GP
rng = np.random.RandomState(4)
X = rng.uniform(0, 5, 10)[:, np.newaxis]
y = np.sin((X[:, 0] - 2.5)**2)
gp.fit(X, y)
# Plot posterior
plt.subplot(2, 1, 2)
mu, sigma = gp.predict(Xstar[:, np.newaxis], return_std=True)
plt.plot(Xstar, mu, 'k', lw=3, zorder=9)
plt.fill_between(Xstar, mu - sigma, mu + sigma, alpha=0.2, color='k')
y_samples = gp.sample_y(Xstar[:, np.newaxis], 10)
plt.plot(Xstar, y_samples, lw=1)
plt.scatter(X[:, 0], y, c='r', s=50, zorder=10, edgecolors=(0, 0, 0))
plt.xlim(0, 5)
plt.ylim(-3, 3)
plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f" %
(gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)),
fontsize=12)
plt.show()
def gp_bootstrap_noise(X, Y, kernel, n_samples=50, alpha=20.0):
# First, we fit a GP model to the data
# (with noise added, represented by the parameter alpha):
m = GaussianProcessRegressor(kernel=kernel, alpha=alpha)
m.fit(X, Y)
gp_samples = m.sample_y(X, n_samples=n_samples).T
boot_samples = []
for si in range(n_samples):
boot_samples.append((X, gp_samples[si][0]))
return boot_samples
class GPDataGen(DataGenerator):
def __init__(self,
params=None,
param_sampler=None,
kernel=RBF(),
mean_fn=None,
xs_sampler=None):
self.params = params
self.param_sampler = None
self.xs_sampler = xs_sampler
self.mean_fn = mean_fn
self.kernel = kernel
self.gp = GaussianProcessRegressor(self.kernel)
def generate_data(self, xs, params=None):
xs = xs.reshape((-1, 1))
return self.gp.sample_y(xs.reshape((-1, 1))).flatten()
def pass_arg(Xx, nsim, tr_size):
print("tr_Size:",tr_size)
tr_size = int(tr_size)
use_YPhy = 0
#List of lakes to choose from
lake = ['mendota' , 'mille_lacs']
lake_num = 0 # 0 : mendota , 1 : mille_lacs
lake_name = lake[lake_num]
# Load features (Xc) and target values (Y)
data_dir = '../../../../data/'
filename = lake_name + '.mat'
mat = spio.loadmat(data_dir + filename, squeeze_me=True,
variable_names=['Y','Xc_doy','Modeled_temp'])
Xc = mat['Xc_doy']
Y = mat['Y']
Xc = Xc[:,:-1]
scaler = preprocessing.StandardScaler()
# train and test data
trainX, testX, trainY, testY = train_test_split(Xc, Y, train_size=tr_size/Xc.shape[0],
test_size=tr_size/Xc.shape[0], random_state=42, shuffle=True)
trainY=trainY[:,np.newaxis]
#trainX = scaler.fit_transform(trainX)
#trainY = scaler.fit_transform(trainY)
kernel = C(5.0, (0.1, 1e2)) * RBF(length_scale = [1] * trainX.shape[1], length_scale_bounds=(1e-1, 1e15))
gp = GaussianProcessRegressor(kernel=kernel, alpha =.1, n_restarts_optimizer=10)
gp.fit(trainX, trainY)
#y_pred1, sigma1 = gp.predict(testX, return_std=True)
# scale the uniform numbers to original space
# max and min value in each column
max_in_column_Xc = np.max(trainX,axis=0)
min_in_column_Xc = np.min(trainX,axis=0)
# Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc)
Xc_org = Xx*(max_in_column_Xc-min_in_column_Xc) + min_in_column_Xc
print(gp.kernel_)
samples = gp.sample_y(Xc_org, n_samples=int(nsim)).T
return np.squeeze(samples)
class DriftSimulator():
def __init__(self, n_samples, rt_range, n_gp_points, gp_params,
data_params):
self.n_samples = n_samples
X = np.linspace(rt_range[0], rt_range[1], n_gp_points)[:, np.newaxis]
y = np.random.normal(data_params[0], data_params[1], n_gp_points)
kernel = gp_params[0] * RBF(length_scale=gp_params[1],
length_scale_bounds=(1e-1, 10.0))
self.gp = GaussianProcessRegressor(kernel=kernel)
self.gp.fit(X, y)
def get_drift(self, model_idx, rt):
samples = self.gp.sample_y(np.array([[rt]]), n_samples=self.n_samples)
return samples[0][model_idx]
def plot_drifts(self):
NotImplementedError()
def test_sample_y_shapes(normalize_y, n_targets):
"""Check the shapes of y_samples in single-output (n_targets=0) and
multi-output settings, including the edge case when n_targets=1, where the
sklearn convention is to squeeze the predictions.
Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/22175
"""
rng = np.random.RandomState(1234)
n_features, n_samples_train = 6, 9
# Number of spatial locations to predict at
n_samples_X_test = 7
# Number of sample predictions per test point
n_samples_y_test = 5
y_train_shape = (n_samples_train, )
if n_targets is not None:
y_train_shape = y_train_shape + (n_targets, )
# By convention single-output data is squeezed upon prediction
if n_targets is not None and n_targets > 1:
y_test_shape = (n_samples_X_test, n_targets, n_samples_y_test)
else:
y_test_shape = (n_samples_X_test, n_samples_y_test)
X_train = rng.randn(n_samples_train, n_features)
X_test = rng.randn(n_samples_X_test, n_features)
y_train = rng.randn(*y_train_shape)
model = GaussianProcessRegressor(normalize_y=normalize_y)
# FIXME: before fitting, the estimator does not have information regarding
# the number of targets and default to 1. This is inconsistent with the shape
# provided after `fit`. This assert should be made once the following issue
# is fixed:
# https://github.com/scikit-learn/scikit-learn/issues/22430
# y_samples = model.sample_y(X_test, n_samples=n_samples_y_test)
# assert y_samples.shape == y_test_shape
model.fit(X_train, y_train)
y_samples = model.sample_y(X_test, n_samples=n_samples_y_test)
assert y_samples.shape == y_test_shape
def generate_slow_background(x, scale=500, npoints=1000):
kernel = 1.0 * RBF(length_scale=scale)
X_ = np.linspace(0, x[-1], npoints)
#print(X_.shape)
gp_bg = GaussianProcessRegressor(kernel=kernel)
y_mean, y_std = gp_bg.predict(X_[:, np.newaxis], return_std=True)
y_samples = gp_bg.sample_y(X_[:, np.newaxis], 1)
#y_samples -= np.min(y_s)
#th=0.5
#y_samples[y_samples <= -th] = -th
#y_samples += th + 1e-7
#print(y_samples.shape)
f2 = interp1d(X_, y_samples[::, 0], kind='cubic', bounds_error=False)
res = f2(np.arange(len(x)))
#res[res<=0] = 0
res -= np.min(res)
return res
def get_gp_prediction(x: np.ndarray,
y: np.ndarray,
n_samples: int,
n_points: int = 100):
"""Fit a GP to observed P( Y | X ) and sample some functions from it.
Args:
x: x-values (features)
y: y-values (targets/labels)
n_samples: The number of GP samples to use as basis functions.
n_points: The number of points to subsample form x and y to fit each GP.
Returns:
a function that takes as input an array either of shape (n,) or (k, n)
and outputs:
if input is 1D -> output: (n, n_samples)
if input is 2D -> output: (k, n, n_samples)
"""
kernel = PairwiseKernel(metric='poly') + RBF()
gp = GaussianProcessRegressor(kernel=kernel,
alpha=0.4,
n_restarts_optimizer=0,
normalize_y=True)
xmin = np.min(x)
xmax = np.max(x)
xx = np.linspace(xmin, xmax, n_points)
y_samples = []
rng = onp.random.RandomState(0)
for i in range(n_samples):
logging.info("Subsample 200 points and fit GP to P(Y|X)...")
idx = rng.choice(len(x), 200, replace=False)
gp.fit(x[idx, np.newaxis], y[idx])
logging.info(f"Get a sample functions from the GP")
y_samples.append(gp.sample_y(xx[:, np.newaxis], 1))
y_samples = np.array(y_samples).squeeze()
logging.info(f"Shape of samples: {y_samples.shape}")
def predict(inputs: np.ndarray) -> np.ndarray:
return interp1d(inputs, xmin, xmax, y_samples).T
return jit(vmap(predict))
class GPRealization(Function):
""" A callable class that is the realization of a gaussian process, f:R->R. """
def __init__(self, kernel=None, x0=0, x1=1, num_init=100):
"""
Parameters
----------
kernel : Kernel
x0 : float
left boundary
x1 : float
right boundary
num_init : int
the number of equidistant samples between x0 and x1 used to build the function
"""
super().__init__()
# RBF(length_scale_bounds=(0.3, 10.0))
self.gp = GaussianProcessRegressor(kernel=kernel)
x = np.linspace(x0, x1, num_init)
x = x[:, np.newaxis]
y = self.gp.sample_y(x,
n_samples=1,
random_state=np.random.RandomState())
y = y[:, 0]
self.gp.fit(x, y)
def reset(self, reset_params=True):
""" parameters are chosen """
self.evals = 0
if reset_params:
raise NotImplementedError(
'No reset of params implemented for GPRealization')
def __call__(self, x):
return self.gp.predict([[x]])[0]
def integral(self, x_0=0, x_1=1):
""" return the integral from x_0 to x_1 """
return quad(self, x_0, x_1)[0]
class GPTSLearner:
def __init__(self,
arms,
sigma,
restart_optimizer=0,
cost_kernel=1,
lenght_scale_kernel=1):
self.name_learner = "GPTS learner"
self.arms = arms
self.means = np.ones(len(arms))
self.sigmas = np.ones(len(arms)) * sigma
self.pulled_arms = list()
self.collected_rewards = list()
self.users_sampled = [0, 0, 0]
kernel = C(1.0, (1e-1, 1e1)) * RBF(1.0, (1e-1, 1e1))
self.gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-10,
normalize_y=True,
n_restarts_optimizer=3,
random_state=17)
def update(self, idx_pulled_arm, reward):
self.pulled_arms.append(self.arms[idx_pulled_arm])
self.collected_rewards.append(reward)
# Split pulled_arms in elements with at least 2 arrays each one.
x = np.atleast_2d(self.pulled_arms).T
y = self.collected_rewards
# X: Training Data
# Y: Target Values
self.gp = self.gp.fit(x, y)
self.means, self.sigmas = self.gp.predict(np.atleast_2d(self.arms).T,
return_std=True)
self.sigmas = np.maximum(self.sigmas, 1e-2)
self.means = np.maximum(self.means, 0)
def sample_arms(self):
values = np.round(self.gp.sample_y(np.atleast_2d(self.arms).T), 3)
return np.where(values > 0, values, 0)
def sample_greedy(df, n_reps=5, n_gp_samples=30):
print("== Sampling for coefficient estimation, greedily...")
logger = CoeffSamplingLogger("greedy")
for _ in range(n_reps):
n_samples = min(len(df) - 1, MAX_N_SAMPLES)
idxs = [np.random.randint(len(df))]
for i in tqdm(range(1, n_samples)):
df_sampled = df.iloc[df.index.isin(idxs), :]
df_rest = df.iloc[~df.index.isin(idxs), :]
logger.tick(df_sampled)
gp = GaussianProcessRegressor(kernel=coeff_est_kernel,
normalize_y=True)
gp.fit(
df_sampled.loc[:, ("lat", "lng", "pred", "electrification")],
df_sampled["true"])
preds = gp.sample_y(df_rest.loc[:, ("lat", "lng", "pred",
"electrification")],
n_samples=n_gp_samples)
idx = df_rest.index[np.argmax(np.std(preds, axis=1))]
idxs.append(idx)
logger.clear_run()
return logger
def gp_fit(df):
# Convert time to float in days
all_times = df.index.values
time_x = df.index.to_julian_date().values
time_x -= time_x.min()
kernel = 1**2 * RBF(length_scale=7,
length_scale_bounds=(1, 20)) + WhiteKernel(
noise_level=0.1**2, noise_level_bounds=(1e-7, 40))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-10,
normalize_y=True,
n_restarts_optimizer=10)
gp.fit(time_x.reshape(-1, 1), df['weight'].values)
y_pred, y_std = gp.predict(time_x.reshape(-1, 1), return_std=True)
gp_samples = gp.sample_y(np.array([time_x[0], time_x[-1]]).reshape(-1, 1),
n_samples=100000)
return y_pred, y_std, gp_samples
def fitGaussianProc(patDXdTdata, patAvgXdata, params):
'''
Fits a GP on the change data (x, dx/dt)
Parameters
----------
patDXdTdata
patAvgXdata
estimNoise
lengthScaleFactors
plotTrajParams
Returns
-------
'''
# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
assert(CTL == 1)
nrBiomk = patDXdTdata.shape[0]
#minX = np.amin(patAvgXdata, axis=0)
#maxX = np.amax(patAvgXdata, axis=0)
minX = np.array([np.nanmin(patAvgXdata[b], axis=0) for b in range(nrBiomk)])
maxX = np.array([np.nanmax(patAvgXdata[b], axis=0) for b in range(nrBiomk)])
assert not any(np.isnan(minX))
assert not any(np.isnan(maxX))
intervalSize = maxX-minX
minX -= intervalSize/0.5
maxX += intervalSize/0.5
#print minX.shape, maxX.shape
nrPointsToEval = 5000
x_pred = np.zeros((nrPointsToEval, nrBiomk),float)
dXdT_pred = np.zeros((nrPointsToEval, nrBiomk),float)
sigma_pred = np.zeros((nrPointsToEval, nrBiomk),float)
nrSamples = 100
posteriorSamples = np.zeros((nrSamples, nrPointsToEval, nrBiomk),float)
# print(avgXdata.shape, diag.shape)
# print(avgXdata[diag == CTL,:].shape)
# ctlXMean = np.nanmean(avgXdata[diag == CTL,:], axis = 0)
# ctlXStd = np.nanstd(avgXdata[diag == CTL,:], axis = 0)
# ctldXdTMean = np.nanmean(dXdTdata[diag == CTL,:], axis = 0)
# ctldXdTStd = np.nanstd(dXdTdata[diag == CTL,:], axis = 0)
# allXMean = np.nanmean(avgXdata, axis = 0)
# allXStd = np.nanstd(avgXdata, axis = 0)
# alldXdTMean = np.nanmean(dXdTdata, axis = 0)
# alldXdTStd = np.nanstd(dXdTdata, axis = 0)
patXMean = np.array([np.nanmean(patAvgXdata[b], axis=0) for b in range(nrBiomk)])
patXStd = np.array([np.nanstd(patAvgXdata[b], axis=0) for b in range(nrBiomk)])
patdXdTMean = np.array([np.nanmean(patDXdTdata[b], axis=0) for b in range(nrBiomk)])
patdXdTStd = np.array([np.nanstd(patDXdTdata[b], axis=0) for b in range(nrBiomk)])
gpList = []
for b in range(nrBiomk):
points = np.linspace(minX[b], maxX[b], nrPointsToEval)
#print points.shape
X = patAvgXdata[b]
Y = patDXdTdata[b]
notNanInd = np.logical_not(np.isnan(X))
X = X[notNanInd]
Y = Y[notNanInd]
X = X.reshape(-1,1)
Y = Y.reshape(-1,1)
# X = (X - allXMean[b]) / allXStd[b] # standardizing the inputs and outputs
# Y = (Y - alldXdTMean[b]) / alldXdTStd[b]
# minX[b] = (minX[b] - allXMean[b]) / allXStd[b]
# maxX[b] = (maxX[b] - allXMean[b]) / allXStd[b]
X = (X - patXMean[b]) / patXStd[b] # standardizing the inputs and outputs
# Y = (Y - patdXdTMean[b]) / patdXdTStd[b]
Y = Y / patdXdTStd[b]
minX[b] = (minX[b] - patXMean[b]) / patXStd[b]
maxX[b] = (maxX[b] - patXMean[b]) / patXStd[b]
#print 'Xshape, Yshape', X.shape, Y.shape
lower, upper = np.abs(1/np.max(X)), np.abs(1/(np.min(X)+1e-6))
if lower > upper:
lower, upper = upper, lower
mid = 1/np.abs(np.mean(X))
# print("X", X[:20],'Y', Y[:20])
# print(minX, maxX)
#lengthScale = (np.max(X)-np.min(X))
lengthScale = params['lengthScaleFactors'][b] * (np.max(X) - np.min(X))/2
estimNoise = np.var(Y)/2 # this should be variance, as it is placed as is on the diagonal of the kernel, which is a covariance matrix
#estimAlpha = np.ravel((np.std(Y))**2)
#estimAlpha = np.var(Y)/2
estimAlpha = np.std(Y)*2
boundsFactor = 2.0
#estimAlpha = 0
#need to specity bounds as the lengthScale is optimised in the fit
rbfKernel = ConstantKernel(1.0, constant_value_bounds="fixed") * RBF(length_scale=lengthScale, length_scale_bounds=(float(lengthScale)/boundsFactor, 1*lengthScale))
whiteKernel = ConstantKernel(1.0, constant_value_bounds="fixed") * WhiteKernel(noise_level=estimNoise, noise_level_bounds=(float(estimNoise)/boundsFactor, boundsFactor*estimNoise))
#rbfKernel = 1 * RBF(length_scale=lengthScale)
#whiteKernel = 1 * WhiteKernel(noise_level=estimNoise)
kernel = rbfKernel + whiteKernel
#kernel = 1.0 * RBF(length_scale=lengthScale)
print('\nbiomk %d lengthScale %f noise %f alpha %f'% (b, lengthScale, estimNoise, estimAlpha))
#print estimAlpha.shape
normalizeYflag = False
#normalizeYflag = True
gp = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer='fmin_l_bfgs_b', n_restarts_optimizer=100, normalize_y=normalizeYflag)
#gp = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer=None, n_restarts_optimizer=100, normalize_y=True)
assert not any(np.isnan(X))
assert not any(np.isnan(Y))
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, Y)
print("optimised kernel", gp.kernel_)#, " theta", gp.kernel_.theta, " bounds", gp.kernel_.bounds)
#gpNonOpt = GaussianProcessRegressor(kernel=rbfKernel, alpha=estimAlpha, optimizer=None, normalize_y=False)
#gpNonOpt.fit(X,Y)
#print("non-optimised kernel", gpNonOpt.kernel_)#, " theta", gpNonOpt.kernel_.theta, " bounds", gpNonOpt.kernel_.bounds)
#gp = gpNonOpt
# Make the prediction on the meshed x-axis (ask for Cov matrix as well)
x_pred[:,b] = np.linspace(minX[b], maxX[b], nrPointsToEval)
assert not any(np.isnan(x_pred[:,b]))
dXdT_predCurr, cov_matrix = gp.predict(x_pred[:,b].reshape(-1,1), return_cov=True)
# make sure dXdT is not too low, otherwise truncate the [minX, maxX] interval
dXdTthresh = 1e-10
tooLowMask = np.abs(np.ravel(dXdT_predCurr)) < dXdTthresh
print(tooLowMask.shape)
if np.sum(tooLowMask) > nrPointsToEval/10:
print("Warning dXdT is too low, will restict the [minxX, maxX] interval")
goodIndicesMask = np.logical_not(tooLowMask)
#print(x_pred.shape, goodIndicesMask.shape)
#print(x_pred[goodIndicesMask, b])
minX[b] = min(x_pred[goodIndicesMask,b])
maxX[b] = max(x_pred[goodIndicesMask,b])
x_pred[:, b] = np.linspace(minX[b], maxX[b], nrPointsToEval)
dXdT_predCurr, cov_matrix = gp.predict(x_pred[:,b].reshape(-1,1), return_cov=True)
MSE = np.diagonal(cov_matrix)
dXdT_pred[:,b] = np.ravel(dXdT_predCurr)
sigma_pred[:,b] = np.ravel(np.sqrt(MSE))
samples = gp.sample_y(x_pred[:,b].reshape(-1,1), n_samples=nrSamples, random_state=0)
posteriorSamples[:,:,b] = np.squeeze(samples).T
# renormalize the Xs and Ys
# x_pred[:,b] = x_pred[:,b] * allXStd[b] + allXMean[b]
# dXdT_pred[:,b] = dXdT_pred[:,b] * alldXdTStd[b] + alldXdTMean[b]
# sigma_pred[:,b] = sigma_pred[:,b] * alldXdTStd[b]
# posteriorSamples[:,:,b] = posteriorSamples[:,:,b]*alldXdTStd[b] + alldXdTMean[b]
# renormalize the Xs and Ys
# x_pred[:, b] = x_pred[:, b] * patXStd[b] + patXMean[b]
# dXdT_pred[:, b] = dXdT_pred[:, b] * patdXdTStd[b] + patdXdTMean[b]
# sigma_pred[:, b] = sigma_pred[:, b] * patdXdTStd[b]
# posteriorSamples[:, :, b] = posteriorSamples[:, :, b] * patdXdTStd[b] + patdXdTMean[b]
x_pred[:, b] = x_pred[:, b] * patXStd[b] + patXMean[b]
dXdT_pred[:, b] = dXdT_pred[:, b] * patdXdTStd[b]
sigma_pred[:, b] = sigma_pred[:, b] * patdXdTStd[b]
posteriorSamples[:, :, b] = posteriorSamples[:, :, b] * patdXdTStd[b]
# diagCol = plotTrajParams['diagColors']
# fig = pl.figure(1)
# nrDiags = np.unique(diag).shape[0]
# for diagNr in range(1, nrDiags + 1):
# print(avgXdata.shape, diag.shape, dXdTdata.shape, diagCol, diagNr)
# pl.scatter(avgXdata[diag == diagNr, b], dXdTdata[diag == diagNr, b], color = diagCol[diagNr - 1])
#
# modelCol = 'r' # red
# pl.plot(x_pred[:, b], dXdT_pred[:, b], '%s-' % modelCol, label = u'Prediction')
# pl.fill(np.concatenate([x_pred[:, b], x_pred[::-1, b]]), np.concatenate(
# [dXdT_pred[:, b] - 1.9600 * sigma_pred[:, b], (dXdT_pred[:, b] + 1.9600 * sigma_pred[:, b])[::-1]]), alpha = .5,
# fc = modelCol, ec = 'None', label = '95% confidence interval')
# for s in range(nrSamples):
# pl.plot(x_pred[:, b], posteriorSamples[s, :, b])
# fig.show()
gpParams = gp.get_params(deep=True)
#print 'kernel', gp.kernel
#print 'gpParams', gpParams
gpList.append(gp)
#print(adsa)
return x_pred, dXdT_pred, sigma_pred, gpList, posteriorSamples
from sklearn.gaussian_process.kernels import (RBF, ExpSineSquared)
kernel = RBF(length_scale=0.5, length_scale_bounds='fixed') #ExpSineSquared(length_scale=1.0, periodicity=1.0, length_scale_bounds='fixed', periodicity_bounds='fixed')
gp = GaussianProcessRegressor(kernel=kernel)
samples = 5
# Plot prior
plt.figure(figsize=(10,10))
ax = plt.subplot(1, 1, 1)
X_ = np.linspace(-5, 5, 500)
y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
plt.plot(X_, y_mean, 'k--', lw=1, zorder=9)
plt.fill_between(X_, y_mean - 2*y_std, y_mean + 2*y_std,
alpha=0.1, color='k')
y_samples = gp.sample_y(X_[:, np.newaxis], samples)
plt.plot(X_, y_samples, lw=1)
plt.xlim(-5, 5)
plt.ylim(-3, 3)
#plt.title("Prior (kernel: %s)" % kernel, fontsize=12)
plt.xticks([5], [r"$x$"], rotation=0, fontsize=30)
plt.yticks([3], [r"$f(x)$"], rotation=0, fontsize=30)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.tick_params(axis=u'both', which=u'both',length=0)
# Generate data and fit GP
rng = np.random.RandomState(11)
def _random_function(self,
node: str,
parent_values: Dict[str, Tensor],
seed=None) -> Tensor:
# Ordering
parent_values = collections.OrderedDict([
(par_node, parent_values[par_node])
for par_node in self.model[node]['parents']
])
parent_values = list(parent_values.values())
if len(parent_values) == 0:
return torch.tensor(0.0)
old_parent_values = self.model[node]['stacked_parent_values']
old_random_effects = self.model[node]['stacked_random_effects']
parent_values = torch.stack(parent_values).T
gp = GaussianProcessRegressor(kernel=RBF(length_scale=self.bandwidth),
optimizer=None)
if old_parent_values is None and old_random_effects is None: # Sampling for the first time
node_values = torch.tensor(
gp.sample_y(parent_values, random_state=seed)).reshape(-1)
self.model[node]['stacked_random_effects'] = node_values
self.model[node]['stacked_parent_values'] = parent_values
return node_values
else: # Sampling, when random function was already evaluated
if len(old_random_effects) >= self.interpolation_switch:
if self.model[node]['interpolator'] is None:
logger.warning(
'Using interpolation instead of Gaussian process!')
if old_parent_values.shape[1] == 1:
interpolator = interp1d(old_parent_values.squeeze(),
old_random_effects,
fill_value=0.0,
bounds_error=False)
else:
interpolator = LinearNDInterpolator(
old_parent_values.squeeze(),
old_random_effects,
fill_value=0.0)
self.model[node]['interpolator'] = interpolator
# else:
# interpolator = LinearNDInterpolator(old_parent_values, old_random_effects, fill_value=0.0)
return torch.tensor(self.model[node]['interpolator'](
parent_values.squeeze()))
else:
gp.fit(old_parent_values, old_random_effects)
node_values = torch.tensor(
gp.sample_y(parent_values, random_state=seed)).reshape(-1)
self.model[node]['stacked_random_effects'] = torch.cat(
[old_random_effects, node_values])
self.model[node]['stacked_parent_values'] = torch.cat(
[old_parent_values, parent_values], dim=0)
return node_values
# Visualization of prior
pylab.figure(0, figsize=(10, 8))
X_nn = gp.kernel.k2._project_manifold(X_[:, None])
pylab.subplot(3, 2, 1)
for i in range(X_nn.shape[1]):
pylab.plot(X_, X_nn[:, i], label="Manifold-dim %d" % i)
pylab.legend(loc="best")
pylab.xlim(-7.5, 7.5)
pylab.title("Prior mapping to manifold")
pylab.subplot(3, 2, 2)
y_mean, y_std = gp.predict(X_[:, None], return_std=True)
pylab.plot(X_, y_mean, 'k', lw=3, zorder=9, label="mean")
pylab.fill_between(X_, y_mean - y_std, y_mean + y_std,
alpha=0.5, color='k')
y_samples = gp.sample_y(X_[:, None], 10)
pylab.plot(X_, y_samples, color='b', lw=1)
pylab.plot(X_, y_samples[:, 0], color='b', lw=1, label="samples") # just for the legend
pylab.legend(loc="best")
pylab.xlim(-7.5, 7.5)
pylab.ylim(-4, 3)
pylab.title("Prior samples")
# Generate data and fit GP
X = np.random.uniform(-5, 5, 40)[:, None]
y = np.sin(X[:, 0]) + (X[:, 0] > 0)
gp.fit(X, y)
# Visualization of posterior
X_nn = gp.kernel_.k2._project_manifold(X_[:, None])
class GaussianProcessFilter(FilteringModel):
def __init__(self,
kernel: Optional[Kernel] = None,
**kwargs):
""" GaussianProcessFilter model
This model uses the GaussianProcessRegressor of scikit-learn to fit a Gaussian Process to the
supplied TimeSeries. This can then be used to obtain samples or the mean values of the
Gaussian Process at the times of the TimeSeries.
Parameters
----------
kernel : sklearn.gaussian_process.kernels.Kernel, default: None
The kernel specifying the covariance function of the Gaussian Process. If None is passed,
the default in scikit-learn is used. Note that the kernel hyperparameters are optimized
during fitting unless the bounds are marked as “fixed”.
**kwargs
Additional keyword arguments passed to `sklearn.gaussian_process.GaussianProcessRegressor`.
"""
super().__init__()
self.model = GaussianProcessRegressor(kernel=kernel, **kwargs)
def filter(self,
series: TimeSeries,
num_samples: int = 1) -> TimeSeries:
"""
Fits the Gaussian Process on the observations and returns samples from the Gaussian Process,
or its mean values if `num_samples` is set to 1.
Parameters
----------
series : TimeSeries
The series of observations used to infer the values according to the specified Gaussian Process.
This must be a deterministic series (containing one sample).
num_samples: int, default: 1
Number of times a prediction is sampled from the Gaussian Process. If set to 1,
instead the mean values will be returned.
Returns
-------
TimeSeries
A stochastic `TimeSeries` sampled from the Gaussian Process, or its mean
if `num_samples` is set to 1.
"""
raise_if_not(series.is_deterministic, 'The input series for the Gaussian Process filter must be '
'deterministic (observations).')
super().filter(series)
values = series.values(copy=False)
if series.has_datetime_index:
times = np.arange(series.n_timesteps).reshape(-1, 1)
else:
times = series.time_index.values.reshape(-1, 1)
not_nan_mask = np.all(~np.isnan(values), axis=1)
self.model.fit(times[not_nan_mask, :], values[not_nan_mask, :])
if num_samples == 1:
filtered_values = self.model.predict(times)
else:
filtered_values = self.model.sample_y(times, n_samples=num_samples)
return TimeSeries.from_times_and_values(series.time_index, filtered_values)
class WrappedGaussianProcess(MultiOutputMixin, RegressorMixin, BaseEstimator):
r"""Wrapped Gaussian Process.
The implementation is based on the algorithm 4 of [1].
Parameters
----------
space : Manifold
Manifold.
metric : RiemannianMetric
Riemannian metric.
prior : function
Associate to each input a manifold valued point.
kernel : kernel instance, default=None
The kernel specifying the covariance function of the GP. If None is
passed, the kernel ``ConstantKernel(1.0, constant_value_bounds="fixed")
* RBF(1.0, length_scale_bounds="fixed")`` is used as default. Note that
the kernel hyperparameters are optimized during fitting unless the
bounds are marked as "fixed".
alpha : float or ndarray of shape (n_samples,), default=1e-10
Value added to the diagonal of the kernel matrix during fitting.
This can prevent a potential numerical issue during fitting, by
ensuring that the calculated values form a positive definite matrix.
It can also be interpreted as the variance of additional Gaussian
measurement noise on the training observations. Note that this is
different from using a `WhiteKernel`. If an array is passed, it must
have the same number of entries as the data used for fitting and is
used as datapoint-dependent noise level. Allowing to specify the
noise level directly as a parameter is mainly for convenience and
for consistency with :class:`~sklearn.linear_model.Ridge`.
optimizer : "fmin_l_bfgs_b" or callable, default="fmin_l_bfgs_b"
Can either be one of the internally supported optimizers for optimizing
the kernel's parameters, specified by a string, or an externally
defined optimizer passed as a callable. If a callable is passed, it
must have the signature::
def optimizer(obj_func, initial_theta, bounds):
# * 'obj_func': the objective function to be minimized, which
# takes the hyperparameters theta as a parameter and an
# optional flag eval_gradient, which determines if the
# gradient is returned additionally to the function value
# * 'initial_theta': the initial value for theta, which can be
# used by local optimizers
# * 'bounds': the bounds on the values of theta
....
# Returned are the best found hyperparameters theta and
# the corresponding value of the target function.
return theta_opt, func_min
Per default, the L-BFGS-B algorithm from `scipy.optimize.minimize`
is used. If None is passed, the kernel's parameters are kept fixed.
Available internal optimizers are: `{'fmin_l_bfgs_b'}`.
n_restarts_optimizer : int, default=0
The number of restarts of the optimizer for finding the kernel's
parameters which maximize the log-marginal likelihood. The first run
of the optimizer is performed from the kernel's initial parameters,
the remaining ones (if any) from thetas sampled log-uniform randomly
from the space of allowed theta-values. If greater than 0, all bounds
must be finite. Note that `n_restarts_optimizer == 0` implies that one
run is performed.
copy_X_train : bool, default=True
If True, a persistent copy of the training data is stored in the
object. Otherwise, just a reference to the training data is stored,
which might cause predictions to change if the data is modified
externally.
random_state : int, RandomState instance or None, default=None
Determines random number generation used to initialize the centers.
Pass an int for reproducible results across multiple function calls.
[1] Mallasto, A. and Feragen, A. Wrapped gaussian process
regression on riemannian manifolds. In 2018 IEEE/CVF
Conference on Computer Vision and Pattern Recognition
"""
def __init__(
self,
space,
metric,
prior,
kernel=None,
*,
alpha=1e-10,
optimizer="fmin_l_bfgs_b",
n_restarts_optimizer=0,
copy_X_train=True,
random_state=None,
):
if metric is None:
metric = space.metric
self.metric = metric
self.space = space
self.prior = prior
self.copy_X_train = copy_X_train
self.y_train_ = None
self.tangent_y_train_ = None
self.y_train_shape_ = None
self._euclidean_gpr = GaussianProcessRegressor(
kernel=kernel,
alpha=alpha,
optimizer=optimizer,
n_restarts_optimizer=n_restarts_optimizer,
normalize_y=False,
copy_X_train=copy_X_train,
random_state=random_state,
)
self.__dict__.update(self._euclidean_gpr.__dict__)
self.log_marginal_likelihood = self._euclidean_gpr.log_marginal_likelihood
def _get_tangent_targets(self, X, y):
"""Compute the tangent targets, using the provided prior.
Parameters
----------
X : array-like of shape (n_samples, n_features) or list of object
Feature vectors or other representations of training data.
y : array-like of shape (n_samples,) or (n_samples, n_targets)
or (n_samples, n1_targets, n2_targets) for
matrix-valued targets.
Target values. The target must belongs to the manifold space
Returns
-------
tangent_y : array-like of shape (n_samples,) or (n_samples, n_targets)
or (n_samples, n1_targets, n2_targets)
Target projected on the associated (by the prior) tangent space.
"""
base_points = self.prior(X)
return self.metric.log(y, base_point=base_points)
def fit(self, X, y):
"""Fit Wrapped Gaussian process regression model.
The Wrapped Gaussian process is fit through the following steps:
- Compute the tangent dataset using the prior
- Fit a Gaussian process regression on the tangent dataset
- Store the resulting euclidean Gaussian process
Parameters
----------
X : array-like of shape (n_samples, n_features) or list of object
Feature vectors or other representations of training data.
y : array-like of shape (n_samples,) or (n_samples, n_targets)
or (n_samples, n1_targets, n2_targets)
Target values. The target must belongs to the manifold space
Returns
-------
self : object
WrappedGaussianProcessRegressor class instance.
"""
if not gs.all(self.space.belongs(y)):
raise AttributeError(
"The target values must belongs to the given space")
# compute the tangent dataset using the prior
tangent_y = self._get_tangent_targets(X, y)
self.y_train_shape_ = y.shape[
1:] # this is really useful when the samples are matrices, or tensor of dim>1
tangent_y = gs.reshape(tangent_y,
(y.shape[0], -1)) # flatten the samples.
# fit a gpr on the tangent dataset
self._euclidean_gpr.fit(X, tangent_y)
# update the attributes of the wgpr using the new attributes of the gpr
self.__dict__.update(self._euclidean_gpr.__dict__)
self.y_train_ = y
self.tangent_y_train_ = tangent_y # = self._euclidean_gpr.y_train_
return self
def predict(self, X, return_tangent_std=False, return_tangent_cov=False):
"""Predict using the Gaussian process regression model.
A fitted Wrapped Gaussian process can be use to predict values
through the following steps:
- Use the stored Gaussian process regression on the dataset to
return tangent predictions
- Compute the base-points using the prior
- Map the tangent predictions on the manifold via the metric's exp
with the base-points yielded by the prior
We can also predict based on an unfitted model by using the GP prior.
In addition to the mean of the predictive distribution, optionally also
returns its standard deviation (`return_std=True`) or covariance
(`return_cov=True`). Note that at most one of the two can be requested.
Parameters
----------
X : array-like of shape (n_samples, n_features) or list of object
Query points where the GP is evaluated.
return_tangent_std : bool, default=False
If True, the standard-deviation of the predictive distribution on at
the query points in the tangent space is returned along with the mean.
return_tangent_cov : bool, default=False
If True, the covariance of the joint predictive distribution at
the query points in the tangent space is returned along with the mean.
Returns
-------
y_mean : ndarray of shape (n_samples,) or (n_samples, n_targets)
Mean of predictive distribution a query points.
y_std : ndarray of shape (n_samples,) or (n_samples, n_targets), optional
Standard deviation of predictive distribution at query points in
the tangent space.
Only returned when `return_std` is True.
y_cov : ndarray of shape (n_samples, n_samples) or \
(n_samples, n_samples, n_targets), optional
Covariance of joint predictive distribution a query points
in the tangent space.
Only returned when `return_cov` is True.
In the case where the target is matrix valued,
return the covariance of the vectorized prediction.
"""
euc_result = self._euclidean_gpr.predict(X,
return_cov=return_tangent_cov,
return_std=return_tangent_std)
return_multiple = return_tangent_std or return_tangent_cov
tangent_means = euc_result[0] if return_multiple else euc_result
base_points = self.prior(X)
tangent_means = gs.reshape(
gs.cast(tangent_means, dtype=X.dtype),
(X.shape[0], *self.y_train_shape_),
)
y_mean = self.metric.exp(tangent_means, base_point=base_points)
if return_multiple:
tangent_std_cov = gs.cast(euc_result[1], dtype=X.dtype)
return (y_mean, tangent_std_cov)
return y_mean
def sample_y(self, X, n_samples=1, random_state=0):
"""Draw samples from Wrapped Gaussian process and evaluate at X.
A fitted Wrapped Gaussian process can be use to sample
values through the following steps:
- Use the stored Gaussian process regression on the dataset
to sample tangent values
- Compute the base-points using the prior
- Flatten (and repeat if needed) both the base-points and the
tangent samples to benefit from vectorized computation.
- Map the tangent samples on the manifold via the metric's exp with the
flattened and repeated base-points yielded by the prior
Parameters
----------
X : array-like of shape (n_samples_X, n_features) or list of object
Query points where the WGP is evaluated.
n_samples : int, default=1
Number of samples drawn from the Wrapped Gaussian process per query point.
random_state : int, RandomState instance or None, default=0
Determines random number generation to randomly draw samples.
Pass an int for reproducible results across multiple function
calls.
Returns
-------
y_samples : ndarray of shape (n_samples_X, n_samples), or \
(n_samples_X, *target_shape, n_samples)
Values of n_samples samples drawn from wrapped Gaussian process and
evaluated at query points.
"""
tangent_samples = self._euclidean_gpr.sample_y(X, n_samples,
random_state)
tangent_samples = gs.cast(tangent_samples, dtype=X.dtype)
if gs.ndim(tangent_samples) > 2:
tangent_samples = gs.moveaxis(tangent_samples, -2, -1)
flat_tangent_samples = gs.reshape(tangent_samples,
(-1, *self.y_train_shape_))
base_points = gs.repeat(self.prior(X), n_samples, axis=0)
flat_y_samples = self.metric.exp(flat_tangent_samples,
base_point=base_points)
y_samples = gs.reshape(flat_y_samples,
(X.shape[0], n_samples, *self.y_train_shape_))
if gs.ndim(tangent_samples) > 2:
y_samples = gs.moveaxis(y_samples, 1, -1)
return y_samples
#X_ = np.array([[0], [1.1], [1.3], [2.2], [2.8], [3.6], [3.7], [4.6], [4.7], [4.8]])
#Y_ = np.array([[-0.1], [0.9], [1], [0.1], [0], [0.7], [0.8], [-1.1], [-1], [-0.8]])
X = np.linspace(0, 5, 100)[:, None]
# RBF
kernel = C(1.0) * RBF(length_scale=1)
gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-5,
n_restarts_optimizer=10)
pylab.figure(0, figsize=(14, 12))
pylab.subplot(3, 2, 1)
ymean, y_std = gp.predict(X, return_std=True)
pylab.plot(X, ymean, 'k', lw=3, zorder=9, label="mean")
pylab.fill_between(X[:, 0], ymean - y_std, ymean + y_std, alpha=0.5, color='k')
y_samples = gp.sample_y(X, 10)
pylab.plot(X, y_samples, color='b', lw=2)
pylab.plot(X, y_samples[:, 0], color='b', lw=2, label="sample")
pylab.legend(loc="best")
pylab.xlim(0, 5)
pylab.ylim(-3, 3)
pylab.title("Prior Samples")
#Matern
kernel = C(1.0) * Matern(length_scale=1, nu=1.5)
gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-5,
n_restarts_optimizer=10)
pylab.subplot(3, 2, 2)
ymean, y_std = gp.predict(X, return_std=True)
plt.plot(x, y_pred, 'b-', label=u'Prediction')
#plt.fill(np.concatenate([x, x[::-1]]),
# np.concatenate([y_pred - 1.9600 * sigma,
# (y_pred + 1.9600 * sigma)[::-1]]),
# alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.fill(np.concatenate([x, x[::-1]]),
np.concatenate([y_pred - 1.9600 * sigma,
(y_pred + 1.9600 * sigma)[::-1]]),
alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')
fig = plt.figure()
y_samples = gp.sample_y(x, 10)
plt.plot(x, y_samples, lw=1)
plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
plt.plot(X, y, 'r.', markersize=10, label=u'Observations')
#plt.plot(x, y_pred, 'b-', label=u'Prediction')
#plt.fill(np.concatenate([x, x[::-1]]),
# np.concatenate([y_pred - 1.9600 * sigma,
# (y_pred + 1.9600 * sigma)[::-1]]),
# alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')
fig = plt.figure()
#y_samples = gp.sample_y(x, 10)
alpha=0.01, # float or array-like of shape(n_sample), default=1e-10
optimizer=
"fmin_l_bfgs_b", # "fmin_l_bfgs_b” or callable, default="fmin_l_bfgs_b"
n_restarts_optimizer=0, # int, default=0
normalize_y=False, # boolean, optional (default: False)
copy_X_train=True, # bool, default=True
random_state=None, # int or RandomState, default=None
)
model.fit(train_x, train_y)
y_pred, y_std = model.predict(x.reshape(-1, 1), return_std=True)
# y_pred, y_std = model.predict(x.reshape(-1, 1), return_std=True)
log_marginal_likelihood = model.log_marginal_likelihood() # 対数周辺尤度
params = model.get_params() # 設定パラメータの取得(辞書)
scores = model.score(train_x, train_y) # 決定係数R^2
# params = model.set_params() # 設定パラメータの設定(辞書)
k_samples = model.sample_y(train_x, n_samples=5) # 事後分布のカーネル関数をランダムに5つサンプリング
X_train = model.X_train_
y_train = model.y_train_
kernel = model.kernel_ # 予測に使用されたカーネル(最適化済みで最初に設定したパラメータとは異なる)
L = model.L_
alpha = model.alpha_
log_marginal_likelihood_value = model.log_marginal_likelihood_value_ # 対数周辺尤度
# plot
fig = plt.figure(figsize=(6, 4))
ax1 = fig.add_subplot(111)
for i in range(k_samples.shape[1]):
ax1.plot(train_x, k_samples[:, i])
ax1.scatter(train_x,
train_y,
# Plot posterior
""" @BUG line below is what we need, but it gives an error """
y_mean, y_std = gpr.predict(X_[:, np.newaxis], return_std=True)
""" @TOTO be able to remove 2 following lines and have above @BUG solved """
# y_mean = gpr.predict(X_[:, np.newaxis], return_std=False)
# y_std = np.zeros(len(y_mean))
y_mean = y_mean + continuous_from_array(X_, np_priors)[:, np.newaxis]
plt.plot(X_, y_mean, 'r', zorder=9)
plt.fill_between(X_, (y_mean - y_std[:, np.newaxis]).flatten(), (y_mean + y_std[:, np.newaxis]).flatten(),
alpha=0.4, color='b')
plt.fill_between(X_, (y_mean - 3 * y_std[:, np.newaxis]).flatten(), (y_mean + 3 * y_std[:, np.newaxis]).flatten(),
alpha=0.1, color='b')
y_samples = gpr.sample_y(X_[:, np.newaxis], 10)
'''
plt.plot(X_, y_samples, lw=1)
'''
plt.scatter(target[:, 0], target[:, 1], marker='+', c='m', s=50, zorder=10, edgecolors=(0, 0, 0))
plt.scatter(X[:, 0], target_y_train[:, np.newaxis], marker='+', c='g', s=50, zorder=10, edgecolors=(0, 0, 0))
plt.title("Posterior (kernel: %s)\n Log-Likelihood: %.3f"
% (gpr.kernel_, gpr.log_marginal_likelihood(gpr.kernel_.theta)),
fontsize=12)
plt.ylim(-1, 1.2 * np.max(target[:, 1]))
time_stop_np_gp = time.clock()
print(f'Execution numpy GP : {time_stop_np_gp - time_start_np_gp}')
import matplotlib.pyplot as plt
from scipy.stats import norm, multivariate_normal
x = np.linspace(0, 1, 100)
x1 = np.linspace(-5, 0, 100)
x2 = np.linspace(0, 5, 100)
K = norm.pdf(10 * np.abs(np.subtract(*np.meshgrid(x, x))))
plt.figure()
for _ in range(10):
plt.plot(x, multivariate_normal.rvs(np.zeros(100, dtype=np.float), K))
#X, y = make_friedman2(n_samples=500, noise=0, random_state=0)
kernel = RBF(0.1) #DotProduct() + WhiteKernel()
gpr = GaussianProcessRegressor(kernel=kernel, random_state=0,
optimizer=None) #.fit(X, y)
plt.figure()
for i in range(10):
y1 = gpr.sample_y(x[:, None], random_state=i).squeeze()
#gpr.fit(x1[:, None], y1)
plt.plot(x, y1)
#plt.plot(x2, gpr.sample_y(x2[:, None], random_state=i).squeeze())
plt.show()
#print(gpr.score(X, y))
#print(gpr.predict(X[:2,:], return_std=True))
print(gpr.get_params())
class GaussianProcess(oneDcurve):
def __init__(self, x, y, dy, mask=None, **args):
'''Fit a GP (Gaussian Process) spline to the data. [args] can be any argument
recognized by Matern kernel'''
oneDcurve.__init__(self, x, y, dy, mask)
self.pars = {
'diff_degree':None,
'scale':None,
'amp':None}
for key in args:
if key not in self.pars and key != "mean":
raise TypeError("%s is an invalid keyword argument for this method" % key)
if key != "mean": self.pars[key] = args[key]
if 'mean' in args:
self.mean = args['mean']
else:
self.mean = lambda x: x*0 + num.median(self.y)
# Make sure the data conform to the spine requirements
self.median = num.median(self.y)
self._setup()
self.realization = None
def __str__(sef):
return "Gaussian Process"
def help(self):
print("scale: Scale over which the function varies")
print("amp: Amplitude of typical function variations")
print("diff_degree: Roughly, the degree of differentiability")
def _setup(self):
'''Given the current set of params, setup the interpolator.'''
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import Matern, ConstantKernel
globals()['GaussianProcessRegressor'] = GaussianProcessRegressor
globals()['Matern'] = Matern
globals()['ConstantKernel'] = ConstantKernel
x,y,dy = self._regularize()
if self.diff_degree is None:
self.diff_degree = 2
if self.amp is None:
self.amp = num.std(y - self.mean(x))
if self.scale is None:
#self.scale = (self.x.max() - self.x.min())/2
self.scale = 30
self.kernel = ConstantKernel(self.amp, constant_value_bounds='fixed')*\
Matern(length_scale=self.scale, nu=self.diff_degree+0.5,
length_scale_bounds='fixed')
Y = y - self.mean(x)
X = num.array([x]).T
self.gpr = GaussianProcessRegressor(kernel=self.kernel,
alpha=dy*dy).fit(X,Y)
self.setup = True
self.realization = None
def __call__(self, x):
'''Interpolate at point [x]. Returns a 3-tuple: (y, mask) where [y]
is the interpolated point, and [mask] is a boolean array with the same
shape as [x] and is True where interpolated and False where extrapolated'''
if not self.setup:
self._setup()
if len(num.shape(x)) < 1:
scalar = True
else:
scalar = False
x = num.atleast_1d(x)
if self.realization is not None:
#res = self.realization(x.reshape(-1,1),random_state=self._seed)[:,0]
res = splev(x, self.realization)
else:
res = self.gpr.predict(x.reshape(-1,1))
res = res + self.mean(x)
if scalar:
return res[0],self.x.min() <= x[0] <= self.x.max()
else:
return res,num.greater_equal(x, self.x.min())*\
num.less_equal(x, self.x.max())
def domain(self):
return (self.x.min(),self.x.max())
def error(self, x):
'''Returns the error in the interpolator at points [x].'''
if not self.setup:
self._setup()
if len(num.shape(x)) < 1:
scalar = True
else:
scalar = False
x = num.atleast_1d(x)
res,sigma = self.gpr.predict(x.reshape(-1,1), return_std=True)
if scalar:
return sigma[0]
else:
return sigma
def draw(self):
'''Generate a random realization of the spline, based on the data.'''
if not self.setup:
self._setup()
# scikit-learn seems to have a bug. We have to make a realization
# and make a smoothing spline to approximate it.
tmin,tmax = self.domain()
t = num.arange(tmin, tmax+1, 1.0)
seed = num.random.randint(2**32)
y = self.gpr.sample_y(t.reshape(-1,1), random_state=seed)[:,0]
self.realization = splrep(t, y, k=3, s=0)
def reset_mean(self):
self.realization = None
def rchisquare(self):
chisq = self.chisquare()
if len(self.x) < 5:
return -1
return chisq/(len(self.x) - 4)
def deriv(self, x, n=1):
'''Returns the nth derivative of the function at x.'''
if len(num.shape(x)) < 1:
scalar = True
else:
scalar = False
xs = num.atleast_1d(x)
f = lambda x: self.__call__(x)[0]
res = deriv(f, xs, dx=self.scale/100., n=n)
if scalar:
return res[0]
else:
return res
def find_extrema(self, xmin=None, xmax=None):
'''Find the position and values of the maxima/minima. Returns a tuple:
(roots,vals,ypps) where roots are the x-values where the extrema
occur, vals are the y-values at these points, and ypps are the
2nd derivatives. Optionally, only search for roots between
xmin and xmax'''
#evaluate the 1st derivative at sacle/10 intervals (that should be
# enough)
if xmin is None: xmin = self.x.min()
if xmax is None: xmax = self.x.max()
dx = min(self.scale*1.0/20, (xmax-xmin)/5.0)
xs = num.arange(xmin, xmax, dx)
dys = self.deriv(xs, n=1)
pids = num.greater(dys, 0)
inds = num.nonzero(num.logical_xor(pids[1:],pids[:-1]))[0]
if len(inds) == 0:
return (num.array([]), num.array([]), num.array([]))
ret = []
for i in range(len(inds)):
try:
res = brentq(self.deriv, xs[inds[i]-1], xs[inds[i]+1])
ret.append(res)
except:
continue
if len(ret) == 0:
return (num.array([]), num.array([]), num.array([]))
ret = num.array(ret)
vals = self.__call__(ret)[0]
curvs = self.deriv(ret, n=2)
curvs = num.where(curvs > 0, 1, curvs)
curvs = num.where(curvs < 0, -1, curvs)
return ret,vals,curvs
def intercept(self, y):
'''Find the value of x for which the interpolator goes through [y]'''
xs = num.arange(self.x.min(), self.x.max(), self.scale/10)
f = lambda x: self.__call__(x)[0] - y
ys = f(xs)
pids = num.greater(ys, 0)
if num.alltrue(pids) or num.alltrue(-pids):
return None
ret = []
inds = num.nonzero(pids[1:] - pids[:-1])[0]
for i in range(len(inds)):
ret.append(brentq(f, xs[inds[i]], xs[inds[i]+1]))
ret = num.array(ret)
return ret
def pass_arg_upd(Xx, nsim, tr_size, pre_trained_hyperparamters):
print("tr_Size:", tr_size)
#pre_tr_size = 100
tr_size = int(tr_size)
#List of lakes to choose from
lake = ['mendota', 'mille_lacs']
lake_num = 0 # 0 : mendota , 1 : mille_lacs
lake_name = lake[lake_num]
# Load features (Xc) and target values (Y)
data_dir = '../../../../data/'
filename = lake_name + '.mat'
mat = spio.loadmat(data_dir + filename,
squeeze_me=True,
variable_names=['Y', 'Xc_doy', 'Modeled_temp'])
Xc = mat['Xc_doy']
Y = mat['Y']
Xc = Xc[:, :-1]
# train and test data
trainX, testX, trainY, testY = train_test_split(
Xc,
Y,
train_size=tr_size / Xc.shape[0],
test_size=tr_size / Xc.shape[0],
random_state=42,
shuffle=True)
## train and test data
#trainX, trainY = Xc[:tr_size,:-1], Y[:tr_size]
#testX, testY = Xc[-50:,:-1], Y[-50:]
# # Loading unsupervised data
# unsup_filename = lake_name + '_sampled.mat'
# unsup_mat = spio.loadmat(data_dir+unsup_filename, squeeze_me=True,
# variable_names=['Xc_doy1','Xc_doy2'])
# uX1 = unsup_mat['Xc_doy1'] # Xc at depth i for every pair of consecutive depth values
# uX2 = unsup_mat['Xc_doy2'] # Xc at depth i + 1 for every pair of consecutive depth values
# uX1 = uX1[:pre_tr_size,:-1]
# uX2 = uX2[:pre_tr_size,:-1]
# uY1 = uX1[:pre_tr_size,-1:]
# uY2 = uX2[:pre_tr_size,-1:]
# kernel = C(5.0, (1e-2, 1e3)) * RBF(length_scale = [1] * trainX.shape[1], length_scale_bounds=(1e-3, 1e4))
# gp1 = GaussianProcessRegressor(kernel=kernel, alpha =1.2, n_restarts_optimizer=0)
# gp1.fit(uX1, uY1)
# # pre-trained model parameters
# pre_trained_hyperparamters = gp1.kernel_
# Updated model
gp2 = GaussianProcessRegressor(kernel=pre_trained_hyperparamters,
alpha=1.5,
n_restarts_optimizer=10)
gp2.fit(trainX, trainY)
# scale the uniform numbers to original space
# max and min value in each column
max_in_column_Xc = np.max(trainX, axis=0)
min_in_column_Xc = np.min(trainX, axis=0)
# Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc)
Xc_org = Xx * (max_in_column_Xc - min_in_column_Xc) + min_in_column_Xc
samples = gp2.sample_y(Xc_org, n_samples=int(nsim)).T
return samples
1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0),
nu=1.5)]
for fig_index, kernel in enumerate(kernels):
# Specify Gaussian Process
gp = GaussianProcessRegressor(kernel=kernel)
# Plot prior
plt.figure(fig_index, figsize=(8, 8))
plt.subplot(2, 1, 1)
X_ = np.linspace(0, 5, 100)
y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - y_std, y_mean + y_std,
alpha=0.2, color='k')
y_samples = gp.sample_y(X_[:, np.newaxis], 10)
plt.plot(X_, y_samples, lw=1)
plt.xlim(0, 5)
plt.ylim(-3, 3)
plt.title("Prior (kernel: %s)" % kernel, fontsize=12)
# Generate data and fit GP
rng = np.random.RandomState(4)
X = rng.uniform(0, 5, 10)[:, np.newaxis]
y = np.sin((X[:, 0] - 2.5) ** 2)
gp.fit(X, y)
# Plot posterior
plt.subplot(2, 1, 2)
X_ = np.linspace(0, 5, 100)
y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
