def predict(self, xs):
""" Predict the outputs and thier variance, y* and V[y*], given new
inputs, x*.
Arguments:
xs: [DxN] test points for prediction (x*)
Returns:
Ems: array of N predictions of m*
Vms: array of N predictions of the variance of m*, V[m*].
"""
# Check we have trained
D, N = self._check_prediction_inputs(xs)
# Pre-allocate
Ems, Vms = np.zeros(N), np.zeros(N)
# Evaluate test kernel vectors and do a stable inversion
ks = self.kfunc(self.x, np.atleast_2d(xs), *self.kparams)
Kinvks = cholsolve(jitchol(self.C), ks)
for n, xn in enumerate(xs.T):
# Evaluate the test kernel vectors
kss = self.kfunc(np.atleast_2d(xn).T, np.atleast_2d(xn).T,
*self.kparams)
# Predict the latent function
Ems[n] = (Kinvks[:, n].T).dot(self.m)
Vms[n] = kss - Kinvks[:, n].T.dot(ks[:, n])
return Ems, Vms
def __gplearn(self, y, K, delta=None, maxit=None, verbose=False):
""" Parameter learning for a basic GP. Called by the _learn method. """
N = y.shape[0]
# Make posterior GP
C = K + np.eye(N) * self.ynoise**2
Cchol = jitchol(C)
# Calculate the log-marginal-likelihood
lml = -0.5 * (logdet(Cchol) + y.T.dot(cholsolve(Cchol, y)) + N *
np.log(2 * np.pi))
return y, C, lml
def __taylin(self, y, K, delta=1e-6, maxit=200, rate=0.75, maxsteps=25,
verbose=False):
""" Posterior parameter learning using 1st order Taylor series
expansion of the nonlinear forward model.
Arguments:
y: N array of training outputs (dimensionality of 1)
K: the [NxN] covariance matrix with the current hyper parameter
estimates.
delta: [optional] the convergence threshold for the objective
function (free energy).
dparams: [optional] the convergence threshold for the
hyperparameter values.
maxit: [optional] maximum number of iterations for learning the
posterior parameters.
rate: [optional] the learning rate of the line search used in
each of the Gauss-Newton style iterations for the posterior
mean.
maxsteps: [optional] the maximum number of line-search steps.
verbose: [optional] whether or not to display current learning
progress to the terminal.
Returns:
The final objective function value (free energy), and the
posterior mean (m) and Covariance (C).
"""
# Establish some parameters
N = y.shape[0]
m = np.zeros(N)
# Make sigma points in latent space
Kchol = jitchol(K)
# Bootstrap iterations
obj = np.finfo(float).min
endcond = "maxit"
# Pre-allocate
a = np.zeros(N)
H = np.zeros((N, N))
for i in range(maxit):
# Store old values in case of divergence, and for "line search"
objo, mo, ao, Ho = obj, m.copy(), a.copy(), H.copy()
# Taylor series linearisation
gm = self._passnlfunc(self.nlfunc, mo)
a = self._passnlfunc(self.dnlfunc, mo)
# Kalmain gain
AK = (a * K).T
AKAsig = a * AK
AKAsig[np.diag_indices(N)] += self.ynoise**2
H = cholsolve(jitchol(AKAsig), AK).T
# Do a bit of a heuristic line search for best step length
for j in range(maxsteps):
step = rate**j
m = (1 - step) * mo + step * H.dot(y - gm + a * mo)
# MAP objective
ygm = y - self._passnlfunc(self.nlfunc, m)
obj = -0.5 * (m.T.dot(cholsolve(Kchol, m))
+ (ygm**2).sum()/self.ynoise**2)
if obj >= objo:
break
dobj = abs((objo - obj) / obj)
# Divergence, use previous result
if (obj < objo) & (dobj > delta):
m, obj, a, H = mo, objo, ao, Ho # recover old values
endcond = "diverge"
break
# Convergence, use latest result
if dobj <= delta:
endcond = "converge"
break
if verbose is True:
print("iters: {}, endcond = {}, MAP = {}, "
"delta = {:.2e}".format(i, endcond, obj, dobj))
# Useful equations for Free energy
C = K - H.dot((a * K).T)
gm = self._passnlfunc(self.nlfunc, m)
# Calculate Free Energy
Feng = -0.5 * (N * np.log(np.pi * 2 * self.ynoise**2)
+ ((y - gm)**2).sum() / self.ynoise**2
+ m.T.dot(cholsolve(Kchol, m)) - logdet(C, dochol=True)
+ logdet(Kchol))
return m, C, Feng
def __taylin(self,
y,
K,
delta=1e-6,
maxit=200,
rate=0.75,
maxsteps=25,
verbose=False):
""" Posterior parameter learning using 1st order Taylor series
expansion of the nonlinear forward model.
Arguments:
y: N array of training outputs (dimensionality of 1)
K: the [NxN] covariance matrix with the current hyper parameter
estimates.
delta: [optional] the convergence threshold for the objective
function (free energy).
dparams: [optional] the convergence threshold for the
hyperparameter values.
maxit: [optional] maximum number of iterations for learning the
posterior parameters.
rate: [optional] the learning rate of the line search used in
each of the Gauss-Newton style iterations for the posterior
mean.
maxsteps: [optional] the maximum number of line-search steps.
verbose: [optional] whether or not to display current learning
progress to the terminal.
Returns:
The final objective function value (free energy), and the
posterior mean (m) and Covariance (C).
"""
# Establish some parameters
N = y.shape[0]
m = np.random.randn(N) / 100.0
# Make sigma points in latent space
Kchol = jitchol(K)
# Bootstrap iterations
obj = np.finfo(float).min
endcond = "maxit"
# Pre-allocate
a = np.zeros(N)
H = np.zeros((N, N))
for i in range(maxit):
# Store old values in case of divergence, and for "line search"
objo, mo, ao, Ho = obj, m.copy(), a.copy(), H.copy()
# Taylor series linearisation
gm = self._passnlfunc(self.nlfunc, mo)
a = self._passnlfunc(self.dnlfunc, mo)
# Kalmain gain
AK = (a * K).T
AKAsig = a * AK
AKAsig[np.diag_indices(N)] += self.ynoise**2
H = cholsolve(jitchol(AKAsig), AK).T
# Do a bit of a heuristic line search for best step length
for j in range(maxsteps):
step = rate**j
m = (1 - step) * mo + step * H.dot(y - gm + a * mo)
# MAP objective
ygm = y - self._passnlfunc(self.nlfunc, m)
obj = -0.5 * (m.T.dot(cholsolve(Kchol, m)) +
(ygm**2).sum() / self.ynoise**2)
if obj >= objo:
break
dobj = abs((objo - obj) / obj)
# Divergence, use previous result
if (obj < objo) & (dobj > delta):
m, obj, a, H = mo, objo, ao, Ho # recover old values
endcond = "diverge"
break
# Convergence, use latest result
if dobj <= delta:
endcond = "converge"
break
if verbose is True:
print("iters: {}, endcond = {}, MAP = {}, "
"delta = {:.2e}".format(i, endcond, obj, dobj))
# Useful equations for Free energy
C = K - H.dot((a * K).T)
gm = self._passnlfunc(self.nlfunc, m)
# Calculate Free Energy
Feng = -0.5 * (N * np.log(np.pi * 2 * self.ynoise**2) +
((y - gm)**2).sum() / self.ynoise**2 +
m.T.dot(cholsolve(Kchol, m)) - logdet(C, dochol=True) +
logdet(Kchol))
return m, C, Feng
def _learn(self, learnfunc, x, y, kparams, ynoise, nlparams, dobj, dparams,
maxit, verbose):
""" Generic optimisation method for this and derived Gaussian Process
algorithms.
Essentially this manages the call to NLopt, establishes upper and
lower bounds on the hyperparameters, and sets the internal
parameters and hyperparameters to their learned values.
Arguments:
learnfunc: the learning function to call to actually learn the
*parameters* of the GP, i.e. the posterior mean and
covariance, it should have the following minimal form:
def learnfunc(y, K, delta=None, maxit=None, verbose=False):
... do some calcs
return m, C, obj
where K is the prior [NxN] covariance matrix built using
the current estimates of the hyperparameters, m is the
posterior mean of the GP, C is the posterior covariance of
the GP and obj is the final value of the objective function
(log marginal likelihood or some proxy). Also:
delta: is the convergence threshold, and is passed
dobj/10
maxit: the maximum number of iterations to perform
(passed maxit from this function)
verbose: toggle verbose output, also routed from this
function.
x: [DxN] array of N input samples with a dimensionality of D.
y: N array of training outputs (dimensionality of 1)
kparams: a tuple of initial values corresponding to the kernel
hyperparameters of the kernel function input to the
constructor.
ynoise: a scalar initial value for the observation (y) noise.
dobj: the convergence threshold for the objective function
(log-marginal likelihood) used by NLopt.
dparams: the convergence threshold for the hyperparameter
values.
maxit: maximum numbr of iterations for learning the
hyperparameters.
verbose: whether or not to display current learning progress to
the terminal.
Returns:
The final objective function value (log marginal likelihood).
Also internally all of the final parameters and hyperparameters
are stored.
Note:
This stops learning as soon as one of the convergence criterion
is met (objective, hyperparameters or iterations).
"""
# Check arguments
self.__wipedata()
D, N = self.__check_training_inputs(x, y)
# Check bounds with parameters to learn
lbounds, ubounds = self.__checkbounds(kparams, nlparams, ynoise)
# Make log-marginal-likelihood closure for optimisation
def objective(params, grad):
# Make sure grad is empty
assert not grad, "Grad is not empty!"
# Extract hyperparameters
self.__extractparams(params, kparams, nlparams, ynoise)
K = self.kfunc(self.x, self.x, *self.kparams)
m, C, obj = learnfunc(y, K, delta=dobj/10, maxit=maxit,
verbose=verbose)
if obj > self.obj:
self.m, self.C, self.obj = m, C, obj
if verbose is True:
print("\tObjective: {}, params: {}".format(obj, params))
return obj
# Get initial hyper-parameters
params = self.__catparams(kparams, nlparams, ynoise)
nparams = len(params)
# Set up optimiser with objective function and bounds
opt = nlopt.opt(nlopt.LN_BOBYQA, nparams)
opt.set_max_objective(objective)
opt.set_lower_bounds(lbounds)
opt.set_upper_bounds(ubounds)
opt.set_maxeval(maxit)
opt.set_ftol_rel(dobj)
opt.set_xtol_rel(dparams)
# Run the optimisation
params = opt.optimize(params)
optfval = opt.last_optimize_result()
if verbose is True:
print("Optimiser finish criterion: {0}".format(optfval))
# Store learned parameters (these have been over-written by nlopt)
self.__extractparams(params, kparams, nlparams, ynoise)
self.Kchol = jitchol(self.kfunc(self.x, self.x, *self.kparams))
return self.obj
def __statlin(self, y, K, delta=1e-6, maxit=200, rate=0.75, maxsteps=25,
verbose=False):
""" Posterior parameter learning using the unscented transform and
statistical linearisation.
Arguments:
y: N array of training outputs (dimensionality of 1)
K: the [NxN] covariance matrix with the current hyper parameter
estimates.
delta: [optional] the convergence threshold for the objective
function (free energy).
dparams: [optional] the convergence threshold for the
hyperparameter values.
maxit: [optional] maximum number of iterations for learning the
posterior parameters.
rate: [optional] the learning rate of the line search used in
each of the Gauss-Newton style iterations for the posterior
mean.
maxsteps: [optional] the maximum number of line-search steps.
verbose: [optional] whether or not to display current learning
progress to the terminal.
Returns:
The final objective function value (free energy), and the
posterior mean (m) and Covariance (C).
"""
# Establish some parameters
N = y.shape[0]
m = np.zeros(N)
# Make sigma points in latent space
C = K.copy()
Kchol = jitchol(K)
Ms = self.__makesigmas1D(K)
# Bootstrap iterations
obj = np.finfo(float).min
endcond = "maxit"
ybar = y.copy()
for i in range(maxit):
# Store old values in case of divergence, and for "line search"
objo, mo, ybaro = obj, m.copy(), ybar.copy()
# Sigma points in obs. space and sigma point stats
Ys = self._passnlfunc(self.nlfunc, m[:, np.newaxis] + Ms)
ybar = (self.W * Ys).sum(axis=1)
Sym = (self.W * (Ys - ybar[:, np.newaxis]) * Ms).sum(axis=1)
# Statistical linearisation
a = Sym / C.diagonal()
# Kalmain gain
AK = (a * K).T
AKAsig = a * AK
AKAsig[np.diag_indices(N)] += self.ynoise**2
H = cholsolve(jitchol(AKAsig), AK).T
# Do a bit of a heuristic line search for best step length
for j in range(maxsteps):
step = rate**j
m = (1 - step) * mo + step * H.dot(y - ybar + a * mo)
# MAP objective
ygm = y - self._passnlfunc(self.nlfunc, m)
obj = -0.5 * (m.T.dot(cholsolve(Kchol, m))
+ (ygm**2).sum()/self.ynoise**2)
if obj >= objo:
break
dobj = abs((objo - obj) / obj)
# Divergence, use previous result
if (obj < objo) & (dobj > delta):
m, ybar, obj = mo, ybaro, objo # recover old values
endcond = "diverge"
break
# Make posterior C if m has not diverged
C = K - H.dot(AK)
Ms = self.__makesigmas1D(C)
# Convergence, use latest result
if dobj <= delta:
endcond = "converge"
break
if verbose is True:
print("iters: {}, endcond = {}, MAP = {}, "
"delta = {:.2e}".format(i, endcond, obj, dobj))
# Calculate Free Energy
Feng = -0.5 * (N * np.log(np.pi * 2 * self.ynoise**2)
+ ((y - ybar)**2).sum() / self.ynoise**2
+ m.T.dot(cholsolve(Kchol, m)) - logdet(C, dochol=True)
+ logdet(Kchol))
return m, C, Feng
plt.legend(loc=3, fontsize=18)
plt.autoscale(tight=True)
ax.set_yticklabels(ax.get_yticks(), size=16)
ax.set_xticklabels(ax.get_xticks(), size=16)
plt.grid(True)
#plt.savefig('test.pdf', bbox_inches='tight')
#plt.show()
#true Latent function evaluation
#k_C,k_B, k_S, k_l
Kfu = lfmkernels.kern_ode_Kfu(xt[:, np.newaxis], xt[:, np.newaxis], k_C, k_B,
k_S, k_l)
Kff = lfmkernels.kern_ode(xt[:, np.newaxis], xt[:, np.newaxis], k_C, k_B, k_S,
k_l)
Kffinvfs = cholsolve(jitchol(Kff), ft[:, np.newaxis])
#True mean of u(t)
mu = np.dot(Kfu.T, Kffinvfs)
#estimated latent function u(t)
Kuu = lfmkernels.kern_ode_Kuu(xt[:, np.newaxis], gp.kparams[3])
Kfu = lfmkernels.kern_ode_Kfu(xt[:, np.newaxis], xt[:, np.newaxis],
*gp.kparams)
KffinvKfu = cholsolve(gp.Kchol, Kfu)
mus = (KffinvKfu.T).dot(gp.m)
Vms = np.diag(Kuu - KffinvKfu.T.dot(Kfu - gp.C.dot(KffinvKfu)))
plt.figure(figsize=(11, 6))
ax = plt.subplot(111)
plt.plot(xt, mu * k_S, label='True process, $u_{true}$')
def __statlin(self,
y,
K,
delta=1e-6,
maxit=200,
rate=0.75,
maxsteps=25,
verbose=False):
""" Posterior parameter learning using the unscented transform and
statistical linearisation.
Arguments:
y: N array of training outputs (dimensionality of 1)
K: the [NxN] covariance matrix with the current hyper parameter
estimates.
delta: [optional] the convergence threshold for the objective
function (free energy).
dparams: [optional] the convergence threshold for the
hyperparameter values.
maxit: [optional] maximum number of iterations for learning the
posterior parameters.
rate: [optional] the learning rate of the line search used in
each of the Gauss-Newton style iterations for the posterior
mean.
maxsteps: [optional] the maximum number of line-search steps.
verbose: [optional] whether or not to display current learning
progress to the terminal.
Returns:
The final objective function value (free energy), and the
posterior mean (m) and Covariance (C).
"""
# Establish some parameters
N = y.shape[0]
m = np.random.randn(N) / 100.0
# Make sigma points in latent space
C = K.copy()
Kchol = jitchol(K)
Ms = self.__makesigmas1D(K)
# Bootstrap iterations
obj = np.finfo(float).min
endcond = "maxit"
ybar = y.copy()
for i in range(maxit):
# Store old values in case of divergence, and for "line search"
objo, mo, ybaro = obj, m.copy(), ybar.copy()
# Sigma points in obs. space and sigma point stats
Ys = self._passnlfunc(self.nlfunc, m[:, np.newaxis] + Ms)
ybar = (self.W * Ys).sum(axis=1)
Sym = (self.W * (Ys - ybar[:, np.newaxis]) * Ms).sum(axis=1)
# Statistical linearisation
a = Sym / C.diagonal()
# Kalmain gain
AK = (a * K).T
AKAsig = a * AK
AKAsig[np.diag_indices(N)] += self.ynoise**2
H = cholsolve(jitchol(AKAsig), AK).T
# Do a bit of a heuristic line search for best step length
for j in range(maxsteps):
step = rate**j
m = (1 - step) * mo + step * H.dot(y - ybar + a * mo)
# MAP objective
ygm = y - self._passnlfunc(self.nlfunc, m)
obj = -0.5 * (m.T.dot(cholsolve(Kchol, m)) +
(ygm**2).sum() / self.ynoise**2)
if obj >= objo:
break
dobj = abs((objo - obj) / obj)
# Divergence, use previous result
if (obj < objo) & (dobj > delta):
m, ybar, obj = mo, ybaro, objo # recover old values
endcond = "diverge"
break
# Make posterior C if m has not diverged
C = K - H.dot(AK)
Ms = self.__makesigmas1D(C)
# Convergence, use latest result
if dobj <= delta:
endcond = "converge"
break
if verbose is True:
print("iters: {}, endcond = {}, MAP = {}, "
"delta = {:.2e}".format(i, endcond, obj, dobj))
# Calculate Free Energy
Feng = -0.5 * (N * np.log(np.pi * 2 * self.ynoise**2) +
((y - ybar)**2).sum() / self.ynoise**2 +
m.T.dot(cholsolve(Kchol, m)) - logdet(C, dochol=True) +
logdet(Kchol))
return m, C, Feng