def __init__(self, x, xp, y, Sigmay=None, covariance_function='sqexp'):
"""
:param x: an array of arrays holding inputs i.e. x = [[x_1], [x_2], ..., [x_3]] where x_1 in R^{N_1} etc.
:param xp: an array of arrays holding targets i.e. xp = [[xp_1], [xp_2], ..., [xp_3]] where xp_1 in R^{Np_1} etc.
:param y: an array of arrays holding data i.e. y = [[y_1], [y_2], ..., [y_3]] where y_1 in R^{N_1} etc.
:param Sigmay: an array of arrays holding variances i.e. Sigmay = [[Sigmay_1], [Sigmay_2], ..., [Sigmay_3]]
where Sigmay_1 in R^{N_1} etc.
:param covariance_function: a list holding the covariance functions to use for each dimension
"""
self.D = x.shape[0]
self.N = kt.kron_N(x)
self.x = x
self.Np = kt.kron_N(xp)
self.xp = xp
self.y = y
if Sigmay is not None:
self.Sigmay = Sigmay
else:
self.Sigmay = None
self.covariance_function = covariance_function
# initialise the individual diff square grids
self.XX = []
self.XXp = []
self.XXpp = []
for i in xrange(self.D):
self.XX.append(abs_diff.abs_diff(self.x[i], self.x[i]))
self.XXp.append(abs_diff.abs_diff(self.x[i], self.xp[i]))
self.XXpp.append(abs_diff.abs_diff(self.xp[i], self.xp[i]))
self.XX = np.asarray(self.XX)
self.XXp = np.asarray(self.XXp)
self.XXpp = np.asarray(self.XXpp)
# set the kernels
from GP.kernels import exponential_squared # only using this one for now
self.kernels = []
self.kernels_transformed = []
for i in xrange(self.D):
if self.Sigmay is not None:
self.kernels.append(exponential_squared.sqexp(Sigmay=self.Sigmay[i], mode='kron'))
#self.kernels_transformed.append(exponential_squared.sqexp(mode='kron'))
else:
self.kernels.append(exponential_squared.sqexp(mode='kron'))
# Instantiate posterior mean, cov and evidence classes
self.meano = posterior_mean.meanf(self.x, self.xp, self.y, self.kernels, mode="kron",
XX=self.XX, XXp=self.XXp)
self.meanf = lambda theta: self.meano.give_mean(theta)
self.covo = posterior_covariance.covf(self.kernels, mode="kron", XX=self.XX, XXp=self.XXp, XXpp=self.XXpp)
self.covf = lambda theta: self.covo.give_covariance(theta)
self.logpo = marginal_posterior.evidence(self.x, self.y, self.kernels, mode="kron", XX=self.XX)
self.logp = lambda theta: self.logpo.logL(theta)
def draw_samples_ND_grid(x, theta, Nsamps, meanf=None):
"""
Draw N dimensional samples on a Euclidean grid
:param meanf: mean function
:param x: array of arrays containing targets [x_1, x_2, ..., x_D]
:param theta: array of arrays containing [theta_1, theta_2, ..., theta_D]
:param Nsamps: number of samples to draw
:return: array containing samples [Nsamps, N_1, N_2, ..., N_D]
"""
D = x.shape[0]
Ns = []
K = np.empty(D, dtype=object)
Kcov = expsq.sqexp()
Ntot = 1
for i in xrange(D):
Ns.append(x[i].size)
XX = abs_diff.abs_diff(x[i], x[i])
print Ns[i], theta[i], K[i]
K[i] = Kcov.cov_func(theta[i], XX, noise=False) + 1e-13 * np.eye(Ns[i])
Ntot *= Ns[i]
L = kt.kron_cholesky(K)
samps = np.zeros([Nsamps] + Ns, dtype=np.float64)
for i in xrange(Nsamps):
xi = np.random.randn(Ntot)
if meanf is not None:
samps[i] = meanf(x) + kt.kron_matvec(L, xi).reshape(Ns)
else:
samps[i] = kt.kron_matvec(L, xi).reshape(Ns)
return samps
def draw_spatio_temporal_samples(meanf, x, t, theta_x, theta_t, Nsamps):
Nt = t.size
Nx = x.size # assumes 1D
Ntot = Nt * Nx
XX = abs_diff.abs_diff(x, x)
TT = abs_diff.abs_diff(t, t)
Kcov = expsq.sqexp()
Kx = Kcov.cov_func(theta_x, XX, noise=False) + 1e-13 * np.eye(Nx)
Kt = Kcov.cov_func(theta_t, TT, noise=False) + 1e-13 * np.eye(Nt)
K = np.array([Kt, Kx])
L = kt.kron_cholesky(K)
samps = np.zeros([Nsamps, Nt, Nx])
for i in xrange(Nsamps):
xi = np.random.randn(Ntot)
samps[i] = meanf + kt.kron_matvec(L, xi).reshape(Nt, Nx)
return samps
:return: y the result
"""
# broadcast to circulant
N, M = x.shape
c = np.append(t, t[np.arange(N)[1:-1][::-1]].conj())
x_aug = np.append(x, np.zeros([N - 2, M]), axis=0)
tmp = FFT_circmat(c, x_aug)
return tmp[0:N, :]
if __name__ == "__main__":
from GP.tools import abs_diff
from GP.kernels import exponential_squared
# get Toeplitz matrix
kernel = exponential_squared.sqexp()
Nx = 1024
thetax = np.array([1.0, 0.5])
x = np.linspace(-10, 10, Nx)
xx = abs_diff.abs_diff(x, x)
Kx = kernel.cov_func(thetax, xx, noise=False)
# broadcast to circulant row
row1 = np.append(Kx[0, :], Kx[0, np.arange(Nx)[1:-1][::-1]].conj())
# create FFTW objects
FFT = pyfftw.builders.rfft
iFFT = pyfftw.builders.irfft
# test 1D FFT
b = np.random.randn(Nx)
def __init__(self,
x,
xp,
y,
Sigmay=None,
prior_mean=None,
covariance_function='sqexp',
mode="Full",
nu=2.5,
basis="Rect",
M=12,
L=5.0,
grid_regular=False):
"""
:param x: vector of inputs
:param xp: vector of targets
:param y: vector of data
:param prior_mean: a prior mean function (must be callable)
:param covariance_function: the kind of covariance function to use (currently 'sqexp' or 'mattern')
:param mode: whether to do a full GPR or a reduced rank GPR (currently "Full" or "RR")
:param nu: specifies the kind of Mattern function to use (must be a half integer)
:param basis: specifies the class of basis functions to use (currently only "Rect")
:param M: integer specifying the number of basis functions to use
:param L: float specifying the boundary of the domain
"""
# set inputs and targets enforcing correct sizes
self.N = x.shape[0]
try:
self.D = x.shape[1]
self.x = x
except:
self.D = 1
self.x = x.reshape(self.N, self.D)
if self.D > 1:
raise Exception("Must pass 1D input to temporal_GP")
self.Np = xp.size
self.xp = xp.reshape(self.Np, self.D)
self.y = y.reshape(self.N, self.D)
# check that prior mean is callable and set it
try:
ym = prior_mean(self.x)
self.prior_mean = prior_mean
del ym
except:
raise Exception("Prior mean function must be callable")
# set covariance
if covariance_function == "sqexp":
from GP.kernels import exponential_squared
# Initialise kernel
self.kernel = exponential_squared.sqexp(Sigmay=Sigmay)
elif covariance_function == "mattern":
from GP.kernels import mattern
# Initialise kernel
self.kernel = mattern.mattern(p=int(nu))
else:
raise Exception('Kernel %s not supported yet' %
covariance_function)
# Get inputs required to evaluate covariance function
self.mode = mode
if mode == "Full": # here we need the differences squared
from GP.tools import abs_diff
# Initialise absolute differences
self.XX = abs_diff.abs_diff(self.x, self.x)
self.XXp = abs_diff.abs_diff(self.x, xp)
self.XXpp = abs_diff.abs_diff(xp, xp)
# Instantiate posterior mean, cov and evidence classes
self.meano = posterior_mean.meanf(self.x,
self.xp,
self.y,
self.kernel,
mode=self.mode,
prior_mean=self.prior_mean,
XX=self.XX,
XXp=self.XXp)
self.meanf = lambda theta: self.meano.give_mean(theta)
self.covo = posterior_covariance.covf(self.kernel,
mode=self.mode,
XX=self.XX,
XXp=self.XXp,
XXpp=self.XXpp)
self.covf = lambda theta: self.covo.give_covariance(theta)
self.logpo = marginal_posterior.evidence(self.x,
self.y,
self.kernel,
mode=self.mode,
XX=self.XX)
self.logp = lambda theta: self.logpo.logL(theta)
elif mode == "RR": # here we need to evaluate the basis functions
from GP.tools import make_basis
# check consistency of inputs
if np.asarray(M).size == 1: # 1D so only a single value allowed
self.M = np.array([
M
]) # cast as array to make it compatible with make_basis class
else:
raise Exception(
'Inconsistent dimensions specified for M. Must be 1D.')
if np.asarray(L).size == 1: # 1D so only a single value allowed
self.L = np.array([
L
]) # cast as array to make it compatible with make_basis class
else:
raise Exception(
'Inconsistent dimensions specified for L. Must be 1D.')
# Construct the basis functions
self.basis = basis
if self.basis == "Rect":
from GP.basisfuncs import rectangular
self.Phi = make_basis.get_eigenvectors(self.x, self.M, self.L,
rectangular.phi)
self.Phip = make_basis.get_eigenvectors(
self.xp, self.M, self.L, rectangular.phi)
self.Lambda = make_basis.get_eigenvals(self.M, self.L,
rectangular.Lambda)
else:
print "%s basis functions not supported yet" % basis
# Precompute some terms that never change
# If inputs are on a regular grid we only need the diagonal
self.grid_regular = grid_regular
if self.grid_regular:
self.PhiTPhi = np.einsum(
'ij,ji->i', self.Phi.T,
self.Phi) # computes only the diagonal entries
else:
self.PhiTPhi = np.dot(self.Phi.T, self.Phi)
self.s = np.sqrt(self.Lambda)
self.meano = posterior_mean.meanf(self.x,
self.xp,
self.y,
self.kernel,
mode=self.mode,
Phi=self.Phi,
Phip=self.Phip,
PhiTPhi=self.PhiTPhi,
s=self.s,
grid_regular=self.grid_regular,
prior_mean=self.prior_mean)
self.meanf = lambda theta: self.meano.give_mean(theta)
self.covo = posterior_covariance.covf(
self.kernel,
mode=self.mode,
Phi=self.Phi,
Phip=self.Phip,
PhiTPhi=self.PhiTPhi,
s=self.s,
grid_regular=self.grid_regular)
self.covf = lambda theta: self.covo.give_covariance(theta)
self.logpo = marginal_posterior.evidence(
self.x,
self.y,
self.kernel,
mode=self.mode,
Phi=self.Phi,
PhiTPhi=self.PhiTPhi,
s=self.s,
grid_regular=self.grid_regular)
self.logp = lambda theta: self.logpo.logL(theta)
else:
raise Exception('Mode %s not supported yet' % mode)
self.has_posterior = False #flag to indicate if the posterior has been computed
def __init__(self,
x,
xp,
covariance_function='sqexp',
mode="Full",
nu=2.5,
basis="Rect",
M=12,
L=5.0):
"""
:param x: vector of inputs
:param xp: vector of targets
:param covariance_function: the kind of covariance function to use (currently 'sqexp' or 'mattern')
:param mode: whether to do a full GPR or a reduced rank GPR (currently "Full" or "RR")
:param nu: specifies the kind of Mattern function to use (must be a half integer)
:param basis: specifies the class of basis functions to use (currently only "Rect")
"""
# set inputs and targets
try:
self.N, self.D = x.shape
self.Np = xp.shape[0]
except:
self.N = x.size
self.D = 1
self.Np = xp.size
self.x = x
self.xp = xp
# set covariance
if covariance_function == "sqexp":
from GP.kernels import exponential_squared
# Initialise kernel
self.kernel = exponential_squared.sqexp()
elif covariance_function == "mattern":
from GP.kernels import mattern
# Initialise kernel
self.kernel = mattern.mattern(p=int(nu))
# Get inputs required to evaluate covariance function
self.mode = mode
if mode == "Full": # here we need the differences squared
from GP.tools import abs_diff
# Initialise absolute differences
self.XX = abs_diff.abs_diff(x, x)
self.XXp = abs_diff.abs_diff(x, xp)
self.XXpp = abs_diff.abs_diff(xp, xp)
elif mode == "RR": # here we need to evaluate the basis functions
from GP.tools import make_basis
# check consistency of inputs
if np.asarray(
M
).size == 1: # if single value is specified for M we use the same number of basis functions for each dimension
self.M = np.tile(M, self.D)
elif np.asarray(M).size == self.D:
self.M = M
else:
raise Exception('Inconsistent dimensions specified for M')
if np.asarray(
L
).size == 1: # if single value is specified for L we use square domain (hopefully also circular in future)
self.L = np.tile(L, self.D)
elif np.asarray(L).size == self.D:
self.L = L
else:
raise Exception('Inconsistent dimensions specified for L')
# Construct the basis functions
if basis == "Rect":
from GP.basisfuncs import rectangular
self.Phi = make_basis.get_eigenvectors(self.x, self.M, self.L,
rectangular.phi)
self.Phip = make_basis.get_eigenvectors(
self.xp, self.M, self.L, rectangular.phi)
self.Lambda = make_basis.get_eigenvals(self.M, self.L,
rectangular.Lambda)
else:
print "%s basis functions not supported yet" % basis
# Precompute some terms that never change
self.PhiTPhi = np.dot(self.Phi.T, self.Phi)
self.s = np.sqrt(self.Lambda)
def __init__(self, t, theta0, solve_mode="full", M=25, L=1.0, jit=1e-4):
"""
This is a LinearOperator representation of the covariance matrix
Input:
t - inputs
theta0 - the vector of hyperparameters
do_inverse - if True computes the Cholesky factor. If false inverse is computed with preconditioned conjugate gradient.
As a rule of thumb this should be False whenever N>1000
M - the number of basis functions to use for the preconditioner
L - the support of the basis functions
jit - jitter for numerical stability of inverse
"""
# set number of data points
self.N = t.size
#self.shape = (self.N, self.N)
self.Nhypers = theta0.size
# set inputs
self.t = t
from GP.kernels import exponential_squared
self.kernel = exponential_squared.sqexp()
#from GP.kernels import mattern
#self.kernel = mattern.mattern(p=2, D=1)
# set covariance function and evaluate
self.theta = theta0
# set initial jitter factor (for numerical stability of inverse)
self.jit = jit
# set derivative
#self.dcov_func = lambda theta, mode: self.kernel.dcov_func(theta, self.tt, self.val, mode=mode)
self.solve_mode = solve_mode
if solve_mode == "full":
# create vector of differences
self.tt = np.tile(self.t,
(self.N, 1)).T - np.tile(self.t, (self.N, 1))
# evaluate covaraince matrix
self.val = self.kernel.cov_func(self.theta, self.tt, noise=False)
#self.val2 = self.kernel.cov_func2(self.theta, self.tt, noise=False)
# get cholesky decomp
self.L = np.linalg.cholesky(
self.val +
self.jit * np.eye(self.N)) #add jitter for numerical stability
# get log determinant (could use entropic trace estimator here)
self.logdet = 2 * np.sum(np.diag(self.L)).real
# get the inverse
self.Linv = np.linalg.inv(self.L)
# elif solve_mode=="approx" or solve_mode=="RR":
# # set up RR basis
# from GP.tools import make_basis
# from GP.basisfuncs import rectangular
# self.M = np.array([M])
# self.L = np.array([L])
# self.Phi = make_basis.get_eigenvectors(self.t.reshape(self.N, 1), self.M, self.L, rectangular.phi)
# Lambda = make_basis.get_eigenvals(self.M, self.L, rectangular.Lambda)
# self.s = np.sqrt(Lambda)
# # get spectral density
# tmp = self.kernel.spectral_density(self.theta, self.s)
# self.Lambdainv = diags(1.0/tmp)
# self.Sigmainv = diags(np.ones(self.N)/self.jit)
#
# self.Mop = LinearOperator((self.N, self.N), matvec=self.M_func)
#
# elif solve_mode=="debug":
# # get cholesky decomp
# self.L = np.linalg.cholesky(self.val + self.jit * np.eye(self.N)) # add jitter for numerical stability
#
# # get log determinant (could use entropic trace estimator here)
# self.logdet = 2 * np.sum(np.diag(self.L)).real
#
# # get the inverse
# self.Linv = np.linalg.inv(self.L)
#
# self.valinv = np.dot(self.Linv.T, self.Linv)
#
# # this is for the preconditioning
# from GP.tools import make_basis
# from GP.basisfuncs import rectangular
# self.M = np.array([M])
# self.L = np.array([L])
# self.Phi = make_basis.get_eigenvectors(self.t.reshape(self.N, 1), self.M, self.L, rectangular.phi)
# Lambda = make_basis.get_eigenvals(self.M, self.L, rectangular.Lambda)
# self.s = np.sqrt(Lambda)
# # get spectral density
# tmp = self.kernel.spectral_density(self.theta, self.s) # figure out where factor of 2.5 comes from!!!!!!!!!!!!!!!!!!!!!!!!
# self.Lambdainv = np.diag(1.0/tmp)
# self.Lambda = np.diag(tmp)
# self.Sigmainv = np.diag(np.ones(self.N)/self.jit)
#
# self.val2 = self.Phi.dot(self.Lambda.dot(self.Phi.T))
#
# Z = self.Lambdainv + self.Phi.T.dot(self.Sigmainv.dot(self.Phi))
# LZ = np.linalg.cholesky(Z)
# LZinv = np.linalg.inv(LZ)
# Zinv = np.dot(LZinv.T, LZinv)
# self.valinv2 = self.Sigmainv - self.Sigmainv.dot(self.Phi.dot(Zinv.dot(self.Phi.T.dot(self.Sigmainv))))
#
# self.Mop = LinearOperator((self.N, self.N), matvec=self.M_func)
else:
raise NotImplementedError("Still working on faster solve_mode")