def generatePolynomialPreconditioner(X, sf, offset, degree, lparam):
"""
computes the preconditioner for a polynomial kernel using TensorSketch
"""
# TODO: use CraftMAps
s = findStableRank(X)
(n, d) = X.shape
augXt = np.vstack((np.sqrt(sf) * X.transpose(), np.sqrt(offset) * np.ones(
(1, n))))
Z = np.ones((s, n))
for iter in range(degree):
C = np.zeros((s, d + 1))
rows = numpy.random.choice(s, size=d + 1, replace=True)
signs = numpy.random.choice([-1.0, 1.0], size=d + 1, replace=True)
for col in range(d + 1):
C[rows[col], col] = signs[col]
Z = Z * fft(C.dot(augXt), axis=0)
Z = ifft(Z, axis=0).real.transpose()
#K = self.kernel(X, X)
#print(numpy.linalg.norm(K), numpy.linalg.norm(Z.dot(Z.transpose())))
precondMat = Z.transpose().dot(Z) + lparam * np.identity(s)
L = dpotrf(precondMat)[0].transpose()
return numpy.linalg.solve(L, Z.transpose())
def cholesky(cov, max_tries=5):
"""
Computest the Cholesky decomposition L of the matrix cov: L*L^T = cov
:param cov: np.array(nxn)
:param max_tries: int
:return: L
"""
cov = np.ascontiguousarray(cov)
L, info = lapack.dpotrf(cov, lower=1)
if info == 0:
return L
diag_cov = np.diag(cov)
if np.any(diag_cov <= 0.):
raise linalg.LinAlgError("not positive definite matrix")
jitter = diag_cov.mean() * 1e-6
n_tries = 1
while n_tries <= max_tries and np.isfinite(jitter):
try:
L = linalg.cholesky(cov + np.eye(cov.shape[0]) * jitter,
lower=True)
return L
except:
jitter *= 10
finally:
n_tries += 1
raise linalg.LinAlgError("not positive definite, even with jitter.")
def jitchol(A, maxtries=5):
"""
Calculates the Cholesky decomposition of ``A``. In the case that it is not possible to calculate the Cholesky,
a jitter will be added to ``A``.
Note
----
This method is adopted from the GPy package
"""
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise JitChol, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
while maxtries > 0 and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
return L
except:
jitter *= 10
finally:
maxtries -= 1
raise JitChol, "not positive definite, even with jitter."
def jitchol(A, maxtries=10):
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError("not pd: non-positive diagonal elements")
jitter = diagA.mean() * 1e-4
num_tries = 1
while num_tries <= maxtries and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
return L
except:
jitter *= 10
finally:
num_tries += 1
Aretr = np.eye(A.shape[0])
Aretr.flags['F_CONTIGUOUS'] = True
import traceback
try:
raise
except:
logging.warning('\n'.join([
'Added jitter of {:.10e}'.format(jitter),
' in ' + traceback.format_list(
traceback.extract_stack(limit=3)[-2:-1])[0][2:]
]))
return L
def jitchol(A, maxtries=5):
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
num_tries = 0
while num_tries < maxtries and np.isfinite(jitter):
try:
print jitter
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True)
return L
except:
jitter *= 10
finally:
num_tries += 1
raise linalg.LinAlgError, "not positive definite, even with jitter."
import traceback
try: raise
except:
logging.warning('\n'.join(['Added jitter of {:.10e}'.format(jitter),
' in '+traceback.format_list(traceback.extract_stack(limit=2)[-2:-1])[0][2:]]))
import ipdb;ipdb.set_trace()
return L
def jitchol(A, maxtries=5):
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
while maxtries > 0 and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
except:
jitter *= 10
finally:
maxtries -= 1
raise linalg.LinAlgError, "not positive definite, even with jitter."
import traceback
try:
raise
except:
logging.warning('\n'.join([
'Added jitter of {:.10e}'.format(jitter),
' in ' + traceback.format_list(
traceback.extract_stack(limit=2)[-2:-1])[0][2:]
]))
import ipdb
ipdb.set_trace()
return L
def jitchol(A, maxtries=5):
"""
Calculates the Cholesky decomposition of ``A``. In the case that it is not possible to calculate the Cholesky,
a jitter will be added to ``A``.
Note
----
This method is adopted from the GPy package
"""
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise JitChol, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
while maxtries > 0 and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True)
return L
except:
jitter *= 10
finally:
maxtries -= 1
raise JitChol, "not positive definite, even with jitter."
def __init__(self, X, Y, tuning):
self.X = X
self.Y = Y
self.tuning = tuning
self.K_uf = tuning(X).T
KK = self.K_uf.dot(self.K_uf.T)
Ky = self.K_uf.dot(Y)
self.w_mean = dpotrs(dpotrf(KK)[0],
Ky)[0] # faster than np.linalg.solve(KK, Ky)
if np.any(np.isnan(self.w_mean)):
try:
self.w_mean = np.linalg.solve(KK, Ky)
except:
jitter = np.diag(KK).mean() * 1e-6
num_tries = 1
while num_tries <= 5 and np.isfinite(jitter):
try:
self.w_mean = np.linalg.solve(
KK + np.eye(KK.shape[0]) * jitter, Ky)
except:
jitter *= 10
finally:
num_tries += 1
self.wb_var = np.transpose([
scipy.optimize.nnls(
np.vstack([self._SNRinv(X), np.ones(len(X))]).T,
(self.mean(X) - Y)[:, a]**2)[0] for a in range(Y.shape[1])
])
self.Gaussian_noise = Foo()
self.variance, self.Gaussian_noise.variance = self.wb_var
def _cholesky(A, lower=True):
'''
Computes the Cholesky decomposition of `A` using the routine `dpotrf`.
Parameters
----------
A : (n, n) float array
lower : bool, optional
Returns
-------
(n, n) float array
'''
# handle rank zero matrix
if A.shape == (0, 0):
return np.zeros((0, 0), dtype=float)
L, info = dpotrf(A, lower=lower)
if info > 0:
raise np.linalg.LinAlgError(
'The leading minor of order %s is not positive definite, and the '
'factorization could not be completed. ' % info)
elif info < 0:
raise ValueError('The %s-th argument has an illegal value.' % -info)
return L
def pd_solve(a, b):
""" Fast matrix solve for positive definite matrix a"""
L, info = dpotrf(a)
if info == 0:
return dpotrs(L, b)[0]
else:
return np.linalg.solve(a, b)
def inv_c(M):
A = np.ascontiguousarray(M)
L_M, info = lapack.dpotrf(A, lower=1)
#L_M = np.linalg.cholesky(M)
iM, _ = dpotri(L_M)
return iM
def inv_logDet(M):
A = np.ascontiguousarray(M)
L_M, info = lapack.dpotrf(A, lower=1)
iM, _ = dpotri(L_M)
logDetM = 2 * sum(np.log(np.diag(L_M)))
return iM, logDetM
def cholSafe(self, A):
# if lapack cholesky decomp fails then consecutively add more jitter until we get
# a working cholesky (ie ensuring we have a positive definite matrix as it can sometimes
# due to floating point errors end up not positive definite).
A = np.ascontiguousarray(A)
L, status = dpotrf(A, lower=1)
if status == 0:
return L
else:
L = linalg.cholesky(A + np.eye(A.shape[0]) * 1e-5, lower=True)
return L
def _cholesky(A, lower=True):
'''
Computes the Cholesky decomposition of `A` using `dpotrf`
'''
if A.shape == (0, 0):
return np.zeros((0, 0), dtype=float)
L, info = dpotrf(A, lower=lower)
if info < 0:
raise ValueError('The %s-th argument has an illegal value.' % -info)
elif info > 0:
raise np.linalg.LinAlgError('Matrix not positive definite')
return L
def cholesky(A, maxtries=5):
"""
Compute the cholesky of A. If the matrix is singular make `maxtries`
additional attempts to invert it by adding small amounts to the diagonal
before giving up.
"""
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
d = A.diagonal()
if np.any(d <= 0):
raise LinAlgError('Matrix has non-positive diagonal elements')
j = d.mean() * 1e-6
for _ in xrange(maxtries):
L, info = lapack.dpotrf(add_diagonal(A, j, True), lower=1)
if info == 0:
message = 'jitter of {:s} required to compute the cholesky'
warnings.warn(message.format(str(j)), stacklevel=2)
return L
else:
j *= 10
raise LinAlgError('Matrix is not positive definite, even with jitter')
def cholesky(A, maxtries=5):
"""
Compute the cholesky of A. If the matrix is singular make `maxtries`
additional attempts to invert it by adding small amounts to the diagonal
before giving up.
"""
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
d = A.diagonal()
if np.any(d <= 0):
raise LinAlgError('Matrix has non-positive diagonal elements')
j = d.mean() * 1e-6
for _ in xrange(maxtries):
L, info = lapack.dpotrf(add_diagonal(A, j, True), lower=1)
if info == 0:
message = 'jitter of {:s} required to compute the cholesky'
warnings.warn(message.format(str(j)), stacklevel=2)
return L
else:
j *= 10
raise LinAlgError('Matrix is not positive definite, even with jitter')
def jitchol(A, maxtries=5):
''' Copyright (c) 2012, GPy authors (James Hensman, Nicolo Fusi, Ricardo Andrade,
Nicolas Durrande, Alan Saul, Max Zwiessele, Neil D. Lawrence).
All rights reserved
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the <organization> nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
:param A: the matrixed to be decomposited
:param int maxtries: number of iterations of adding jitters
'''
A = np.asfortranarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise np.linalg.LinAlgError, "kernel matrix not positive definite: non-positive diagonal elements"
jitter = diagA.mean() * 1e-9
while maxtries > 0 and np.isfinite(jitter):
print 'Warning: adding jitter of {:.10e} to diagnol of kernel matrix for numerical stability'.format(
jitter)
try:
return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter,
lower=True)
except:
jitter *= 10
finally:
maxtries -= 1
raise np.linalg.LinAlgError, "kernel matrix not positive definite, even with jitter."
def jit_cholesky(A, maxtries=5):
"""
Performs Jittered Cholesky Decomposition
Performs a Jittered Cholesky decomposition, adding noise to the diagonal of the matrix as needed
in order to ensure that the matrix can be inverted. Adapted from code in GPy.
On occasion, the matrix that needs to be inverted in fitting a GP is nearly singular. This arises
when the training samples are very close to one another, and can be averted by adding a noise term
to the diagonal of the matrix. This routine performs an exact Cholesky decomposition if it can
be done, and if it cannot it successively adds noise to the diagonal (starting with 1.e-6 times
the mean of the diagonal and incrementing by a factor of 10 each time) until the matrix can be
decomposed or the algorithm reaches ``maxtries`` attempts. The routine returns the lower
triangular matrix and the amount of noise necessary to stabilize the decomposition.
:param A: The matrix to be inverted as an array of shape ``(n,n)``. Must be a symmetric positive
definite matrix.
:type A: ndarray
:param maxtries: (optional) Maximum allowable number of attempts to stabilize the Cholesky
Decomposition. Must be a positive integer (default = 5)
:type maxtries: int
:returns: Lower-triangular factored matrix (shape ``(n,n)`` and the noise that was added to
the diagonal to achieve that result.
:rtype: tuple containing an ndarray and a float
"""
A = check_cholesky_inputs(A)
assert int(maxtries) > 0, "maxtries must be a positive integer"
A = np.ascontiguousarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L, 0.
else:
diagA = np.diag(A)
jitter = diagA.mean() * 1e-6
num_tries = 1
while num_tries <= maxtries and np.isfinite(jitter):
try:
L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter,
lower=True)
return L, jitter
except:
jitter *= 10
finally:
num_tries += 1
raise linalg.LinAlgError("not positive definite, even with jitter.")
return L, jitter
def jitchol(A,maxtries=5):
''' Copyright (c) 2012, GPy authors (James Hensman, Nicolo Fusi, Ricardo Andrade,
Nicolas Durrande, Alan Saul, Max Zwiessele, Neil D. Lawrence).
All rights reserved
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the <organization> nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
:param A: the matrixed to be decomposited
:param int maxtries: number of iterations of adding jitters
'''
A = np.asfortranarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise np.linalg.LinAlgError, "kernel matrix not positive definite: non-positive diagonal elements"
jitter = diagA.mean() * 1e-9
while maxtries > 0 and np.isfinite(jitter):
print 'Warning: adding jitter of {:.10e} to diagnol of kernel matrix for numerical stability'.format(jitter)
try:
return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True)
except:
jitter *= 10
finally:
maxtries -= 1
raise np.linalg.LinAlgError, "kernel matrix not positive definite, even with jitter."
def chol_inv(x: np.array):
"""Calculates invserse of matrix using Cholesky decomposition.
Keyword arguments:
x -- A matrix.
Returns:
x^-1.
"""
c, info = lapack.dpotrf(x)
if info:
raise np.linalg.LinAlgError
lapack.dpotri(c, overwrite_c=1)
c += c.T
np.fill_diagonal(c, c.diagonal() / 2)
return c
def jitchol(A, maxtries=5):
A = np.asfortranarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
while maxtries > 0 and np.isfinite(jitter):
print 'Warning: adding jitter of {:.10e}'.format(jitter)
try:
return linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True)
except:
jitter *= 10
finally:
maxtries -= 1
raise linalg.LinAlgError, "not positive definite, even with jitter."
def fast_pd_inverse(m: 'numpy array') -> 'numpy array':
'''
This method calculates the inverse of a A real symmetric positive definite (n × n)-matrix
It is much faster than Numpy's "np.linalg.inv" method for example.
'''
try:
cholesky, info = lapack.dpotrf(m)
if info != 0:
raise ValueError('dpotrf failed on input {}'.format(m))
#print("cas 1")
inv, info = lapack.dpotri(cholesky)
if info != 0:
raise ValueError('dpotri failed on input {}'.format(cholesky))
#print("cas 2")
except:
inv = np.linalg.inv(m)
uppertriangular_2_symmetric(inv)
return inv
def jitchol(A, maxtries=5):
A = np.asfortranarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
jitter = diagA.mean() * 1e-6
while maxtries > 0 and np.isfinite(jitter):
print 'Warning: adding jitter of {:.10e}'.format(jitter)
try:
return linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True)
except:
jitter *= 10
finally:
maxtries -= 1
raise linalg.LinAlgError, "not positive definite, even with jitter."
def mvn_loglike(y, cov):
"""
Evaluate the multivariate-normal log-likelihood for difference vector `y`
and covariance matrix `cov`:
log_p = -1/2*[(y^T).(C^-1).y + log(det(C))] + const.
The likelihood is NOT NORMALIZED, since this does not affect MCMC. The
normalization const = -n/2*log(2*pi), where n is the dimensionality.
Arguments `y` and `cov` MUST be np.arrays with dtype == float64 and shapes
(n) and (n, n), respectively. These requirements are NOT CHECKED.
The calculation follows algorithm 2.1 in Rasmussen and Williams (Gaussian
Processes for Machine Learning).
"""
# Compute the Cholesky decomposition of the covariance.
# Use bare LAPACK function to avoid scipy.linalg wrapper overhead.
L, info = lapack.dpotrf(cov, clean=False)
if info < 0:
raise ValueError(
'lapack dpotrf error: '
'the {}-th argument had an illegal value'.format(-info)
)
elif info < 0:
raise np.linalg.LinAlgError(
'lapack dpotrf error: '
'the leading minor of order {} is not positive definite'
.format(info)
)
# Solve for alpha = cov^-1.y using the Cholesky decomp.
alpha, info = lapack.dpotrs(L, y)
if info != 0:
raise ValueError(
'lapack dpotrs error: '
'the {}-th argument had an illegal value'.format(-info)
)
return -.5*np.dot(y, alpha) - np.log(L.diagonal()).sum()
def train(self, X, Y):
"""Trains the machine on the supplied set of inputs and outputs."""
assert X.shape[0] == Y.shape[0]
assert len(X.shape) == len(Y.shape) == 2
assert X.shape[1] == self.n
assert Y.shape[1] == self.p
self.X = X
self.Y = Y
self.Phi = self.mapping.evaluate(self.X)
m, n = self.Phi.shape
sigma_n2 = self.sigma_n * self.sigma_n
self.L = numpy.dot(self.Phi.T, self.Phi)
idx = numpy.diag_indices(self.Phi.shape[1])
self.L[idx] += sigma_n2
self.up = 0
self.L, info = dpotrf(self.L, lower=self.up, clean=1, overwrite_a=1)
self.B = numpy.dot(self.Phi.T, self.Y)
self.LPhiY, info = dgetrs(self.L,
numpy.arange(self.L.shape[0]),
self.B,
trans=1,
overwrite_b=1)
assert info == 0, 'scipy.linalg.lapack.dgetrs(L, B)'
self.W, info = dgetrs(self.L,
numpy.arange(self.L.shape[0]),
self.LPhiY,
trans=0,
overwrite_b=0)
assert info == 0, 'scipy.linalg.lapack.dgetrs(L, LPhiY)'
self.lml = -0.5 * ((1. / (sigma_n2)) * (inner1d(self.Y.T, self.Y.T) - inner1d(self.LPhiY.T, self.LPhiY.T)) +
(m - n) * numpy.log(sigma_n2) + m * numpy.log(2. * numpy.pi)) - \
numpy.log(self.L.diagonal()).sum()
def __init__(self, X, Y, theta):
theta = theta ** [2,1,2]
[sf2, l2, sn2] = theta
# evaluate RBF kernel for our given X
r = dist.pdist(X) / l2
K = dist.squareform(sf2 * np.exp(-0.5 * r**2))
np.fill_diagonal(K, sf2)
# add in Gaussian noise (+ a bit for numerical stability)
Ky = K.copy()
np.fill_diagonal(Ky, sf2 + sn2 + 1e-8)
# compute the Cholesky factorization of our covariance matrix
LW, info = lapack.dpotrf(Ky, lower=True)
assert info == 0
# calculate lower half of inverse of K (assumes real symmetric positive definite)
Wi, info = lapack.dpotri(LW, lower=True)
assert info == 0
# make symmetric by filling in the upper half
Wi += np.tril(Wi,-1).T
# and solve
alpha, info = lapack.dpotrs(LW, Y, lower=True)
assert info == 0
# save these for later
self.X = X
self.Y = Y
self.theta = theta
self.r = r
self.K = K
self.Ky = Ky
self.LW = LW
self.Wi = Wi
self.alpha = alpha
def _loss_and_grad(self, params):
V = self.V
X = self.X
X.fill(0)
#X = np.zeros((self.n,self.n))
X[self._mask] = params
X += X.T
np.fill_diagonal(X, 1)
zz, info0 = dpotrf(X, False, False)
iX, info1 = dpotri(zz)
iX = np.triu(iX) + np.triu(iX, k=1).T
if info0 != 0 or info1 != 0:
#print('checkpt')
return self._loss * 100, np.zeros_like(params)
loss = np.sum(iX * V)
G = -iX @ V @ iX
g = G[self._mask] + G.T[self._mask]
self._loss = loss
#print(np.sqrt(loss / self.W.shape[0]))
return loss, g #G.flatten()
def cholesky(A, lower=True):
'''
Computes the Cholesky decomposition of *A* using the routine
*dpotrf*.
Parameters
----------
A : (N,N) float array
lower : bool, optional
'''
if A.shape == (0, 0):
return np.zeros((0, 0), dtype=float)
L, info = dpotrf(A, lower=lower)
if info > 0:
raise np.linalg.LinAlgError(
'The leading minor of order %s is not positive definite, and '
'the factorization could not be completed. ' % info)
elif info < 0:
raise ValueError('The %s-th argument has an illegal value.' % (-info))
return L
def mvn_loglike(y, cov):
"""
Calculate multi-varaite-normal log-likelihood
log_p = -1/2 * [(y^T).(C^-1).y + log(det(C))] + const
To normalize the likelihood, const = -n/2*log(2*pi), which is omitted here
Args:
y -- (n)
cov -- shape (n, n)
Returns:
log_p
"""
L, info = lapack.dpotrf(cov, clean=False)
if info != 0:
raise ValueError('lapack dpotrf error: illegal value for info!')
alpha, info = lapack.dpotrs(L, y)
if info != 0:
raise ValueError(
'lapack dpotrf error: illegal value for info! {}'.format(info))
return -.5 * np.dot(y, alpha) - np.log(L.diagonal()).sum()
def __init__(self, X, Y, inducing_inputs, lengthscale):
self.X = X
self.Y = Y
self.inducing_inputs = inducing_inputs
self.lengthscale = lengthscale
self.rectify = rectify
self.kern = GPy.kern.RBF(input_dim=X.shape[1],
variance=1.,
lengthscale=lengthscale,
ARD=True)
K_uf = self.kern.K(inducing_inputs, X)
KK = K_uf.dot(K_uf.T)
Ky = K_uf.dot(Y)
self.w_mean = dpotrs(dpotrf(KK)[0],
Ky)[0] # faster than np.linalg.solve(KK, Ky)
if np.any(np.isnan(self.w_mean)):
try:
self.w_mean = np.linalg.solve(KK, Ky)
except:
jitter = np.diag(KK).mean() * 1e-6
num_tries = 1
while num_tries <= 5 and np.isfinite(jitter):
try:
self.w_mean = np.linalg.solve(
KK + np.eye(KK.shape[0]) * jitter, Ky)
except:
jitter *= 10
finally:
num_tries += 1
self.wb_var = np.transpose([
scipy.optimize.nnls(
np.vstack([self._SNRinv(X), np.ones(len(X))]).T,
(self.mean(X) - Y)[:, a]**2)[0] for a in range(Y.shape[1])
])
self.Gaussian_noise = Foo()
self.variance, self.Gaussian_noise.variance = self.wb_var
def potrf(A, MKLProc):
from scipy.linalg.lapack import dpotrf
import os
os.environ['MKL_NUM_THREADS'] = str(MKLProc)
A = dpotrf(A, lower=True)[0]
return A
def lnLL(dy, cov):
L, info = lapack.dpotrf(cov, clean=False)
alpha, info = lapack.dpotrs(L, dy)
return -.5 * np.dot(dy, alpha) - np.log(L.diagonal()).sum()
def kalman_filter(model, return_loglike=False):
# Parameters
dtype = model.dtype
# Kalman filter properties
filter_method = model.filter_method
inversion_method = model.inversion_method
stability_method = model.stability_method
conserve_memory = model.conserve_memory
tolerance = model.tolerance
loglikelihood_burn = model.loglikelihood_burn
# Check for acceptable values
if not filter_method == FILTER_CONVENTIONAL:
warn('The pure Python version of the kalman filter only supports the'
' conventional Kalman filter')
implemented_inv_methods = INVERT_NUMPY | INVERT_UNIVARIATE | SOLVE_CHOLESKY
if not inversion_method & implemented_inv_methods:
warn('The pure Python version of the kalman filter only performs'
' inversion using `numpy.linalg.inv`.')
if not tolerance == 0:
warn('The pure Python version of the kalman filter does not check'
' for convergence.')
# Convergence (this implementation does not consider convergence)
converged = False
period_converged = 0
# Dimensions
nobs = model.nobs
k_endog = model.k_endog
k_states = model.k_states
k_posdef = model.k_posdef
# Allocate memory for variables
filtered_state = np.zeros((k_states, nobs), dtype=dtype)
filtered_state_cov = np.zeros((k_states, k_states, nobs), dtype=dtype)
predicted_state = np.zeros((k_states, nobs+1), dtype=dtype)
predicted_state_cov = np.zeros((k_states, k_states, nobs+1), dtype=dtype)
forecast = np.zeros((k_endog, nobs), dtype=dtype)
forecast_error = np.zeros((k_endog, nobs), dtype=dtype)
forecast_error_cov = np.zeros((k_endog, k_endog, nobs), dtype=dtype)
loglikelihood = np.zeros((nobs+1,), dtype=dtype)
# Selected state covariance matrix
selected_state_cov = (
np.dot(
np.dot(model.selection[:, :, 0],
model.state_cov[:, :, 0]),
model.selection[:, :, 0].T
)
)
# Initial values
if model.initialization == 'known':
initial_state = model._initial_state.astype(dtype)
initial_state_cov = model._initial_state_cov.astype(dtype)
elif model.initialization == 'approximate_diffuse':
initial_state = np.zeros((k_states,), dtype=dtype)
initial_state_cov = (
np.eye(k_states).astype(dtype) * model._initial_variance
)
elif model.initialization == 'stationary':
initial_state = np.zeros((k_states,), dtype=dtype)
initial_state_cov = solve_discrete_lyapunov(
np.array(model.transition[:, :, 0], dtype=dtype),
np.array(selected_state_cov[:, :], dtype=dtype),
)
else:
raise RuntimeError('Statespace model not initialized.')
# Copy initial values to predicted
predicted_state[:, 0] = initial_state
predicted_state_cov[:, :, 0] = initial_state_cov
# print(predicted_state_cov[:, :, 0])
# Setup indices for possibly time-varying matrices
design_t = 0
obs_intercept_t = 0
obs_cov_t = 0
transition_t = 0
state_intercept_t = 0
selection_t = 0
state_cov_t = 0
# Iterate forwards
time_invariant = model.time_invariant
for t in range(nobs):
# Get indices for possibly time-varying arrays
if not time_invariant:
if model.design.shape[2] > 1: design_t = t
if model.obs_intercept.shape[1] > 1: obs_intercept_t = t
if model.obs_cov.shape[2] > 1: obs_cov_t = t
if model.transition.shape[2] > 1: transition_t = t
if model.state_intercept.shape[1] > 1: state_intercept_t = t
if model.selection.shape[2] > 1: selection_t = t
if model.state_cov.shape[2] > 1: state_cov_t = t
# Selected state covariance matrix
if model.selection.shape[2] > 1 or model.state_cov.shape[2] > 1:
selected_state_cov = (
np.dot(
np.dot(model.selection[:, :, selection_t],
model.state_cov[:, :, state_cov_t]),
model.selection[:, :, selection_t].T
)
)
# #### Forecast for time t
# `forecast` $= Z_t a_t + d_t$
#
# *Note*: $a_t$ is given from the initialization (for $t = 0$) or
# from the previous iteration of the filter (for $t > 0$).
forecast[:, t] = (
np.dot(model.design[:, :, design_t], predicted_state[:, t]) +
model.obs_intercept[:, obs_intercept_t]
)
# *Intermediate calculation* (used just below and then once more)
# `tmp1` array used here, dimension $(m \times p)$
# $\\#_1 = P_t Z_t'$
# $(m \times p) = (m \times m) (p \times m)'$
tmp1 = np.dot(predicted_state_cov[:, :, t],
model.design[:, :, design_t].T)
# #### Forecast error for time t
# `forecast_error` $\equiv v_t = y_t -$ `forecast`
forecast_error[:, t] = model.obs[:, t] - forecast[:, t]
# #### Forecast error covariance matrix for time t
# $F_t \equiv Z_t P_t Z_t' + H_t$
forecast_error_cov[:, :, t] = (
np.dot(model.design[:, :, design_t], tmp1) +
model.obs_cov[:, :, obs_cov_t]
)
# Store the inverse
if k_endog == 1 and inversion_method & INVERT_UNIVARIATE:
forecast_error_cov_inv = 1.0 / forecast_error_cov[0, 0, t]
determinant = forecast_error_cov[0, 0, t]
tmp2 = forecast_error_cov_inv * forecast_error[:, t]
tmp3 = forecast_error_cov_inv * model.design[:, :, design_t]
elif inversion_method & SOLVE_CHOLESKY:
U, info = lapack.dpotrf(forecast_error_cov[:, :, t])
determinant = np.product(U.diagonal())**2
tmp2, info = lapack.dpotrs(U, forecast_error[:, t])
tmp3, info = lapack.dpotrs(U, model.design[:, :, design_t])
else:
forecast_error_cov_inv = np.linalg.inv(forecast_error_cov[:, :, t])
determinant = np.linalg.det(forecast_error_cov[:, :, t])
tmp2 = np.dot(forecast_error_cov_inv, forecast_error[:, t])
tmp3 = np.dot(forecast_error_cov_inv, model.design[:, :, design_t])
# #### Filtered state for time t
# $a_{t|t} = a_t + P_t Z_t' F_t^{-1} v_t$
# $a_{t|t} = 1.0 * \\#_1 \\#_2 + 1.0 a_t$
filtered_state[:, t] = (
predicted_state[:, t] + np.dot(tmp1, tmp2)
)
# #### Filtered state covariance for time t
# $P_{t|t} = P_t - P_t Z_t' F_t^{-1} Z_t P_t$
# $P_{t|t} = P_t - \\#_1 \\#_3 P_t$
filtered_state_cov[:, :, t] = (
predicted_state_cov[:, :, t] -
np.dot(
np.dot(tmp1, tmp3),
predicted_state_cov[:, :, t]
)
)
# #### Loglikelihood
loglikelihood[t] = -0.5 * (
np.log((2*np.pi)**k_endog * determinant) +
np.dot(forecast_error[:, t], tmp2)
)
# #### Predicted state for time t+1
# $a_{t+1} = T_t a_{t|t} + c_t$
predicted_state[:, t+1] = (
np.dot(model.transition[:, :, transition_t],
filtered_state[:, t]) +
model.state_intercept[:, state_intercept_t]
)
# #### Predicted state covariance matrix for time t+1
# $P_{t+1} = T_t P_{t|t} T_t' + Q_t^*$
predicted_state_cov[:, :, t+1] = (
np.dot(
np.dot(model.transition[:, :, transition_t],
filtered_state_cov[:, :, t]),
model.transition[:, :, transition_t].T
) + selected_state_cov
)
# Enforce symmetry of predicted covariance matrix
predicted_state_cov[:, :, t+1] = (
predicted_state_cov[:, :, t+1] + predicted_state_cov[:, :, t+1].T
) / 2
if return_loglike:
return np.array(loglikelihood)
else:
kwargs = dict(
(k, v) for k, v in locals().items()
if k in _kalman_filter._fields
)
kwargs['model'] = _statespace(
initial_state=initial_state, initial_state_cov=initial_state_cov
)
kfilter = _kalman_filter(**kwargs)
return FilterResults(model, kfilter)
def jitchol(A, attempts=5):
'''
JITCHOL Do a Cholesky decomposition with jitter.
Description:
U = JITCHOL(A, MAXTRIES) attempts a Cholesky decomposition on the
given matrix, if matrix isn't positive definite the function gives a
warning, adds 'jitter' and tries again. At the first attempt the
amount of jitter added is 1e-6 times the mean of the diagonal.
Thereafter the amount of jitter is multiplied by 10 each time it is
added again. This is continued for a maximum of 10 times.
Returns:
U - the Cholesky decomposition for the matrix.
Arguments:
A - the matrix for which the Cholesky decomposition is required.
MAXTRIES - the maximum number of times that jitter is added before
giving up (default 10).
[U, JITTER] = JITCHOL(A, MAXTRIES) attempts a Cholesky decomposition
on the given matrix, if matrix isn't positive definite the function
adds 'jitter' and tries again. Thereafter the amount of jitter is
multiplied by 10 each time it is added again. This is continued for
a maximum of 10 times. The amount of jitter added is returned.
Returns:
U - the Cholesky decomposition for the matrix.
JITTER - the amount of jitter that was added to the matrix.
Arguments:
A - the matrix for which the Cholesky decomposition is required.
MAXTRIES - the maximum number of times that jitter is added before
giving up (default 10)
:param A: the matrixed to be decomposited
:param int maxtries: number of iterations of adding jitters
'''
A = np.asfortranarray(A)
L, info = lapack.dpotrf(A, lower=1)
if info == 0:
return L
else:
diagA = np.diag(A)
if np.any(diagA <= 0.):
raise np.linalg.LinAlgError("kernel matrix not positive definite: "
"non-positive diagonal elements")
jitter = diagA.mean() * np.finfo(float).tiny #1e-9
while attempts > 0 and np.isfinite(jitter):
#logging.getLogger(__name__).warning('adding jitter of {:.10e} to '
#'diagnol of kernel matrix for '
#'numerical stability'.format(jitter))
try:
return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter,
lower=True)
except:
jitter *= 10
finally:
attempts -= 1
raise np.linalg.LinAlgError(
"kernel matrix not positive definite, even with jitter.")
def kalman_filter(model, return_loglike=False):
# Parameters
dtype = model.dtype
# Kalman filter properties
filter_method = model.filter_method
inversion_method = model.inversion_method
stability_method = model.stability_method
conserve_memory = model.conserve_memory
tolerance = model.tolerance
loglikelihood_burn = model.loglikelihood_burn
# Check for acceptable values
if not filter_method == FILTER_CONVENTIONAL:
warn('The pure Python version of the kalman filter only supports the'
' conventional Kalman filter')
implemented_inv_methods = INVERT_NUMPY | INVERT_UNIVARIATE | SOLVE_CHOLESKY
if not inversion_method & implemented_inv_methods:
warn('The pure Python version of the kalman filter only performs'
' inversion using `numpy.linalg.inv`.')
if not tolerance == 0:
warn('The pure Python version of the kalman filter does not check'
' for convergence.')
# Convergence (this implementation does not consider convergence)
converged = False
period_converged = 0
# Dimensions
nobs = model.nobs
k_endog = model.k_endog
k_states = model.k_states
k_posdef = model.k_posdef
# Allocate memory for variables
filtered_state = np.zeros((k_states, nobs), dtype=dtype)
filtered_state_cov = np.zeros((k_states, k_states, nobs), dtype=dtype)
predicted_state = np.zeros((k_states, nobs + 1), dtype=dtype)
predicted_state_cov = np.zeros((k_states, k_states, nobs + 1), dtype=dtype)
forecast = np.zeros((k_endog, nobs), dtype=dtype)
forecast_error = np.zeros((k_endog, nobs), dtype=dtype)
forecast_error_cov = np.zeros((k_endog, k_endog, nobs), dtype=dtype)
loglikelihood = np.zeros((nobs + 1, ), dtype=dtype)
# Selected state covariance matrix
selected_state_cov = (np.dot(
np.dot(model.selection[:, :, 0], model.state_cov[:, :, 0]),
model.selection[:, :, 0].T))
# Initial values
if model.initialization == 'known':
initial_state = model._initial_state.astype(dtype)
initial_state_cov = model._initial_state_cov.astype(dtype)
elif model.initialization == 'approximate_diffuse':
initial_state = np.zeros((k_states, ), dtype=dtype)
initial_state_cov = (np.eye(k_states).astype(dtype) *
model._initial_variance)
elif model.initialization == 'stationary':
initial_state = np.zeros((k_states, ), dtype=dtype)
initial_state_cov = solve_discrete_lyapunov(
np.array(model.transition[:, :, 0], dtype=dtype),
np.array(selected_state_cov[:, :], dtype=dtype),
)
else:
raise RuntimeError('Statespace model not initialized.')
# Copy initial values to predicted
predicted_state[:, 0] = initial_state
predicted_state_cov[:, :, 0] = initial_state_cov
# print(predicted_state_cov[:, :, 0])
# Setup indices for possibly time-varying matrices
design_t = 0
obs_intercept_t = 0
obs_cov_t = 0
transition_t = 0
state_intercept_t = 0
selection_t = 0
state_cov_t = 0
# Iterate forwards
time_invariant = model.time_invariant
for t in range(nobs):
# Get indices for possibly time-varying arrays
if not time_invariant:
if model.design.shape[2] > 1: design_t = t
if model.obs_intercept.shape[1] > 1: obs_intercept_t = t
if model.obs_cov.shape[2] > 1: obs_cov_t = t
if model.transition.shape[2] > 1: transition_t = t
if model.state_intercept.shape[1] > 1: state_intercept_t = t
if model.selection.shape[2] > 1: selection_t = t
if model.state_cov.shape[2] > 1: state_cov_t = t
# Selected state covariance matrix
if model.selection.shape[2] > 1 or model.state_cov.shape[2] > 1:
selected_state_cov = (np.dot(
np.dot(model.selection[:, :, selection_t],
model.state_cov[:, :, state_cov_t]),
model.selection[:, :, selection_t].T))
# #### Forecast for time t
# `forecast` $= Z_t a_t + d_t$
#
# *Note*: $a_t$ is given from the initialization (for $t = 0$) or
# from the previous iteration of the filter (for $t > 0$).
forecast[:, t] = (
np.dot(model.design[:, :, design_t], predicted_state[:, t]) +
model.obs_intercept[:, obs_intercept_t])
# *Intermediate calculation* (used just below and then once more)
# `tmp1` array used here, dimension $(m \times p)$
# $\\#_1 = P_t Z_t'$
# $(m \times p) = (m \times m) (p \times m)'$
tmp1 = np.dot(predicted_state_cov[:, :, t], model.design[:, :,
design_t].T)
# #### Forecast error for time t
# `forecast_error` $\equiv v_t = y_t -$ `forecast`
forecast_error[:, t] = model.obs[:, t] - forecast[:, t]
# #### Forecast error covariance matrix for time t
# $F_t \equiv Z_t P_t Z_t' + H_t$
forecast_error_cov[:, :,
t] = (np.dot(model.design[:, :, design_t], tmp1) +
model.obs_cov[:, :, obs_cov_t])
# Store the inverse
if k_endog == 1 and inversion_method & INVERT_UNIVARIATE:
forecast_error_cov_inv = 1.0 / forecast_error_cov[0, 0, t]
determinant = forecast_error_cov[0, 0, t]
tmp2 = forecast_error_cov_inv * forecast_error[:, t]
tmp3 = forecast_error_cov_inv * model.design[:, :, design_t]
elif inversion_method & SOLVE_CHOLESKY:
U, info = lapack.dpotrf(forecast_error_cov[:, :, t])
determinant = np.product(U.diagonal())**2
tmp2, info = lapack.dpotrs(U, forecast_error[:, t])
tmp3, info = lapack.dpotrs(U, model.design[:, :, design_t])
else:
forecast_error_cov_inv = np.linalg.inv(forecast_error_cov[:, :, t])
determinant = np.linalg.det(forecast_error_cov[:, :, t])
tmp2 = np.dot(forecast_error_cov_inv, forecast_error[:, t])
tmp3 = np.dot(forecast_error_cov_inv, model.design[:, :, design_t])
# #### Filtered state for time t
# $a_{t|t} = a_t + P_t Z_t' F_t^{-1} v_t$
# $a_{t|t} = 1.0 * \\#_1 \\#_2 + 1.0 a_t$
filtered_state[:, t] = (predicted_state[:, t] + np.dot(tmp1, tmp2))
# #### Filtered state covariance for time t
# $P_{t|t} = P_t - P_t Z_t' F_t^{-1} Z_t P_t$
# $P_{t|t} = P_t - \\#_1 \\#_3 P_t$
filtered_state_cov[:, :, t] = (
predicted_state_cov[:, :, t] -
np.dot(np.dot(tmp1, tmp3), predicted_state_cov[:, :, t]))
# #### Loglikelihood
loglikelihood[t] = -0.5 * (np.log((2 * np.pi)**k_endog * determinant) +
np.dot(forecast_error[:, t], tmp2))
# #### Predicted state for time t+1
# $a_{t+1} = T_t a_{t|t} + c_t$
predicted_state[:,
t + 1] = (np.dot(model.transition[:, :, transition_t],
filtered_state[:, t]) +
model.state_intercept[:, state_intercept_t])
# #### Predicted state covariance matrix for time t+1
# $P_{t+1} = T_t P_{t|t} T_t' + Q_t^*$
predicted_state_cov[:, :, t + 1] = (np.dot(
np.dot(model.transition[:, :, transition_t],
filtered_state_cov[:, :, t]),
model.transition[:, :, transition_t].T) + selected_state_cov)
# Enforce symmetry of predicted covariance matrix
predicted_state_cov[:, :,
t + 1] = (predicted_state_cov[:, :, t + 1] +
predicted_state_cov[:, :, t + 1].T) / 2
if return_loglike:
return np.array(loglikelihood)
else:
kwargs = dict(
(k, v) for k, v in locals().items() if k in _kalman_filter._fields)
kwargs['model'] = _statespace(initial_state=initial_state,
initial_state_cov=initial_state_cov)
kfilter = _kalman_filter(**kwargs)
return FilterResults(model, kfilter)
