def generatePolynomialPreconditioner(X, sf, offset, degree, lparam): """ computes the preconditioner for a polynomial kernel using TensorSketch """ # TODO: use CraftMAps s = findStableRank(X) (n, d) = X.shape augXt = np.vstack((np.sqrt(sf) * X.transpose(), np.sqrt(offset) * np.ones( (1, n)))) Z = np.ones((s, n)) for iter in range(degree): C = np.zeros((s, d + 1)) rows = numpy.random.choice(s, size=d + 1, replace=True) signs = numpy.random.choice([-1.0, 1.0], size=d + 1, replace=True) for col in range(d + 1): C[rows[col], col] = signs[col] Z = Z * fft(C.dot(augXt), axis=0) Z = ifft(Z, axis=0).real.transpose() #K = self.kernel(X, X) #print(numpy.linalg.norm(K), numpy.linalg.norm(Z.dot(Z.transpose()))) precondMat = Z.transpose().dot(Z) + lparam * np.identity(s) L = dpotrf(precondMat)[0].transpose() return numpy.linalg.solve(L, Z.transpose())
def cholesky(cov, max_tries=5): """ Computest the Cholesky decomposition L of the matrix cov: L*L^T = cov :param cov: np.array(nxn) :param max_tries: int :return: L """ cov = np.ascontiguousarray(cov) L, info = lapack.dpotrf(cov, lower=1) if info == 0: return L diag_cov = np.diag(cov) if np.any(diag_cov <= 0.): raise linalg.LinAlgError("not positive definite matrix") jitter = diag_cov.mean() * 1e-6 n_tries = 1 while n_tries <= max_tries and np.isfinite(jitter): try: L = linalg.cholesky(cov + np.eye(cov.shape[0]) * jitter, lower=True) return L except: jitter *= 10 finally: n_tries += 1 raise linalg.LinAlgError("not positive definite, even with jitter.")
def jitchol(A, maxtries=5): """ Calculates the Cholesky decomposition of ``A``. In the case that it is not possible to calculate the Cholesky, a jitter will be added to ``A``. Note ---- This method is adopted from the GPy package """ A = np.ascontiguousarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise JitChol, "not pd: non-positive diagonal elements" jitter = diagA.mean() * 1e-6 while maxtries > 0 and np.isfinite(jitter): try: L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True) return L except: jitter *= 10 finally: maxtries -= 1 raise JitChol, "not positive definite, even with jitter."
def jitchol(A, maxtries=10): A = np.ascontiguousarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise linalg.LinAlgError("not pd: non-positive diagonal elements") jitter = diagA.mean() * 1e-4 num_tries = 1 while num_tries <= maxtries and np.isfinite(jitter): try: L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True) return L except: jitter *= 10 finally: num_tries += 1 Aretr = np.eye(A.shape[0]) Aretr.flags['F_CONTIGUOUS'] = True import traceback try: raise except: logging.warning('\n'.join([ 'Added jitter of {:.10e}'.format(jitter), ' in ' + traceback.format_list( traceback.extract_stack(limit=3)[-2:-1])[0][2:] ])) return L
def jitchol(A, maxtries=5): A = np.ascontiguousarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise linalg.LinAlgError, "not pd: non-positive diagonal elements" jitter = diagA.mean() * 1e-6 num_tries = 0 while num_tries < maxtries and np.isfinite(jitter): try: print jitter L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True) return L except: jitter *= 10 finally: num_tries += 1 raise linalg.LinAlgError, "not positive definite, even with jitter." import traceback try: raise except: logging.warning('\n'.join(['Added jitter of {:.10e}'.format(jitter), ' in '+traceback.format_list(traceback.extract_stack(limit=2)[-2:-1])[0][2:]])) import ipdb;ipdb.set_trace() return L
def jitchol(A, maxtries=5): A = np.ascontiguousarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise linalg.LinAlgError, "not pd: non-positive diagonal elements" jitter = diagA.mean() * 1e-6 while maxtries > 0 and np.isfinite(jitter): try: L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True) except: jitter *= 10 finally: maxtries -= 1 raise linalg.LinAlgError, "not positive definite, even with jitter." import traceback try: raise except: logging.warning('\n'.join([ 'Added jitter of {:.10e}'.format(jitter), ' in ' + traceback.format_list( traceback.extract_stack(limit=2)[-2:-1])[0][2:] ])) import ipdb ipdb.set_trace() return L
def __init__(self, X, Y, tuning): self.X = X self.Y = Y self.tuning = tuning self.K_uf = tuning(X).T KK = self.K_uf.dot(self.K_uf.T) Ky = self.K_uf.dot(Y) self.w_mean = dpotrs(dpotrf(KK)[0], Ky)[0] # faster than np.linalg.solve(KK, Ky) if np.any(np.isnan(self.w_mean)): try: self.w_mean = np.linalg.solve(KK, Ky) except: jitter = np.diag(KK).mean() * 1e-6 num_tries = 1 while num_tries <= 5 and np.isfinite(jitter): try: self.w_mean = np.linalg.solve( KK + np.eye(KK.shape[0]) * jitter, Ky) except: jitter *= 10 finally: num_tries += 1 self.wb_var = np.transpose([ scipy.optimize.nnls( np.vstack([self._SNRinv(X), np.ones(len(X))]).T, (self.mean(X) - Y)[:, a]**2)[0] for a in range(Y.shape[1]) ]) self.Gaussian_noise = Foo() self.variance, self.Gaussian_noise.variance = self.wb_var
def _cholesky(A, lower=True): ''' Computes the Cholesky decomposition of `A` using the routine `dpotrf`. Parameters ---------- A : (n, n) float array lower : bool, optional Returns ------- (n, n) float array ''' # handle rank zero matrix if A.shape == (0, 0): return np.zeros((0, 0), dtype=float) L, info = dpotrf(A, lower=lower) if info > 0: raise np.linalg.LinAlgError( 'The leading minor of order %s is not positive definite, and the ' 'factorization could not be completed. ' % info) elif info < 0: raise ValueError('The %s-th argument has an illegal value.' % -info) return L
def pd_solve(a, b): """ Fast matrix solve for positive definite matrix a""" L, info = dpotrf(a) if info == 0: return dpotrs(L, b)[0] else: return np.linalg.solve(a, b)
def inv_c(M): A = np.ascontiguousarray(M) L_M, info = lapack.dpotrf(A, lower=1) #L_M = np.linalg.cholesky(M) iM, _ = dpotri(L_M) return iM
def inv_logDet(M): A = np.ascontiguousarray(M) L_M, info = lapack.dpotrf(A, lower=1) iM, _ = dpotri(L_M) logDetM = 2 * sum(np.log(np.diag(L_M))) return iM, logDetM
def cholSafe(self, A): # if lapack cholesky decomp fails then consecutively add more jitter until we get # a working cholesky (ie ensuring we have a positive definite matrix as it can sometimes # due to floating point errors end up not positive definite). A = np.ascontiguousarray(A) L, status = dpotrf(A, lower=1) if status == 0: return L else: L = linalg.cholesky(A + np.eye(A.shape[0]) * 1e-5, lower=True) return L
def _cholesky(A, lower=True): ''' Computes the Cholesky decomposition of `A` using `dpotrf` ''' if A.shape == (0, 0): return np.zeros((0, 0), dtype=float) L, info = dpotrf(A, lower=lower) if info < 0: raise ValueError('The %s-th argument has an illegal value.' % -info) elif info > 0: raise np.linalg.LinAlgError('Matrix not positive definite') return L
def cholesky(A, maxtries=5): """ Compute the cholesky of A. If the matrix is singular make `maxtries` additional attempts to invert it by adding small amounts to the diagonal before giving up. """ L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: d = A.diagonal() if np.any(d <= 0): raise LinAlgError('Matrix has non-positive diagonal elements') j = d.mean() * 1e-6 for _ in xrange(maxtries): L, info = lapack.dpotrf(add_diagonal(A, j, True), lower=1) if info == 0: message = 'jitter of {:s} required to compute the cholesky' warnings.warn(message.format(str(j)), stacklevel=2) return L else: j *= 10 raise LinAlgError('Matrix is not positive definite, even with jitter')
def jitchol(A, maxtries=5): ''' Copyright (c) 2012, GPy authors (James Hensman, Nicolo Fusi, Ricardo Andrade, Nicolas Durrande, Alan Saul, Max Zwiessele, Neil D. Lawrence). All rights reserved Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the <organization> nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. :param A: the matrixed to be decomposited :param int maxtries: number of iterations of adding jitters ''' A = np.asfortranarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise np.linalg.LinAlgError, "kernel matrix not positive definite: non-positive diagonal elements" jitter = diagA.mean() * 1e-9 while maxtries > 0 and np.isfinite(jitter): print 'Warning: adding jitter of {:.10e} to diagnol of kernel matrix for numerical stability'.format( jitter) try: return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True) except: jitter *= 10 finally: maxtries -= 1 raise np.linalg.LinAlgError, "kernel matrix not positive definite, even with jitter."
def jit_cholesky(A, maxtries=5): """ Performs Jittered Cholesky Decomposition Performs a Jittered Cholesky decomposition, adding noise to the diagonal of the matrix as needed in order to ensure that the matrix can be inverted. Adapted from code in GPy. On occasion, the matrix that needs to be inverted in fitting a GP is nearly singular. This arises when the training samples are very close to one another, and can be averted by adding a noise term to the diagonal of the matrix. This routine performs an exact Cholesky decomposition if it can be done, and if it cannot it successively adds noise to the diagonal (starting with 1.e-6 times the mean of the diagonal and incrementing by a factor of 10 each time) until the matrix can be decomposed or the algorithm reaches ``maxtries`` attempts. The routine returns the lower triangular matrix and the amount of noise necessary to stabilize the decomposition. :param A: The matrix to be inverted as an array of shape ``(n,n)``. Must be a symmetric positive definite matrix. :type A: ndarray :param maxtries: (optional) Maximum allowable number of attempts to stabilize the Cholesky Decomposition. Must be a positive integer (default = 5) :type maxtries: int :returns: Lower-triangular factored matrix (shape ``(n,n)`` and the noise that was added to the diagonal to achieve that result. :rtype: tuple containing an ndarray and a float """ A = check_cholesky_inputs(A) assert int(maxtries) > 0, "maxtries must be a positive integer" A = np.ascontiguousarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L, 0. else: diagA = np.diag(A) jitter = diagA.mean() * 1e-6 num_tries = 1 while num_tries <= maxtries and np.isfinite(jitter): try: L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True) return L, jitter except: jitter *= 10 finally: num_tries += 1 raise linalg.LinAlgError("not positive definite, even with jitter.") return L, jitter
def jitchol(A,maxtries=5): ''' Copyright (c) 2012, GPy authors (James Hensman, Nicolo Fusi, Ricardo Andrade, Nicolas Durrande, Alan Saul, Max Zwiessele, Neil D. Lawrence). All rights reserved Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the <organization> nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. :param A: the matrixed to be decomposited :param int maxtries: number of iterations of adding jitters ''' A = np.asfortranarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise np.linalg.LinAlgError, "kernel matrix not positive definite: non-positive diagonal elements" jitter = diagA.mean() * 1e-9 while maxtries > 0 and np.isfinite(jitter): print 'Warning: adding jitter of {:.10e} to diagnol of kernel matrix for numerical stability'.format(jitter) try: return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True) except: jitter *= 10 finally: maxtries -= 1 raise np.linalg.LinAlgError, "kernel matrix not positive definite, even with jitter."
def chol_inv(x: np.array): """Calculates invserse of matrix using Cholesky decomposition. Keyword arguments: x -- A matrix. Returns: x^-1. """ c, info = lapack.dpotrf(x) if info: raise np.linalg.LinAlgError lapack.dpotri(c, overwrite_c=1) c += c.T np.fill_diagonal(c, c.diagonal() / 2) return c
def jitchol(A, maxtries=5): A = np.asfortranarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise linalg.LinAlgError, "not pd: non-positive diagonal elements" jitter = diagA.mean() * 1e-6 while maxtries > 0 and np.isfinite(jitter): print 'Warning: adding jitter of {:.10e}'.format(jitter) try: return linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True) except: jitter *= 10 finally: maxtries -= 1 raise linalg.LinAlgError, "not positive definite, even with jitter."
def fast_pd_inverse(m: 'numpy array') -> 'numpy array': ''' This method calculates the inverse of a A real symmetric positive definite (n × n)-matrix It is much faster than Numpy's "np.linalg.inv" method for example. ''' try: cholesky, info = lapack.dpotrf(m) if info != 0: raise ValueError('dpotrf failed on input {}'.format(m)) #print("cas 1") inv, info = lapack.dpotri(cholesky) if info != 0: raise ValueError('dpotri failed on input {}'.format(cholesky)) #print("cas 2") except: inv = np.linalg.inv(m) uppertriangular_2_symmetric(inv) return inv
def mvn_loglike(y, cov): """ Evaluate the multivariate-normal log-likelihood for difference vector `y` and covariance matrix `cov`: log_p = -1/2*[(y^T).(C^-1).y + log(det(C))] + const. The likelihood is NOT NORMALIZED, since this does not affect MCMC. The normalization const = -n/2*log(2*pi), where n is the dimensionality. Arguments `y` and `cov` MUST be np.arrays with dtype == float64 and shapes (n) and (n, n), respectively. These requirements are NOT CHECKED. The calculation follows algorithm 2.1 in Rasmussen and Williams (Gaussian Processes for Machine Learning). """ # Compute the Cholesky decomposition of the covariance. # Use bare LAPACK function to avoid scipy.linalg wrapper overhead. L, info = lapack.dpotrf(cov, clean=False) if info < 0: raise ValueError( 'lapack dpotrf error: ' 'the {}-th argument had an illegal value'.format(-info) ) elif info < 0: raise np.linalg.LinAlgError( 'lapack dpotrf error: ' 'the leading minor of order {} is not positive definite' .format(info) ) # Solve for alpha = cov^-1.y using the Cholesky decomp. alpha, info = lapack.dpotrs(L, y) if info != 0: raise ValueError( 'lapack dpotrs error: ' 'the {}-th argument had an illegal value'.format(-info) ) return -.5*np.dot(y, alpha) - np.log(L.diagonal()).sum()
def train(self, X, Y): """Trains the machine on the supplied set of inputs and outputs.""" assert X.shape[0] == Y.shape[0] assert len(X.shape) == len(Y.shape) == 2 assert X.shape[1] == self.n assert Y.shape[1] == self.p self.X = X self.Y = Y self.Phi = self.mapping.evaluate(self.X) m, n = self.Phi.shape sigma_n2 = self.sigma_n * self.sigma_n self.L = numpy.dot(self.Phi.T, self.Phi) idx = numpy.diag_indices(self.Phi.shape[1]) self.L[idx] += sigma_n2 self.up = 0 self.L, info = dpotrf(self.L, lower=self.up, clean=1, overwrite_a=1) self.B = numpy.dot(self.Phi.T, self.Y) self.LPhiY, info = dgetrs(self.L, numpy.arange(self.L.shape[0]), self.B, trans=1, overwrite_b=1) assert info == 0, 'scipy.linalg.lapack.dgetrs(L, B)' self.W, info = dgetrs(self.L, numpy.arange(self.L.shape[0]), self.LPhiY, trans=0, overwrite_b=0) assert info == 0, 'scipy.linalg.lapack.dgetrs(L, LPhiY)' self.lml = -0.5 * ((1. / (sigma_n2)) * (inner1d(self.Y.T, self.Y.T) - inner1d(self.LPhiY.T, self.LPhiY.T)) + (m - n) * numpy.log(sigma_n2) + m * numpy.log(2. * numpy.pi)) - \ numpy.log(self.L.diagonal()).sum()
def __init__(self, X, Y, theta): theta = theta ** [2,1,2] [sf2, l2, sn2] = theta # evaluate RBF kernel for our given X r = dist.pdist(X) / l2 K = dist.squareform(sf2 * np.exp(-0.5 * r**2)) np.fill_diagonal(K, sf2) # add in Gaussian noise (+ a bit for numerical stability) Ky = K.copy() np.fill_diagonal(Ky, sf2 + sn2 + 1e-8) # compute the Cholesky factorization of our covariance matrix LW, info = lapack.dpotrf(Ky, lower=True) assert info == 0 # calculate lower half of inverse of K (assumes real symmetric positive definite) Wi, info = lapack.dpotri(LW, lower=True) assert info == 0 # make symmetric by filling in the upper half Wi += np.tril(Wi,-1).T # and solve alpha, info = lapack.dpotrs(LW, Y, lower=True) assert info == 0 # save these for later self.X = X self.Y = Y self.theta = theta self.r = r self.K = K self.Ky = Ky self.LW = LW self.Wi = Wi self.alpha = alpha
def _loss_and_grad(self, params): V = self.V X = self.X X.fill(0) #X = np.zeros((self.n,self.n)) X[self._mask] = params X += X.T np.fill_diagonal(X, 1) zz, info0 = dpotrf(X, False, False) iX, info1 = dpotri(zz) iX = np.triu(iX) + np.triu(iX, k=1).T if info0 != 0 or info1 != 0: #print('checkpt') return self._loss * 100, np.zeros_like(params) loss = np.sum(iX * V) G = -iX @ V @ iX g = G[self._mask] + G.T[self._mask] self._loss = loss #print(np.sqrt(loss / self.W.shape[0])) return loss, g #G.flatten()
def cholesky(A, lower=True): ''' Computes the Cholesky decomposition of *A* using the routine *dpotrf*. Parameters ---------- A : (N,N) float array lower : bool, optional ''' if A.shape == (0, 0): return np.zeros((0, 0), dtype=float) L, info = dpotrf(A, lower=lower) if info > 0: raise np.linalg.LinAlgError( 'The leading minor of order %s is not positive definite, and ' 'the factorization could not be completed. ' % info) elif info < 0: raise ValueError('The %s-th argument has an illegal value.' % (-info)) return L
def mvn_loglike(y, cov): """ Calculate multi-varaite-normal log-likelihood log_p = -1/2 * [(y^T).(C^-1).y + log(det(C))] + const To normalize the likelihood, const = -n/2*log(2*pi), which is omitted here Args: y -- (n) cov -- shape (n, n) Returns: log_p """ L, info = lapack.dpotrf(cov, clean=False) if info != 0: raise ValueError('lapack dpotrf error: illegal value for info!') alpha, info = lapack.dpotrs(L, y) if info != 0: raise ValueError( 'lapack dpotrf error: illegal value for info! {}'.format(info)) return -.5 * np.dot(y, alpha) - np.log(L.diagonal()).sum()
def __init__(self, X, Y, inducing_inputs, lengthscale): self.X = X self.Y = Y self.inducing_inputs = inducing_inputs self.lengthscale = lengthscale self.rectify = rectify self.kern = GPy.kern.RBF(input_dim=X.shape[1], variance=1., lengthscale=lengthscale, ARD=True) K_uf = self.kern.K(inducing_inputs, X) KK = K_uf.dot(K_uf.T) Ky = K_uf.dot(Y) self.w_mean = dpotrs(dpotrf(KK)[0], Ky)[0] # faster than np.linalg.solve(KK, Ky) if np.any(np.isnan(self.w_mean)): try: self.w_mean = np.linalg.solve(KK, Ky) except: jitter = np.diag(KK).mean() * 1e-6 num_tries = 1 while num_tries <= 5 and np.isfinite(jitter): try: self.w_mean = np.linalg.solve( KK + np.eye(KK.shape[0]) * jitter, Ky) except: jitter *= 10 finally: num_tries += 1 self.wb_var = np.transpose([ scipy.optimize.nnls( np.vstack([self._SNRinv(X), np.ones(len(X))]).T, (self.mean(X) - Y)[:, a]**2)[0] for a in range(Y.shape[1]) ]) self.Gaussian_noise = Foo() self.variance, self.Gaussian_noise.variance = self.wb_var
def potrf(A, MKLProc): from scipy.linalg.lapack import dpotrf import os os.environ['MKL_NUM_THREADS'] = str(MKLProc) A = dpotrf(A, lower=True)[0] return A
def lnLL(dy, cov): L, info = lapack.dpotrf(cov, clean=False) alpha, info = lapack.dpotrs(L, dy) return -.5 * np.dot(dy, alpha) - np.log(L.diagonal()).sum()
def kalman_filter(model, return_loglike=False): # Parameters dtype = model.dtype # Kalman filter properties filter_method = model.filter_method inversion_method = model.inversion_method stability_method = model.stability_method conserve_memory = model.conserve_memory tolerance = model.tolerance loglikelihood_burn = model.loglikelihood_burn # Check for acceptable values if not filter_method == FILTER_CONVENTIONAL: warn('The pure Python version of the kalman filter only supports the' ' conventional Kalman filter') implemented_inv_methods = INVERT_NUMPY | INVERT_UNIVARIATE | SOLVE_CHOLESKY if not inversion_method & implemented_inv_methods: warn('The pure Python version of the kalman filter only performs' ' inversion using `numpy.linalg.inv`.') if not tolerance == 0: warn('The pure Python version of the kalman filter does not check' ' for convergence.') # Convergence (this implementation does not consider convergence) converged = False period_converged = 0 # Dimensions nobs = model.nobs k_endog = model.k_endog k_states = model.k_states k_posdef = model.k_posdef # Allocate memory for variables filtered_state = np.zeros((k_states, nobs), dtype=dtype) filtered_state_cov = np.zeros((k_states, k_states, nobs), dtype=dtype) predicted_state = np.zeros((k_states, nobs+1), dtype=dtype) predicted_state_cov = np.zeros((k_states, k_states, nobs+1), dtype=dtype) forecast = np.zeros((k_endog, nobs), dtype=dtype) forecast_error = np.zeros((k_endog, nobs), dtype=dtype) forecast_error_cov = np.zeros((k_endog, k_endog, nobs), dtype=dtype) loglikelihood = np.zeros((nobs+1,), dtype=dtype) # Selected state covariance matrix selected_state_cov = ( np.dot( np.dot(model.selection[:, :, 0], model.state_cov[:, :, 0]), model.selection[:, :, 0].T ) ) # Initial values if model.initialization == 'known': initial_state = model._initial_state.astype(dtype) initial_state_cov = model._initial_state_cov.astype(dtype) elif model.initialization == 'approximate_diffuse': initial_state = np.zeros((k_states,), dtype=dtype) initial_state_cov = ( np.eye(k_states).astype(dtype) * model._initial_variance ) elif model.initialization == 'stationary': initial_state = np.zeros((k_states,), dtype=dtype) initial_state_cov = solve_discrete_lyapunov( np.array(model.transition[:, :, 0], dtype=dtype), np.array(selected_state_cov[:, :], dtype=dtype), ) else: raise RuntimeError('Statespace model not initialized.') # Copy initial values to predicted predicted_state[:, 0] = initial_state predicted_state_cov[:, :, 0] = initial_state_cov # print(predicted_state_cov[:, :, 0]) # Setup indices for possibly time-varying matrices design_t = 0 obs_intercept_t = 0 obs_cov_t = 0 transition_t = 0 state_intercept_t = 0 selection_t = 0 state_cov_t = 0 # Iterate forwards time_invariant = model.time_invariant for t in range(nobs): # Get indices for possibly time-varying arrays if not time_invariant: if model.design.shape[2] > 1: design_t = t if model.obs_intercept.shape[1] > 1: obs_intercept_t = t if model.obs_cov.shape[2] > 1: obs_cov_t = t if model.transition.shape[2] > 1: transition_t = t if model.state_intercept.shape[1] > 1: state_intercept_t = t if model.selection.shape[2] > 1: selection_t = t if model.state_cov.shape[2] > 1: state_cov_t = t # Selected state covariance matrix if model.selection.shape[2] > 1 or model.state_cov.shape[2] > 1: selected_state_cov = ( np.dot( np.dot(model.selection[:, :, selection_t], model.state_cov[:, :, state_cov_t]), model.selection[:, :, selection_t].T ) ) # #### Forecast for time t # `forecast` $= Z_t a_t + d_t$ # # *Note*: $a_t$ is given from the initialization (for $t = 0$) or # from the previous iteration of the filter (for $t > 0$). forecast[:, t] = ( np.dot(model.design[:, :, design_t], predicted_state[:, t]) + model.obs_intercept[:, obs_intercept_t] ) # *Intermediate calculation* (used just below and then once more) # `tmp1` array used here, dimension $(m \times p)$ # $\\#_1 = P_t Z_t'$ # $(m \times p) = (m \times m) (p \times m)'$ tmp1 = np.dot(predicted_state_cov[:, :, t], model.design[:, :, design_t].T) # #### Forecast error for time t # `forecast_error` $\equiv v_t = y_t -$ `forecast` forecast_error[:, t] = model.obs[:, t] - forecast[:, t] # #### Forecast error covariance matrix for time t # $F_t \equiv Z_t P_t Z_t' + H_t$ forecast_error_cov[:, :, t] = ( np.dot(model.design[:, :, design_t], tmp1) + model.obs_cov[:, :, obs_cov_t] ) # Store the inverse if k_endog == 1 and inversion_method & INVERT_UNIVARIATE: forecast_error_cov_inv = 1.0 / forecast_error_cov[0, 0, t] determinant = forecast_error_cov[0, 0, t] tmp2 = forecast_error_cov_inv * forecast_error[:, t] tmp3 = forecast_error_cov_inv * model.design[:, :, design_t] elif inversion_method & SOLVE_CHOLESKY: U, info = lapack.dpotrf(forecast_error_cov[:, :, t]) determinant = np.product(U.diagonal())**2 tmp2, info = lapack.dpotrs(U, forecast_error[:, t]) tmp3, info = lapack.dpotrs(U, model.design[:, :, design_t]) else: forecast_error_cov_inv = np.linalg.inv(forecast_error_cov[:, :, t]) determinant = np.linalg.det(forecast_error_cov[:, :, t]) tmp2 = np.dot(forecast_error_cov_inv, forecast_error[:, t]) tmp3 = np.dot(forecast_error_cov_inv, model.design[:, :, design_t]) # #### Filtered state for time t # $a_{t|t} = a_t + P_t Z_t' F_t^{-1} v_t$ # $a_{t|t} = 1.0 * \\#_1 \\#_2 + 1.0 a_t$ filtered_state[:, t] = ( predicted_state[:, t] + np.dot(tmp1, tmp2) ) # #### Filtered state covariance for time t # $P_{t|t} = P_t - P_t Z_t' F_t^{-1} Z_t P_t$ # $P_{t|t} = P_t - \\#_1 \\#_3 P_t$ filtered_state_cov[:, :, t] = ( predicted_state_cov[:, :, t] - np.dot( np.dot(tmp1, tmp3), predicted_state_cov[:, :, t] ) ) # #### Loglikelihood loglikelihood[t] = -0.5 * ( np.log((2*np.pi)**k_endog * determinant) + np.dot(forecast_error[:, t], tmp2) ) # #### Predicted state for time t+1 # $a_{t+1} = T_t a_{t|t} + c_t$ predicted_state[:, t+1] = ( np.dot(model.transition[:, :, transition_t], filtered_state[:, t]) + model.state_intercept[:, state_intercept_t] ) # #### Predicted state covariance matrix for time t+1 # $P_{t+1} = T_t P_{t|t} T_t' + Q_t^*$ predicted_state_cov[:, :, t+1] = ( np.dot( np.dot(model.transition[:, :, transition_t], filtered_state_cov[:, :, t]), model.transition[:, :, transition_t].T ) + selected_state_cov ) # Enforce symmetry of predicted covariance matrix predicted_state_cov[:, :, t+1] = ( predicted_state_cov[:, :, t+1] + predicted_state_cov[:, :, t+1].T ) / 2 if return_loglike: return np.array(loglikelihood) else: kwargs = dict( (k, v) for k, v in locals().items() if k in _kalman_filter._fields ) kwargs['model'] = _statespace( initial_state=initial_state, initial_state_cov=initial_state_cov ) kfilter = _kalman_filter(**kwargs) return FilterResults(model, kfilter)
def jitchol(A, attempts=5): ''' JITCHOL Do a Cholesky decomposition with jitter. Description: U = JITCHOL(A, MAXTRIES) attempts a Cholesky decomposition on the given matrix, if matrix isn't positive definite the function gives a warning, adds 'jitter' and tries again. At the first attempt the amount of jitter added is 1e-6 times the mean of the diagonal. Thereafter the amount of jitter is multiplied by 10 each time it is added again. This is continued for a maximum of 10 times. Returns: U - the Cholesky decomposition for the matrix. Arguments: A - the matrix for which the Cholesky decomposition is required. MAXTRIES - the maximum number of times that jitter is added before giving up (default 10). [U, JITTER] = JITCHOL(A, MAXTRIES) attempts a Cholesky decomposition on the given matrix, if matrix isn't positive definite the function adds 'jitter' and tries again. Thereafter the amount of jitter is multiplied by 10 each time it is added again. This is continued for a maximum of 10 times. The amount of jitter added is returned. Returns: U - the Cholesky decomposition for the matrix. JITTER - the amount of jitter that was added to the matrix. Arguments: A - the matrix for which the Cholesky decomposition is required. MAXTRIES - the maximum number of times that jitter is added before giving up (default 10) :param A: the matrixed to be decomposited :param int maxtries: number of iterations of adding jitters ''' A = np.asfortranarray(A) L, info = lapack.dpotrf(A, lower=1) if info == 0: return L else: diagA = np.diag(A) if np.any(diagA <= 0.): raise np.linalg.LinAlgError("kernel matrix not positive definite: " "non-positive diagonal elements") jitter = diagA.mean() * np.finfo(float).tiny #1e-9 while attempts > 0 and np.isfinite(jitter): #logging.getLogger(__name__).warning('adding jitter of {:.10e} to ' #'diagnol of kernel matrix for ' #'numerical stability'.format(jitter)) try: return np.linalg.cholesky(A + np.eye(A.shape[0]).T * jitter, lower=True) except: jitter *= 10 finally: attempts -= 1 raise np.linalg.LinAlgError( "kernel matrix not positive definite, even with jitter.")
def kalman_filter(model, return_loglike=False): # Parameters dtype = model.dtype # Kalman filter properties filter_method = model.filter_method inversion_method = model.inversion_method stability_method = model.stability_method conserve_memory = model.conserve_memory tolerance = model.tolerance loglikelihood_burn = model.loglikelihood_burn # Check for acceptable values if not filter_method == FILTER_CONVENTIONAL: warn('The pure Python version of the kalman filter only supports the' ' conventional Kalman filter') implemented_inv_methods = INVERT_NUMPY | INVERT_UNIVARIATE | SOLVE_CHOLESKY if not inversion_method & implemented_inv_methods: warn('The pure Python version of the kalman filter only performs' ' inversion using `numpy.linalg.inv`.') if not tolerance == 0: warn('The pure Python version of the kalman filter does not check' ' for convergence.') # Convergence (this implementation does not consider convergence) converged = False period_converged = 0 # Dimensions nobs = model.nobs k_endog = model.k_endog k_states = model.k_states k_posdef = model.k_posdef # Allocate memory for variables filtered_state = np.zeros((k_states, nobs), dtype=dtype) filtered_state_cov = np.zeros((k_states, k_states, nobs), dtype=dtype) predicted_state = np.zeros((k_states, nobs + 1), dtype=dtype) predicted_state_cov = np.zeros((k_states, k_states, nobs + 1), dtype=dtype) forecast = np.zeros((k_endog, nobs), dtype=dtype) forecast_error = np.zeros((k_endog, nobs), dtype=dtype) forecast_error_cov = np.zeros((k_endog, k_endog, nobs), dtype=dtype) loglikelihood = np.zeros((nobs + 1, ), dtype=dtype) # Selected state covariance matrix selected_state_cov = (np.dot( np.dot(model.selection[:, :, 0], model.state_cov[:, :, 0]), model.selection[:, :, 0].T)) # Initial values if model.initialization == 'known': initial_state = model._initial_state.astype(dtype) initial_state_cov = model._initial_state_cov.astype(dtype) elif model.initialization == 'approximate_diffuse': initial_state = np.zeros((k_states, ), dtype=dtype) initial_state_cov = (np.eye(k_states).astype(dtype) * model._initial_variance) elif model.initialization == 'stationary': initial_state = np.zeros((k_states, ), dtype=dtype) initial_state_cov = solve_discrete_lyapunov( np.array(model.transition[:, :, 0], dtype=dtype), np.array(selected_state_cov[:, :], dtype=dtype), ) else: raise RuntimeError('Statespace model not initialized.') # Copy initial values to predicted predicted_state[:, 0] = initial_state predicted_state_cov[:, :, 0] = initial_state_cov # print(predicted_state_cov[:, :, 0]) # Setup indices for possibly time-varying matrices design_t = 0 obs_intercept_t = 0 obs_cov_t = 0 transition_t = 0 state_intercept_t = 0 selection_t = 0 state_cov_t = 0 # Iterate forwards time_invariant = model.time_invariant for t in range(nobs): # Get indices for possibly time-varying arrays if not time_invariant: if model.design.shape[2] > 1: design_t = t if model.obs_intercept.shape[1] > 1: obs_intercept_t = t if model.obs_cov.shape[2] > 1: obs_cov_t = t if model.transition.shape[2] > 1: transition_t = t if model.state_intercept.shape[1] > 1: state_intercept_t = t if model.selection.shape[2] > 1: selection_t = t if model.state_cov.shape[2] > 1: state_cov_t = t # Selected state covariance matrix if model.selection.shape[2] > 1 or model.state_cov.shape[2] > 1: selected_state_cov = (np.dot( np.dot(model.selection[:, :, selection_t], model.state_cov[:, :, state_cov_t]), model.selection[:, :, selection_t].T)) # #### Forecast for time t # `forecast` $= Z_t a_t + d_t$ # # *Note*: $a_t$ is given from the initialization (for $t = 0$) or # from the previous iteration of the filter (for $t > 0$). forecast[:, t] = ( np.dot(model.design[:, :, design_t], predicted_state[:, t]) + model.obs_intercept[:, obs_intercept_t]) # *Intermediate calculation* (used just below and then once more) # `tmp1` array used here, dimension $(m \times p)$ # $\\#_1 = P_t Z_t'$ # $(m \times p) = (m \times m) (p \times m)'$ tmp1 = np.dot(predicted_state_cov[:, :, t], model.design[:, :, design_t].T) # #### Forecast error for time t # `forecast_error` $\equiv v_t = y_t -$ `forecast` forecast_error[:, t] = model.obs[:, t] - forecast[:, t] # #### Forecast error covariance matrix for time t # $F_t \equiv Z_t P_t Z_t' + H_t$ forecast_error_cov[:, :, t] = (np.dot(model.design[:, :, design_t], tmp1) + model.obs_cov[:, :, obs_cov_t]) # Store the inverse if k_endog == 1 and inversion_method & INVERT_UNIVARIATE: forecast_error_cov_inv = 1.0 / forecast_error_cov[0, 0, t] determinant = forecast_error_cov[0, 0, t] tmp2 = forecast_error_cov_inv * forecast_error[:, t] tmp3 = forecast_error_cov_inv * model.design[:, :, design_t] elif inversion_method & SOLVE_CHOLESKY: U, info = lapack.dpotrf(forecast_error_cov[:, :, t]) determinant = np.product(U.diagonal())**2 tmp2, info = lapack.dpotrs(U, forecast_error[:, t]) tmp3, info = lapack.dpotrs(U, model.design[:, :, design_t]) else: forecast_error_cov_inv = np.linalg.inv(forecast_error_cov[:, :, t]) determinant = np.linalg.det(forecast_error_cov[:, :, t]) tmp2 = np.dot(forecast_error_cov_inv, forecast_error[:, t]) tmp3 = np.dot(forecast_error_cov_inv, model.design[:, :, design_t]) # #### Filtered state for time t # $a_{t|t} = a_t + P_t Z_t' F_t^{-1} v_t$ # $a_{t|t} = 1.0 * \\#_1 \\#_2 + 1.0 a_t$ filtered_state[:, t] = (predicted_state[:, t] + np.dot(tmp1, tmp2)) # #### Filtered state covariance for time t # $P_{t|t} = P_t - P_t Z_t' F_t^{-1} Z_t P_t$ # $P_{t|t} = P_t - \\#_1 \\#_3 P_t$ filtered_state_cov[:, :, t] = ( predicted_state_cov[:, :, t] - np.dot(np.dot(tmp1, tmp3), predicted_state_cov[:, :, t])) # #### Loglikelihood loglikelihood[t] = -0.5 * (np.log((2 * np.pi)**k_endog * determinant) + np.dot(forecast_error[:, t], tmp2)) # #### Predicted state for time t+1 # $a_{t+1} = T_t a_{t|t} + c_t$ predicted_state[:, t + 1] = (np.dot(model.transition[:, :, transition_t], filtered_state[:, t]) + model.state_intercept[:, state_intercept_t]) # #### Predicted state covariance matrix for time t+1 # $P_{t+1} = T_t P_{t|t} T_t' + Q_t^*$ predicted_state_cov[:, :, t + 1] = (np.dot( np.dot(model.transition[:, :, transition_t], filtered_state_cov[:, :, t]), model.transition[:, :, transition_t].T) + selected_state_cov) # Enforce symmetry of predicted covariance matrix predicted_state_cov[:, :, t + 1] = (predicted_state_cov[:, :, t + 1] + predicted_state_cov[:, :, t + 1].T) / 2 if return_loglike: return np.array(loglikelihood) else: kwargs = dict( (k, v) for k, v in locals().items() if k in _kalman_filter._fields) kwargs['model'] = _statespace(initial_state=initial_state, initial_state_cov=initial_state_cov) kfilter = _kalman_filter(**kwargs) return FilterResults(model, kfilter)