def solve(X,y,tolerance=1e-6) : """ Solve the regression problem ||Xw - y||^2 using the QR factorization of X. Returns: beta, variance estimates for parameters, SSE TODO: use an implementation that uses ||Q'Xw - Q'y||^2 to avoid returning Q when it is not needed. Somwhere in Lapack there is probably something we could use. """ n = X.shape[0] p = X.shape[1] Q,R,permutation = linalg.qr(X,pivoting=True,mode='economic') R_diag = np.diag(R) rank = np.where(np.abs(R_diag) < tolerance)[0] rank = R_diag.shape[0] if rank.shape[0] == 0 else rank[0] b = np.dot(y,Q)[:rank] beta = linalg.solve_triangular(R[:rank,:rank],b,lower=False) if rank < R.shape[0] : beta = np.append(beta,np.zeros(R_diag.shape[0] - rank)) beta[permutation] = beta.copy() #non-atomic operation requires copy R_small = R[:rank,:rank] #Computing the inverse from R in a numericaly stable way requires a Cholesky #based approach. I discovered this while testing against statsmodel's #Langley data set which has very high condition number XXn1,_ = dpotri(R_small) XXn1_diag = np.append(np.diag(XXn1),np.zeros(R.shape[1] - R_small.shape[1])) XXn1_diag[permutation] = XXn1_diag.copy() betaSigmaSq = XXn1_diag resid = y - np.dot(X,beta) SSE = np.dot(resid,resid) return beta, betaSigmaSq, SSE, rank,Q
def dpotri(A, lower=1): A = force_F_ordered(A) R, info = lapack.dpotri( A, lower=lower) #needs to be zero here, seems to be a scipy bug symmetrify(R) return R, info
def pdinv(A, *args): """ :param A: A DxD pd numpy array :rval Ai: the inverse of A :rtype Ai: np.ndarray :rval L: the Cholesky decomposition of A :rtype L: np.ndarray :rval Li: the Cholesky decomposition of Ai :rtype Li: np.ndarray :rval logdet: the log of the determinant of A :rtype logdet: float64 """ try: Ai = np.linalg.inv(A) logdet = np.linalg.slogdet(A)[1] L = np.linalg.cholesky(A) Li = chol_inv(L) except linalg.LinAlgError: L = jitchol(A, *args) logdet = 2. * np.sum(np.log(np.diag(L))) Li = chol_inv(L) Ai, _ = lapack.dpotri(L) # Ai = np.tril(Ai) + np.tril(Ai,-1).T symmetrify(Ai) return Ai, L, Li, logdet
def LML_se(self,theta,returnGradients=False): self.setTheta(theta) K,r = self.cov(self.X,retr=True) Ky = K.copy() Ky += np.eye(self.X.shape[0])*self.var_n + np.eye(self.X.shape[0])*1e-8 L = self.cholSafe(Ky) WlogDet = 2.*np.sum(np.log(np.diag(L))) alpha, status = dpotrs(L, self.Y, lower=1) dataFit = - np.sum(alpha * self.Y) modelComplexity = -self.Y.shape[1] * WlogDet normalizer = -self.Y.size * log2pi logMarginalLikelihood = 0.5*(dataFit + modelComplexity + normalizer) if returnGradients == False: return logMarginalLikelihood else: Wi, status = dpotri(-L, lower=1) Wi = np.asarray(Wi) # copy bottom triangle to top triangle triu = np.triu_indices_from(Wi,k=1) Wi[triu] = Wi.T[triu] # dL = change in LML, dK is change in Kernel(K) dL_dK = 0.5 * (np.dot(alpha,alpha.T) - self.Y.shape[1] * Wi) dL_dVarn = np.diag(dL_dK).sum() varfGradient = np.sum(K* dL_dK)/self.var_f dK_dr = -r*K dL_dr = dK_dr * dL_dK lengthscaleGradient = -np.sum(dL_dr*r)/self.charLen grads = np.array([varfGradient, lengthscaleGradient, dL_dVarn]) return logMarginalLikelihood, grads
def pdinv(A, *args): """ :param A: A DxD pd numpy array :rval Ai: the inverse of A :rtype Ai: np.ndarray :rval L: the Cholesky decomposition of A :rtype L: np.ndarray :rval Li: the Cholesky decomposition of Ai :rtype Li: np.ndarray :rval logdet: the log of the determinant of A :rtype logdet: float64 """ try: Ai = np.linalg.inv(A) logdet = np.linalg.slogdet(A)[1] L = np.linalg.cholesky(A) Li = chol_inv(L) except linalg.LinAlgError: L = jitchol(A, *args) logdet = 2.*np.sum(np.log(np.diag(L))) Li = chol_inv(L) Ai, _ = lapack.dpotri(L) # Ai = np.tril(Ai) + np.tril(Ai,-1).T symmetrify(Ai) return Ai, L, Li, logdet
def invpd(A, return_chol=False): L = np.linalg.cholesky(A) Ainv = lapack.dpotri(L, lower=True)[0] copy_lower_to_upper(Ainv) if return_chol: return Ainv, L else: return Ainv
def dpotri(A, lower=0): """Wrapper for lapack dpotri function :param A: Matrix A :param lower: is matrix lower (true) or upper (false) :returns: """ return lapack.dpotri(A, lower=lower)
def dpotri(A, lower=0): """ Wrapper for lapack dpotri function :param A: Matrix A :param lower: is matrix lower (true) or upper (false) :returns: A inverse """ return lapack.dpotri(A, lower=lower)
def invPsd(A, AChol=None, returnChol=False): # https://github.com/mattjj/pybasicbayes/blob/9c00244b2d6fd767549de6ab5d0436cec4e06818/pybasicbayes/util/general.py L = np.linalg.cholesky(A) if AChol is None else AChol Ainv = lapack.dpotri(L, lower=True)[0] Ainv += np.tril(Ainv, k=-1).T if returnChol: return Ainv, L return Ainv
def chol_inv(x: np.array): """Calculates invserse of matrix using Cholesky decomposition. Keyword arguments: x -- A matrix. Returns: x^-1. """ c, info = lapack.dpotrf(x) if info: raise np.linalg.LinAlgError lapack.dpotri(c, overwrite_c=1) c += c.T np.fill_diagonal(c, c.diagonal() / 2) return c
def cholesky_inverse(A): """ Compute the inverse of the matrix AA' given its cholesky decomposition A. """ X, info = lapack.dpotri(A, lower=1) if info != 0: raise LinAlgError('Matrix is not invertible') triu = np.triu_indices_from(X, k=1) X[triu] = X.T[triu] return X
def inv_psd(A, return_chol=False): L = np.linalg.cholesky(A) Ainv = lapack.dpotri(L, lower=True)[0] copy_lower_to_upper(Ainv) # if not np.allclose(Ainv, np.linalg.inv(A), rtol=1e-5, atol=1e-5): # import ipdb; ipdb.set_trace() if return_chol: return Ainv, L else: return Ainv
def cholesky_inv(M): # Lower-triangular Cholesky factorization of M #L = np.linalg.cholesky(M) # Call LAPACK dpotri to get inverse (only lower triangle populated) Minv0 = lapack.dpotri(M, lower=True)[0] # Copy lower triangle to upper triangle Minv = Minv0 + Minv0.T - np.diag(Minv0.diagonal()) return Minv
def multiple_pdinv(A): """ :param A: A DxDxN numpy array (each A[:,:,i] is pd) :rval invs: the inverses of A :rtype invs: np.ndarray :rval hld: 0.5* the log of the determinants of A :rtype hld: np.array """ N = A.shape[-1] chols = [jitchol(A[:, :, i]) for i in range(N)] halflogdets = [np.sum(np.log(np.diag(L[0]))) for L in chols] invs = [lapack.dpotri(L[0], True)[0] for L in chols] invs = [np.triu(I) + np.triu(I, 1).T for I in invs] return np.dstack(invs), np.array(halflogdets)
def cholesky_inv(M): """ Returns inverse of positive-definite matrix M using Cholesky decomposition """ # Lower-triangular Cholesky factorization of M L = linalg.cholesky(M, lower=True) # Call LAPACK dpotri to get inverse (only lower triangle populated) Minv0 = lapack.dpotri(L, lower=True)[0] # Copy lower triangle to upper triangle Minv = Minv0 + Minv0.T - np.diag(Minv0.diagonal()) return Minv
def grammy_uncertainty(class_prob, prob_est, classifications): """ Parameters ---------- class_prob: np.ndarry probabilities of classification as each class for each species prob_est: np.ndarray estimated fragment proportions - output of grammy fitting procedure classifications: np.ndarray is the read data (so a vector with the classification of each read) Returns ------- tuple output is a tuple first element is 95% lower bound, second element 95% upper bound """ # todo what do all these represent?! lp = prob_est.size lphat = prob_est.size - 1 phat = prob_est[:-1] K = len(classifications) IO = np.zeros((lphat, lphat)) for i in range(lphat): for j in range(lphat): IOij = [0] * K for k in range(K): IOij1 = ( class_prob[i, classifications[k]] - class_prob[lphat, classifications[k]] ) IOij2 = ( class_prob[j, classifications[k]] - class_prob[lphat, classifications[k]] ) IOij3 = IOij1 * IOij2 IOij4 = np.sum(phat * class_prob[:lphat, classifications[k]]) IOij5 = (1 - np.sum(phat)) * class_prob[lphat, classifications[k]] IOij6 = (IOij4 + IOij5) ** 2 IOij7 = IOij3 / IOij6 IOij[k] = IOij7 IO[i, j] = sum(IOij) IO = np.asfortranarray(IO) IOinv = lapack.dpotri(np.linalg.cholesky(IO).T)[0] est_se = np.sqrt(np.diag(IOinv)) up_CI = phat + 1.96 * est_se low_CI = phat - 1.96 * est_se return (low_CI, up_CI)
def dpotri(A, lower=1): """ Wrapper for lapack dpotri function DPOTRI - compute the inverse of a real symmetric positive definite matrix A using the Cholesky factorization A = U**T*U or A = L*L**T computed by DPOTRF :param A: Matrix A :param lower: is matrix lower (true) or upper (false) :returns: A inverse """ A = force_F_ordered(A) R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug symmetrify(R) return R, info
def fast_pd_inverse(m: 'numpy array') -> 'numpy array': ''' This method calculates the inverse of a A real symmetric positive definite (n × n)-matrix It is much faster than Numpy's "np.linalg.inv" method for example. ''' try: cholesky, info = lapack.dpotrf(m) if info != 0: raise ValueError('dpotrf failed on input {}'.format(m)) #print("cas 1") inv, info = lapack.dpotri(cholesky) if info != 0: raise ValueError('dpotri failed on input {}'.format(cholesky)) #print("cas 2") except: inv = np.linalg.inv(m) uppertriangular_2_symmetric(inv) return inv
def partial_correlation(x, return_pvalue=False): """ x: samples X features """ n, gp = x.shape df = np.max(n - np.linalg.matrix_rank(x), 0) cvx = np.cov(x.T) r = la.cholesky(cvx, overwrite_a=True) icvx, info = lapack.dpotri(r, overwrite_c=True) pcor = -_cov2cor(icvx) pcor += pcor.T np.fill_diagonal(pcor, 1.0) if return_pvalue: statistic = pcor*np.sqrt(df/(1-pcor**2)) pvalue = 2*stats.t.cdf(-abs(statistic), df) np.fill_diagonal(pvalue, 0.0) return pcor, pvalue return pcor
def partial_correlation(x, return_pvalue=False): """ x: samples X features """ n, gp = x.shape df = np.max(n - np.linalg.matrix_rank(x), 0) cvx = np.cov(x.T) r = la.cholesky(cvx, overwrite_a=True) icvx, info = lapack.dpotri(r, overwrite_c=True) pcor = -_cov2cor(icvx) pcor += pcor.T np.fill_diagonal(pcor, 1.0) if return_pvalue: statistic = pcor * np.sqrt(df / (1 - pcor**2)) pvalue = 2 * stats.t.cdf(-abs(statistic), df) np.fill_diagonal(pvalue, 0.0) return pcor, pvalue return pcor
def multiple_dpotri_old(Ls): M, _, D = Ls.shape Kis = np.rollaxis(Ls, -1).copy() [dpotri(Kis[i,:,:], overwrite_c=1, lower=1) for i in range(D)] code = """ for(int d=0; d<D; d++) { for(int m=0; m<M; m++) { for(int mm=0; mm<m; mm++) { Kis[d*M*M + mm*M + m ] = Kis[d*M*M + m*M + mm]; } } } """ weave.inline(code, ['Kis', 'D', 'M']) Kis = np.rollaxis(Kis, 0, 3) #wtf rollaxis? return Kis
def dpotri(A, lower=1): """ Wrapper for lapack dpotri function DPOTRI - compute the inverse of a real symmetric positive definite matrix A using the Cholesky factorization A = U**T*U or A = L*L**T computed by DPOTRF :param A: Matrix A :param lower: is matrix lower (true) or upper (false) :returns: A inverse """ if _fix_dpotri_scipy_bug: assert lower==1, "scipy linalg behaviour is very weird. please use lower, fortran ordered arrays" lower = 0 A = force_F_ordered(A) R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug symmetrify(R) return R, info
def solveStepwiseInit(X,y,active,tolerance=1e-6) : """ Stepwise regression implementation requires we solve for the lower bound of the search space. So how is this different from the non-stepwise case? 1. Treatment of rank deficiency To treat rank deficiency, discover it using rank-revealing QR and then flag deficient columns for non-inclusion in a separate pass of full rank QR 2. Creation of an augmented R which has two properties: a. columns forming the active set form an upper triangular R matrix b. the remaining columns have been multiplied by the set of householder matrices sufficient to transform those same active set columns into upper triangular form """ n = X.shape[0] p = X.shape[1] #We start the computation with a rank-revealing QR (Scipy's QR routine) Q,R,permutation = linalg.qr(X,pivoting=True,mode='economic') R_diag = np.diag(R) rank = np.where(R_diag < tolerance)[0] rank = R_diag.shape[0] if rank.shape[0] == 0 else rank[0] allowed_active = np.zeros(active.shape,dtype=np.bool) #We use allowed_active to filter out starting set variables that are linear combinations #of other starting variables allowed_active[permutation[:rank]] = True #Having completed this analysis, we will want an un-permuted full-rank QR decopomposition. #For use in the stepwise procedure. Numpy's QR is a full rank variant and that is the one #we apply below (in contrast to scipy's used in the non-stepwise case). df = np.where(allowed_active)[0].shape[0] Q,R = np.linalg.qr(X[:,allowed_active],mode='full') #numpy's method differs from scipy's! b = np.dot(Q.T,y) beta = linalg.solve_triangular(R,b,lower=False) XXn1 = dpotri(R) var = np.diag(XXn1) resid = y - np.dot(X[:,allowed_active],beta) SSE = np.dot(resid,resid) return beta, var, SSE, df, np.dot(Q.T,X)
def __init__(self, X, Y, theta): theta = theta ** [2,1,2] [sf2, l2, sn2] = theta # evaluate RBF kernel for our given X r = dist.pdist(X) / l2 K = dist.squareform(sf2 * np.exp(-0.5 * r**2)) np.fill_diagonal(K, sf2) # add in Gaussian noise (+ a bit for numerical stability) Ky = K.copy() np.fill_diagonal(Ky, sf2 + sn2 + 1e-8) # compute the Cholesky factorization of our covariance matrix LW, info = lapack.dpotrf(Ky, lower=True) assert info == 0 # calculate lower half of inverse of K (assumes real symmetric positive definite) Wi, info = lapack.dpotri(LW, lower=True) assert info == 0 # make symmetric by filling in the upper half Wi += np.tril(Wi,-1).T # and solve alpha, info = lapack.dpotrs(LW, Y, lower=True) assert info == 0 # save these for later self.X = X self.Y = Y self.theta = theta self.r = r self.K = K self.Ky = Ky self.LW = LW self.Wi = Wi self.alpha = alpha
def _loss_and_grad(self, params): V = self.V X = self.X X.fill(0) #X = np.zeros((self.n,self.n)) X[self._mask] = params X += X.T np.fill_diagonal(X, 1) zz, info0 = dpotrf(X, False, False) iX, info1 = dpotri(zz) iX = np.triu(iX) + np.triu(iX, k=1).T if info0 != 0 or info1 != 0: #print('checkpt') return self._loss * 100, np.zeros_like(params) loss = np.sum(iX * V) G = -iX @ V @ iX g = G[self._mask] + G.T[self._mask] self._loss = loss #print(np.sqrt(loss / self.W.shape[0])) return loss, g #G.flatten()
def dpotri(A, lower=1): """ Wrapper for lapack dpotri function DPOTRI - compute the inverse of a real symmetric positive definite matrix A using the Cholesky factorization A = U**T*U or A = L*L**T computed by DPOTRF :param A: Matrix A :param lower: is matrix lower (true) or upper (false) :returns: A inverse """ if _fix_dpotri_scipy_bug: assert lower == 1, "scipy linalg behaviour is very weird. please use lower, fortran ordered arrays" lower = 0 A = force_F_ordered(A) R, info = lapack.dpotri( A, lower=lower) #needs to be zero here, seems to be a scipy bug symmetrify(R) return R, info