def solve(X,y,tolerance=1e-6) :
"""
Solve the regression problem ||Xw - y||^2 using the QR factorization of X.
Returns: beta, variance estimates for parameters, SSE
TODO: use an implementation that uses ||Q'Xw - Q'y||^2 to avoid returning Q when it is not
needed. Somwhere in Lapack there is probably something we could use.
"""
n = X.shape[0]
p = X.shape[1]
Q,R,permutation = linalg.qr(X,pivoting=True,mode='economic')
R_diag = np.diag(R)
rank = np.where(np.abs(R_diag) < tolerance)[0]
rank = R_diag.shape[0] if rank.shape[0] == 0 else rank[0]
b = np.dot(y,Q)[:rank]
beta = linalg.solve_triangular(R[:rank,:rank],b,lower=False)
if rank < R.shape[0] : beta = np.append(beta,np.zeros(R_diag.shape[0] - rank))
beta[permutation] = beta.copy() #non-atomic operation requires copy
R_small = R[:rank,:rank]
#Computing the inverse from R in a numericaly stable way requires a Cholesky
#based approach. I discovered this while testing against statsmodel's
#Langley data set which has very high condition number
XXn1,_ = dpotri(R_small)
XXn1_diag = np.append(np.diag(XXn1),np.zeros(R.shape[1] - R_small.shape[1]))
XXn1_diag[permutation] = XXn1_diag.copy()
betaSigmaSq = XXn1_diag
resid = y - np.dot(X,beta)
SSE = np.dot(resid,resid)
return beta, betaSigmaSq, SSE, rank,Q
def dpotri(A, lower=1):
A = force_F_ordered(A)
R, info = lapack.dpotri(
A, lower=lower) #needs to be zero here, seems to be a scipy bug
symmetrify(R)
return R, info
def pdinv(A, *args):
"""
:param A: A DxD pd numpy array
:rval Ai: the inverse of A
:rtype Ai: np.ndarray
:rval L: the Cholesky decomposition of A
:rtype L: np.ndarray
:rval Li: the Cholesky decomposition of Ai
:rtype Li: np.ndarray
:rval logdet: the log of the determinant of A
:rtype logdet: float64
"""
try:
Ai = np.linalg.inv(A)
logdet = np.linalg.slogdet(A)[1]
L = np.linalg.cholesky(A)
Li = chol_inv(L)
except linalg.LinAlgError:
L = jitchol(A, *args)
logdet = 2. * np.sum(np.log(np.diag(L)))
Li = chol_inv(L)
Ai, _ = lapack.dpotri(L)
# Ai = np.tril(Ai) + np.tril(Ai,-1).T
symmetrify(Ai)
return Ai, L, Li, logdet
def LML_se(self,theta,returnGradients=False):
self.setTheta(theta)
K,r = self.cov(self.X,retr=True)
Ky = K.copy()
Ky += np.eye(self.X.shape[0])*self.var_n + np.eye(self.X.shape[0])*1e-8
L = self.cholSafe(Ky)
WlogDet = 2.*np.sum(np.log(np.diag(L)))
alpha, status = dpotrs(L, self.Y, lower=1)
dataFit = - np.sum(alpha * self.Y)
modelComplexity = -self.Y.shape[1] * WlogDet
normalizer = -self.Y.size * log2pi
logMarginalLikelihood = 0.5*(dataFit + modelComplexity + normalizer)
if returnGradients == False:
return logMarginalLikelihood
else:
Wi, status = dpotri(-L, lower=1)
Wi = np.asarray(Wi)
# copy bottom triangle to top triangle
triu = np.triu_indices_from(Wi,k=1)
Wi[triu] = Wi.T[triu]
# dL = change in LML, dK is change in Kernel(K)
dL_dK = 0.5 * (np.dot(alpha,alpha.T) - self.Y.shape[1] * Wi)
dL_dVarn = np.diag(dL_dK).sum()
varfGradient = np.sum(K* dL_dK)/self.var_f
dK_dr = -r*K
dL_dr = dK_dr * dL_dK
lengthscaleGradient = -np.sum(dL_dr*r)/self.charLen
grads = np.array([varfGradient, lengthscaleGradient, dL_dVarn])
return logMarginalLikelihood, grads
def pdinv(A, *args):
"""
:param A: A DxD pd numpy array
:rval Ai: the inverse of A
:rtype Ai: np.ndarray
:rval L: the Cholesky decomposition of A
:rtype L: np.ndarray
:rval Li: the Cholesky decomposition of Ai
:rtype Li: np.ndarray
:rval logdet: the log of the determinant of A
:rtype logdet: float64
"""
try:
Ai = np.linalg.inv(A)
logdet = np.linalg.slogdet(A)[1]
L = np.linalg.cholesky(A)
Li = chol_inv(L)
except linalg.LinAlgError:
L = jitchol(A, *args)
logdet = 2.*np.sum(np.log(np.diag(L)))
Li = chol_inv(L)
Ai, _ = lapack.dpotri(L)
# Ai = np.tril(Ai) + np.tril(Ai,-1).T
symmetrify(Ai)
return Ai, L, Li, logdet
def invpd(A, return_chol=False):
L = np.linalg.cholesky(A)
Ainv = lapack.dpotri(L, lower=True)[0]
copy_lower_to_upper(Ainv)
if return_chol:
return Ainv, L
else:
return Ainv
def dpotri(A, lower=0):
"""Wrapper for lapack dpotri function
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns:
"""
return lapack.dpotri(A, lower=lower)
def dpotri(A, lower=0):
"""
Wrapper for lapack dpotri function
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns: A inverse
"""
return lapack.dpotri(A, lower=lower)
def invPsd(A, AChol=None, returnChol=False):
# https://github.com/mattjj/pybasicbayes/blob/9c00244b2d6fd767549de6ab5d0436cec4e06818/pybasicbayes/util/general.py
L = np.linalg.cholesky(A) if AChol is None else AChol
Ainv = lapack.dpotri(L, lower=True)[0]
Ainv += np.tril(Ainv, k=-1).T
if returnChol:
return Ainv, L
return Ainv
def chol_inv(x: np.array):
"""Calculates invserse of matrix using Cholesky decomposition.
Keyword arguments:
x -- A matrix.
Returns:
x^-1.
"""
c, info = lapack.dpotrf(x)
if info:
raise np.linalg.LinAlgError
lapack.dpotri(c, overwrite_c=1)
c += c.T
np.fill_diagonal(c, c.diagonal() / 2)
return c
def cholesky_inverse(A):
"""
Compute the inverse of the matrix AA' given its cholesky decomposition A.
"""
X, info = lapack.dpotri(A, lower=1)
if info != 0:
raise LinAlgError('Matrix is not invertible')
triu = np.triu_indices_from(X, k=1)
X[triu] = X.T[triu]
return X
def cholesky_inverse(A):
"""
Compute the inverse of the matrix AA' given its cholesky decomposition A.
"""
X, info = lapack.dpotri(A, lower=1)
if info != 0:
raise LinAlgError('Matrix is not invertible')
triu = np.triu_indices_from(X, k=1)
X[triu] = X.T[triu]
return X
def inv_psd(A, return_chol=False):
L = np.linalg.cholesky(A)
Ainv = lapack.dpotri(L, lower=True)[0]
copy_lower_to_upper(Ainv)
# if not np.allclose(Ainv, np.linalg.inv(A), rtol=1e-5, atol=1e-5):
# import ipdb; ipdb.set_trace()
if return_chol:
return Ainv, L
else:
return Ainv
def inv_psd(A, return_chol=False):
L = np.linalg.cholesky(A)
Ainv = lapack.dpotri(L, lower=True)[0]
copy_lower_to_upper(Ainv)
# if not np.allclose(Ainv, np.linalg.inv(A), rtol=1e-5, atol=1e-5):
# import ipdb; ipdb.set_trace()
if return_chol:
return Ainv, L
else:
return Ainv
def cholesky_inv(M):
# Lower-triangular Cholesky factorization of M
#L = np.linalg.cholesky(M)
# Call LAPACK dpotri to get inverse (only lower triangle populated)
Minv0 = lapack.dpotri(M, lower=True)[0]
# Copy lower triangle to upper triangle
Minv = Minv0 + Minv0.T - np.diag(Minv0.diagonal())
return Minv
def multiple_pdinv(A):
"""
:param A: A DxDxN numpy array (each A[:,:,i] is pd)
:rval invs: the inverses of A
:rtype invs: np.ndarray
:rval hld: 0.5* the log of the determinants of A
:rtype hld: np.array
"""
N = A.shape[-1]
chols = [jitchol(A[:, :, i]) for i in range(N)]
halflogdets = [np.sum(np.log(np.diag(L[0]))) for L in chols]
invs = [lapack.dpotri(L[0], True)[0] for L in chols]
invs = [np.triu(I) + np.triu(I, 1).T for I in invs]
return np.dstack(invs), np.array(halflogdets)
def multiple_pdinv(A):
"""
:param A: A DxDxN numpy array (each A[:,:,i] is pd)
:rval invs: the inverses of A
:rtype invs: np.ndarray
:rval hld: 0.5* the log of the determinants of A
:rtype hld: np.array
"""
N = A.shape[-1]
chols = [jitchol(A[:, :, i]) for i in range(N)]
halflogdets = [np.sum(np.log(np.diag(L[0]))) for L in chols]
invs = [lapack.dpotri(L[0], True)[0] for L in chols]
invs = [np.triu(I) + np.triu(I, 1).T for I in invs]
return np.dstack(invs), np.array(halflogdets)
def cholesky_inv(M):
"""
Returns inverse of positive-definite matrix M using Cholesky decomposition
"""
# Lower-triangular Cholesky factorization of M
L = linalg.cholesky(M, lower=True)
# Call LAPACK dpotri to get inverse (only lower triangle populated)
Minv0 = lapack.dpotri(L, lower=True)[0]
# Copy lower triangle to upper triangle
Minv = Minv0 + Minv0.T - np.diag(Minv0.diagonal())
return Minv
def grammy_uncertainty(class_prob, prob_est, classifications):
"""
Parameters
----------
class_prob: np.ndarry
probabilities of classification as each class for each species
prob_est: np.ndarray
estimated fragment proportions - output of grammy fitting procedure
classifications: np.ndarray
is the read data (so a vector with the classification of each read)
Returns
-------
tuple
output is a tuple first element is 95% lower bound, second element 95% upper bound
"""
# todo what do all these represent?!
lp = prob_est.size
lphat = prob_est.size - 1
phat = prob_est[:-1]
K = len(classifications)
IO = np.zeros((lphat, lphat))
for i in range(lphat):
for j in range(lphat):
IOij = [0] * K
for k in range(K):
IOij1 = (
class_prob[i, classifications[k]]
- class_prob[lphat, classifications[k]]
)
IOij2 = (
class_prob[j, classifications[k]]
- class_prob[lphat, classifications[k]]
)
IOij3 = IOij1 * IOij2
IOij4 = np.sum(phat * class_prob[:lphat, classifications[k]])
IOij5 = (1 - np.sum(phat)) * class_prob[lphat, classifications[k]]
IOij6 = (IOij4 + IOij5) ** 2
IOij7 = IOij3 / IOij6
IOij[k] = IOij7
IO[i, j] = sum(IOij)
IO = np.asfortranarray(IO)
IOinv = lapack.dpotri(np.linalg.cholesky(IO).T)[0]
est_se = np.sqrt(np.diag(IOinv))
up_CI = phat + 1.96 * est_se
low_CI = phat - 1.96 * est_se
return (low_CI, up_CI)
def dpotri(A, lower=1):
"""
Wrapper for lapack dpotri function
DPOTRI - compute the inverse of a real symmetric positive
definite matrix A using the Cholesky factorization A =
U**T*U or A = L*L**T computed by DPOTRF
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns: A inverse
"""
A = force_F_ordered(A)
R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug
symmetrify(R)
return R, info
def dpotri(A, lower=1):
"""
Wrapper for lapack dpotri function
DPOTRI - compute the inverse of a real symmetric positive
definite matrix A using the Cholesky factorization A =
U**T*U or A = L*L**T computed by DPOTRF
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns: A inverse
"""
A = force_F_ordered(A)
R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug
symmetrify(R)
return R, info
def fast_pd_inverse(m: 'numpy array') -> 'numpy array':
'''
This method calculates the inverse of a A real symmetric positive definite (n × n)-matrix
It is much faster than Numpy's "np.linalg.inv" method for example.
'''
try:
cholesky, info = lapack.dpotrf(m)
if info != 0:
raise ValueError('dpotrf failed on input {}'.format(m))
#print("cas 1")
inv, info = lapack.dpotri(cholesky)
if info != 0:
raise ValueError('dpotri failed on input {}'.format(cholesky))
#print("cas 2")
except:
inv = np.linalg.inv(m)
uppertriangular_2_symmetric(inv)
return inv
def partial_correlation(x, return_pvalue=False):
""" x: samples X features """
n, gp = x.shape
df = np.max(n - np.linalg.matrix_rank(x), 0)
cvx = np.cov(x.T)
r = la.cholesky(cvx, overwrite_a=True)
icvx, info = lapack.dpotri(r, overwrite_c=True)
pcor = -_cov2cor(icvx)
pcor += pcor.T
np.fill_diagonal(pcor, 1.0)
if return_pvalue:
statistic = pcor*np.sqrt(df/(1-pcor**2))
pvalue = 2*stats.t.cdf(-abs(statistic), df)
np.fill_diagonal(pvalue, 0.0)
return pcor, pvalue
return pcor
def partial_correlation(x, return_pvalue=False):
""" x: samples X features """
n, gp = x.shape
df = np.max(n - np.linalg.matrix_rank(x), 0)
cvx = np.cov(x.T)
r = la.cholesky(cvx, overwrite_a=True)
icvx, info = lapack.dpotri(r, overwrite_c=True)
pcor = -_cov2cor(icvx)
pcor += pcor.T
np.fill_diagonal(pcor, 1.0)
if return_pvalue:
statistic = pcor * np.sqrt(df / (1 - pcor**2))
pvalue = 2 * stats.t.cdf(-abs(statistic), df)
np.fill_diagonal(pvalue, 0.0)
return pcor, pvalue
return pcor
def multiple_dpotri_old(Ls):
M, _, D = Ls.shape
Kis = np.rollaxis(Ls, -1).copy()
[dpotri(Kis[i,:,:], overwrite_c=1, lower=1) for i in range(D)]
code = """
for(int d=0; d<D; d++)
{
for(int m=0; m<M; m++)
{
for(int mm=0; mm<m; mm++)
{
Kis[d*M*M + mm*M + m ] = Kis[d*M*M + m*M + mm];
}
}
}
"""
weave.inline(code, ['Kis', 'D', 'M'])
Kis = np.rollaxis(Kis, 0, 3) #wtf rollaxis?
return Kis
def dpotri(A, lower=1):
"""
Wrapper for lapack dpotri function
DPOTRI - compute the inverse of a real symmetric positive
definite matrix A using the Cholesky factorization A =
U**T*U or A = L*L**T computed by DPOTRF
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns: A inverse
"""
if _fix_dpotri_scipy_bug:
assert lower==1, "scipy linalg behaviour is very weird. please use lower, fortran ordered arrays"
lower = 0
A = force_F_ordered(A)
R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug
symmetrify(R)
return R, info
def solveStepwiseInit(X,y,active,tolerance=1e-6) :
"""
Stepwise regression implementation requires we solve for the lower bound of the
search space.
So how is this different from the non-stepwise case?
1. Treatment of rank deficiency
To treat rank deficiency, discover it using rank-revealing QR and then flag
deficient columns for non-inclusion in a separate pass of full rank QR
2. Creation of an augmented R which has two properties:
a. columns forming the active set form an upper triangular R matrix
b. the remaining columns have been multiplied by the set of householder matrices
sufficient to transform those same active set columns into upper triangular form
"""
n = X.shape[0]
p = X.shape[1]
#We start the computation with a rank-revealing QR (Scipy's QR routine)
Q,R,permutation = linalg.qr(X,pivoting=True,mode='economic')
R_diag = np.diag(R)
rank = np.where(R_diag < tolerance)[0]
rank = R_diag.shape[0] if rank.shape[0] == 0 else rank[0]
allowed_active = np.zeros(active.shape,dtype=np.bool)
#We use allowed_active to filter out starting set variables that are linear combinations
#of other starting variables
allowed_active[permutation[:rank]] = True
#Having completed this analysis, we will want an un-permuted full-rank QR decopomposition.
#For use in the stepwise procedure. Numpy's QR is a full rank variant and that is the one
#we apply below (in contrast to scipy's used in the non-stepwise case).
df = np.where(allowed_active)[0].shape[0]
Q,R = np.linalg.qr(X[:,allowed_active],mode='full') #numpy's method differs from scipy's!
b = np.dot(Q.T,y)
beta = linalg.solve_triangular(R,b,lower=False)
XXn1 = dpotri(R)
var = np.diag(XXn1)
resid = y - np.dot(X[:,allowed_active],beta)
SSE = np.dot(resid,resid)
return beta, var, SSE, df, np.dot(Q.T,X)
def __init__(self, X, Y, theta):
theta = theta ** [2,1,2]
[sf2, l2, sn2] = theta
# evaluate RBF kernel for our given X
r = dist.pdist(X) / l2
K = dist.squareform(sf2 * np.exp(-0.5 * r**2))
np.fill_diagonal(K, sf2)
# add in Gaussian noise (+ a bit for numerical stability)
Ky = K.copy()
np.fill_diagonal(Ky, sf2 + sn2 + 1e-8)
# compute the Cholesky factorization of our covariance matrix
LW, info = lapack.dpotrf(Ky, lower=True)
assert info == 0
# calculate lower half of inverse of K (assumes real symmetric positive definite)
Wi, info = lapack.dpotri(LW, lower=True)
assert info == 0
# make symmetric by filling in the upper half
Wi += np.tril(Wi,-1).T
# and solve
alpha, info = lapack.dpotrs(LW, Y, lower=True)
assert info == 0
# save these for later
self.X = X
self.Y = Y
self.theta = theta
self.r = r
self.K = K
self.Ky = Ky
self.LW = LW
self.Wi = Wi
self.alpha = alpha
def _loss_and_grad(self, params):
V = self.V
X = self.X
X.fill(0)
#X = np.zeros((self.n,self.n))
X[self._mask] = params
X += X.T
np.fill_diagonal(X, 1)
zz, info0 = dpotrf(X, False, False)
iX, info1 = dpotri(zz)
iX = np.triu(iX) + np.triu(iX, k=1).T
if info0 != 0 or info1 != 0:
#print('checkpt')
return self._loss * 100, np.zeros_like(params)
loss = np.sum(iX * V)
G = -iX @ V @ iX
g = G[self._mask] + G.T[self._mask]
self._loss = loss
#print(np.sqrt(loss / self.W.shape[0]))
return loss, g #G.flatten()
def dpotri(A, lower=1):
"""
Wrapper for lapack dpotri function
DPOTRI - compute the inverse of a real symmetric positive
definite matrix A using the Cholesky factorization A =
U**T*U or A = L*L**T computed by DPOTRF
:param A: Matrix A
:param lower: is matrix lower (true) or upper (false)
:returns: A inverse
"""
if _fix_dpotri_scipy_bug:
assert lower == 1, "scipy linalg behaviour is very weird. please use lower, fortran ordered arrays"
lower = 0
A = force_F_ordered(A)
R, info = lapack.dpotri(
A, lower=lower) #needs to be zero here, seems to be a scipy bug
symmetrify(R)
return R, info
