def PLS2D_getD(theta, tinds, rinds, cinds, sigma2):
# Obtain Lambda
Lambda = mapping2D(theta, tinds, rinds, cinds)
# Obtain Lambda'
Lambdat = spmatrix.trans(Lambda)
# Get D
D = Lambda * Lambdat * sigma2
return (D)
def PLS2D_getBeta(theta, ZtX, ZtY, XtX, ZtZ, XtY, YtX, YtZ, XtZ, YtY, n, P,
tinds, rinds, cinds):
# Obtain Lambda
Lambda = mapping2D(theta, tinds, rinds, cinds)
# Obtain Lambda'
Lambdat = spmatrix.trans(Lambda)
# Obtain Lambda'Z'Y and Lambda'Z'X
LambdatZtY = Lambdat * ZtY
LambdatZtX = Lambdat * ZtX
# Set the factorisation to use LL' instead of LDL'
cholmod.options['supernodal'] = 2
# Obtain the cholesky decomposition
LambdatZtZLambda = Lambdat * ZtZ * Lambda
I = spmatrix(1.0, range(Lambda.size[0]), range(Lambda.size[0]))
chol_dict = sparse_chol2D(LambdatZtZLambda + I,
perm=P,
retF=True,
retP=False,
retL=False)
F = chol_dict['F']
# Obtain C_u (annoyingly solve writes over the second argument,
# whereas spsolve outputs)
Cu = LambdatZtY[P, :]
cholmod.solve(F, Cu, sys=4)
# Obtain RZX
RZX = LambdatZtX[P, :]
cholmod.solve(F, RZX, sys=4)
# Obtain RXtRX
RXtRX = XtX - matrix.trans(RZX) * RZX
# Obtain beta estimates (note: gesv also replaces the second
# argument)
betahat = XtY - matrix.trans(RZX) * Cu
try:
lapack.posv(RXtRX, betahat)
except:
lapack.gesv(RXtRX, betahat)
return (betahat)
def main():
#================================================================================
# Scalars
#================================================================================
# Number of factors, random integer between 1 and 3
r = 1#np.random.randint(2,4)#np.random.randint(1,4)
#print("Number of grouping factors for random effects:")
#print(r)
# Number of levels, random number between 2 and 8
nlevels = np.array([20])#np.random.randint(2,8,size=(r))
# Let the first number of levels be a little larger (typically like subjects)
#nlevels[0] = np.random.randint(2,35,size=1)
#nlevels = np.sort(nlevels)[::-1]
#print("Number of levels for each factor:")
#print(nlevels)
# Number of parameters, random number between 1 and 5
nparams = np.array([2])#np.random.randint(1,6,size=(r))
#print("Number of parameters for each factor:")
#print(nparams)
# Dimension of D
#print("Dimension of D, q:")
q = np.sum(nlevels*nparams)
#print(q)
# Number of fixed effects, random number between 6 and 30
p = 5#np.random.randint(6,31)
#print("Number of fixed effects:")
#print(p)
# Number of subjects, n
n = 1000
#print("Number of subjects:")
#print(n)
# Voxel dimensions
dimv = [20,20,20]
nv = np.prod(dimv)
#print("Number of voxels:")
#print(nv)
#================================================================================
# Design matrix
#================================================================================
# Initialize empty x
X = np.zeros((n,p))
# First column is intercept
X[:,0] = 1
# Rest of the columns we will make random noise
X[:,1:] = np.random.randn(n*(p-1)).reshape((n,(p-1)))
#================================================================================
# Random Effects Design matrix
#================================================================================
# We need to create a block of Z for each level of each factor
for i in np.arange(r):
Zdata_factor = np.random.randn(n,nparams[i])
if i==0:
#The first factor should be block diagonal, so the factor indices are grouped
factorVec = np.repeat(np.arange(nlevels[i]), repeats=np.floor(n/max(nlevels[i],1)))
if len(factorVec) < n:
# Quick fix incase rounding leaves empty columns
factorVecTmp = np.zeros(n)
factorVecTmp[0:len(factorVec)] = factorVec
factorVecTmp[len(factorVec):n] = nlevels[i]-1
factorVec = np.int64(factorVecTmp)
# Crop the factor vector - otherwise have a few too many
factorVec = factorVec[0:n]
# Give the data an intercept
#Zdata_factor[:,0]=1
else:
# The factor is randomly arranged across subjects
factorVec = np.random.randint(0,nlevels[i],size=n)
# Build a matrix showing where the elements of Z should be
indicatorMatrix_factor = np.zeros((n,nlevels[i]))
indicatorMatrix_factor[np.arange(n),factorVec] = 1
# Need to repeat for each parameter the factor has
indicatorMatrix_factor = np.repeat(indicatorMatrix_factor, nparams[i], axis=1)
# Enter the Z values
indicatorMatrix_factor[indicatorMatrix_factor==1]=Zdata_factor.reshape(Zdata_factor.shape[0]*Zdata_factor.shape[1])
# Make sparse
Zfactor = scipy.sparse.csr_matrix(indicatorMatrix_factor)
# Put all the factors together
if i == 0:
Z = Zfactor
else:
Z = scipy.sparse.hstack((Z, Zfactor))
#================================================================================
# Smoothed beta
#================================================================================
# Random 4D matrix (unsmoothed)
beta_us = np.random.randn(nv*p).reshape(dimv[0],dimv[1],dimv[2],p)
#beta_us[3:5,3:5,3:5,3] = beta_us[3:5,3:5,3:5,3] + 100
t1 = time.time()
# Some random affine, not important for this simulation
affine = np.diag([1, 1, 1, 1])
beta_us_nii = nib.Nifti1Image(beta_us, affine)
# Smoothed beta nifti
beta_s_nii = nilearn.image.smooth_img(beta_us_nii, 5)
# Final beta
beta = beta_s_nii.get_fdata()
#================================================================================
# Smoothed b
#================================================================================
# Random 4D matrix (unsmoothed)
b_us = np.random.randn(nv*q).reshape(dimv[0],dimv[1],dimv[2],q)
# Some random affine, not important for this simulation
affine = np.diag([1, 1, 1, 1])
b_us_nii = nib.Nifti1Image(b_us, affine)
# Smoothed beta nifti
b_s_nii = nilearn.image.smooth_img(b_us_nii, 5)
# Final beta
b = b_s_nii.get_fdata()
#================================================================================
# Response
#================================================================================
# Reshape X
X = X.reshape(1, X.shape[0], X.shape[1])
# Reshape beta
beta = beta.reshape(beta.shape[0]*beta.shape[1]*beta.shape[2],beta.shape[3],1)
beta_True = beta
# Reshape Z (note: This step is slow because of the sparse to dense conversion;
# it could probably be made quicker but this is only for one simulation at current)
Ztmp = Z.toarray().reshape(1, Z.shape[0], Z.shape[1])
# Reshape b
b = b.reshape(b.shape[0]*b.shape[1]*b.shape[2],b.shape[3],1)
# Generate Y
Y = np.matmul(X,beta)+np.matmul(Ztmp,b) + np.random.randn(n,1)
#================================================================================
# Transpose products
#================================================================================
# X'Z\Z'X
XtZ = np.matmul(X.transpose(0,2,1),Ztmp)
ZtX = XtZ.transpose(0,2,1)
# Z'Y\Y'Z
YtZ = np.matmul(Y.transpose(0,2,1),Ztmp)
ZtY = YtZ.transpose(0,2,1)
# Y'X/X'Y
YtX = np.matmul(Y.transpose(0,2,1),X)
XtY = YtX.transpose(0,2,1)
# YtY
YtY = np.matmul(Y.transpose(0,2,1),Y)
# ZtZ
ZtZ = np.matmul(Ztmp.transpose(0,2,1),Ztmp)
# X'X
XtX = np.matmul(X.transpose(0,2,1),X)
#================================================================================
# Looping
#================================================================================
# Initialize empty estimates
beta_est = np.zeros(beta.shape)
# Initialize temporary X'X, X'Z, Z'X and Z'Z
XtZtmp = matrix(XtZ[0,:,:])
ZtXtmp = matrix(ZtX[0,:,:])
ZtZtmp = cvxopt.sparse(matrix(ZtZ[0,:,:]))
XtXtmp = matrix(XtX[0,:,:])
# Initial theta value. Bates (2005) suggests using [vech(I_q1),...,vech(I_qr)] where I is the identity matrix
theta0 = np.array([])
for i in np.arange(r):
theta0 = np.hstack((theta0, mat2vech2D(np.eye(nparams[i])).reshape(np.int64(nparams[i]*(nparams[i]+1)/2))))
# Obtain a random Lambda matrix with the correct sparsity for the permutation vector
tinds,rinds,cinds=get_mapping2D(nlevels, nparams)
Lam=mapping2D(np.random.randn(theta0.shape[0]),tinds,rinds,cinds)
# Obtain Lambda'Z'ZLambda
LamtZtZLam = spmatrix.trans(Lam)*cvxopt.sparse(matrix(ZtZtmp))*Lam
# Identity (Actually quicker to calculate outside of estimation)
I = spmatrix(1.0, range(Lam.size[0]), range(Lam.size[0]))
# Obtaining permutation for PLS
P=cvxopt.amd.order(LamtZtZLam)
runningtime = 0
beta_runningsum = 0
b_runningsum = 0
for i in np.arange(nv):
XtYtmp = matrix(XtY[i,:,:])
ZtYtmp = matrix(ZtY[i,:,:])
YtYtmp = matrix(YtY[i,:,:])
YtZtmp = matrix(YtZ[i,:,:])
YtXtmp = matrix(YtX[i,:,:])
t1 = time.time()
theta_est = minimize(PLS2D, theta0, args=(ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P, I, tinds, rinds, cinds), method='L-BFGS-B', tol=1e-7)
runningtime = runningtime + time.time()-t1
nit = theta_est.nit
theta_est = theta_est.x
# Get current beta
beta_est = np.array(PLS2D_getBeta(theta_est, ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P,tinds, rinds, cinds))
sigma2_est = PLS2D_getSigma2(theta_est, ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P, I, tinds, rinds, cinds)
D_est = np.array(matrix(PLS2D_getD(theta_est, tinds, rinds, cinds, sigma2_est)))
DinvIplusZtZD = D_est @ np.linalg.inv(np.eye(q) + np.array(ZtZ[0,:,:]) @ D_est)
Zte = np.array(ZtYtmp) - np.array(ZtX[0,:,:]) @ beta_est
b_est = (DinvIplusZtZD @ Zte)
b_true = b[i,:]
beta_runningsum = beta_runningsum + np.sum(np.abs(beta_True[i,:] - beta_est))
b_runningsum = b_runningsum + np.sum(np.abs(b_true - b_est))
print(beta_runningsum/(nv*p))
print(b_runningsum/(nv*q))
print(runningtime)
def test2D():
#===============================================================================
# Setup
#===============================================================================
# Generate test data
Y,X,Z,nlevels,nraneffs,beta,sigma2,b,D = genTestData2D()
# Work out number of observations, parameters, random effects, etc
n = X.shape[0]
p = X.shape[1]
q = np.sum(nraneffs*nlevels)
qu = np.sum(nraneffs*(nraneffs+1)//2)
r = nlevels.shape[0]
# Tolerance
tol = 1e-6
# Work out factor indices.
facInds = np.cumsum(nraneffs*nlevels)
facInds = np.insert(facInds,0,0)
# Convert D to dict
Ddict=dict()
for k in np.arange(len(nlevels)):
Ddict[k] = D[facInds[k]:(facInds[k]+nraneffs[k]),facInds[k]:(facInds[k]+nraneffs[k])]
# Get the product matrices
XtX, XtY, XtZ, YtX, YtY, YtZ, ZtX, ZtY, ZtZ = prodMats2D(Y,Z,X)
# -----------------------------------------------------------------------------
# Display parameters:
# -----------------------------------------------------------------------------
print('--------------------------------------------------------------------------')
print('Test Settings:')
print('--------------------------------------------------------------------------')
print('nlevels: ', nlevels)
print('nraneffs: ', nraneffs)
print('n: ', n, ', p: ', p, ', r: ', r, ', q: ', q, ', tol: ', tol)
print('--------------------------------------------------------------------------')
# -----------------------------------------------------------------------------
# Create empty data frame for results:
# -----------------------------------------------------------------------------
# Row indices
indexVec = np.array(['Time', 'nit', 'llh'])
for i in np.arange(p):
indexVec = np.append(indexVec, 'beta'+str(i+1))
# Sigma2
indexVec = np.append(indexVec, 'sigma2')
# Dk
for k in np.arange(r):
for j in np.arange(nraneffs[k]*(nraneffs[k]+1)//2):
indexVec = np.append(indexVec, 'D'+str(k+1)+','+str(j+1))
# Sigma2*Dk
for k in np.arange(r):
for j in np.arange(nraneffs[k]*(nraneffs[k]+1)//2):
indexVec = np.append(indexVec, 'sigma2*D'+str(k+1)+','+str(j+1))
# Construct dataframe
results = pd.DataFrame(index=indexVec, columns=['Truth', 'PeLS', 'FS', 'pFS', 'SFS', 'pSFS', 'cSFS'])
# ------------------------------------------------------------------------------------
# Truth
# ------------------------------------------------------------------------------------
# Default time and number of iterations
results.at['Time','Truth']=0
results.at['nit','Truth']=0
# Construct parameter vector
paramVec_true = beta[:]
paramVec_true = np.concatenate((paramVec_true,np.array(sigma2).reshape(1,1)),axis=0)
# Add D to parameter vector
facInds = np.cumsum(nraneffs*nlevels)
facInds = np.insert(facInds,0,0)
# Convert D to vector
for k in np.arange(len(nlevels)):
vechD = mat2vech2D(D[facInds[k]:(facInds[k]+nraneffs[k]),facInds[k]:(facInds[k]+nraneffs[k])])/sigma2
paramVec_true = np.concatenate((paramVec_true,vechD),axis=0)
# Add results to parameter vector
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'Truth']=paramVec_true[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'Truth']=paramVec_true[p,0]*paramVec_true[i-3,0]
# Matrices needed for
Zte = ZtY - ZtX @ beta
ete = ssr2D(YtX, YtY, XtX, beta)
DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + ZtZ @ D)
# True log likelihood
llh = llh2D(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D)[0,0]-n/2*np.log(np.pi)
results.at['llh','Truth']=llh
print('Truth Saved')
#===============================================================================
# pSFS
#===============================================================================
# Get the indices for the individual random factor covariance parameters.
DkInds = np.zeros(len(nlevels)+1)
DkInds[0]=np.int(p+1)
for k in np.arange(len(nlevels)):
DkInds[k+1] = np.int(DkInds[k] + nraneffs[k]*(nraneffs[k]+1)//2)
# Run Pseudo Simplified Fisher Scoring
t1 = time.time()
paramVector_pSFS,_,nit,llh = pSFS2D(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nraneffs, tol, n, init_paramVector=None)
t2 = time.time()
# Record Time and number of iterations
results.at['Time','pSFS']=t2-t1
results.at['nit','pSFS']=nit
results.at['llh','pSFS']=llh-n/2*np.log(np.pi)
# Record parameters
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'pSFS']=paramVector_pSFS[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'pSFS']=paramVector_pSFS[p,0]*paramVector_pSFS[i-3,0]
print('pSFS Saved')
#===============================================================================
# PeLS
#===============================================================================
# Convert matrices to cvxopt format.
XtZtmp = matrix(XtZ)
ZtXtmp = matrix(ZtX)
ZtZtmp = cvxopt.sparse(matrix(ZtZ))
XtXtmp = matrix(XtX)
XtYtmp = matrix(XtY)
ZtYtmp = matrix(ZtY)
YtYtmp = matrix(YtY)
YtZtmp = matrix(YtZ)
YtXtmp = matrix(YtX)
# Initial theta value. Bates (2005) suggests using [vech(I_q1),...,vech(I_qr)] where I is the identity matrix
theta0 = np.array([])
for i in np.arange(len(nraneffs)):
theta0 = np.hstack((theta0, mat2vech2D(np.eye(nraneffs[i])).reshape(np.int64(nraneffs[i]*(nraneffs[i]+1)/2))))
# Obtain a random Lambda matrix with the correct sparsity for the permutation vector
tinds,rinds,cinds=get_mapping2D(nlevels, nraneffs)
Lam=mapping2D(np.random.randn(theta0.shape[0]),tinds,rinds,cinds)
# Obtain Lambda'Z'ZLambda
LamtZtZLam = spmatrix.trans(Lam)*cvxopt.sparse(matrix(ZtZtmp))*Lam
# Identity (Actually quicker to calculate outside of estimation)
I = spmatrix(1.0, range(Lam.size[0]), range(Lam.size[0]))
# Obtaining permutation for PeLS
P=cvxopt.amd.order(LamtZtZLam)
# Run Penalized Least Squares
t1 = time.time()
estimation = minimize(PeLS2D, theta0, args=(ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P, I, tinds, rinds, cinds), method='L-BFGS-B', tol=tol)
# llh
llh = -estimation['fun']
# Theta parameters
theta = estimation['x']
# Number of iterations
nit = estimation['nit']
# Obtain Beta, sigma2 and D
beta_pls = PeLS2D_getBeta(theta, ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P, tinds, rinds, cinds)
sigma2_pls = PeLS2D_getSigma2(theta, ZtXtmp, ZtYtmp, XtXtmp, ZtZtmp, XtYtmp, YtXtmp, YtZtmp, XtZtmp, YtYtmp, n, P, I, tinds, rinds, cinds)
D_pls = PeLS2D_getD(theta, tinds, rinds, cinds, sigma2_pls)
t2 = time.time()
# Record time, number of iterations and log likelihood
results.at['Time','PeLS']=t2-t1
results.at['nit','PeLS']=nit
results.at['llh','PeLS']=llh
# Record beta parameters
for i in range(p):
results.at[indexVec[i+3],'PeLS']=beta_pls[i]
# Record sigma2 parameter
results.at['sigma2','PeLS']=sigma2_pls[0]
# Indices corresponding to random factors.
Dinds = np.cumsum(nraneffs*(nraneffs+1)//2)+p+4
Dinds = np.insert(Dinds,0,p+4)
# Work out vechDk for each random factor
for k in np.arange(len(nlevels)):
vechDk = mat2vech2D(np.array(matrix(D_pls[facInds[k]:(facInds[k]+nraneffs[k]),facInds[k]:(facInds[k]+nraneffs[k])])))
# Save parameters
for j in np.arange(len(vechDk)):
results.at[indexVec[Dinds[k]+j],'PeLS']=vechDk[j,0]/sigma2_pls[0]
results.at[indexVec[Dinds[k]+qu+j],'PeLS']=vechDk[j,0]
print('PeLS Saved')
#===============================================================================
# cSFS
#===============================================================================
# Run Cholesky Simplified Fisher Scoring
t1 = time.time()
paramVector_cSFS,_,nit,llh = cSFS2D(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nraneffs, tol, n, init_paramVector=None)
t2 = time.time()
# Record time and number of iterations
results.at['Time','cSFS']=t2-t1
results.at['nit','cSFS']=nit
results.at['llh','cSFS']=llh-n/2*np.log(np.pi)
# Save parameters
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'cSFS']=paramVector_cSFS[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'cSFS']=paramVector_cSFS[p,0]*paramVector_cSFS[i-3,0]
print('cSFS Saved')
#===============================================================================
# FS
#===============================================================================
# Run Fisher Scoring
t1 = time.time()
paramVector_FS,_,nit,llh = FS2D(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nraneffs, tol, n, init_paramVector=None)
t2 = time.time()
# Record time and number of iterations
results.at['Time','FS']=t2-t1
results.at['nit','FS']=nit
results.at['llh','FS']=llh-n/2*np.log(np.pi)
# Save parameters
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'FS']=paramVector_FS[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'FS']=paramVector_FS[p,0]*paramVector_FS[i-3,0]
print('FS Saved')
#===============================================================================
# SFS
#===============================================================================
# Run Simplified Fisher Scoring
t1 = time.time()
paramVector_SFS,_,nit,llh = SFS2D(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nraneffs, tol, n, init_paramVector=None)
t2 = time.time()
# Record time and number of iterations
results.at['Time','SFS']=t2-t1
results.at['nit','SFS']=nit
results.at['llh','SFS']=llh-n/2*np.log(np.pi)
# Save parameters
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'SFS']=paramVector_SFS[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'SFS']=paramVector_SFS[p,0]*paramVector_SFS[i-3,0]
print('SFS Saved')
#===============================================================================
# pFS
#===============================================================================
# Run Pseudo Fisher Scoring
t1 = time.time()
paramVector_pFS,_,nit,llh = pFS2D(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nraneffs, tol, n, init_paramVector=None)
t2 = time.time()
# Record time and number of iterations
results.at['Time','pFS']=t2-t1
results.at['nit','pFS']=nit
results.at['llh','pFS']=llh-n/2*np.log(np.pi)
# Save parameters
for i in np.arange(3,p+qu+4):
results.at[indexVec[i],'pFS']=paramVector_pFS[i-3,0]
# Record D*sigma2
for i in np.arange(4+p,p+qu+4):
results.at[indexVec[i+qu],'pFS']=paramVector_pFS[p,0]*paramVector_pFS[i-3,0]
print('pFS Saved')
# Print results
print(results.to_string())
# Return results
return(results)
def PLS2D_getSigma2(theta, ZtX, ZtY, XtX, ZtZ, XtY, YtX, YtZ, XtZ, YtY, n, P,
I, tinds, rinds, cinds):
# Obtain Lambda
#t1 = time.time()
Lambda = mapping2D(theta, tinds, rinds, cinds)
#t2 = time.time()
#print(t2-t1)#3.170967102050781e-05 9
# Obtain Lambda'
#t1 = time.time()
Lambdat = spmatrix.trans(Lambda)
#t2 = time.time()
#print(t2-t1)# 3.5762786865234375e-06
# Obtain Lambda'Z'Y and Lambda'Z'X
#t1 = time.time()
LambdatZtY = Lambdat * ZtY
LambdatZtX = Lambdat * ZtX
#t2 = time.time()
#print(t2-t1)#1.049041748046875e-05 13
# Obtain the cholesky decomposition
#t1 = time.time()
LambdatZtZLambda = Lambdat * (ZtZ * Lambda)
#t2 = time.time()
#print(t2-t1)#3.790855407714844e-05 2
#t1 = time.time()
chol_dict = sparse_chol2D(LambdatZtZLambda + I,
perm=P,
retF=True,
retP=False,
retL=False)
F = chol_dict['F']
#t2 = time.time()
#print(t2-t1)#0.0001342296600341797 1
# Obtain C_u (annoyingly solve writes over the second argument,
# whereas spsolve outputs)
#t1 = time.time()
Cu = LambdatZtY[P, :]
cholmod.solve(F, Cu, sys=4)
#t2 = time.time()
#print(t2-t1)#1.5974044799804688e-05 5
# Obtain RZX
#t1 = time.time()
RZX = LambdatZtX[P, :]
cholmod.solve(F, RZX, sys=4)
#t2 = time.time()
#print(t2-t1)#1.2159347534179688e-05 7
# Obtain RXtRX
#t1 = time.time()
RXtRX = XtX - matrix.trans(RZX) * RZX
#t2 = time.time()
#print(t2-t1)#9.775161743164062e-06 11
# Obtain beta estimates (note: gesv also replaces the second
# argument)
#t1 = time.time()
betahat = XtY - matrix.trans(RZX) * Cu
try:
lapack.posv(RXtRX, betahat)
except:
lapack.gesv(RXtRX, betahat)
#t2 = time.time()
#print(t2-t1)#1.7404556274414062e-05 6
# Obtain u estimates
#t1 = time.time()
uhat = Cu - RZX * betahat
cholmod.solve(F, uhat, sys=5)
cholmod.solve(F, uhat, sys=8)
#t2 = time.time()
#print(t2-t1)#1.2874603271484375e-05 8
# Obtain b estimates
#t1 = time.time()
bhat = Lambda * uhat
#t2 = time.time()
#print(t2-t1)#2.86102294921875e-06 15
# Obtain residuals sum of squares
#t1 = time.time()
resss = YtY - 2 * YtX * betahat - 2 * YtZ * bhat + 2 * matrix.trans(
betahat) * XtZ * bhat + matrix.trans(
betahat) * XtX * betahat + matrix.trans(bhat) * ZtZ * bhat
#t2 = time.time()
#print(t2-t1)#3.409385681152344e-05 4
# Obtain penalised residual sum of squares
#t1 = time.time()
pss = resss + matrix.trans(uhat) * uhat
return (pss / n)
def SW_lmerTest(theta3D,L,nlevels,nparams,ZtX,ZtY,XtX,ZtZ,XtY,YtX,YtZ,XtZ,YtY,n,beta):# TODO inputs
#================================================================================
# Initial theta
#================================================================================
theta0 = np.array([])
r = np.amax(nlevels.shape)
for i in np.arange(r):
theta0 = np.hstack((theta0, mat2vech2D(np.eye(nparams[i])).reshape(np.int64(nparams[i]*(nparams[i]+1)/2))))
#================================================================================
# Sparse Permutation, P
#================================================================================
tinds,rinds,cinds=get_mapping2D(nlevels, nparams)
tmp = np.random.randn(theta0.shape[0])
Lam=mapping2D(tmp,tinds,rinds,cinds)
# Obtain Lambda'Z'ZLambda
LamtZtZLam = spmatrix.trans(Lam)*cvxopt.sparse(matrix(ZtZ[0,:,:]))*Lam
# Obtaining permutation for PLS
cholmod.options['supernodal']=2
P=amd.order(LamtZtZLam)
# Identity
I = spmatrix(1.0, range(Lam.size[0]), range(Lam.size[0]))
# These are not spatially varying
XtX_current = cvxopt.matrix(XtX[0,:,:])
XtZ_current = cvxopt.matrix(XtZ[0,:,:])
ZtX_current = cvxopt.matrix(ZtX[0,:,:])
ZtZ_current = cvxopt.sparse(cvxopt.matrix(ZtZ[0,:,:]))
df = np.zeros(YtY.shape[0])
# Get the sigma^2 and D estimates.
for i in np.arange(theta3D.shape[0]):
# Get current theta
theta = theta3D[i,:]
# Convert product matrices to CVXopt form
XtY_current = cvxopt.matrix(XtY[i,:,:])
YtX_current = cvxopt.matrix(YtX[i,:,:])
YtY_current = cvxopt.matrix(YtY[i,:,:])
YtZ_current = cvxopt.matrix(YtZ[i,:,:])
ZtY_current = cvxopt.matrix(ZtY[i,:,:])
# Convert to gamma form
gamma = theta2gamma(theta, ZtX_current, ZtY_current, XtX_current, ZtZ_current, XtY_current, YtX_current, YtZ_current, XtZ_current, YtY_current, n, P, I, tinds, rinds, cinds)
# Estimate hessian
H = nd.Hessian(llh_gamma)(gamma, beta[i,:,:], np.array(ZtX_current), np.array(ZtY_current), np.array(XtX_current), np.array(matrix(ZtZ_current)), np.array(XtY_current), np.array(YtX_current), np.array(YtZ_current), np.array(XtZ_current), np.array(YtY_current), nlevels, nparams, n, P, tinds, rinds, cinds)
# Estimate Jacobian
J = nd.Jacobian(S2_gammavec)(gamma, L, np.array(ZtX_current), np.array(ZtY_current), np.array(XtX_current), np.array(matrix(ZtZ_current)), np.array(XtY_current), np.array(YtX_current), np.array(YtZ_current), np.array(XtZ_current), np.array(YtY_current), nparams, nlevels)
# print('J shape')
# print(J.shape)
# Calulcate S^2
S2 = S2_gamma(gamma, L, ZtX_current, ZtY_current, XtX_current, ZtZ_current, XtY_current, YtX_current,
YtZ_current, XtZ_current, YtY_current, n, P, I, tinds, rinds, cinds)
# Calculate the degrees of freedom
df[i] = 2*(S2**2)/(J @ np.linalg.pinv(H) @ J.transpose())
return(df)
def PLS(theta, ZtX, ZtY, XtX, ZtZ, XtY, YtX, YtZ, XtZ, YtY, P, tinds, rinds,
cinds):
#t1 = time.time()
# Obtain Lambda from theta
Lambda = mapping(theta, tinds, rinds, cinds)
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
# Obtain Lambda'
Lambdat = spmatrix.trans(Lambda)
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
LambdatZtY = Lambdat * ZtY
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
LambdatZtX = Lambdat * ZtX
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
# Set the factorisation to use LL' instead of LDL'
cholmod.options['supernodal'] = 2
#t2 = time.time()
#print(t2-t1)
# Obtain L
#t1 = time.time()
LambdatZtZLambda = Lambdat * ZtZ * Lambda
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
I = spmatrix(1.0, range(Lambda.size[0]), range(Lambda.size[0]))
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
chol_dict = sparse_chol(LambdatZtZLambda + I,
perm=P,
retF=True,
retP=False,
retL=False)
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
F = chol_dict['F']
#t2 = time.time()
#print(t2-t1)
# Obtain C_u (annoyingly solve writes over the second argument,
# whereas spsolve outputs)
#t1 = time.time()
Cu = LambdatZtY[P, :]
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
cholmod.solve(F, Cu, sys=4)
#t2 = time.time()
#print(t2-t1)
# Obtain RZX
#t1 = time.time()
RZX = LambdatZtX[P, :]
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
cholmod.solve(F, RZX, sys=4)
#t2 = time.time()
#print(t2-t1)
# Obtain RXtRX
#t1 = time.time()
RXtRX = XtX - matrix.trans(RZX) * RZX
#t2 = time.time()
#print(t2-t1)
#print(RXtRX.size)
#print(X.size)
#print(Y.size)
#print(RZX.size)
#print(Cu.size)
# Obtain beta estimates (note: gesv also replaces the second
# argument)
#t1 = time.time()
betahat = XtY - matrix.trans(RZX) * Cu
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
lapack.posv(RXtRX, betahat)
#t2 = time.time()
#print(t2-t1)
# Obtain u estimates
#t1 = time.time()
uhat = Cu - RZX * betahat
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
cholmod.solve(F, uhat, sys=5)
#t2 = time.time()
#print(t2-t1)
#t1 = time.time()
cholmod.solve(F, uhat, sys=8)
#t2 = time.time()
#print(t2-t1)
# Obtain b estimates
#t1 = time.time()
bhat = Lambda * uhat
#t2 = time.time()
#print(t2-t1)
# Obtain residuals sum of squares
#t1 = time.time()
resss = YtY - 2 * YtX * betahat - 2 * YtZ * bhat + 2 * matrix.trans(
betahat) * XtZ * bhat + matrix.trans(
betahat) * XtX * betahat + matrix.trans(bhat) * ZtZ * bhat
#t2 = time.time()
#print(t2-t1)
# Obtain penalised residual sum of squares
#t1 = time.time()
pss = resss + matrix.trans(uhat) * uhat
#t2 = time.time()
#print(t2-t1)
# Obtain Log(|L|^2)
#t1 = time.time()
logdet = 2 * sum(cvxopt.log(
cholmod.diag(F))) # this method only works for symm decomps
# Need to do tr(R_X)^2 for rml
#t2 = time.time()
#print(t2-t1)
# Obtain log likelihood
logllh = -logdet / 2 - X.size[0] / 2 * (1 + np.log(2 * np.pi * pss) -
np.log(X.size[0]))
#print(L[::(L.size[0]+1)]) # gives diag
#print(logllh[0,0])
#print(theta)
return (-logllh[0, 0])
# Calculate lambda for R example
tmp = pd.read_csv('estd_rfxvar.csv', header=None).values
rfxvarest = spmatrix(tmp[tmp != 0], [0, 0, 1, 1, 2, 2, 3, 3],
[0, 1, 0, 1, 2, 3, 2, 3])
f = sparse_chol(rfxvarest)
theta = inv_mapping(f['L'])
nlevels = np.array([20, 3])
nparams = np.array([2, 2])
tinds, rinds, cinds = get_mapping(theta, nlevels, nparams)
Lam = mapping(theta, tinds, rinds, cinds)
#cvxopt.printing.options['width'] = -1
# Obtaining permutation for PLS
# Obtain Lambda'Z'ZLambda
LamtZt = spmatrix.trans(Lam) * spmatrix.trans(Z)
LamtZtZLam = LamtZt * spmatrix.trans(LamtZt)
#f=sparse_chol(LamtZtZLam)
#P = f['P']
P = cvxopt.amd.order(LamtZtZLam)
Y = matrix(pd.read_csv('Y.csv', header=None).values)
X = matrix(pd.read_csv('X.csv', header=None).values)
ZtX = cvxopt.spmatrix.trans(Z) * X
ZtY = cvxopt.spmatrix.trans(Z) * Y
XtX = cvxopt.matrix.trans(X) * X
ZtZ = cvxopt.spmatrix.trans(Z) * Z
XtY = cvxopt.matrix.trans(X) * Y
YtX = cvxopt.matrix.trans(Y) * X
YtZ = cvxopt.matrix.trans(Y) * Z
def PeLS2D(theta, ZtX, ZtY, XtX, ZtZ, XtY, YtX, YtZ, XtZ, YtY, n, P, I, tinds,
rinds, cinds):
# Obtain Lambda
Lambda = mapping2D(theta, tinds, rinds, cinds)
# Obtain Lambda'
Lambdat = spmatrix.trans(Lambda)
# Obtain Lambda'Z'Y and Lambda'Z'X
LambdatZtY = Lambdat * ZtY
LambdatZtX = Lambdat * ZtX
# Obtain the cholesky decomposition
LambdatZtZLambda = Lambdat * (ZtZ * Lambda)
chol_dict = sparse_chol2D(LambdatZtZLambda + I,
perm=P,
retF=True,
retP=False,
retL=False)
F = chol_dict['F']
# Obtain C_u (annoyingly solve writes over the second argument,
# whereas spsolve outputs)
Cu = LambdatZtY[P, :]
cholmod.solve(F, Cu, sys=4)
# Obtain RZX
RZX = LambdatZtX[P, :]
cholmod.solve(F, RZX, sys=4)
# Obtain RXtRX
RXtRX = XtX - matrix.trans(RZX) * RZX
# Obtain beta estimates (note: gesv also replaces the second
# argument)
betahat = XtY - matrix.trans(RZX) * Cu
try:
lapack.posv(RXtRX, betahat)
except:
lapack.gesv(RXtRX, betahat)
# Obtain u estimates
uhat = Cu - RZX * betahat
cholmod.solve(F, uhat, sys=5)
cholmod.solve(F, uhat, sys=8)
# Obtain b estimates
bhat = Lambda * uhat
# Obtain residuals sum of squares
resss = YtY - 2 * YtX * betahat - 2 * YtZ * bhat + 2 * matrix.trans(
betahat) * XtZ * bhat + matrix.trans(
betahat) * XtX * betahat + matrix.trans(bhat) * ZtZ * bhat
# Obtain penalised residual sum of squares
pss = resss + matrix.trans(uhat) * uhat
# Obtain Log(|L|^2)
logdet = 2 * sum(cvxopt.log(cholmod.diag(F)))
# Obtain log likelihood
logllh = -logdet / 2 - n / 2 * (1 + np.log(2 * np.pi * pss[0, 0]) -
np.log(n))
return (-logllh)
