def linReg(X, t, Lambda=0):
d = np.size(X, 0)
idx = np.transpose(np.array(range(0, d)))
if len(idx) == 0:
idx = 0
dg = sub2ind((d, d), idx, idx)
xbar = np.mean(X, 1)
tbar = np.mean(t, 1)
X = X - xbar
t = t - tbar
XX = np.matrix(X) * np.matrix(np.transpose(X))
XX[dg] = XX[dg] + Lambda
U = scipy.linalg.cholesky(XX)
Xt = np.matrix(X) * np.matrix(np.transpose(t))
Unum = mldivide(np.transpose(U), Xt)
w = mldivide(U, Unum)
w0 = tbar - np.dot(w, xbar)
beta = 1 / np.mean(
np.power(t - np.matrix(np.transpose(w)) * np.matrix(X), 2))
return model(w, w0, xbar, beta, U)
def rvmRegEm(X, t, alpha=0.02, beta=0.5):
(d,n)=X.shape
xbar = np.mean(X,1)
tbar = np.mean(t,1)
# if(X.shape[1]!=1):
# xbar = xbar.reshape((xbar.shape[0],1))
# xbar = np.matlib.repmat(xbar,1,X.shape[1])
xbar = xbar.reshape((xbar.shape[0],1))
X = X-xbar
t = t-tbar
XX = X.dot(X.transpose())
Xt = X.dot(t.transpose())
alpha = np.array([alpha])*np.ones((d,))
beta = np.array([beta])
tol = 1e-3
maxiter = 500
inf = 1000000
llh = -inf*np.ones(maxiter)
index = np.array(range(0,d))
for iter in range(1,maxiter):
# remove zeros
nz = (1/alpha) > tol
nz = np.nonzero(nz)
index = index[nz]
alpha = alpha[nz]
XXa = XX[nz[0],:]
XX = XXa[:,nz[0]]
Xt = Xt[nz]
X = X[nz]
# E-step
U = scipy.linalg.cholesky(beta*XX+np.diag(alpha))
m = beta*mldivide(U,mldivide(U.transpose(),X.dot(t.transpose())))
m2 = m*m
e2 = np.sum((t-np.transpose(m).dot(X))*(t-np.transpose(m).dot(X)))
logdetS = 2*np.sum(np.log(np.diag(U)))
llh[iter] = 0.5*(np.sum(np.log(alpha))+n*np.log(beta)-beta*e2-logdetS-np.multiply(alpha,m2.reshape(alpha.shape)).sum()-n*np.log(2*np.pi))
if (np.abs(llh[iter]-llh[iter-1])<tol*np.abs(llh[iter-1])):
break
# M-step
V=np.linalg.inv(U)
dgS = np.multiply(V,V).sum(axis=1).reshape(m2.shape)
alpha = (1/(m2+dgS))
alpha = alpha.reshape(alpha.shape[0],)
UX = mldivide(U.transpose(),X)
trXSX = np.multiply(UX,UX).sum()
beta = n/(e2+trXSX)
llh = llh[1:iter+1]
model.index = index
model.w0 = tbar-np.multiply(m,xbar[nz]).sum()
model.w = m
model.alpha = alpha
model.beta = beta
# optional for bayesian probabilistic prediction purpose
model.xbar = xbar
model.U = U
return model,llh
def rvmRegFp(X, t, alpha=0.02, beta=0.5):
(d,n)=X.shape
xbar = np.mean(X,1).reshape((X.shape[0],1))
tbar = np.mean(t,1).reshape((t.shape[0],1))
X = X-xbar
t = t-tbar
XX = X.dot(X.transpose())
Xt = X.dot(t.transpose())
alpha = np.array([alpha])*np.ones((d,))
beta = np.array([beta])
tol = 1e-3
maxiter = 500
inf = 1000000
llh = -inf*np.ones(maxiter)
index = np.array(range(0,d))
for iter in range(1,maxiter):
# remove zeros
nz = (1/alpha) > tol
nz = np.nonzero(nz)
index = index[nz]
alpha = alpha[nz]
XXa = XX[nz[0],:]
XX = XXa[:,nz[0]]
Xt = Xt[nz]
X = X[nz]
U = scipy.linalg.cholesky(beta*XX+np.diag(alpha))
m = beta*mldivide(U,mldivide(U.transpose(),X.dot(t.transpose())))
m2 = (m*m).reshape(alpha.shape)
e = np.sum((t-np.transpose(m).dot(X))*(t-np.transpose(m).dot(X)))
logdetS = 2*np.sum(np.log(np.diag(U)))
llh[iter] = 0.5*(np.sum(np.log(alpha))+n*np.log(beta)-beta*e-logdetS-np.multiply(alpha,m2).sum()-n*np.log(2*np.pi))
if (np.abs(llh[iter]-llh[iter-1])<tol*np.abs(llh[iter-1])):
break
V=np.linalg.inv(U)
dgSigma = np.multiply(V,V).sum(axis=1).reshape(alpha.shape)
gamma = 1-alpha*dgSigma
alpha = gamma/m2
alpha = alpha.reshape(alpha.shape[0],)
beta = (n-np.sum(gamma))/e
llh = llh[1:iter+1]
model.index = index
model.w0 = tbar-np.multiply(m,xbar[nz]).sum()
model.w = m
model.alpha = alpha
model.beta = beta
# optional for bayesian probabilistic prediction purpose
model.xbar = xbar
model.U = U
return model,llh
def linRegEm(X,t,alpha=0.02,beta=0.5):
(d,n)=np.shape(X)
xbar = np.mean(X, 1)
tbar = np.mean(t, 1)
X = X-xbar
t = t-tbar
XX = X.dot(X.transpose())
Xt = X.dot(t.transpose())
tol = 1e-4
maxiter = 100
inf = -10000000
llh = -inf*np.ones((maxiter+1,1))
for iter in range(1,maxiter):
A = beta*XX + alpha*np.eye(d)
U = scipy.linalg.cholesky(A)
m = beta*(mldivide(U,mldivide(U.transpose(),Xt)))
m2 = np.multiply(m,m).sum(axis=0) # dot(m,m) in matlab
e2 = np.power(t-m.transpose().dot(X),2).sum(axis=1)
logdetA = 2*sum(np.log(np.diag(U))) #
llh[iter] = 0.5*(d*np.log(alpha)+n*np.log(beta)-alpha*m2-beta*e2-logdetA-n*np.log(2*np.pi))
if(np.abs(llh[iter]-llh[iter-1]) < tol*np.abs(llh[iter-1])):
break
V = scipy.linalg.inv(U)
trS = np.multiply(V,V).sum(axis=0) # A = inv(S)
alpha = d/(m2+trS)
UX = mldivide(U.transpose(),X)
trXSX = np.multiply(UX,UX).sum()
beta = n/(e2+trXSX)
w0 = tbar - np.multiply(m,xbar).sum(axis=0)
llh = llh[1:iter]
model.w0 = w0
model.w = m
# optional for bayesian probabilistic inference purpose
model.alpha = alpha
model.beta = beta
model.xbar = xbar
model.U = U
return model,llh
def linRegPred(model, X, t, nargout=2):
w = model.w
w0 = model.w0
y = np.transpose(w) * X + w0 # X and w already are matrix type
# probability prediction
beta = model.beta
if (
hasattr(model, "U")
): # python3 will have no problem, but python2 is dangerous to use hasattr()
U = model.U
Xo = X - model.xbar
XU = mldivide(np.transpose(U), Xo)
sigma = np.sqrt((1 + np.multiply(XU, XU).sum(axis=0)) / beta)
else:
if X.ndim == 2:
sigma = np.sqrt(1 / beta) * np.ones((1, np.size(X, 1)))
elif X.ndim == 1:
sigma = np.sqrt(1 / beta) * np.ones((1, np.size(X, 0)))
# try:
# U = model.U
# except AttributeError:
# sigma = np.sqrt(1/beta)*np.ones((1,np.size(X,1)))
# print("no U!")
if nargout == 3:
p = np.exp(logGauss(t, y, sigma))
return y, sigma, p
return y, sigma
def logitBin(X, t, Lambda, w):
# Logistic regression
(d, n) = np.shape(X)
tol = 1e-4
maxiter = 100
inf = 1000000
llh = np.ones(maxiter) * (-inf)
idx = np.array(range(0, d))
if len(idx) == 0:
idx = 0
dg = sub2ind((d, d), idx, idx)
h = np.ones((1, n))
h[t == 0] = -1
a = np.transpose(w).dot(X)
for iter in range(1, maxiter):
y = np.exp(-log1pexp(-a)) # y = sigmoid(a);
r = y * (1 - y)
Xw = X * np.sqrt(r)
H = Xw.dot(Xw.transpose())
H.flatten()[dg] = H.flatten()[dg] + Lambda
U = scipy.linalg.cholesky(H)
g = X.dot((y - t).transpose()).reshape((d, )) + Lambda * w
p = -mldivide(U, mldivide(U.transpose(), g))
wo = w
w = wo + p
a = w.transpose().dot(X)
llh[iter] = -np.sum(log1pexp(
-h * a)) - 0.5 * (Lambda * np.multiply(w, w)).sum()
incr = llh[iter] - llh[iter - 1]
while (incr < 0):
p = p / 2
w = wo + p
a = w.transpose().dot(X)
llh[iter] = -np.sum(log1pexp(
-h * a)) - 0.5 * (Lambda * np.multiply(w, w)).sum()
incr = llh[iter] - llh[iter - 1]
if incr < tol:
break
llh = llh[1:iter]
return w, llh, U
def logGauss(X, mu, sigma):
(d,k) = np.shape(mu)
y = np.array([])
if(np.all(np.array(np.shape(sigma))) and k == 1):
X = X - mu
R = scipy.linalg.cholesky(sigma)
Q = mldivide(np.transpose(R), X)
q = np.multiply(Q,Q).sum(axis=0) # quadratic term ( M distance) equals to q = dot(Q,Q,1) in matlab
c = d*np.log(2*np.pi)+2*np.sum(np.log(np.diag(R))) # normalization constant
y = -0.5*(c+q)
elif(np.shape(sigma)[0] == 1 and np.shape(sigma)[1] == np.shape(mu)[1]) : #k mu and (k or one) scalar sigma
XXdot = np.multiply(X,X).sum(axis=0)
X2 = np.matlib.repmat(np.transpose(XXdot),1,k)
D = X2 - 2*np.transpose(X).dot(mu) + np.multiply(mu,mu).sum(axis=0)
q = D*(1/sigma) # M distance
c = d*np.log(2*np.pi) + 2*np.log(sigma) # normalization constant
y = -0.5*(q+c)
return y
def newtonRaphson(X, t, Lambda):
(d, n) = np.shape(X)
k = np.max(t)
tol = 1e-4
maxiter = 100
inf = 1000000
llh = np.ones(maxiter) * (-inf)
dk = d * k
idx = np.array(range(0, dk))
if len(idx) == 0:
idx = 0
dg = sub2ind((dk, dk), idx, idx)
T = csr_matrix(
(np.ones(n, ), ((t - 1).reshape(n, ), np.array(range(0, n)))),
shape=(k, n)).toarray()
W = np.zeros((d, k))
HT = np.zeros((d, k, d, k))
for iter in range(1, maxiter):
A = W.transpose().dot(X)
logY = A - logsumexp(A, 0)
llh[iter] = np.multiply(
T, logY).sum() - 0.5 * Lambda * np.multiply(W, W).sum()
if (llh[iter] - llh[iter - 1] < tol):
break
Y = np.exp(logY)
for i in range(0, k):
for j in range(0, k):
r = Y[i, ] * ((i == j) - Y[j, ]
) # r has negative value, so cannot use sqrt
HT[:, i, :, j] = (X * r).dot(X.transpose())
G = X.dot((Y - T).transpose()) + Lambda * W
H = np.reshape(HT, (dk, dk))
Hi = H.flatten()
Hi[dg] = Hi[dg] + Lambda
H = Hi.reshape(H.shape)
Wi = W.flatten() - mldivide(H, G.flatten())
W = Wi.reshape(W.shape)
llh = llh[1:iter]
return W, llh
def linRegVb(X, t, Prior=0):
(m, n) = X.shape
if Prior == 0:
a0 = 1e-4
b0 = 1e-4
c0 = 1e-4
d0 = 1e-4
else:
a0 = Prior.a0
b0 = Prior.b0
c0 = Prior.c0
d0 = Prior.d0
I = np.eye(m)
xbar = np.mean(X, 1)
tbar = np.mean(t, 1)
X = X - xbar.reshape(xbar.size, 1)
t = t - tbar.reshape(tbar.size, 1)
XX = X.dot(X.transpose())
Xt = X.dot(t.transpose())
tol = 1e-8
maxiter = 100
inf = 10000000
energy = -inf * np.ones((maxiter + 1, 1))
a = a0 + m / 2
c = c0 + n / 2
Ealpha = 1e-4
Ebeta = 1e-4
for iter in range(1, maxiter):
invS = np.diag([Ealpha]) + Ebeta * XX
U = scipy.linalg.cholesky(invS)
Ew = Ebeta * (mldivide(U, mldivide(U.transpose(), Xt)))
KLw = -np.sum(np.log(np.diag(U)))
w2 = np.multiply(Ew, Ew).sum()
invU = mldivide(U, I)
trS = np.multiply(invU, invU).sum()
b = b0 + 0.5 * (w2 + trS)
Ealpha = a / b
KLalpha = -a * np.log(b)
e2pre = t - Ew.transpose().dot(X)
e2 = np.sum(np.multiply(e2pre, e2pre))
invUX = mldivide(U, X)
trXSX = np.multiply(invUX, invUX).sum()
d = d0 + 0.5 * (e2 + trXSX)
Ebeta = c / d
KLbeta = -c * np.log(d)
energy[iter] = KLalpha + KLbeta + KLw
if energy[iter] - energy[iter - 1] < tol * np.abs(energy[iter - 1]):
break
const = gammaln(a)-gammaln(a0)+gammaln(c)-gammaln(c0)+a0*np.log(b0)\
+ c0*np.log(d0)+0.5*(m-n*np.log(2*np.pi))
energy = energy[1:iter] + const
w0 = tbar - np.multiply(Ew, xbar).sum()
model.w0 = w0
model.w = Ew
model.alpha = Ealpha
model.beta = Ebeta
model.a = a
model.b = b
model.c = c
model.d = d
model.xbar = xbar
return model, energy