def l1regls_mosek(A, b):
"""
Returns the solution of l1-norm regularized least-squares problem
minimize || A*x - b ||_2^2 + e'*u
subject to -u <= x <= u
"""
m, n = A.size
env = mosek.Env()
task = env.Task(0, 0)
task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))
task.appendvars(2 * n) # number of variables
task.appendcons(2 * n) # number of constraints
# input quadratic objective
Q = matrix(0.0, (n, n))
blas.syrk(A, Q, alpha=2.0, trans='T')
I = []
for i in range(n):
I.extend(list(range(i, n)))
J = []
for i in range(n):
J.extend((n - i) * [i])
task.putqobj(I, J, list(Q[matrix(I) + matrix(J) * n]))
task.putclist(list(range(2 * n)),
list(-2 * A.T * b) + n * [1.0]) # setup linear objective
# input constraint matrix row by row
for i in range(n):
task.putarow(i, [i, n + i], [1.0, -1.0])
task.putarow(n + i, [i, n + i], [1.0, 1.0])
# setup bounds on constraints
task.putboundslice(mosek.accmode.con, 0, n, n * [mosek.boundkey.up],
n * [0.0], n * [0.0])
task.putboundslice(mosek.accmode.con, n, 2 * n,
n * [mosek.boundkey.lo], n * [0.0], n * [0.0])
# setup variable bounds
task.putboundslice(mosek.accmode.var, 0, 2 * n,
2 * n * [mosek.boundkey.fr], 2 * n * [0.0],
2 * n * [0.0])
# optimize the task
task.putobjsense(mosek.objsense.minimize)
task.optimize()
task.solutionsummary(mosek.streamtype.log)
x = n * [0.0]
task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)
return matrix(x)
def llt(L):
"""
Supernodal multifrontal Cholesky product:
.. math::
X = LL^T
where :math:`L` is lower-triangular. On exit, the argument `L`
contains the product :math:`X`.
:param L: :py:class:`cspmatrix` (factor)
"""
assert isinstance(L, cspmatrix) and L.is_factor is True, "L must be a cspmatrix factor"
n = L.symb.n
snpost = L.symb.snpost
snptr = L.symb.snptr
chptr = L.symb.chptr
chidx = L.symb.chidx
relptr = L.symb.relptr
relidx = L.symb.relidx
blkptr = L.symb.blkptr
blkval = L.blkval
stack = []
for k in snpost:
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# compute [I; L_{Ak,Nk}]*D_k*[I;L_{Ak,Nk}]'; store in F
blas.trmm(blkval, blkval, side = "R", m = na, n = nn, ldA = nj,\
ldB = nj, offsetA = blkptr[k], offsetB = blkptr[k]+nn)
F = matrix(0.0, (nj, nj))
blas.syrk(blkval, F, n = nj, k = nn, offsetA = blkptr[k], ldA = nj)
# if supernode k has any children, subtract update matrices
for _ in range(chptr[k],chptr[k+1]):
Ui, i = stack.pop()
frontal_add_update(F, Ui, relidx, relptr, i)
# if supernode k is not a root node, push update matrix onto stack
if na > 0:
Uk = matrix(0.0,(na,na))
lapack.lacpy(F, Uk, m = na, n = na, uplo = 'L', offsetA = nn*nj+nn, ldA = nj)
stack.append((Uk,k))
# copy leading Nk columns of F to blkval
lapack.lacpy(F, blkval, m = nj, n = nn, ldA = nj, uplo = 'L',\
ldB = nj, offsetB = blkptr[k])
L._is_factor = False
return
def l1regls_mosek(A, b):
"""
Returns the solution of l1-norm regularized least-squares problem
minimize || A*x - b ||_2^2 + e'*u
subject to -u <= x <= u
"""
m, n = A.size
env = mosek.Env()
task = env.Task(0,0)
task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))
task.appendvars( 2*n) # number of variables
task.appendcons( 2*n) # number of constraints
# input quadratic objective
Q = matrix(0.0, (n,n))
blas.syrk(A, Q, alpha = 2.0, trans='T')
I = []
for i in range(n):
I.extend(range(i,n))
J = []
for i in range(n):
J.extend((n-i)*[i])
task.putqobj(I, J, list(Q[matrix(I) + matrix(J)*n]))
task.putclist(range(2*n), list(-2*A.T*b) + n*[1.0]) # setup linear objective
# input constraint matrix row by row
for i in range(n):
task.putarow( i, [i, n+i], [1.0, -1.0])
task.putarow( n+i, [i, n+i], [1.0, 1.0])
# setup bounds on constraints
task.putboundslice(mosek.accmode.con,
0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])
task.putboundslice(mosek.accmode.con,
n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])
# setup variable bounds
task.putboundslice(mosek.accmode.var,
0, 2*n, 2*n*[mosek.boundkey.fr], 2*n*[0.0], 2*n*[0.0])
# optimize the task
task.putobjsense(mosek.objsense.minimize)
task.optimize()
task.solutionsummary(mosek.streamtype.log)
x = n*[0.0]
task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)
return matrix(x)
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(A, Asc)
for k in range(m):
blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m + 1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \
mul(d1, z[:n] + mul(d3, z[:n])) - \
mul(d2, z[n:] - mul(d3, z[n:])) )
x[:n] = div(x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n])
z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:])
return g
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), sqrt(d1+d2) )
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(A, Asc)
for k in range(m):
blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \
mul(d1, z[:n] + mul(d3, z[:n])) - \
mul(d2, z[n:] - mul(d3, z[n:])) )
x[:n] = div( x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] )
z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] )
return g
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
print 'printing: ', W['di']
# ds is square root of diagonal of D
ds = math.sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
Asc = matrix(0.0, (m, n))
# Asc = A*diag(d)^-1/2
Asc = A * spdiag(ds**-1)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m + 1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * (x[:n] - mul(d3, x[n:]) + mul(
d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - mul(d3, z[n:])))
x[:n] = div(x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n] -
# (D2-D1)*(D1+D2)^-1 * bx[n:] +
# D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
# D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
# zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n])
z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:])
return g
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]),
sqrt(d1+d2) )
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
Asc = A * spdiag(ds**-1)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) +
mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] -
mul(d3, z[n:])) )
x[:n] = div( x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n] -
# (D2-D1)*(D1+D2)^-1 * bx[n:] +
# D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
# D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
# zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] )
z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] )
return g
def Fgp(x = None, z = None):
if x is None: return mnl, matrix(0.0, (n,1))
f = matrix(0.0, (mnl+1,1))
Df = matrix(0.0, (mnl+1,n))
# y = F*x+g
blas.copy(g, y)
base.gemv(F, x, y, beta=1.0)
if z is not None: H = matrix(0.0, (n,n))
for i, start, stop in ind:
# yi := exp(yi) = exp(Fi*x+gi)
ymax = max(y[start:stop])
y[start:stop] = base.exp(y[start:stop] - ymax)
# fi = log sum yi = log sum exp(Fi*x+gi)
ysum = blas.asum(y, n=stop-start, offset=start)
f[i] = ymax + math.log(ysum)
# yi := yi / sum(yi) = exp(Fi*x+gi) / sum(exp(Fi*x+gi))
blas.scal(1.0/ysum, y, n=stop-start, offset=start)
# gradfi := Fi' * yi
# = Fi' * exp(Fi*x+gi) / sum(exp(Fi*x+gi))
base.gemv(F, y, Df, trans='T', m=stop-start, incy=mnl+1,
offsetA=start, offsetx=start, offsety=i)
if z is not None:
# Hi = Fi' * (diag(yi) - yi*yi') * Fi
# = Fisc' * Fisc
# where
# Fisc = diag(yi)^1/2 * (I - 1*yi') * Fi
# = diag(yi)^1/2 * (Fi - 1*gradfi')
Fsc[:K[i], :] = F[start:stop, :]
for k in range(start,stop):
blas.axpy(Df, Fsc, n=n, alpha=-1.0, incx=mnl+1,
incy=Fsc.size[0], offsetx=i, offsety=k-start)
blas.scal(math.sqrt(y[k]), Fsc, inc=Fsc.size[0],
offset=k-start)
# H += z[i]*Hi = z[i] * Fisc' * Fisc
blas.syrk(Fsc, H, trans='T', k=stop-start, alpha=z[i],
beta=1.0)
if z is None:
return f, Df
return f, Df, H
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ]
#
# where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di'].
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (di .* z is
# returned instead of z).
# Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and
# d1 = d[:m].^2, d2 = d[m:].^2.
di = W['di']
d1, d2 = di[:m]**2, di[m:]**2
D = div(mul(d1, d2), d1 + d2)
Ds = spdiag(2 * sqrt(D))
base.gemm(Ds, P, Ps)
blas.syrk(Ps, A, trans='T')
lapack.potrf(A)
def f(x, y, z):
# Solve for x[:n]:
#
# A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:]
# + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ).
blas.copy((mul(div(d1 - d2, d1 + d2), x[n:]) +
mul(2 * D, z[:m] - z[m:])), u)
blas.gemv(P, u, x, beta=1.0, trans='T')
lapack.potrs(A, x)
# x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:]
# + (D1-D2)*P*x[:n])
base.gemv(P, x, u)
x[n:] = div(
x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) + mul(d1 - d2, u),
d1 + d2)
# z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m])
# z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:])
z[:m] = mul(di[:m], u - x[n:] - z[:m])
z[m:] = mul(di[m:], -u - x[n:] - z[m:])
return f
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ]
#
# where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di'].
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (di .* z is
# returned instead of z).
# Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and
# d1 = d[:m].^2, d2 = d[m:].^2.
di = W['di']
d1, d2 = di[:m]**2, di[m:]**2
D = div( mul(d1,d2), d1+d2 )
Ds = spdiag(2 * sqrt(D))
base.gemm(Ds, P, Ps)
blas.syrk(Ps, A, trans = 'T')
lapack.potrf(A)
def f(x, y, z):
# Solve for x[:n]:
#
# A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:]
# + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ).
blas.copy(( mul( div(d1-d2, d1+d2), x[n:]) +
mul( 2*D, z[:m]-z[m:] ) ), u)
blas.gemv(P, u, x, beta = 1.0, trans = 'T')
lapack.potrs(A, x)
# x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:]
# + (D1-D2)*P*x[:n])
base.gemv(P, x, u)
x[n:] = div( x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) +
mul(d1-d2, u), d1+d2 )
# z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m])
# z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:])
z[:m] = mul(di[:m], u-x[n:]-z[:m])
z[m:] = mul(di[m:], -u-x[n:]-z[m:])
return f
def Fkkt(x, z, W):
ds = (2.0 * div(1 + x**2, (1 - x**2)**2))**-0.5
Asc = A * spdiag(ds)
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
a = z[0]
def g(x, y, z):
x[:] = mul(x, ds) / a
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
blas.gemv(Asc, v, x, alpha = -1.0, beta = 1.0, trans = 'T')
x[:] = mul(x, ds)
return g
def F(x=None, z=None):
if x is None: return 0, matrix(1.0, (n,1))
X = V * spdiag(x) * V.T
L = +X
try: lapack.potrf(L)
except ArithmeticError: return None
f = - 2.0 * (log(L[0,0]) + log(L[1,1]))
W = +V
blas.trsm(L, W)
gradf = matrix(-1.0, (1,2)) * W**2
if z is None: return f, gradf
H = matrix(0.0, (n,n))
blas.syrk(W, H, trans='T')
return f, gradf, z[0] * H**2
def Fkkt(x, z, W):
ds = (2.0 * div(1 + x**2, (1 - x**2)**2))**-0.5
Asc = A * spdiag(ds)
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
a = z[0]
def g(x, y, z):
x[:] = mul(x, ds) / a
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
blas.gemv(Asc, v, x, alpha = -1.0, beta = 1.0, trans = 'T')
x[:] = mul(x, ds)
return g
def barrier():
# variables kept same from cvxopt example
MAXITERS = 100
ALPHA = 0.01
BETA = 0.5
x = matrix(0.0, (n, 1))
H = matrix(0.0, (n, n)) # Symmetrix matrix
for iter in range(MAXITERS):
# get the gradient of the function
d = (b - A * x)**-1
g = A.T * d
# print(d[:, n*[0]].size)
# print(A.size)
"""bug here: won't multiply of two matrix of same dimension. code is looking into another dimension?"""
# get Hessian
# lower diaganol multiplied to constraint matrix, n*[0] is first center x^t(0)
h = np.zeros(shape=(m, n))
np.matmul(d[:, n * [0]], A, h)
# use the BLAS solver to get the symmetric matrix and get roots
blas.syrk(h, H, trans='T')
# do Newton's step
v = -g # g is our gradient
# LAPACK solves the matrix and gives us the tep value to transverse with
lapack.posv(H, v)
# Stop condition if exceeding tolerance
lam = blas.dot(g, v)
if sqrt(-lam) < mu:
return x # return the orignal value if we're above tolerance
# Line search to go to optimal using ALPHA and BETA
y = mul(A * v, d)
step = 1.0
while 1 - step * max(y) < 0:
step *= BETA
while True:
if -sum(log(1 - step * y)) < (ALPHA * step * lam):
break
step *= BETA
# increment x by the step times the negative gradient otherwise
x += step * v
def acent(A, b):
"""
Computes analytic center of A*x <= b with A m by n of rank n.
We assume that b > 0 and the feasible set is bounded.
"""
MAXITERS = 100
ALPHA = 0.01
BETA = 0.5
TOL = 1e-8
ntdecrs = []
m, n = A.size
x = matrix(0.0, (n, 1))
H = matrix(0.0, (n, n))
for iter in range(MAXITERS):
# Gradient is g = A^T * (1./(b-A*x)).
d = (b - A * x) ** -1
g = A.T * d
# Hessian is H = A^T * diag(1./(b-A*x))^2 * A.
Asc = mul(d[:, n * [0]], A)
blas.syrk(Asc, H, trans="T")
# Newton step is v = H^-1 * g.
v = -g
lapack.posv(H, v)
# Directional derivative and Newton decrement.
lam = blas.dot(g, v)
ntdecrs += [sqrt(-lam)]
print("%2d. Newton decr. = %3.3e" % (iter, ntdecrs[-1]))
if ntdecrs[-1] < TOL:
return x, ntdecrs
# Backtracking line search.
y = mul(A * v, d)
step = 1.0
while 1 - step * max(y) < 0:
step *= BETA
while True:
if -sum(log(1 - step * y)) < ALPHA * step * lam:
break
step *= BETA
x += step * v
def acent(A, b):
"""
Computes analytic center of A*x <= b with A m by n of rank n.
We assume that b > 0 and the feasible set is bounded.
"""
MAXITERS = 100
ALPHA = 0.01
BETA = 0.5
TOL = 1e-8
ntdecrs = []
m, n = A.size
x = matrix(0.0, (n, 1))
H = matrix(0.0, (n, n))
for iter in range(MAXITERS):
# Gradient is g = A^T * (1./(b-A*x)).
d = (b - A * x)**-1
g = A.T * d
# Hessian is H = A^T * diag(1./(b-A*x))^2 * A.
Asc = mul(d[:, n * [0]], A)
blas.syrk(Asc, H, trans='T')
# Newton step is v = H^-1 * g.
v = -g
lapack.posv(H, v)
# Directional derivative and Newton decrement.
lam = blas.dot(g, v)
ntdecrs += [sqrt(-lam)]
print("%2d. Newton decr. = %3.3e" % (iter, ntdecrs[-1]))
if ntdecrs[-1] < TOL: return x, ntdecrs
# Backtracking line search.
y = mul(A * v, d)
step = 1.0
while 1 - step * max(y) < 0:
step *= BETA
while True:
if -sum(log(1 - step * y)) < ALPHA * step * lam: break
step *= BETA
x += step * v
def F(W):
"""
Create a solver for the linear equations
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - Dl^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - r*r' * [ ] * r*r' = bzs
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
where Dl = diag(W['l']), r = W['r'][0].
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ].
1. Compute matrices V1, V2 such that (with T = r*r')
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and S = [ diag(s); 0 ], s a positive q-vector.
2. Factor the mapping X -> X + S * X' * S:
X + S * X' * S = L( L'( X )).
3. Compute scaled mappings: a matrix As with as its columns the
coefficients of the scaled mapping
L^-1( V2' * A() * V1' )
and the matrix Gs = Dl^-1 * G.
4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As.
"""
# 1. Compute V1, V2, s.
r = W['r'][0]
# LQ factorization R[:q, :] = L1 * Q1.
lapack.lacpy(r, Q1, m=q)
lapack.gelqf(Q1, tau1)
lapack.lacpy(Q1, L1, n=q, uplo='L')
lapack.orglq(Q1, tau1)
# LQ factorization R[q:, :] = L2 * Q2.
lapack.lacpy(r, Q2, m=p, offsetA=q)
lapack.gelqf(Q2, tau2)
lapack.lacpy(Q2, L2, n=p, uplo='L')
lapack.orglq(Q2, tau2)
# V2, V1, s are computed from an SVD: if
#
# Q2 * Q1' = U * diag(s) * V',
#
# then V1 = V' * L1^-1 and V2 = L2^-T * U.
# T21 = Q2 * Q1.T
blas.gemm(Q2, Q1, T21, transB='T')
# SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1.
lapack.gesvd(T21, s, jobu='A', jobvt='A', U=V2, Vt=V1)
# # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2
# this will requires lapack.ormlq or lapack.unmlq
# V2 = L2^-T * U
blas.trsm(L2, V2, transA='T')
# V1 = V' * L1^-1
blas.trsm(L1, V1, side='R')
# 2. Factorization X + S * X' * S = L( L'( X )).
#
# The factor L is stored as a diagonal matrix D and a sparse lower
# triangular matrix P, such that
#
# L(X)[:] = D**-1 * (I + P) * X[:]
# L^-1(X)[:] = D * (I - P) * X[:].
# SS is q x q with SS[i,j] = si*sj.
blas.scal(0.0, SS)
blas.syr(s, SS)
# For a p x q matrix X, P*X[:] is Y[:] where
#
# Yij = si * sj * Xji if i < j
# = 0 otherwise.
#
P.V = SS[Itril2]
# For a p x q matrix X, D*X[:] is Y[:] where
#
# Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j
# = Xii / sqrt( 1 + si^2 ) if i = j
# = Xij otherwise.
#
DV[Idiag] = sqrt(1.0 + SS[::q + 1])
DV[Itriu] = sqrt(1.0 - SS[Itril3]**2)
D.V = DV**-1
# 3. Scaled linear mappings
# Ask := V2' * Ask * V1'
blas.scal(0.0, As)
base.axpy(A, As)
for i in xrange(n):
# tmp := V2' * As[i, :]
blas.gemm(V2,
As,
tmp,
transA='T',
m=p,
n=q,
k=p,
ldB=p,
offsetB=i * p * q)
# As[:,i] := tmp * V1'
blas.gemm(tmp,
V1,
As,
transB='T',
m=p,
n=q,
k=q,
ldC=p,
offsetC=i * p * q)
# As := D * (I - P) * As
# = L^-1 * As.
blas.copy(As, As2)
base.gemm(P, As, As2, alpha=-1.0, beta=1.0)
base.gemm(D, As2, As)
# Gs := Dl^-1 * G
blas.scal(0.0, Gs)
base.axpy(G, Gs)
for k in xrange(n):
blas.tbmv(W['di'], Gs, n=m, k=0, ldA=1, offsetx=k * m)
# 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As.
blas.syrk(As, H, trans='T', alpha=2.0)
blas.syrk(Gs, H, trans='T', beta=1.0)
base.axpy(C, H)
lapack.potrf(H)
def f(x, y, z):
"""
Solve
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - T * [ ] * T = bzs.
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ].
Define X = uzs21, Z = T * uzs * T:
C * ux + G' * uzl - 2*A'(X) = bx
[ 0 X' ] [ bX1 0 ]
T * [ ] * T - Z = T * [ ] * T
[ X 0 ] [ 0 bX2 ]
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs
[ -A(ux) -uX2 ] [ Z21 Z22 ]
Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ].
We use the congruence transformation
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and the factorization
X + S * X' * S = L( L'(X) )
to write this as
C * ux + G' * uzl - 2*A'(X) = bx
L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs,
[ -A(ux) -uX2 ] [ Z21 Z22 ]
or
C * ux + Gs' * uuzl - 2*As'(XX) = bx
XX - ZZ21 = bX
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - ZZ21 = bbzs_21
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
if we introduce scaled variables
uuzl = D * uzl
XX = L'(V2^-1 * X * V1^-1)
= L'(V2^-1 * uzs21 * V1^-1)
ZZ21 = L^-1(V2' * Z21 * V1')
and define
bbzs_21 = L^-1(V2' * bzs_21 * V1')
[ bX1 0 ]
bX = L^-1( V2' * (T * [ ] * T)_21 * V1').
[ 0 bX2 ]
Eliminating Z21 gives
C * ux + Gs' * uuzl - 2*As'(XX) = bx
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
and eliminating uuzl and XX gives
H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21)
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22.
In summary, we can use the following algorithm:
1. bXX := bX - bbzs21
[ bX1 0 ]
= L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1')
[ 0 bX2 ]
2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
3. From ux, compute
uuzl = Gs*ux - D^-1 * bzl and
X = V2 * L^-T(-As(ux) + bXX) * V1.
4. Return ux, uuzl,
rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r
and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22.
"""
# Save bzs_11, bzs_22, bzs_21.
lapack.lacpy(z, bz11, uplo='L', m=q, n=q, ldA=p + q, offsetA=m)
lapack.lacpy(z, bz21, m=p, n=q, ldA=p + q, offsetA=m + q)
lapack.lacpy(z,
bz22,
uplo='L',
m=p,
n=p,
ldA=p + q,
offsetA=m + (p + q + 1) * q)
# zl := D^-1 * zl
# = D^-1 * bzl
blas.tbmv(W['di'], z, n=m, k=0, ldA=1)
# zs := r' * [ bX1, 0; 0, bX2 ] * r.
# zs := [ bX1, 0; 0, bX2 ]
blas.scal(0.0, z, offset=m)
lapack.lacpy(x[1], z, uplo='L', m=q, n=q, ldB=p + q, offsetB=m)
lapack.lacpy(x[2],
z,
uplo='L',
m=p,
n=p,
ldB=p + q,
offsetB=m + (p + q + 1) * q)
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc=p + q + 1, offset=m)
# a := tril(zs)*r
blas.copy(r, a)
blas.trmm(z,
a,
side='L',
m=p + q,
n=p + q,
ldA=p + q,
ldB=p + q,
offsetA=m)
# zs := a'*r + r'*a
blas.syr2k(r,
a,
z,
trans='T',
n=p + q,
k=p + q,
ldB=p + q,
ldC=p + q,
offsetC=m)
# bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1')
#
# [ bX1 0 ]
# = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1').
# [ 0 bX2 ]
# a = [ r21 r22 ] * z
# = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r
# = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r
blas.symm(z,
r,
a,
side='R',
m=p,
n=p + q,
ldA=p + q,
ldC=p + q,
offsetB=q)
# bz21 := -bz21 + a * [ r11, r12 ]'
# = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21
blas.gemm(a,
r,
bz21,
transB='T',
m=p,
n=q,
k=p + q,
beta=-1.0,
ldA=p + q,
ldC=p)
# bz21 := V2' * bz21 * V1'
# = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1'
blas.gemm(V2, bz21, tmp, transA='T', m=p, n=q, k=p, ldB=p)
blas.gemm(tmp, V1, bz21, transB='T', m=p, n=q, k=q, ldC=p)
# bz21[:] := D * (I-P) * bz21[:]
# = L^-1 * bz21[:]
# = bXX[:]
blas.copy(bz21, tmp)
base.gemv(P, bz21, tmp, alpha=-1.0, beta=1.0)
base.gemv(D, tmp, bz21)
# Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
# x[0] := x[0] + Gs'*zl + 2*As'(bz21)
# = bx + G' * D^-1 * bzl + 2 * As'(bXX)
blas.gemv(Gs, z, x[0], trans='T', alpha=1.0, beta=1.0)
blas.gemv(As, bz21, x[0], trans='T', alpha=2.0, beta=1.0)
# x[0] := H \ x[0]
# = ux
lapack.potrs(H, x[0])
# uuzl = Gs*ux - D^-1 * bzl
blas.gemv(Gs, x[0], z, alpha=1.0, beta=-1.0)
# bz21 := V2 * L^-T(-As(ux) + bz21) * V1
# = X
blas.gemv(As, x[0], bz21, alpha=-1.0, beta=1.0)
blas.tbsv(DV, bz21, n=p * q, k=0, ldA=1)
blas.copy(bz21, tmp)
base.gemv(P, tmp, bz21, alpha=-1.0, beta=1.0, trans='T')
blas.gemm(V2, bz21, tmp)
blas.gemm(tmp, V1, bz21)
# zs := -zs + r' * [ 0, X'; X, 0 ] * r
# = r' * [ -bX1, X'; X, -bX2 ] * r.
# a := bz21 * [ r11, r12 ]
# = X * [ r11, r12 ]
blas.gemm(bz21, r, a, m=p, n=p + q, k=q, ldA=p, ldC=p + q)
# z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ]
# = rti' * uzs * rti
blas.syr2k(r,
a,
z,
trans='T',
beta=-1.0,
n=p + q,
k=p,
offsetA=q,
offsetC=m,
ldB=p + q,
ldC=p + q)
# uX1 = -Z11 - bzs_11
# = -(r*zs*r')_11 - bzs_11
# uX2 = -Z22 - bzs_22
# = -(r*zs*r')_22 - bzs_22
blas.copy(bz11, x[1])
blas.copy(bz22, x[2])
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc=p + q + 1, offset=m)
# a := r*tril(zs)
blas.copy(r, a)
blas.trmm(z,
a,
side='R',
m=p + q,
n=p + q,
ldA=p + q,
ldB=p + q,
offsetA=m)
# x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]'
# = -bzs_11 - (r*zs*r')_11
blas.syr2k(a, r, x[1], n=q, alpha=-1.0, beta=-1.0)
# x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]'
# = -bzs_22 - (r*zs*r')_22
blas.syr2k(a,
r,
x[2],
n=p,
alpha=-1.0,
beta=-1.0,
offsetA=q,
offsetB=q)
# scale diagonal of zs by 1/2
blas.scal(2.0, z, inc=p + q + 1, offset=m)
return f
def F(W):
"""
Create a solver for the linear equations
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - Dl^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - r*r' * [ ] * r*r' = bzs
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
where Dl = diag(W['l']), r = W['r'][0].
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ].
1. Compute matrices V1, V2 such that (with T = r*r')
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and S = [ diag(s); 0 ], s a positive q-vector.
2. Factor the mapping X -> X + S * X' * S:
X + S * X' * S = L( L'( X )).
3. Compute scaled mappings: a matrix As with as its columns the
coefficients of the scaled mapping
L^-1( V2' * A() * V1' )
and the matrix Gs = Dl^-1 * G.
4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As.
"""
# 1. Compute V1, V2, s.
r = W['r'][0]
# LQ factorization R[:q, :] = L1 * Q1.
lapack.lacpy(r, Q1, m = q)
lapack.gelqf(Q1, tau1)
lapack.lacpy(Q1, L1, n = q, uplo = 'L')
lapack.orglq(Q1, tau1)
# LQ factorization R[q:, :] = L2 * Q2.
lapack.lacpy(r, Q2, m = p, offsetA = q)
lapack.gelqf(Q2, tau2)
lapack.lacpy(Q2, L2, n = p, uplo = 'L')
lapack.orglq(Q2, tau2)
# V2, V1, s are computed from an SVD: if
#
# Q2 * Q1' = U * diag(s) * V',
#
# then V1 = V' * L1^-1 and V2 = L2^-T * U.
# T21 = Q2 * Q1.T
blas.gemm(Q2, Q1, T21, transB = 'T')
# SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1.
lapack.gesvd(T21, s, jobu = 'A', jobvt = 'A', U = V2, Vt = V1)
# # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2
# this will requires lapack.ormlq or lapack.unmlq
# V2 = L2^-T * U
blas.trsm(L2, V2, transA = 'T')
# V1 = V' * L1^-1
blas.trsm(L1, V1, side = 'R')
# 2. Factorization X + S * X' * S = L( L'( X )).
#
# The factor L is stored as a diagonal matrix D and a sparse lower
# triangular matrix P, such that
#
# L(X)[:] = D**-1 * (I + P) * X[:]
# L^-1(X)[:] = D * (I - P) * X[:].
# SS is q x q with SS[i,j] = si*sj.
blas.scal(0.0, SS)
blas.syr(s, SS)
# For a p x q matrix X, P*X[:] is Y[:] where
#
# Yij = si * sj * Xji if i < j
# = 0 otherwise.
#
P.V = SS[Itril2]
# For a p x q matrix X, D*X[:] is Y[:] where
#
# Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j
# = Xii / sqrt( 1 + si^2 ) if i = j
# = Xij otherwise.
#
DV[Idiag] = sqrt(1.0 + SS[::q+1])
DV[Itriu] = sqrt(1.0 - SS[Itril3]**2)
D.V = DV**-1
# 3. Scaled linear mappings
# Ask := V2' * Ask * V1'
blas.scal(0.0, As)
base.axpy(A, As)
for i in xrange(n):
# tmp := V2' * As[i, :]
blas.gemm(V2, As, tmp, transA = 'T', m = p, n = q, k = p,
ldB = p, offsetB = i*p*q)
# As[:,i] := tmp * V1'
blas.gemm(tmp, V1, As, transB = 'T', m = p, n = q, k = q,
ldC = p, offsetC = i*p*q)
# As := D * (I - P) * As
# = L^-1 * As.
blas.copy(As, As2)
base.gemm(P, As, As2, alpha = -1.0, beta = 1.0)
base.gemm(D, As2, As)
# Gs := Dl^-1 * G
blas.scal(0.0, Gs)
base.axpy(G, Gs)
for k in xrange(n):
blas.tbmv(W['di'], Gs, n = m, k = 0, ldA = 1, offsetx = k*m)
# 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As.
blas.syrk(As, H, trans = 'T', alpha = 2.0)
blas.syrk(Gs, H, trans = 'T', beta = 1.0)
base.axpy(C, H)
lapack.potrf(H)
def f(x, y, z):
"""
Solve
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - T * [ ] * T = bzs.
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ].
Define X = uzs21, Z = T * uzs * T:
C * ux + G' * uzl - 2*A'(X) = bx
[ 0 X' ] [ bX1 0 ]
T * [ ] * T - Z = T * [ ] * T
[ X 0 ] [ 0 bX2 ]
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs
[ -A(ux) -uX2 ] [ Z21 Z22 ]
Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ].
We use the congruence transformation
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and the factorization
X + S * X' * S = L( L'(X) )
to write this as
C * ux + G' * uzl - 2*A'(X) = bx
L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs,
[ -A(ux) -uX2 ] [ Z21 Z22 ]
or
C * ux + Gs' * uuzl - 2*As'(XX) = bx
XX - ZZ21 = bX
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - ZZ21 = bbzs_21
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
if we introduce scaled variables
uuzl = D * uzl
XX = L'(V2^-1 * X * V1^-1)
= L'(V2^-1 * uzs21 * V1^-1)
ZZ21 = L^-1(V2' * Z21 * V1')
and define
bbzs_21 = L^-1(V2' * bzs_21 * V1')
[ bX1 0 ]
bX = L^-1( V2' * (T * [ ] * T)_21 * V1').
[ 0 bX2 ]
Eliminating Z21 gives
C * ux + Gs' * uuzl - 2*As'(XX) = bx
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
and eliminating uuzl and XX gives
H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21)
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22.
In summary, we can use the following algorithm:
1. bXX := bX - bbzs21
[ bX1 0 ]
= L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1')
[ 0 bX2 ]
2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
3. From ux, compute
uuzl = Gs*ux - D^-1 * bzl and
X = V2 * L^-T(-As(ux) + bXX) * V1.
4. Return ux, uuzl,
rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r
and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22.
"""
# Save bzs_11, bzs_22, bzs_21.
lapack.lacpy(z, bz11, uplo = 'L', m = q, n = q, ldA = p+q,
offsetA = m)
lapack.lacpy(z, bz21, m = p, n = q, ldA = p+q, offsetA = m+q)
lapack.lacpy(z, bz22, uplo = 'L', m = p, n = p, ldA = p+q,
offsetA = m + (p+q+1)*q)
# zl := D^-1 * zl
# = D^-1 * bzl
blas.tbmv(W['di'], z, n = m, k = 0, ldA = 1)
# zs := r' * [ bX1, 0; 0, bX2 ] * r.
# zs := [ bX1, 0; 0, bX2 ]
blas.scal(0.0, z, offset = m)
lapack.lacpy(x[1], z, uplo = 'L', m = q, n = q, ldB = p+q,
offsetB = m)
lapack.lacpy(x[2], z, uplo = 'L', m = p, n = p, ldB = p+q,
offsetB = m + (p+q+1)*q)
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc = p+q+1, offset = m)
# a := tril(zs)*r
blas.copy(r, a)
blas.trmm(z, a, side = 'L', m = p+q, n = p+q, ldA = p+q, ldB =
p+q, offsetA = m)
# zs := a'*r + r'*a
blas.syr2k(r, a, z, trans = 'T', n = p+q, k = p+q, ldB = p+q,
ldC = p+q, offsetC = m)
# bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1')
#
# [ bX1 0 ]
# = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1').
# [ 0 bX2 ]
# a = [ r21 r22 ] * z
# = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r
# = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r
blas.symm(z, r, a, side = 'R', m = p, n = p+q, ldA = p+q,
ldC = p+q, offsetB = q)
# bz21 := -bz21 + a * [ r11, r12 ]'
# = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21
blas.gemm(a, r, bz21, transB = 'T', m = p, n = q, k = p+q,
beta = -1.0, ldA = p+q, ldC = p)
# bz21 := V2' * bz21 * V1'
# = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1'
blas.gemm(V2, bz21, tmp, transA = 'T', m = p, n = q, k = p,
ldB = p)
blas.gemm(tmp, V1, bz21, transB = 'T', m = p, n = q, k = q,
ldC = p)
# bz21[:] := D * (I-P) * bz21[:]
# = L^-1 * bz21[:]
# = bXX[:]
blas.copy(bz21, tmp)
base.gemv(P, bz21, tmp, alpha = -1.0, beta = 1.0)
base.gemv(D, tmp, bz21)
# Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
# x[0] := x[0] + Gs'*zl + 2*As'(bz21)
# = bx + G' * D^-1 * bzl + 2 * As'(bXX)
blas.gemv(Gs, z, x[0], trans = 'T', alpha = 1.0, beta = 1.0)
blas.gemv(As, bz21, x[0], trans = 'T', alpha = 2.0, beta = 1.0)
# x[0] := H \ x[0]
# = ux
lapack.potrs(H, x[0])
# uuzl = Gs*ux - D^-1 * bzl
blas.gemv(Gs, x[0], z, alpha = 1.0, beta = -1.0)
# bz21 := V2 * L^-T(-As(ux) + bz21) * V1
# = X
blas.gemv(As, x[0], bz21, alpha = -1.0, beta = 1.0)
blas.tbsv(DV, bz21, n = p*q, k = 0, ldA = 1)
blas.copy(bz21, tmp)
base.gemv(P, tmp, bz21, alpha = -1.0, beta = 1.0, trans = 'T')
blas.gemm(V2, bz21, tmp)
blas.gemm(tmp, V1, bz21)
# zs := -zs + r' * [ 0, X'; X, 0 ] * r
# = r' * [ -bX1, X'; X, -bX2 ] * r.
# a := bz21 * [ r11, r12 ]
# = X * [ r11, r12 ]
blas.gemm(bz21, r, a, m = p, n = p+q, k = q, ldA = p, ldC = p+q)
# z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ]
# = rti' * uzs * rti
blas.syr2k(r, a, z, trans = 'T', beta = -1.0, n = p+q, k = p,
offsetA = q, offsetC = m, ldB = p+q, ldC = p+q)
# uX1 = -Z11 - bzs_11
# = -(r*zs*r')_11 - bzs_11
# uX2 = -Z22 - bzs_22
# = -(r*zs*r')_22 - bzs_22
blas.copy(bz11, x[1])
blas.copy(bz22, x[2])
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc = p+q+1, offset = m)
# a := r*tril(zs)
blas.copy(r, a)
blas.trmm(z, a, side = 'R', m = p+q, n = p+q, ldA = p+q, ldB =
p+q, offsetA = m)
# x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]'
# = -bzs_11 - (r*zs*r')_11
blas.syr2k(a, r, x[1], n = q, alpha = -1.0, beta = -1.0)
# x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]'
# = -bzs_22 - (r*zs*r')_22
blas.syr2k(a, r, x[2], n = p, alpha = -1.0, beta = -1.0,
offsetA = q, offsetB = q)
# scale diagonal of zs by 1/2
blas.scal(2.0, z, inc = p+q+1, offset = m)
return f
def custom_kkt(W):
"""
Custom KKT solver for the following conic LP
formulation of the Schur relaxation of the
balance-constrained min/max cut problem
maximize Tr(C,X)
subject to
X_{ii} = 1, i=1...n
sum(X) + x = const
x >= 0, X psd
"""
r = W['rti'][0]
N = r.size[0]
e = matrix(1.0, (N, 1))
# Form and factorize reduced KKT system
H = matrix(0.0, (N + 1, N + 1))
blas.syrk(r, H, n=N, ldC=N + 1)
blas.symv(H, e, H, n=N, ldA=N + 1, offsety=N, incy=N + 1)
H[N, N] = blas.dot(H, e, n=N, offsetx=N, incx=N + 1)
rr = H[:N, :N] # Extract and symmetrize (1,1) block
misc.symm(rr, N) #
q = H[N, :N].T # Extract q = rr*e
H = mul(H, H)
H[N, N] += W['di'][0]**2
lapack.potrf(H)
def fsolve(x, y, z):
"""
Solves the system of equations
[ 0 G'*W^{-1} ] [ ux ] = [ bx ]
[ G -W' ] [ uz ] [ bz ]
"""
# Compute bx := bx + G'*W^{-1}*W^{-T}*bz
v = matrix(0., (N, 1))
for i in range(N):
blas.symv(z, rr, v, ldA=N, offsetA=1, n=N, offsetx=N * i)
x[i] += blas.dot(rr, v, n=N, offsetx=N * i)
blas.symv(z, q, v, ldA=N, offsetA=1, n=N)
x[N] += blas.dot(q, v) + z[0] * W['di'][0]**2
# Solve G'*W^{-1}*W^{-T}*G*ux = bx
lapack.potrs(H, x)
# Compute bz := -W^{-T}*(bz-G*ux)
# z -= G*x
z[1::N + 1] -= x[:-1]
z -= x[-1]
# Apply scaling
z[0] *= -W['di'][0]
blas.scal(0.5, z, n=N, offset=1, inc=N + 1)
tmp = +r
blas.trmm(z, tmp, ldA=N, offsetA=1, n=N, m=N)
blas.syr2k(r,
tmp,
z,
trans='T',
offsetC=1,
ldC=N,
n=N,
k=N,
alpha=-1.0)
return fsolve
def Fkkt(W):
"""
Custom solver:
v := alpha * 2*A'*A * u + beta * v
"""
global mmS
mmS = matrix(0.0, (iR, iR))
global vvV
vvV = matrix(0.0, (iR, 1))
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
mmAsc = matrix(0.0, (iR, iC))
d1, d2 = W['di'][:iC]**2, W['di'][iC:]**2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div(mul(W['di'][:iC], W['di'][iC:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(mmTh, mmAsc)
for k in range(iR):
blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(mmAsc, mmS)
mmS[::iR + 1] += 1.0
lapack.potrf(mmS)
def g(x, y, z):
x[:iC] = 0.5 * ( x[:iC] - mul(d3, x[iC:]) + \
mul(d1, z[:iC] + mul(d3, z[:iC])) - \
mul(d2, z[iC:] - mul(d3, z[iC:])) )
x[:iC] = div(x[:iC], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(mmAsc, x, vvV)
lapack.potrs(mmS, vvV)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans='T')
x[:iC] = div(x[:iC], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[iC:] = div( x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1+d2 )\
- mul( d3, x[:iC] )
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:iC] = mul(W['di'][:iC], x[:iC] - x[iC:] - z[:iC])
z[iC:] = mul(W['di'][iC:], -x[:iC] - x[iC:] - z[iC:])
return g
def F(W):
"""
Generate a solver for
A'(uz0) = bx[0]
-uz0 - uz1 = bx[1]
A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0
- ux[1] - r1*r1' * uz1 * r1*r1' = bz1.
uz0, uz1, bz0, bz1 are symmetric m x m-matrices.
ux[0], bx[0] are n-vectors.
ux[1], bx[1] are symmetric m x m-matrices.
We first calculate a congruence that diagonalizes r0*r0' and r1*r1':
U' * r0 * r0' * U = I, U' * r1 * r1' * U = S.
We then make a change of variables
usx[0] = ux[0],
usx[1] = U' * ux[1] * U
usz0 = U^-1 * uz0 * U^-T
usz1 = U^-1 * uz1 * U^-T
and define
As() = U' * A() * U'
bsx[1] = U^-1 * bx[1] * U^-T
bsz0 = U' * bz0 * U
bsz1 = U' * bz1 * U.
This gives
As'(usz0) = bx[0]
-usz0 - usz1 = bsx[1]
As(usx[0]) - usx[1] - usz0 = bsz0
-usx[1] - S * usz1 * S = bsz1.
1. Eliminate usz0, usz1 using equations 3 and 4,
usz0 = As(usx[0]) - usx[1] - bsz0
usz1 = -S^-1 * (usx[1] + bsz1) * S^-1.
This gives two equations in usx[0] an usx[1].
As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0)
-As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1
= bsx[1] - bsz0 - S^-1 * bsz1 * S^-1.
2. Eliminate usx[1] using equation 2:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j],
usx[1] = ( S * As(usx[0]) * S ) ./ Gamma
+ ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma.
This gives an equation in usx[0].
As'( As(usx[0]) ./ Gamma )
= bx0 + As'(bsz0) +
As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma )
= bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ).
"""
# Calculate U s.t.
#
# U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s).
# Cholesky factorization r0 * r0' = L * L'
blas.syrk(W['r'][0], L)
lapack.potrf(L)
# SVD L^-1 * r1 = U * diag(s) * V'
blas.copy(W['r'][1], U)
blas.trsm(L, U)
lapack.gesvd(U, s, jobu='O')
# s := s**2
s[:] = s**2
# Uti := U
blas.copy(U, Uti)
# U := L^-T * U
blas.trsm(L, U, transA='T')
# Uti := L * Uti = U^-T
blas.trmm(L, Uti)
# Us := U * diag(s)^-1
blas.copy(U, Us)
for i in range(m):
blas.tbsv(s, Us, n=m, k=0, ldA=1, incx=m, offsetx=i)
# S is m x m with lower triangular entries s[i] * s[j]
# sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j])
# Upper triangular entries are undefined but nonzero.
blas.scal(0.0, S)
blas.syrk(s, S)
Gamma = 1.0 + S
sqrtG = sqrt(Gamma)
# Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n
# = Asi ./ sqrt(Gamma)
blas.copy(A, Asc)
misc.scale(
Asc, # only 'r' part of the dictionary is used
{
'dnl': matrix(0.0, (0, 1)),
'dnli': matrix(0.0, (0, 1)),
'd': matrix(0.0, (0, 1)),
'di': matrix(0.0, (0, 1)),
'v': [],
'beta': [],
'r': [U],
'rti': [U]
})
for i in range(n):
blas.tbsv(sqrtG, Asc, n=msq, k=0, ldA=1, offsetx=i * msq)
# Convert columns of Asc to packed storage
misc.pack2(Asc, {'l': 0, 'q': [], 's': [m]})
# Cholesky factorization of Asc' * Asc.
H = matrix(0.0, (n, n))
blas.syrk(Asc, H, trans='T', k=mpckd)
lapack.potrf(H)
def solve(x, y, z):
"""
1. Solve for usx[0]:
Asc'(Asc(usx[0]))
= bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
= bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S )
./ sqrtG)
where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U,
bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1
2. Solve for usx[1]:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
usx[1]
= ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma
Unscale ux[1] = Uti * usx[1] * Uti'
3. Compute usz0, usz1
r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T
r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T
"""
# z0 := U' * z0 * U
# = bsz0
__cngrnc(U, z, trans='T')
# z1 := Us' * bz1 * Us
# = S^-1 * U' * bz1 * U * S^-1
# = S^-1 * bsz1 * S^-1
__cngrnc(Us, z, trans='T', offsetx=msq)
# x[1] := Uti' * x[1] * Uti
# = bsx[1]
__cngrnc(Uti, x[1], trans='T')
# x[1] := x[1] - z[msq:]
# = bsx[1] - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], alpha=-1.0, offsetx=msq)
# x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG
# = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG
# = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG
# in packed storage
blas.copy(x[1], x1)
blas.tbmv(S, x1, n=msq, k=0, ldA=1)
blas.axpy(z, x1, n=msq)
blas.tbsv(sqrtG, x1, n=msq, k=0, ldA=1)
misc.pack2(x1, {'l': 0, 'q': [], 's': [m]})
# x[0] := x[0] + Asc'*x1
# = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
# = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma )
blas.gemv(Asc, x1, x[0], m=mpckd, trans='T', beta=1.0)
# x[0] := H^-1 * x[0]
# = ux[0]
lapack.potrs(H, x[0])
# x1 = Asc(x[0]) .* sqrtG (unpacked)
# = As(x[0])
blas.gemv(Asc, x[0], tmp, m=mpckd)
misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]})
blas.tbmv(sqrtG, x1, n=msq, k=0, ldA=1)
# usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2
# = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1)
# ./ Gamma
# x[1] := x[1] - z[:msq]
# = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], -1.0, n=msq)
# x[1] := x[1] + x1
# = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(x1, x[1])
# x[1] := x[1] / Gammma
# = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma
# = S^-1 * usx[1] * S^-1
blas.tbsv(Gamma, x[1], n=msq, k=0, ldA=1)
# z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1
# := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1
# := -r1' * uz1 * r1
blas.axpy(x[1], z, n=msq, offsety=msq)
blas.scal(-1.0, z, offset=msq)
__cngrnc(U, z, offsetx=msq)
__cngrnc(W['r'][1], z, trans='T', offsetx=msq)
# x[1] := S * x[1] * S
# = usx1
blas.tbmv(S, x[1], n=msq, k=0, ldA=1)
# z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0
# = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0
# = r0' * U' * usz0 * U * r0
# = r0' * uz0 * r0
blas.axpy(x1, z, -1.0, n=msq)
blas.scal(-1.0, z, n=msq)
blas.axpy(x[1], z, -1.0, n=msq)
__cngrnc(U, z)
__cngrnc(W['r'][0], z, trans='T')
# x[1] := Uti * x[1] * Uti'
# = ux[1]
__cngrnc(Uti, x[1])
return solve
def Fkkt(W):
"""
Custom solver:
v := alpha * 2*A'*A * u + beta * v
"""
global mmS
mmS = matrix(0.0, (iR, iR))
global vvV
vvV = matrix(0.0, (iR, 1))
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
mmAsc = matrix(0.0, (iR, iC))
d1, d2 = W["di"][:iC] ** 2, W["di"][iC:] ** 2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div(mul(W["di"][:iC], W["di"][iC:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(mmTh, mmAsc)
for k in range(iR):
blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(mmAsc, mmS)
mmS[:: iR + 1] += 1.0
lapack.potrf(mmS)
def g(x, y, z):
x[:iC] = 0.5 * (
x[:iC] - mul(d3, x[iC:]) + mul(d1, z[:iC] + mul(d3, z[:iC])) - mul(d2, z[iC:] - mul(d3, z[iC:]))
)
x[:iC] = div(x[:iC], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(mmAsc, x, vvV)
lapack.potrs(mmS, vvV)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans="T")
x[:iC] = div(x[:iC], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[iC:] = div(x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1 + d2) - mul(d3, x[:iC])
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:iC] = mul(W["di"][:iC], x[:iC] - x[iC:] - z[:iC])
z[iC:] = mul(W["di"][iC:], -x[:iC] - x[iC:] - z[iC:])
return g
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2*m]**2
dbi = div(1.0,db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2,dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt,mul(W['di'][:m],d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0,(k,1))
base.gemv(E, mul(W['di'][:m],lam), lt, trans = 'T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt,W['di'][:m])
Xs = spmatrix(tmp,range(m),range(m))*X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d,tmp),range(m),range(m))*E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0,(n,n))
Ab[::n+1] = 1.0
base.syrk(Xs,Ab,trans = 'T', beta = 1.0)
Bb = matrix(0.0,(n,k))
Bb = -Xs.T*Es # inefficient!?
D2 = spmatrix(0.0,range(k),range(k))
base.syrk(Es,D2,trans = 'T', partial = True)
d2 = +D2.V
del D2
py = matrix(0.0,(n,1))
for i in range(k):
binvsq = (1.0/W['beta'][i])**2
Ab += binvsq*Pt[i]
dvv = blas.dot(W['v'][i],W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans = 'T', alpha = 1.0, beta = 0.0)
blas.syrk(py, Ab, alpha = 4*binvsq*(dvv+1), beta = 1.0)
Bb[:,i] -= 4*binvsq*W['v'][i][0]*dvv*py
d2[i] += binvsq*(1+4*(W['v'][i][0]**2)*(dvv-1))
d2i = div(1.0,d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam,mul(W['di'][:m],d))
tmp = matrix(0.0,(k,1))
base.gemv(E,mul(W['di'][:m],lam), tmp, trans = 'T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp,tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,M12, alpha = 1.0)
tmp = mul(d,mul(W['di'][:m],lam))
blas.gemv(X,tmp,M12, trans = 'T', alpha = 1.0, beta = 1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt,range(k), range(k))
base.syrk(sBb, Ab, alpha = -1.0, beta = 1.0)
M = matrix([[Ab, M12.T],[M12, a]])
lapack.potrf(M)
def f(x,y,z):
# residuals
rwt = x[:n+k]
rb = x[n+k]
rv = x[n+k+1:n+k+1+m]
iw_rl1 = mul(W['di'][:m],z[:m])
iw_rl2 = mul(W['di'][m:2*m],z[m:2*m])
ri = [z[2*m+i*(n+1):2*m+(i+1)*(n+1)] for i in range(k)]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
rbwt[n+i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans = 'T', alpha = -1.0, beta = 1.0)
tmp = mul(W['di'][:m],iw_rl1)
tmp2 = matrix(0.0,(k,1))
base.gemv(E,tmp,tmp2,trans='T')
rbwt[n:] += tmp2
tmp = mul(d,tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X,tmp,rbwt,trans='T', alpha = -1.0, beta = 1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2*m],iw_rl2) - mul(W['di'][:m],iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi,rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans = 'T', alpha = 1.0, beta = 1.0)
rtw = +rbwt[:n]
tmp = mul(d,tmp)
blas.gemv(X, tmp, rtw, trans = 'T', alpha = -1.0, beta = 1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i,rtt)
tmp2 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,tmp2)
dwdb = matrix([rtw - tmp2,rtb + blas.dot(mul(d2i,lt),rtt)])
lapack.potrs(M,dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0,(k,1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt*dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0,(m,1))
blas.gemv(X,dwdb[:n],dv,alpha = -1.0)
dv = mul(d,dv) - d*dwdb[-1]
base.gemv(E, dt, dv, beta = 1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2,dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = - mul(W['di'][m:2*m],dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0,(n,1))
for i in range(k):
blas.gemv(P[i],dwdb[:n],tmp, alpha = -1.0, beta = 0.0)
tmp1 = matrix([-dt[i],tmp])
blas.axpy(ri[i],tmp1,alpha = -1.0)
qscal(tmp1,W['beta'][i],W['v'][i],inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n+k] = dt
x[n+k] = dwdb[-1]
x[n+k+1:] = dv
z[:m] = wdz1
z[m:2*m] = wdz2
for i in range(k):
z[2*m+i*(n+1):2*m+(i+1)*(n+1)] = wdzi[i]
return f
def kernel_matrix(X,
kernel,
sigma=1.0,
theta=1.0,
degree=1,
V=None,
width=None):
"""
Computes the kernel matrix or a partial kernel matrix.
Input arguments.
X is an N x n matrix.
kernel is a string with values 'linear', 'rfb', 'poly', or 'tanh'.
'linear': k(u,v) = u'*v/sigma.
'rbf': k(u,v) = exp(-||u - v||^2 / (2*sigma)).
'poly': k(u,v) = (u'*v/sigma)**degree.
'tanh': k(u,v) = tanh(u'*v/sigma - theta). kernel is a
sigma and theta are positive numbers.
degree is a positive integer.
V is an N x N sparse matrix (default is None).
width is a positive integer (default is None).
Output.
Q, an N x N matrix or sparse matrix.
If V is a sparse matrix, a partial kernel matrix with the sparsity
pattern V is returned.
If width is specified and V = 'band', a partial kernel matrix
with band sparsity is returned (width is the half-bandwidth).
a, an N x 1 matrix with the products <xi,xi>/sigma.
"""
N, n = X.size
#### dense (full) kernel matrix
if V is None:
if verbose: print("building kernel matrix ..")
# Qij = xi'*xj / sigma
Q = matrix(0.0, (N, N))
blas.syrk(X, Q, alpha=1.0 / sigma)
a = Q[::N + 1] # ai = ||xi||**2 / sigma
if kernel == 'linear':
pass
elif kernel == 'rbf':
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
ones = matrix(1.0, (N, 1))
blas.syr2(a, ones, Q, alpha=-0.5)
Q = exp(Q)
elif kernel == 'tanh':
Q = exp(Q - theta)
Q = div(Q - Q**-1, Q + Q**-1)
elif kernel == 'poly':
Q = Q**degree
else:
raise ValueError('invalid kernel type')
#### general sparse partial kernel matrix
elif type(V) is cvxopt.base.spmatrix:
if verbose: print("building projected kernel matrix ...")
Q = +V
base.syrk(X, Q, partial=True, alpha=1.0 / sigma)
# ai = ||xi||**2 / sigma
a = matrix(Q[::N + 1], (N, 1))
if kernel == 'linear':
pass
elif kernel == 'rbf':
ones = matrix(1.0, (N, 1))
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
p = chompack.maxcardsearch(V)
symb = chompack.symbolic(Q, p)
Qc = chompack.cspmatrix(symb) + Q
chompack.syr2(Qc, a, ones, alpha=-0.5)
Q = Qc.spmatrix(reordered=False)
Q.V = exp(Q.V)
elif kernel == 'tanh':
v = +Q.V
v = exp(v - theta)
v = div(v - v**-1, v + v**-1)
Q.V = v
elif kernel == 'poly':
Q.V = Q.V**degree
else:
raise ValueError('invalid kernel type')
#### banded partial kernel matrix
elif V == 'band' and width is not None:
# Lower triangular part of band matrix with bandwidth 2*w+1.
if verbose: print("building projected kernel matrix ...")
I = [i for k in range(N) for i in range(k, min(width + k + 1, N))]
J = [k for k in range(N) for i in range(min(width + 1, N - k))]
V = matrix(0.0, (len(I), 1))
oy = 0
for k in range(N): # V[:,k] = Xtrain[k:k+w, :] * Xtrain[k,:].T
m = min(width + 1, N - k)
blas.gemv(X,
X,
V,
m=m,
ldA=N,
incx=N,
offsetA=k,
offsetx=k,
offsety=oy)
oy += m
blas.scal(1.0 / sigma, V)
# ai = ||xi||**2 / sigma
a = matrix(V[[i for i in range(len(I)) if I[i] == J[i]]], (N, 1))
if kernel == 'linear':
Q = spmatrix(V, I, J, (N, N))
elif kernel == 'rbf':
Q = spmatrix(V, I, J, (N, N))
ones = matrix(1.0, (N, 1))
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
symb = chompack.symbolic(Q)
Qc = chompack.cspmatrix(symb) + Q
chompack.syr2(Qc, a, ones, alpha=-0.5)
Q = Qc.spmatrix(reordered=False)
Q.V = exp(Q.V)
elif kernel == 'tanh':
V = exp(V - theta)
V = div(V - V**-1, V + V**-1)
Q = spmatrix(V, I, J, (N, N))
elif kernel == 'poly':
Q = spmatrix(V**degree, I, J, (N, N))
else:
raise ValueError('invalid kernel type')
else:
raise TypeError('invalid type V')
return Q, a
def robsvm(X, d, gamma, P, e):
"""
Solves the following robust SVM training problem:
minimize (1/2) w'*w + gamma*sum(v)
subject to diag(d)*(X*w + b*1) >= 1 - v + E*u
|| S_j*w ||_2 <= u_j, j = 1...t
v >= 0
The variables are w, b, v, and u. The matrix E is a selector
matrix with zeros and one '1' per row. E_ij = 1 means that the
i'th training vector is associated with the j'th uncertainty
ellipsoid.
A custom KKT solver that exploits low-rank structure is used, and
a positive definite system of equations of order n is
formed and solved at each iteration.
ARGUMENTS
X m-by-n matrix with training vectors as rows
d m-vector with training labels (-1,+1)
P list of t symmetric matrices of order n
e m-vector where e[i] is the index of the uncertainty
ellipsoid associated with the i'th training vector
RETURNS
w n-vector
b scalar
u t-vector
v m-vector
iters number of interior-point iterations
"""
m,n = X.size
assert type(P) is list, "P must be a list of t symmtric positive definite matrices of order n."
k = len(P)
if k > 0:
assert e.size == (m,1), "e must be an m-vector."
assert max(e) < k and min(e) >= 0, "e[i] must be in {0,1,...,k-1}."
E = spmatrix(1.,e,range(m),(k,m)).T
d = matrix(d,tc='d')
q = matrix(0.0, (n+k+1+m,1))
q[n+k+1:] = gamma
h = matrix(0.0,(2*m+k*(n+1),1))
h[:m] = -1.0
# linear operators Q and G
def Q(x, y, alpha = 1.0, beta = 0.0, trans = 'N'):
y[:n] = alpha * x[:n] + beta * y[:n]
def G(x, y, alpha = 1.0, beta = 0.0, trans = 'N'):
"""
Implements the linear operator
[ -DX E -d -I ]
[ 0 0 0 -I ]
[ 0 -e_1' 0 0 ]
G = [ -P_1' 0 0 0 ]
[ . . . . ]
[ 0 -e_k' 0 0 ]
[ -P_k' 0 0 0 ]
and its adjoint G'.
"""
if trans == 'N':
tmp = +y[:m]
# y[:m] = alpha*(-DXw + Et - d*b - v) + beta*y[:m]
base.gemv(E, x[n:n+k], tmp, alpha = alpha, beta = beta)
blas.axpy(x[n+k+1:], tmp, alpha = -alpha)
blas.axpy(d, tmp, alpha = -alpha*x[n+k])
y[:m] = tmp
base.gemv(X, x[:n], tmp, alpha = alpha, beta = 0.0)
tmp = mul(d,tmp)
y[:m] -= tmp
# y[m:2*m] = -v
y[m:2*m] = -alpha * x[n+k+1:] + beta * y[m:2*m]
# SOC 1,...,k
for i in range(k):
l = 2*m+i*(n+1)
y[l] = -alpha * x[n+i] + beta * y[l]
y[l+1:l+1+n] = -alpha * P[i] * x[:n] + beta * y[l+1:l+1+n];
else:
tmp1 = mul(d,x[:m])
tmp2 = y[:n]
blas.gemv(X, tmp1, tmp2, trans = 'T', alpha = -alpha, beta = beta)
for i in range(k):
l = 2*m+1+i*(n+1)
blas.gemv(P[i], x[l:l+n], tmp2, trans = 'T', alpha = -alpha, beta = 1.0)
y[:n] = tmp2
tmp2 = y[n:n+k]
base.gemv(E, x[:m], tmp2, trans = 'T', alpha = alpha, beta = beta)
blas.axpy(x[2*m:2*m+k*(1+n):n+1], tmp2, alpha = -alpha)
y[n:n+k] = tmp2
y[n+k] = -alpha * blas.dot(d,x[:m]) + beta * y[n+k]
y[n+k+1:] = -alpha * (x[:m] + x[m:2*m]) + beta * y[n+k+1:]
# precompute products Pi'*Pi
Pt = []
for p in P:
y = matrix(0.0, (n,n))
blas.syrk(p, y, trans = 'T')
Pt.append(y)
# scaled hyperbolic Householder transformations
def qscal(u, beta, v, inv = False):
"""
Transforms the vector u as
u := beta * (2*v*v' - J) * u
if 'inv' is False and as
u := (1/beta) * (2*J*v*v'*J - J) * u
if 'inv' is True.
"""
if not inv:
tmp = blas.dot(u,v)
u[0] *= -1
u += 2 * v * tmp
u *= beta
else:
u[0] *= -1.0
tmp = blas.dot(v,u)
u[0] -= 2*v[0] * tmp
u[1:] += 2*v[1:] * tmp
u /= beta
# custom KKT solver
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2*m]**2
dbi = div(1.0,db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2,dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt,mul(W['di'][:m],d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0,(k,1))
base.gemv(E, mul(W['di'][:m],lam), lt, trans = 'T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt,W['di'][:m])
Xs = spmatrix(tmp,range(m),range(m))*X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d,tmp),range(m),range(m))*E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0,(n,n))
Ab[::n+1] = 1.0
base.syrk(Xs,Ab,trans = 'T', beta = 1.0)
Bb = matrix(0.0,(n,k))
Bb = -Xs.T*Es # inefficient!?
D2 = spmatrix(0.0,range(k),range(k))
base.syrk(Es,D2,trans = 'T', partial = True)
d2 = +D2.V
del D2
py = matrix(0.0,(n,1))
for i in range(k):
binvsq = (1.0/W['beta'][i])**2
Ab += binvsq*Pt[i]
dvv = blas.dot(W['v'][i],W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans = 'T', alpha = 1.0, beta = 0.0)
blas.syrk(py, Ab, alpha = 4*binvsq*(dvv+1), beta = 1.0)
Bb[:,i] -= 4*binvsq*W['v'][i][0]*dvv*py
d2[i] += binvsq*(1+4*(W['v'][i][0]**2)*(dvv-1))
d2i = div(1.0,d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam,mul(W['di'][:m],d))
tmp = matrix(0.0,(k,1))
base.gemv(E,mul(W['di'][:m],lam), tmp, trans = 'T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp,tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,M12, alpha = 1.0)
tmp = mul(d,mul(W['di'][:m],lam))
blas.gemv(X,tmp,M12, trans = 'T', alpha = 1.0, beta = 1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt,range(k), range(k))
base.syrk(sBb, Ab, alpha = -1.0, beta = 1.0)
M = matrix([[Ab, M12.T],[M12, a]])
lapack.potrf(M)
def f(x,y,z):
# residuals
rwt = x[:n+k]
rb = x[n+k]
rv = x[n+k+1:n+k+1+m]
iw_rl1 = mul(W['di'][:m],z[:m])
iw_rl2 = mul(W['di'][m:2*m],z[m:2*m])
ri = [z[2*m+i*(n+1):2*m+(i+1)*(n+1)] for i in range(k)]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
rbwt[n+i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans = 'T', alpha = -1.0, beta = 1.0)
tmp = mul(W['di'][:m],iw_rl1)
tmp2 = matrix(0.0,(k,1))
base.gemv(E,tmp,tmp2,trans='T')
rbwt[n:] += tmp2
tmp = mul(d,tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X,tmp,rbwt,trans='T', alpha = -1.0, beta = 1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2*m],iw_rl2) - mul(W['di'][:m],iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi,rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans = 'T', alpha = 1.0, beta = 1.0)
rtw = +rbwt[:n]
tmp = mul(d,tmp)
blas.gemv(X, tmp, rtw, trans = 'T', alpha = -1.0, beta = 1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i,rtt)
tmp2 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,tmp2)
dwdb = matrix([rtw - tmp2,rtb + blas.dot(mul(d2i,lt),rtt)])
lapack.potrs(M,dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0,(k,1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt*dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0,(m,1))
blas.gemv(X,dwdb[:n],dv,alpha = -1.0)
dv = mul(d,dv) - d*dwdb[-1]
base.gemv(E, dt, dv, beta = 1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2,dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = - mul(W['di'][m:2*m],dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0,(n,1))
for i in range(k):
blas.gemv(P[i],dwdb[:n],tmp, alpha = -1.0, beta = 0.0)
tmp1 = matrix([-dt[i],tmp])
blas.axpy(ri[i],tmp1,alpha = -1.0)
qscal(tmp1,W['beta'][i],W['v'][i],inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n+k] = dt
x[n+k] = dwdb[-1]
x[n+k+1:] = dv
z[:m] = wdz1
z[m:2*m] = wdz2
for i in range(k):
z[2*m+i*(n+1):2*m+(i+1)*(n+1)] = wdzi[i]
return f
# solve cone QP and return solution
sol = solvers.coneqp(Q, q, G, h, dims = {'l':2*m,'q':[n+1 for i in range(k)],'s':[]}, kktsolver = F)
return sol['x'][:n], sol['x'][n+k], sol['x'][n:n+k], sol['x'][n+k+1:], sol['iterations']
def factor(W, H=None, Df=None):
if F['firstcall']:
if type(G) is matrix:
F['Gs'] = matrix(0.0, G.size)
else:
F['Gs'] = spmatrix(0.0, G.I, G.J, G.size)
if mnl:
if type(Df) is matrix:
F['Dfs'] = matrix(0.0, Df.size)
else:
F['Dfs'] = spmatrix(0.0, Df.I, Df.J, Df.size)
if (mnl and type(Df) is matrix) or type(G) is matrix or \
type(H) is matrix:
F['S'] = matrix(0.0, (n, n))
F['K'] = matrix(0.0, (p, p))
else:
F['S'] = spmatrix([], [], [], (n, n), 'd')
F['Sf'] = None
if type(A) is matrix:
F['K'] = matrix(0.0, (p, p))
else:
F['K'] = spmatrix([], [], [], (p, p), 'd')
# Dfs = Wnl^{-1} * Df
if mnl:
base.gemm(spmatrix(W['dnli'], list(range(mnl)),
list(range(mnl))), Df, F['Dfs'], partial=True)
# Gs = Wl^{-1} * G.
base.gemm(spmatrix(W['di'], list(range(ml)), list(range(ml))),
G, F['Gs'], partial=True)
if F['firstcall']:
base.syrk(F['Gs'], F['S'], trans='T')
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0)
if H is not None:
F['S'] += H
try:
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
F['Sf'] = cholmod.symbolic(F['S'])
cholmod.numeric(F['S'], F['Sf'])
except ArithmeticError:
F['singular'] = True
if type(A) is matrix and type(F['S']) is spmatrix:
F['S'] = matrix(0.0, (n, n))
base.syrk(F['Gs'], F['S'], trans='T')
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0)
base.syrk(A, F['S'], trans='T', beta=1.0)
if H is not None:
F['S'] += H
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
F['Sf'] = cholmod.symbolic(F['S'])
cholmod.numeric(F['S'], F['Sf'])
F['firstcall'] = False
else:
base.syrk(F['Gs'], F['S'], trans='T', partial=True)
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0,
partial=True)
if H is not None:
F['S'] += H
if F['singular']:
base.syrk(A, F['S'], trans='T', beta=1.0, partial=True)
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
cholmod.numeric(F['S'], F['Sf'])
if type(F['S']) is matrix:
# Asct := L^{-1}*A'. Factor K = Asct'*Asct.
if type(A) is matrix:
Asct = A.T
else:
Asct = matrix(A.T)
blas.trsm(F['S'], Asct)
blas.syrk(Asct, F['K'], trans='T')
lapack.potrf(F['K'])
else:
# Asct := L^{-1}*P*A'. Factor K = Asct'*Asct.
if type(A) is matrix:
Asct = A.T
cholmod.solve(F['Sf'], Asct, sys=7)
cholmod.solve(F['Sf'], Asct, sys=4)
blas.syrk(Asct, F['K'], trans='T')
lapack.potrf(F['K'])
else:
Asct = cholmod.spsolve(F['Sf'], A.T, sys=7)
Asct = cholmod.spsolve(F['Sf'], Asct, sys=4)
base.syrk(Asct, F['K'], trans='T')
Kf = cholmod.symbolic(F['K'])
cholmod.numeric(F['K'], Kf)
def solve(x, y, z):
# Solve
#
# [ H A' GG'*W^{-1} ] [ ux ] [ bx ]
# [ A 0 0 ] * [ uy ] = [ by ]
# [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
#
# and return ux, uy, W*uz.
#
# If not F['singular']:
#
# K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz ) - by
# S*ux = bx + GG'*W^{-1}*W^{-T}*bz - A'*uy
# W*uz = W^{-T} * ( GG*ux - bz ).
#
# If F['singular']:
#
# K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz + A'*by )
# - by
# S*ux = bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y.
# W*uz = W^{-T} * ( GG*ux - bz ).
# z := W^{-1} * z = W^{-1} * bz
scale(z, W, trans='T', inverse='I')
# If not F['singular']:
# x := L^{-1} * P * (x + GGs'*z)
# = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz)
#
# If F['singular']:
# x := L^{-1} * P * (x + GGs'*z + A'*y))
# = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz + A'*y)
if mnl:
base.gemv(F['Dfs'], z, x, trans='T', beta=1.0)
base.gemv(F['Gs'], z, x, offsetx=mnl, trans='T',
beta=1.0)
if F['singular']:
base.gemv(A, y, x, trans='T', beta=1.0)
if type(F['S']) is matrix:
blas.trsv(F['S'], x)
else:
cholmod.solve(F['Sf'], x, sys=7)
cholmod.solve(F['Sf'], x, sys=4)
# y := K^{-1} * (Asc*x - y)
# = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz) - by)
# (if not F['singular'])
# = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz +
# A'*by) - by)
# (if F['singular']).
base.gemv(Asct, x, y, trans='T', beta=-1.0)
if type(F['K']) is matrix:
lapack.potrs(F['K'], y)
else:
cholmod.solve(Kf, y)
# x := P' * L^{-T} * (x - Asc'*y)
# = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz - A'*y)
# (if not F['singular'])
# = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y)
# (if F['singular'])
base.gemv(Asct, y, x, alpha=-1.0, beta=1.0)
if type(F['S']) is matrix:
blas.trsv(F['S'], x, trans='T')
else:
cholmod.solve(F['Sf'], x, sys=5)
cholmod.solve(F['Sf'], x, sys=8)
# W*z := GGs*x - z = W^{-T} * (GG*x - bz)
if mnl:
base.gemv(F['Dfs'], x, z, beta=-1.0)
base.gemv(F['Gs'], x, z, beta=-1.0, offsety=mnl)
return solve
def mcsvm(X, labels, gamma, kernel='linear', sigma=1.0, degree=1):
"""
Solves the Crammer and Singer multiclass SVM training problem
maximize -(1/2) * tr(U' * Q * U) + tr(E' * U)
subject to U <= gamma * E
U * 1_m = 0.
The variable is an (N x m)-matrix U if N is the number of training
examples and m the number of classes.
Q is a positive definite matrix of order N with Q[i,j] = K(xi, xj)
where K is a kernel function and xi is the ith row of X.
The matrix E is an N x m matrix with E[i,j] = 1 if labels[i] = j
and E[i,j] = 0 otherwise.
Input arguments.
X is a N x n matrix. The rows are the training vectors.
labels is a list of integers of length N with values 0, ..., m-1.
labels[i] is the class of training example i.
gamma is a positive parameter.
kernel is a string with values 'linear' or 'poly'.
'linear': K(u,v) = u'*v.
'poly': K(u,v) = (u'*v / sigma)**degree.
sigma is a positive number.
degree is a positive integer.
Output.
Returns a function classifier(). If Y is M x n then classifier(Y)
returns a list with as its kth element
argmax { j = 0, ..., m-1 | sum_{i=1}^N U[i,j] * K(xi, yk) }
where yk' = Y[k, :], xi' = X[i, :], and U is the optimal solution
of the QP.
"""
N, n = X.size
m = max(labels) + 1
E = matrix(0.0, (N, m))
E[matrix(range(N)) + N * matrix(labels)] = 1.0
def G(x, y, alpha=1.0, beta=0.0, trans='N'):
"""
If trans is 'N', x is an N x m matrix, and y is an N*m-vector.
y := alpha * x[:] + beta * y.
If trans is 'T', x is an N*m vector, and y is an N x m matrix.
y[:] := alpha * x + beta * y[:].
"""
blas.scal(beta, y)
blas.axpy(x, y, alpha)
h = matrix(gamma * E, (N * m, 1))
ones = matrix(1.0, (m, 1))
def A(x, y, alpha=1.0, beta=0.0, trans='N'):
"""
If trans is 'N', x is an N x m matrix and y an N-vector.
y := alpha * x * 1_m + beta y.
If trans is 'T', x is an N vector and y an N x m matrix.
y := alpha * x * 1_m' + beta y.
"""
if trans == 'N':
blas.gemv(x, ones, y, alpha=alpha, beta=beta)
else:
blas.scal(beta, y)
blas.ger(x, ones, y, alpha=alpha)
b = matrix(0.0, (N, 1))
if kernel == 'linear' and N > n:
def P(x, y, alpha=1.0, beta=0.0):
"""
x and y are N x m matrices.
y = alpha * X * X' * x + beta * y.
"""
z = matrix(0.0, (n, m))
blas.gemm(X, x, z, transA='T')
blas.gemm(X, z, y, alpha=alpha, beta=beta)
else:
if kernel == 'linear':
# Q = X * X'
Q = matrix(0.0, (N, N))
blas.syrk(X, Q)
elif kernel == 'poly':
# Q = (X * X' / sigma) ** degree
Q = matrix(0.0, (N, N))
blas.syrk(X, Q, alpha=1.0 / sigma)
Q = Q**degree
else:
raise ValueError("invalid kernel type")
def P(x, y, alpha=1.0, beta=0.0):
"""
x and y are N x m matrices.
y = alpha * Q * x + beta * y.
"""
blas.symm(Q, x, y, alpha=alpha, beta=beta)
if kernel == 'linear' and N > n: # add separate code for n <= N <= m*n
H = [matrix(0.0, (n, n)) for k in range(m)]
S = matrix(0.0, (m * n, m * n))
Xs = matrix(0.0, (N, n))
wnm = matrix(0.0, (m * n, 1))
wN = matrix(0.0, (N, 1))
D = matrix(0.0, (N, 1))
def kkt(W):
"""
KKT solver for
X*X' * ux + uy * 1_m' + mat(uz) = bx
ux * 1_m = by
ux - d.^2 .* mat(uz) = mat(bz).
ux and bx are N x m matrices.
uy and by are N-vectors.
uz and bz are N*m-vectors. mat(uz) is the N x m matrix that
satisfies mat(uz)[:] = uz.
d = mat(W['d']) a positive N x m matrix.
If we eliminate uz from the last equation using
mat(uz) = (ux - mat(bz)) ./ d.^2
we get two equations in ux, uy:
X*X' * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2
ux * 1_m = by.
From the 1st equation,
uxk = (X*X' + Dk^-2)^-1 * (-uy + bxk + Dk^-2 * bzk)
= Dk * (I + Xk*Xk')^-1 * Dk * (-uy + bxk + Dk^-2 * bzk)
for k = 1, ..., m, where Dk = diag(d[:,k]), Xk = Dk * X,
uxk is column k of ux, and bzk is column k of mat(bz).
We use the matrix inversion lemma
( I + Xk * Xk' )^-1 = I - Xk * (I + Xk' * Xk)^-1 * Xk'
= I - Xk * Hk^-1 * Xk'
= I - Xk * Lk^-T * Lk^-1 * Xk'
where Hk = I + Xk' * Xk = Lk * Lk' to write this as
uxk = Dk * (I - Xk * Hk^-1 * Xk') * Dk *
(-uy + bxk + Dk^-2 * bzk)
= (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk).
Substituting this in the second equation gives an equation
for uy:
sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 * bzk ),
i.e., with D = (sum_k Dk^2)^1/2, Yk = D^-1 * Dk^2 * X * Lk^-T,
D * ( I - sum_k Yk * Yk' ) * D * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 *bzk ).
Another application of the matrix inversion lemma gives
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k ( Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) *
( bxk + Dk^-2 *bzk ) )
with S = I - Y' * Y, Y = [ Y1 ... Ym ].
Summary:
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
d = matrix(W['d'], (N, m))
dsq = matrix(W['d']**2, (N, m))
# Factor the matrices
#
# H[k] = I + Xk' * Xk
# = I + X' * Dk^2 * X.
#
# Dk = diag(d[:,k]).
for k in range(m):
# H[k] = I
blas.scal(0.0, H[k])
H[k][::n + 1] = 1.0
# Xs = Dk * X
# = diag(d[:,k]]) * X
blas.copy(X, Xs)
for j in range(n):
blas.tbmv(d,
Xs,
n=N,
k=0,
ldA=1,
offsetA=k * N,
offsetx=j * N)
# H[k] := H[k] + Xs' * Xs
# = I + Xk' * Xk
blas.syrk(Xs, H[k], trans='T', beta=1.0)
# Factorization H[k] = Lk * Lk'
lapack.potrf(H[k])
###
utime, stime = cputime()
print("Factor Hk's: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# diag(D) = ( sum_k d[:,k]**2 ) ** 1/2
# = ( sum_k Dk^2) ** 1/2.
blas.gemv(dsq, ones, D)
D[:] = sqrt(D)
###
# utime, stime = cputime()
# print("Compute D: utime = %.2f, stime = %.2f" \
# %(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = I - Y'* Y is an m x m block matrix.
# The i,j block of Y' * Y is
#
# Yi' * Yj = Li^-1 * X' * Di^2 * D^-1 * Dj^2 * X * Lj^-T.
#
# We compute only the lower triangular blocks in Y'*Y.
blas.scal(0.0, S)
for i in range(m):
for j in range(i + 1):
# Xs = Di * Dj * D^-1 * X
blas.copy(X, Xs)
blas.copy(d, wN, n=N, offsetx=i * N)
blas.tbmv(d, wN, n=N, k=0, ldA=1, offsetA=j * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
for k in range(n):
blas.tbmv(wN, Xs, n=N, k=0, ldA=1, offsetx=k * N)
# block i, j of S is Xs' * Xs (as nonsymmetric matrix so we
# get the correct multiple after scaling with Li, Lj)
blas.gemm(Xs,
Xs,
S,
transA='T',
ldC=m * n,
offsetC=(j * n) * m * n + i * n)
###
utime, stime = cputime()
print("Form S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
for i in range(m):
# multiply block row i of S on the left with Li^-1
blas.trsm(H[i],
S,
m=n,
n=(i + 1) * n,
ldB=m * n,
offsetB=i * n)
# multiply block column i of S on the right with Li^-T
blas.trsm(H[i],
S,
side='R',
transA='T',
m=(m - i) * n,
n=n,
ldB=m * n,
offsetB=i * n * (m * n + 1))
blas.scal(-1.0, S)
S[::(m * n + 1)] += 1.0
###
utime, stime = cputime()
print("Form S (2): utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = L*L'
lapack.potrf(S)
###
utime, stime = cputime()
print("Factor S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
def f(x, y, z):
"""
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
# xk := Dk^2 * xk + zk
# = Dk^2 * bxk + bzk
blas.tbmv(dsq, x, n=N * m, k=0, ldA=1)
blas.axpy(z, x)
# y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk
# = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk
# y := -y + x*ones
blas.gemv(x, ones, y, alpha=1.0, beta=-1.0)
# wnm = X' * x (wnm interpreted as an n x m matrix)
blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n)
# wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix)
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=k * n)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=n * k)
# wN = Dk^2 * wN
blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1)
# y := y - wN
blas.axpy(wN, y, -1.0)
# y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y
#
# Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T.
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# wnm = Y' * y (interpreted as an Nm vector)
# = [ L1^-1 * X' * D1^2 * D^-1 * y;
# L2^-1 * X' * D2^2 * D^-1 * y;
# ...
# Lm^-1 * X' * Dm^2 * D^-1 * y ]
for k in range(m):
# wN = D^-1 * Dk^2 * y
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# wnm[:,k] = X' * wN
blas.gemv(X, wN, wnm, trans='T', offsety=k * n)
# wnm[:,k] = Lk^-1 * wnm[:,k]
blas.trsv(H[k], wnm, offsetx=k * n)
# wnm := S^-1 * wnm (an mn-vector)
lapack.potrs(S, wnm)
# y := y + Y * wnm
# = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T]
# * wnm
for k in range(m):
# wnm[:,k] = Lk^-T * wnm[:,k]
blas.trsv(H[k], wnm, trans='T', offsetx=k * n)
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = D^-1 * Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# y += wN
blas.axpy(wN, y)
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# For k = 1, ..., m:
#
# xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk)
# x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix)
for k in range(m):
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.axpy(wN, x, -1.0, offsety=k * N)
# wnm = X' * x (as an n x m matrix)
blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n)
# wnm[:,k] = Hk^-1 * wnm[:,k]
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=n * k)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
# x[:,k] := x[:,k] - wN
blas.axpy(wN, x, -1.0, n=N, offsety=k * N)
# z := ( x - z ) ./ d
blas.axpy(x, z, -1.0)
blas.scal(-1.0, z)
blas.tbsv(d, z, n=N * m, k=0, ldA=1)
###
utime, stime = cputime()
print("Solve: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
###
return f
else:
H = [matrix(0.0, (N, N)) for k in range(m)]
S = matrix(0.0, (N, N))
def kkt(W):
"""
KKT solver for
Q * ux + uy * 1_m' + mat(uz) = bx
ux * 1_m = by
ux - d.^2 .* mat(uz) = mat(bz).
ux and bx are N x m matrices.
uy and by are N-vectors.
uz and bz are N*m-vectors. mat(uz) is the N x m matrix that
satisfies mat(uz)[:] = uz.
d = mat(W['d']) a positive N x m matrix.
If we eliminate uz from the last equation using
mat(uz) = (ux - mat(bz)) ./ d.^2
we get two equations in ux, uy:
Q * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2
ux * 1_m = by.
From the 1st equation
uxk = -(Q + Dk)^-1 * uy + (Q + Dk)^-1 * (bxk + Dk * bzk)
where uxk is column k of ux, Dk = diag(d[:,k].^-2), and bzk is
column k of mat(bz). Substituting this in the second equation
gives an equation for uy.
1. Solve for uy
sum_k (Q + Dk)^-1 * uy =
sum_k (Q + Dk)^-1 * (bxk + Dk * bzk) - by.
2. Solve for ux (column by column)
Q * ux + ux ./ d.^2 = bx + mat(bz) ./ d.^2 - uy * 1_m'.
3. Solve for uz
mat(uz) = ( ux - mat(bz) ) ./ d.^2.
Return ux, uy, d .* uz.
"""
# D = d.^-2
D = matrix(W['di']**2, (N, m))
blas.scal(0.0, S)
for k in range(m):
# Hk := Q + Dk
blas.copy(Q, H[k])
H[k][::N + 1] += D[:, k]
# Hk := Hk^-1
# = (Q + Dk)^-1
lapack.potrf(H[k])
lapack.potri(H[k])
# S := S + Hk
# = S + (Q + Dk)^-1
blas.axpy(H[k], S)
# Factor S = sum_k (Q + Dk)^-1
lapack.potrf(S)
def f(x, y, z):
# z := mat(z)
# = mat(bz)
z.size = N, m
# x := x + D .* z
# = bx + mat(bz) ./ d.^2
x += mul(D, z)
# y := y - sum_k (Q + Dk)^-1 * X[:,k]
# = by - sum_k (Q + Dk)^-1 * (bxk + Dk * bzk)
for k in range(m):
blas.symv(H[k], x[:, k], y, alpha=-1.0, beta=1.0)
# y := H^-1 * y
# = -uy
lapack.potrs(S, y)
# x[:,k] := H[k] * (x[:,k] + y)
# = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y)
# = ux[:,k]
w = matrix(0.0, (N, 1))
for k in range(m):
# x[:,k] := x[:,k] + y
blas.axpy(y, x, offsety=N * k, n=N)
# w := H[k] * x[:,k]
# = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y)
blas.symv(H[k], x, w, offsetx=N * k)
# x[:,k] := w
# = ux[:,k]
blas.copy(w, x, offsety=N * k)
# y := -y
# = uy
blas.scal(-1.0, y)
# z := (x - z) ./ d
blas.axpy(x, z, -1.0)
blas.tbsv(W['d'], z, n=m * N, k=0, ldA=1)
blas.scal(-1.0, z)
z.size = N * m, 1
return f
utime0, stime0 = cputime()
# solvers.options['debug'] = True
# solvers.options['maxiters'] = 1
solvers.options['refinement'] = 1
sol = solvers.coneqp(P,
-E,
G,
h,
A=A,
b=b,
kktsolver=kkt,
xnewcopy=matrix,
xdot=blas.dot,
xaxpy=blas.axpy,
xscal=blas.scal)
utime, stime = cputime()
utime -= utime0
stime -= stime0
print("utime = %.2f, stime = %.2f" % (utime, stime))
U = sol['x']
if kernel == 'linear':
# W = X' * U
W = matrix(0.0, (n, m))
blas.gemm(X, U, W, transA='T')
def classifier(Y):
# return [ argmax of Y[k,:] * W for k in range(M) ]
M = Y.size[0]
S = Y * W
c = []
for i in range(M):
a = zip(list(S[i, :]), range(m))
a.sort(reverse=True)
c += [a[0][1]]
return c
elif kernel == 'poly':
def classifier(Y):
M = Y.size[0]
# K = Y * X' / sigma
K = matrix(0.0, (M, N))
blas.gemm(Y, X, K, transB='T', alpha=1.0 / sigma)
S = K**degree * U
c = []
for i in range(M):
a = zip(list(S[i, :]), range(m))
a.sort(reverse=True)
c += [a[0][1]]
return c
else:
pass
return classifier #, utime, sol['iterations']
def kkt(W):
"""
KKT solver for
X*X' * ux + uy * 1_m' + mat(uz) = bx
ux * 1_m = by
ux - d.^2 .* mat(uz) = mat(bz).
ux and bx are N x m matrices.
uy and by are N-vectors.
uz and bz are N*m-vectors. mat(uz) is the N x m matrix that
satisfies mat(uz)[:] = uz.
d = mat(W['d']) a positive N x m matrix.
If we eliminate uz from the last equation using
mat(uz) = (ux - mat(bz)) ./ d.^2
we get two equations in ux, uy:
X*X' * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2
ux * 1_m = by.
From the 1st equation,
uxk = (X*X' + Dk^-2)^-1 * (-uy + bxk + Dk^-2 * bzk)
= Dk * (I + Xk*Xk')^-1 * Dk * (-uy + bxk + Dk^-2 * bzk)
for k = 1, ..., m, where Dk = diag(d[:,k]), Xk = Dk * X,
uxk is column k of ux, and bzk is column k of mat(bz).
We use the matrix inversion lemma
( I + Xk * Xk' )^-1 = I - Xk * (I + Xk' * Xk)^-1 * Xk'
= I - Xk * Hk^-1 * Xk'
= I - Xk * Lk^-T * Lk^-1 * Xk'
where Hk = I + Xk' * Xk = Lk * Lk' to write this as
uxk = Dk * (I - Xk * Hk^-1 * Xk') * Dk *
(-uy + bxk + Dk^-2 * bzk)
= (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk).
Substituting this in the second equation gives an equation
for uy:
sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 * bzk ),
i.e., with D = (sum_k Dk^2)^1/2, Yk = D^-1 * Dk^2 * X * Lk^-T,
D * ( I - sum_k Yk * Yk' ) * D * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 *bzk ).
Another application of the matrix inversion lemma gives
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k ( Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) *
( bxk + Dk^-2 *bzk ) )
with S = I - Y' * Y, Y = [ Y1 ... Ym ].
Summary:
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
d = matrix(W['d'], (N, m))
dsq = matrix(W['d']**2, (N, m))
# Factor the matrices
#
# H[k] = I + Xk' * Xk
# = I + X' * Dk^2 * X.
#
# Dk = diag(d[:,k]).
for k in range(m):
# H[k] = I
blas.scal(0.0, H[k])
H[k][::n + 1] = 1.0
# Xs = Dk * X
# = diag(d[:,k]]) * X
blas.copy(X, Xs)
for j in range(n):
blas.tbmv(d,
Xs,
n=N,
k=0,
ldA=1,
offsetA=k * N,
offsetx=j * N)
# H[k] := H[k] + Xs' * Xs
# = I + Xk' * Xk
blas.syrk(Xs, H[k], trans='T', beta=1.0)
# Factorization H[k] = Lk * Lk'
lapack.potrf(H[k])
###
utime, stime = cputime()
print("Factor Hk's: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# diag(D) = ( sum_k d[:,k]**2 ) ** 1/2
# = ( sum_k Dk^2) ** 1/2.
blas.gemv(dsq, ones, D)
D[:] = sqrt(D)
###
# utime, stime = cputime()
# print("Compute D: utime = %.2f, stime = %.2f" \
# %(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = I - Y'* Y is an m x m block matrix.
# The i,j block of Y' * Y is
#
# Yi' * Yj = Li^-1 * X' * Di^2 * D^-1 * Dj^2 * X * Lj^-T.
#
# We compute only the lower triangular blocks in Y'*Y.
blas.scal(0.0, S)
for i in range(m):
for j in range(i + 1):
# Xs = Di * Dj * D^-1 * X
blas.copy(X, Xs)
blas.copy(d, wN, n=N, offsetx=i * N)
blas.tbmv(d, wN, n=N, k=0, ldA=1, offsetA=j * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
for k in range(n):
blas.tbmv(wN, Xs, n=N, k=0, ldA=1, offsetx=k * N)
# block i, j of S is Xs' * Xs (as nonsymmetric matrix so we
# get the correct multiple after scaling with Li, Lj)
blas.gemm(Xs,
Xs,
S,
transA='T',
ldC=m * n,
offsetC=(j * n) * m * n + i * n)
###
utime, stime = cputime()
print("Form S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
for i in range(m):
# multiply block row i of S on the left with Li^-1
blas.trsm(H[i],
S,
m=n,
n=(i + 1) * n,
ldB=m * n,
offsetB=i * n)
# multiply block column i of S on the right with Li^-T
blas.trsm(H[i],
S,
side='R',
transA='T',
m=(m - i) * n,
n=n,
ldB=m * n,
offsetB=i * n * (m * n + 1))
blas.scal(-1.0, S)
S[::(m * n + 1)] += 1.0
###
utime, stime = cputime()
print("Form S (2): utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = L*L'
lapack.potrf(S)
###
utime, stime = cputime()
print("Factor S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
def f(x, y, z):
"""
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
# xk := Dk^2 * xk + zk
# = Dk^2 * bxk + bzk
blas.tbmv(dsq, x, n=N * m, k=0, ldA=1)
blas.axpy(z, x)
# y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk
# = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk
# y := -y + x*ones
blas.gemv(x, ones, y, alpha=1.0, beta=-1.0)
# wnm = X' * x (wnm interpreted as an n x m matrix)
blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n)
# wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix)
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=k * n)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=n * k)
# wN = Dk^2 * wN
blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1)
# y := y - wN
blas.axpy(wN, y, -1.0)
# y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y
#
# Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T.
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# wnm = Y' * y (interpreted as an Nm vector)
# = [ L1^-1 * X' * D1^2 * D^-1 * y;
# L2^-1 * X' * D2^2 * D^-1 * y;
# ...
# Lm^-1 * X' * Dm^2 * D^-1 * y ]
for k in range(m):
# wN = D^-1 * Dk^2 * y
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# wnm[:,k] = X' * wN
blas.gemv(X, wN, wnm, trans='T', offsety=k * n)
# wnm[:,k] = Lk^-1 * wnm[:,k]
blas.trsv(H[k], wnm, offsetx=k * n)
# wnm := S^-1 * wnm (an mn-vector)
lapack.potrs(S, wnm)
# y := y + Y * wnm
# = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T]
# * wnm
for k in range(m):
# wnm[:,k] = Lk^-T * wnm[:,k]
blas.trsv(H[k], wnm, trans='T', offsetx=k * n)
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = D^-1 * Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# y += wN
blas.axpy(wN, y)
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# For k = 1, ..., m:
#
# xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk)
# x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix)
for k in range(m):
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.axpy(wN, x, -1.0, offsety=k * N)
# wnm = X' * x (as an n x m matrix)
blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n)
# wnm[:,k] = Hk^-1 * wnm[:,k]
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=n * k)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
# x[:,k] := x[:,k] - wN
blas.axpy(wN, x, -1.0, n=N, offsety=k * N)
# z := ( x - z ) ./ d
blas.axpy(x, z, -1.0)
blas.scal(-1.0, z)
blas.tbsv(d, z, n=N * m, k=0, ldA=1)
###
utime, stime = cputime()
print("Solve: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
###
return f
def Fgp(x=None, z=None):
if x is None: return mnl, matrix(0.0, (n, 1))
f = matrix(0.0, (mnl + 1, 1))
Df = matrix(0.0, (mnl + 1, n))
# y = F*x+g
blas.copy(g, y)
base.gemv(F, x, y, beta=1.0)
if z is not None: H = matrix(0.0, (n, n))
for i, start, stop in ind:
# yi := exp(yi) = exp(Fi*x+gi)
ymax = max(y[start:stop])
y[start:stop] = base.exp(y[start:stop] - ymax)
# fi = log sum yi = log sum exp(Fi*x+gi)
ysum = blas.asum(y, n=stop - start, offset=start)
f[i] = ymax + math.log(ysum)
# yi := yi / sum(yi) = exp(Fi*x+gi) / sum(exp(Fi*x+gi))
blas.scal(1.0 / ysum, y, n=stop - start, offset=start)
# gradfi := Fi' * yi
# = Fi' * exp(Fi*x+gi) / sum(exp(Fi*x+gi))
base.gemv(F,
y,
Df,
trans='T',
m=stop - start,
incy=mnl + 1,
offsetA=start,
offsetx=start,
offsety=i)
if z is not None:
# Hi = Fi' * (diag(yi) - yi*yi') * Fi
# = Fisc' * Fisc
# where
# Fisc = diag(yi)^1/2 * (I - 1*yi') * Fi
# = diag(yi)^1/2 * (Fi - 1*gradfi')
Fsc[:K[i], :] = F[start:stop, :]
for k in range(start, stop):
blas.axpy(Df,
Fsc,
n=n,
alpha=-1.0,
incx=mnl + 1,
incy=Fsc.size[0],
offsetx=i,
offsety=k - start)
blas.scal(math.sqrt(y[k]),
Fsc,
inc=Fsc.size[0],
offset=k - start)
# H += z[i]*Hi = z[i] * Fisc' * Fisc
blas.syrk(Fsc,
H,
trans='T',
k=stop - start,
alpha=z[i],
beta=1.0)
if z is None:
return f, Df
return f, Df, H
def completion(X, factored_updates = True):
"""
Supernodal multifrontal maximum determinant positive definite
matrix completion. The routine computes the Cholesky factor
:math:`L` of the inverse of the maximum determinant positive
definite matrix completion of :math:`X`:, i.e.,
.. math::
P( S^{-1} ) = X
where :math:`S = LL^T`. On exit, the argument `X` contains the
lower-triangular Cholesky factor :math:`L`.
The optional argument `factored_updates` can be used to enable (if
True) or disable (if False) updating of intermediate
factorizations.
:param X: :py:class:`cspmatrix`
:param factored_updates: boolean
"""
assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix"
n = X.symb.n
snpost = X.symb.snpost
snptr = X.symb.snptr
chptr = X.symb.chptr
chidx = X.symb.chidx
relptr = X.symb.relptr
relidx = X.symb.relidx
blkptr = X.symb.blkptr
blkval = X.blkval
stack = []
for k in reversed(list(snpost)):
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# allocate F and copy X_{Jk,Nk} to leading columns of F
F = matrix(0.0, (nj,nj))
lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj, m = nj, n = nn, uplo = 'L')
# if supernode k is not a root node:
if na > 0:
# copy Vk to 2,2 block of F
Vk = stack.pop()
lapack.lacpy(Vk, F, offsetB = nn*nj+nn, m = na, n = na, uplo = 'L')
# if supernode k has any children:
for ii in range(chptr[k],chptr[k+1]):
i = chidx[ii]
if factored_updates:
r = relidx[relptr[i]:relptr[i+1]]
stack.append(frontal_get_update_factor(F,r,nn,na))
else:
stack.append(frontal_get_update(F,relidx,relptr,i))
# if supernode k is not a root node:
if na > 0:
if factored_updates:
# In this case we have Vk = Lk'*Lk
trL1 = 'T'
trL2 = 'N'
else:
# factorize Vk
lapack.potrf(F, offsetA = nj*nn+nn, n = na, ldA = nj)
# In this case we have Vk = Lk*Lk'
trL1 = 'N'
trL2 = 'T'
# compute L_{Ak,Nk} and inv(D_{Nk,Nk}) = S_{Nk,Nk} - S_{Ak,Nk}'*L_{Ak,Nk}
lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL1,\
offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn)
blas.syrk(blkval, blkval, n = nn, k = na, trans= 'T', alpha = -1.0, beta = 1.0,
offsetA = blkptr[k]+nn, offsetC = blkptr[k], ldA = nj, ldC = nj)
lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL2,\
offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn)
for i in range(nn):
blas.scal(-1.0, blkval, n = na, offset = blkptr[k] + i*nj + nn)
# factorize inv(D_{Nk,Nk}) as R*R' so that D_{Nk,Nk} = L*L' with L = inv(R)'
lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj,\
ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX!
F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX!
lapack.potrf(F, ldA = nj, n = nn, uplo = 'U') # factorize
F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX!
lapack.lacpy(F, blkval, offsetB = blkptr[k], ldA = nj,\
ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX!
# compute L = inv(R')
lapack.trtri(blkval, offsetA = blkptr[k], ldA = nj, n = nn)
X._is_factor = True
return
def robsvm(X, d, gamma, P, e):
"""
Solves the following robust SVM training problem:
minimize (1/2) w'*w + gamma*sum(v)
subject to diag(d)*(X*w + b*1) >= 1 - v + E*u
|| S_j*w ||_2 <= u_j, j = 1...t
v >= 0
The variables are w, b, v, and u. The matrix E is a selector
matrix with zeros and one '1' per row. E_ij = 1 means that the
i'th training vector is associated with the j'th uncertainty
ellipsoid.
A custom KKT solver that exploits low-rank structure is used, and
a positive definite system of equations of order n is
formed and solved at each iteration.
ARGUMENTS
X m-by-n matrix with training vectors as rows
d m-vector with training labels (-1,+1)
P list of t symmetric matrices of order n
e m-vector where e[i] is the index of the uncertainty
ellipsoid associated with the i'th training vector
RETURNS
w n-vector
b scalar
u t-vector
v m-vector
iters number of interior-point iterations
"""
m, n = X.size
assert type(
P
) is list, "P must be a list of t symmtric positive definite matrices of order n."
k = len(P)
if k > 0:
assert e.size == (m, 1), "e must be an m-vector."
assert max(e) < k and min(e) >= 0, "e[i] must be in {0,1,...,k-1}."
E = spmatrix(1., e, range(m), (k, m)).T
d = matrix(d, tc='d')
q = matrix(0.0, (n + k + 1 + m, 1))
q[n + k + 1:] = gamma
h = matrix(0.0, (2 * m + k * (n + 1), 1))
h[:m] = -1.0
# linear operators Q and G
def Q(x, y, alpha=1.0, beta=0.0, trans='N'):
y[:n] = alpha * x[:n] + beta * y[:n]
def G(x, y, alpha=1.0, beta=0.0, trans='N'):
"""
Implements the linear operator
[ -DX E -d -I ]
[ 0 0 0 -I ]
[ 0 -e_1' 0 0 ]
G = [ -P_1' 0 0 0 ]
[ . . . . ]
[ 0 -e_k' 0 0 ]
[ -P_k' 0 0 0 ]
and its adjoint G'.
"""
if trans == 'N':
tmp = +y[:m]
# y[:m] = alpha*(-DXw + Et - d*b - v) + beta*y[:m]
base.gemv(E, x[n:n + k], tmp, alpha=alpha, beta=beta)
blas.axpy(x[n + k + 1:], tmp, alpha=-alpha)
blas.axpy(d, tmp, alpha=-alpha * x[n + k])
y[:m] = tmp
base.gemv(X, x[:n], tmp, alpha=alpha, beta=0.0)
tmp = mul(d, tmp)
y[:m] -= tmp
# y[m:2*m] = -v
y[m:2 * m] = -alpha * x[n + k + 1:] + beta * y[m:2 * m]
# SOC 1,...,k
for i in range(k):
l = 2 * m + i * (n + 1)
y[l] = -alpha * x[n + i] + beta * y[l]
y[l + 1:l + 1 +
n] = -alpha * P[i] * x[:n] + beta * y[l + 1:l + 1 + n]
else:
tmp1 = mul(d, x[:m])
tmp2 = y[:n]
blas.gemv(X, tmp1, tmp2, trans='T', alpha=-alpha, beta=beta)
for i in range(k):
l = 2 * m + 1 + i * (n + 1)
blas.gemv(P[i],
x[l:l + n],
tmp2,
trans='T',
alpha=-alpha,
beta=1.0)
y[:n] = tmp2
tmp2 = y[n:n + k]
base.gemv(E, x[:m], tmp2, trans='T', alpha=alpha, beta=beta)
blas.axpy(x[2 * m:2 * m + k * (1 + n):n + 1], tmp2, alpha=-alpha)
y[n:n + k] = tmp2
y[n + k] = -alpha * blas.dot(d, x[:m]) + beta * y[n + k]
y[n + k +
1:] = -alpha * (x[:m] + x[m:2 * m]) + beta * y[n + k + 1:]
# precompute products Pi'*Pi
Pt = []
for p in P:
y = matrix(0.0, (n, n))
blas.syrk(p, y, trans='T')
Pt.append(y)
# scaled hyperbolic Householder transformations
def qscal(u, beta, v, inv=False):
"""
Transforms the vector u as
u := beta * (2*v*v' - J) * u
if 'inv' is False and as
u := (1/beta) * (2*J*v*v'*J - J) * u
if 'inv' is True.
"""
if not inv:
tmp = blas.dot(u, v)
u[0] *= -1
u += 2 * v * tmp
u *= beta
else:
u[0] *= -1.0
tmp = blas.dot(v, u)
u[0] -= 2 * v[0] * tmp
u[1:] += 2 * v[1:] * tmp
u /= beta
# custom KKT solver
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2 * m]**2
dbi = div(1.0, db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2, dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt, mul(W['di'][:m], d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), lt, trans='T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt, W['di'][:m])
Xs = spmatrix(tmp, range(m), range(m)) * X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d, tmp), range(m), range(m)) * E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0, (n, n))
Ab[::n + 1] = 1.0
base.syrk(Xs, Ab, trans='T', beta=1.0)
Bb = matrix(0.0, (n, k))
Bb = -Xs.T * Es # inefficient!?
D2 = spmatrix(0.0, range(k), range(k))
base.syrk(Es, D2, trans='T', partial=True)
d2 = +D2.V
del D2
py = matrix(0.0, (n, 1))
for i in range(k):
binvsq = (1.0 / W['beta'][i])**2
Ab += binvsq * Pt[i]
dvv = blas.dot(W['v'][i], W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans='T', alpha=1.0, beta=0.0)
blas.syrk(py, Ab, alpha=4 * binvsq * (dvv + 1), beta=1.0)
Bb[:, i] -= 4 * binvsq * W['v'][i][0] * dvv * py
d2[i] += binvsq * (1 + 4 * (W['v'][i][0]**2) * (dvv - 1))
d2i = div(1.0, d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam, mul(W['di'][:m], d))
tmp = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), tmp, trans='T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp, tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, M12, alpha=1.0)
tmp = mul(d, mul(W['di'][:m], lam))
blas.gemv(X, tmp, M12, trans='T', alpha=1.0, beta=1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt, range(k), range(k))
base.syrk(sBb, Ab, alpha=-1.0, beta=1.0)
M = matrix([[Ab, M12.T], [M12, a]])
lapack.potrf(M)
def f(x, y, z):
# residuals
rwt = x[:n + k]
rb = x[n + k]
rv = x[n + k + 1:n + k + 1 + m]
iw_rl1 = mul(W['di'][:m], z[:m])
iw_rl2 = mul(W['di'][m:2 * m], z[m:2 * m])
ri = [
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)]
for i in range(k)
]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
rbwt[n + i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans='T', alpha=-1.0, beta=1.0)
tmp = mul(W['di'][:m], iw_rl1)
tmp2 = matrix(0.0, (k, 1))
base.gemv(E, tmp, tmp2, trans='T')
rbwt[n:] += tmp2
tmp = mul(d, tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X, tmp, rbwt, trans='T', alpha=-1.0, beta=1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2 * m], iw_rl2) - mul(W['di'][:m], iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi, rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans='T', alpha=1.0, beta=1.0)
rtw = +rbwt[:n]
tmp = mul(d, tmp)
blas.gemv(X, tmp, rtw, trans='T', alpha=-1.0, beta=1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i, rtt)
tmp2 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, tmp2)
dwdb = matrix([rtw - tmp2, rtb + blas.dot(mul(d2i, lt), rtt)])
lapack.potrs(M, dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0, (k, 1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt * dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0, (m, 1))
blas.gemv(X, dwdb[:n], dv, alpha=-1.0)
dv = mul(d, dv) - d * dwdb[-1]
base.gemv(E, dt, dv, beta=1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2, dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = -mul(W['di'][m:2 * m], dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0, (n, 1))
for i in range(k):
blas.gemv(P[i], dwdb[:n], tmp, alpha=-1.0, beta=0.0)
tmp1 = matrix([-dt[i], tmp])
blas.axpy(ri[i], tmp1, alpha=-1.0)
qscal(tmp1, W['beta'][i], W['v'][i], inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n + k] = dt
x[n + k] = dwdb[-1]
x[n + k + 1:] = dv
z[:m] = wdz1
z[m:2 * m] = wdz2
for i in range(k):
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)] = wdzi[i]
return f
# solve cone QP and return solution
sol = solvers.coneqp(Q,
q,
G,
h,
dims={
'l': 2 * m,
'q': [n + 1 for i in range(k)],
's': []
},
kktsolver=F)
return sol['x'][:n], sol['x'][
n + k], sol['x'][n:n + k], sol['x'][n + k + 1:], sol['iterations']
def cholesky(X):
"""
Supernodal multifrontal Cholesky factorization:
.. math::
X = LL^T
where :math:`L` is lower-triangular. On exit, the argument :math:`X`
contains the Cholesky factor :math:`L`.
:param X: :py:class:`cspmatrix`
"""
assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix"
n = X.symb.n
snpost = X.symb.snpost
snptr = X.symb.snptr
chptr = X.symb.chptr
chidx = X.symb.chidx
relptr = X.symb.relptr
relidx = X.symb.relidx
blkptr = X.symb.blkptr
blkval = X.blkval
stack = []
for k in snpost:
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# build frontal matrix
F = matrix(0.0, (nj, nj))
lapack.lacpy(blkval, F, offsetA = blkptr[k], m = nj, n = nn, ldA = nj, uplo = 'L')
# add update matrices from children to frontal matrix
for i in range(chptr[k+1]-1,chptr[k]-1,-1):
Ui = stack.pop()
frontal_add_update(F, Ui, relidx, relptr, chidx[i])
# factor L_{Nk,Nk}
lapack.potrf(F, n = nn, ldA = nj)
# if supernode k is not a root node, compute and push update matrix onto stack
if na > 0:
# compute L_{Ak,Nk} := A_{Ak,Nk}*inv(L_{Nk,Nk}')
blas.trsm(F, F, m = na, n = nn, ldA = nj,
ldB = nj, offsetB = nn, transA = 'T', side = 'R')
# compute Uk = Uk - L_{Ak,Nk}*inv(D_{Nk,Nk})*L_{Ak,Nk}'
if nn == 1:
blas.syr(F, F, n = na, offsetx = nn, \
offsetA = nn*nj+nn, ldA = nj, alpha = -1.0)
else:
blas.syrk(F, F, k = nn, n = na, offsetA = nn, ldA = nj,
offsetC = nn*nj+nn, ldC = nj, alpha = -1.0, beta = 1.0)
# compute L_{Ak,Nk} := L_{Ak,Nk}*inv(L_{Nk,Nk})
blas.trsm(F, F, m = na, n = nn,\
ldA = nj, ldB = nj, offsetB = nn, side = 'R')
# add Uk to stack
Uk = matrix(0.0,(na,na))
lapack.lacpy(F, Uk, m = na, n = na, uplo = 'L', offsetA = nn*nj+nn, ldA = nj)
stack.append(Uk)
# copy the leading Nk columns of frontal matrix to blkval
lapack.lacpy(F, blkval, uplo = "L", offsetB = blkptr[k], m = nj, n = nn, ldB = nj)
X.is_factor = True
return
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2 * m]**2
dbi = div(1.0, db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2, dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt, mul(W['di'][:m], d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), lt, trans='T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt, W['di'][:m])
Xs = spmatrix(tmp, range(m), range(m)) * X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d, tmp), range(m), range(m)) * E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0, (n, n))
Ab[::n + 1] = 1.0
base.syrk(Xs, Ab, trans='T', beta=1.0)
Bb = matrix(0.0, (n, k))
Bb = -Xs.T * Es # inefficient!?
D2 = spmatrix(0.0, range(k), range(k))
base.syrk(Es, D2, trans='T', partial=True)
d2 = +D2.V
del D2
py = matrix(0.0, (n, 1))
for i in range(k):
binvsq = (1.0 / W['beta'][i])**2
Ab += binvsq * Pt[i]
dvv = blas.dot(W['v'][i], W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans='T', alpha=1.0, beta=0.0)
blas.syrk(py, Ab, alpha=4 * binvsq * (dvv + 1), beta=1.0)
Bb[:, i] -= 4 * binvsq * W['v'][i][0] * dvv * py
d2[i] += binvsq * (1 + 4 * (W['v'][i][0]**2) * (dvv - 1))
d2i = div(1.0, d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam, mul(W['di'][:m], d))
tmp = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), tmp, trans='T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp, tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, M12, alpha=1.0)
tmp = mul(d, mul(W['di'][:m], lam))
blas.gemv(X, tmp, M12, trans='T', alpha=1.0, beta=1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt, range(k), range(k))
base.syrk(sBb, Ab, alpha=-1.0, beta=1.0)
M = matrix([[Ab, M12.T], [M12, a]])
lapack.potrf(M)
def f(x, y, z):
# residuals
rwt = x[:n + k]
rb = x[n + k]
rv = x[n + k + 1:n + k + 1 + m]
iw_rl1 = mul(W['di'][:m], z[:m])
iw_rl2 = mul(W['di'][m:2 * m], z[m:2 * m])
ri = [
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)]
for i in range(k)
]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
rbwt[n + i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans='T', alpha=-1.0, beta=1.0)
tmp = mul(W['di'][:m], iw_rl1)
tmp2 = matrix(0.0, (k, 1))
base.gemv(E, tmp, tmp2, trans='T')
rbwt[n:] += tmp2
tmp = mul(d, tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X, tmp, rbwt, trans='T', alpha=-1.0, beta=1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2 * m], iw_rl2) - mul(W['di'][:m], iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi, rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans='T', alpha=1.0, beta=1.0)
rtw = +rbwt[:n]
tmp = mul(d, tmp)
blas.gemv(X, tmp, rtw, trans='T', alpha=-1.0, beta=1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i, rtt)
tmp2 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, tmp2)
dwdb = matrix([rtw - tmp2, rtb + blas.dot(mul(d2i, lt), rtt)])
lapack.potrs(M, dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0, (k, 1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt * dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0, (m, 1))
blas.gemv(X, dwdb[:n], dv, alpha=-1.0)
dv = mul(d, dv) - d * dwdb[-1]
base.gemv(E, dt, dv, beta=1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2, dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = -mul(W['di'][m:2 * m], dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0, (n, 1))
for i in range(k):
blas.gemv(P[i], dwdb[:n], tmp, alpha=-1.0, beta=0.0)
tmp1 = matrix([-dt[i], tmp])
blas.axpy(ri[i], tmp1, alpha=-1.0)
qscal(tmp1, W['beta'][i], W['v'][i], inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n + k] = dt
x[n + k] = dwdb[-1]
x[n + k + 1:] = dv
z[:m] = wdz1
z[m:2 * m] = wdz2
for i in range(k):
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)] = wdzi[i]
return f
def F(W):
# SVD R[j] = U[j] * diag(sig[j]) * Vt[j]
lapack.gesvd(+W['r'][0], sv, jobu='A', jobvt='A', U=U, Vt=Vt)
# Vt[j] := diag(sig[j])^-1 * Vt[j]
for k in xrange(ns):
blas.tbsv(sv, Vt, n=ns, k=0, ldA=1, offsetx=k * ns)
# Gamma[j] is an ns[j] x ns[j] symmetric matrix
#
# (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2)
# S = sig[j] * sig[j]'
S = matrix(0.0, (ns, ns))
blas.syrk(sv, S)
Gamma = div(S, sqrt(1.0 + rho * S**2))
symmetrize(Gamma, ns)
# As represents the scaled mapping
#
# As(x) = A(u * (Gamma .* x) * u')
# As'(y) = Gamma .* (u' * A'(y) * u)
#
# stored in a similar format as A, except that we use packed
# storage for the columns of As[i][j].
if type(A) is spmatrix:
blas.scal(0.0, As)
try:
As[VecAIndex] = +A['s'][VecAIndex]
except:
As[VecAIndex] = +A[VecAIndex]
else:
blas.copy(A, As)
# As[i][j][:,k] = diag( diag(Gamma[j]))*As[i][j][:,k]
# As[i][j][l,:] = Gamma[j][l,l]*As[i][j][l,:]
for k in xrange(ms):
cngrnc(U, As, trans='T', offsetx=k * (ns2))
blas.tbmv(Gamma, As, n=ns2, k=0, ldA=1, offsetx=k * (ns2))
misc.pack(As, Aspkd, {'l': 0, 'q': [], 's': [ns] * ms})
# H is an m times m block matrix with i, k block
#
# Hik = sum_j As[i,j]' * As[k,j]
#
# of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j]
# are zero for all j
H = matrix(0.0, (ms, ms))
blas.syrk(Aspkd, H, trans='T', beta=1.0, k=ns * (ns + 1) / 2)
lapack.potrf(H)
def solve(x, y, z):
"""
Returns solution of
rho * ux + A'(uy) - r^-T * uz * r^-1 = bx
A(ux) = by
-ux - r * uz * r' = bz.
On entry, x = bx, y = by, z = bz.
On exit, x = ux, y = uy, z = uz.
"""
# bz is a copy of z in the format of x
blas.copy(z, bz)
blas.axpy(bz, x, alpha=rho)
# x := Gamma .* (u' * x * u)
# = Gamma .* (u' * (bx + rho * bz) * u)
cngrnc(U, x, trans='T', offsetx=0)
blas.tbmv(Gamma, x, n=ns2, k=0, ldA=1, offsetx=0)
# y := y - As(x)
# := by - As( Gamma .* u' * (bx + rho * bz) * u)
#blas.copy(x,xp)
#pack_ip(xp,n = ns,m=1,nl=nl)
misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]})
blas.gemv(Aspkd, xp, y, trans = 'T',alpha = -1.0, beta = 1.0, \
m = ns*(ns+1)/2, n = ms,offsetx = 0)
# y := -y - A(bz)
# = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u)
Af(bz, y, alpha=-1.0, beta=-1.0)
# y := H^-1 * y
# = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) )
# = uy
blas.trsv(H, y)
blas.trsv(H, y, trans='T')
# bz = Vt' * vz * Vt
# = uz where
# vz := Gamma .* ( As'(uy) - x )
# = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) )
# = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ).
#blas.copy(x,xp)
#pack_ip(xp,n=ns,m=1,nl=nl)
misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]})
blas.scal(-1.0, xp)
blas.gemv(Aspkd,
y,
xp,
alpha=1.0,
beta=1.0,
m=ns * (ns + 1) / 2,
n=ms,
offsety=0)
# bz[j] is xp unpacked and multiplied with Gamma
misc.unpack(xp, bz, {'l': 0, 'q': [], 's': [ns]})
blas.tbmv(Gamma, bz, n=ns2, k=0, ldA=1, offsetx=0)
# bz = Vt' * bz * Vt
# = uz
cngrnc(Vt, bz, trans='T', offsetx=0)
symmetrize(bz, ns, offset=0)
# x = -bz - r * uz * r'
# z contains r.h.s. bz; copy to x
blas.copy(z, x)
blas.copy(bz, z)
cngrnc(W['r'][0], bz, offsetx=0)
blas.axpy(bz, x)
blas.scal(-1.0, x)
return solve
def F(W):
for j in xrange(N):
# SVD R[j] = U[j] * diag(sig[j]) * Vt[j]
lapack.gesvd(+W['r'][j],
sv[j],
jobu='A',
jobvt='A',
U=U[j],
Vt=Vt[j])
# Vt[j] := diag(sig[j])^-1 * Vt[j]
for k in xrange(ns[j]):
blas.tbsv(sv[j], Vt[j], n=ns[j], k=0, ldA=1, offsetx=k * ns[j])
# Gamma[j] is an ns[j] x ns[j] symmetric matrix
#
# (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2)
# S = sig[j] * sig[j]'
S = matrix(0.0, (ns[j], ns[j]))
blas.syrk(sv[j], S)
Gamma[j][:] = div(S, sqrt(1.0 + rho * S**2))[:]
symmetrize(Gamma[j], ns[j])
# As represents the scaled mapping
#
# As(x) = A(u * (Gamma .* x) * u')
# As'(y) = Gamma .* (u' * A'(y) * u)
#
# stored in a similar format as A, except that we use packed
# storage for the columns of As[i][j].
for i in xrange(M):
if (type(A[i][j]) is matrix) or (type(A[i][j]) is spmatrix):
# As[i][j][:,k] = vec(
# (U[j]' * mat( A[i][j][:,k] ) * U[j]) .* Gamma[j])
copy(A[i][j], As[i][j])
As[i][j] = matrix(As[i][j])
for k in xrange(ms[i]):
cngrnc(U[j],
As[i][j],
trans='T',
offsetx=k * (ns[j]**2),
n=ns[j])
blas.tbmv(Gamma[j],
As[i][j],
n=ns[j]**2,
k=0,
ldA=1,
offsetx=k * (ns[j]**2))
# pack As[i][j] in place
#pack_ip(As[i][j], ns[j])
for k in xrange(As[i][j].size[1]):
tmp = +As[i][j][:, k]
misc.pack2(tmp, {'l': 0, 'q': [], 's': [ns[j]]})
As[i][j][:, k] = tmp
else:
As[i][j] = 0.0
# H is an m times m block matrix with i, k block
#
# Hik = sum_j As[i,j]' * As[k,j]
#
# of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j]
# are zero for all j
H = spmatrix([], [], [], (sum(ms), sum(ms)))
rowid = 0
for i in xrange(M):
colid = 0
for k in xrange(i + 1):
sparse_block = True
Hik = matrix(0.0, (ms[i], ms[k]))
for j in xrange(N):
if (type(As[i][j]) is matrix) and \
(type(As[k][j]) is matrix):
sparse_block = False
# Hik += As[i,j]' * As[k,j]
if i == k:
blas.syrk(As[i][j],
Hik,
trans='T',
beta=1.0,
k=ns[j] * (ns[j] + 1) / 2,
ldA=ns[j]**2)
else:
blas.gemm(As[i][j],
As[k][j],
Hik,
transA='T',
beta=1.0,
k=ns[j] * (ns[j] + 1) / 2,
ldA=ns[j]**2,
ldB=ns[j]**2)
if not (sparse_block):
H[rowid:rowid+ms[i], colid:colid+ms[k]] \
= sparse(Hik)
colid += ms[k]
rowid += ms[i]
HF = cholmod.symbolic(H)
cholmod.numeric(H, HF)
def solve(x, y, z):
"""
Returns solution of
rho * ux + A'(uy) - r^-T * uz * r^-1 = bx
A(ux) = by
-ux - r * uz * r' = bz.
On entry, x = bx, y = by, z = bz.
On exit, x = ux, y = uy, z = uz.
"""
# bz is a copy of z in the format of x
blas.copy(z, bz)
# x := x + rho * bz
# = bx + rho * bz
blas.axpy(bz, x, alpha=rho)
# x := Gamma .* (u' * x * u)
# = Gamma .* (u' * (bx + rho * bz) * u)
offsetj = 0
for j in xrange(N):
cngrnc(U[j], x, trans='T', offsetx=offsetj, n=ns[j])
blas.tbmv(Gamma[j], x, n=ns[j]**2, k=0, ldA=1, offsetx=offsetj)
offsetj += ns[j]**2
# y := y - As(x)
# := by - As( Gamma .* u' * (bx + rho * bz) * u)
blas.copy(x, xp)
offsetj = 0
for j in xrange(N):
misc.pack2(xp, {'l': offsetj, 'q': [], 's': [ns[j]]})
offsetj += ns[j]**2
offseti = 0
for i in xrange(M):
offsetj = 0
for j in xrange(N):
if type(As[i][j]) is matrix:
blas.gemv(As[i][j],
xp,
y,
trans='T',
alpha=-1.0,
beta=1.0,
m=ns[j] * (ns[j] + 1) / 2,
n=ms[i],
ldA=ns[j]**2,
offsetx=offsetj,
offsety=offseti)
offsetj += ns[j]**2
offseti += ms[i]
# y := -y - A(bz)
# = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u)
Af(bz, y, alpha=-1.0, beta=-1.0)
# y := H^-1 * y
# = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) )
# = uy
cholmod.solve(HF, y)
# bz = Vt' * vz * Vt
# = uz where
# vz := Gamma .* ( As'(uy) - x )
# = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) )
# = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ).
blas.copy(x, xp)
offsetj = 0
for j in xrange(N):
# xp is -x[j] = -Gamma .* (u' * (bx + rho*bz) * u)
# in packed storage
misc.pack2(xp, {'l': offsetj, 'q': [], 's': [ns[j]]})
offsetj += ns[j]**2
blas.scal(-1.0, xp)
offsetj = 0
for j in xrange(N):
# xp += As'(uy)
offseti = 0
for i in xrange(M):
if type(As[i][j]) is matrix:
blas.gemv(As[i][j], y, xp, alpha = 1.0,
beta = 1.0, m = ns[j]*(ns[j]+1)/2, \
n = ms[i],ldA = ns[j]**2, \
offsetx = offseti, offsety = offsetj)
offseti += ms[i]
# bz[j] is xp unpacked and multiplied with Gamma
#unpack(xp, bz[j], ns[j])
misc.unpack(xp,
bz, {
'l': 0,
'q': [],
's': [ns[j]]
},
offsetx=offsetj,
offsety=offsetj)
blas.tbmv(Gamma[j],
bz,
n=ns[j]**2,
k=0,
ldA=1,
offsetx=offsetj)
# bz = Vt' * bz * Vt
# = uz
cngrnc(Vt[j], bz, trans='T', offsetx=offsetj, n=ns[j])
symmetrize(bz, ns[j], offset=offsetj)
offsetj += ns[j]**2
# x = -bz - r * uz * r'
blas.copy(z, x)
blas.copy(bz, z)
offsetj = 0
for j in xrange(N):
cngrnc(+W['r'][j], bz, offsetx=offsetj, n=ns[j])
offsetj += ns[j]**2
blas.axpy(bz, x)
blas.scal(-1.0, x)
return solve
def F(W):
"""
Generate a solver for
A'(uz0) = bx[0]
-uz0 - uz1 = bx[1]
A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0
- ux[1] - r1*r1' * uz1 * r1*r1' = bz1.
uz0, uz1, bz0, bz1 are symmetric m x m-matrices.
ux[0], bx[0] are n-vectors.
ux[1], bx[1] are symmetric m x m-matrices.
We first calculate a congruence that diagonalizes r0*r0' and r1*r1':
U' * r0 * r0' * U = I, U' * r1 * r1' * U = S.
We then make a change of variables
usx[0] = ux[0],
usx[1] = U' * ux[1] * U
usz0 = U^-1 * uz0 * U^-T
usz1 = U^-1 * uz1 * U^-T
and define
As() = U' * A() * U'
bsx[1] = U^-1 * bx[1] * U^-T
bsz0 = U' * bz0 * U
bsz1 = U' * bz1 * U.
This gives
As'(usz0) = bx[0]
-usz0 - usz1 = bsx[1]
As(usx[0]) - usx[1] - usz0 = bsz0
-usx[1] - S * usz1 * S = bsz1.
1. Eliminate usz0, usz1 using equations 3 and 4,
usz0 = As(usx[0]) - usx[1] - bsz0
usz1 = -S^-1 * (usx[1] + bsz1) * S^-1.
This gives two equations in usx[0] an usx[1].
As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0)
-As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1
= bsx[1] - bsz0 - S^-1 * bsz1 * S^-1.
2. Eliminate usx[1] using equation 2:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j],
usx[1] = ( S * As(usx[0]) * S ) ./ Gamma
+ ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma.
This gives an equation in usx[0].
As'( As(usx[0]) ./ Gamma )
= bx0 + As'(bsz0) +
As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma )
= bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ).
"""
# Calculate U s.t.
#
# U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s).
# Cholesky factorization r0 * r0' = L * L'
blas.syrk(W['r'][0], L)
lapack.potrf(L)
# SVD L^-1 * r1 = U * diag(s) * V'
blas.copy(W['r'][1], U)
blas.trsm(L, U)
lapack.gesvd(U, s, jobu = 'O')
# s := s**2
s[:] = s**2
# Uti := U
blas.copy(U, Uti)
# U := L^-T * U
blas.trsm(L, U, transA = 'T')
# Uti := L * Uti = U^-T
blas.trmm(L, Uti)
# Us := U * diag(s)^-1
blas.copy(U, Us)
for i in range(m):
blas.tbsv(s, Us, n = m, k = 0, ldA = 1, incx = m, offsetx = i)
# S is m x m with lower triangular entries s[i] * s[j]
# sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j])
# Upper triangular entries are undefined but nonzero.
blas.scal(0.0, S)
blas.syrk(s, S)
Gamma = 1.0 + S
sqrtG = sqrt(Gamma)
# Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n
# = Asi ./ sqrt(Gamma)
blas.copy(A, Asc)
misc.scale(Asc, # only 'r' part of the dictionary is used
{'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)),
'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)),
'v': [], 'beta': [], 'r': [ U ], 'rti': [ U ]})
for i in range(n):
blas.tbsv(sqrtG, Asc, n = msq, k = 0, ldA = 1, offsetx = i*msq)
# Convert columns of Asc to packed storage
misc.pack2(Asc, {'l': 0, 'q': [], 's': [ m ]})
# Cholesky factorization of Asc' * Asc.
H = matrix(0.0, (n, n))
blas.syrk(Asc, H, trans = 'T', k = mpckd)
lapack.potrf(H)
def solve(x, y, z):
"""
1. Solve for usx[0]:
Asc'(Asc(usx[0]))
= bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
= bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S )
./ sqrtG)
where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U,
bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1
2. Solve for usx[1]:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
usx[1]
= ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma
Unscale ux[1] = Uti * usx[1] * Uti'
3. Compute usz0, usz1
r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T
r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T
"""
# z0 := U' * z0 * U
# = bsz0
__cngrnc(U, z, trans = 'T')
# z1 := Us' * bz1 * Us
# = S^-1 * U' * bz1 * U * S^-1
# = S^-1 * bsz1 * S^-1
__cngrnc(Us, z, trans = 'T', offsetx = msq)
# x[1] := Uti' * x[1] * Uti
# = bsx[1]
__cngrnc(Uti, x[1], trans = 'T')
# x[1] := x[1] - z[msq:]
# = bsx[1] - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], alpha = -1.0, offsetx = msq)
# x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG
# = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG
# = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG
# in packed storage
blas.copy(x[1], x1)
blas.tbmv(S, x1, n = msq, k = 0, ldA = 1)
blas.axpy(z, x1, n = msq)
blas.tbsv(sqrtG, x1, n = msq, k = 0, ldA = 1)
misc.pack2(x1, {'l': 0, 'q': [], 's': [m]})
# x[0] := x[0] + Asc'*x1
# = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
# = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma )
blas.gemv(Asc, x1, x[0], m = mpckd, trans = 'T', beta = 1.0)
# x[0] := H^-1 * x[0]
# = ux[0]
lapack.potrs(H, x[0])
# x1 = Asc(x[0]) .* sqrtG (unpacked)
# = As(x[0])
blas.gemv(Asc, x[0], tmp, m = mpckd)
misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]})
blas.tbmv(sqrtG, x1, n = msq, k = 0, ldA = 1)
# usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2
# = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1)
# ./ Gamma
# x[1] := x[1] - z[:msq]
# = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], -1.0, n = msq)
# x[1] := x[1] + x1
# = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(x1, x[1])
# x[1] := x[1] / Gammma
# = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma
# = S^-1 * usx[1] * S^-1
blas.tbsv(Gamma, x[1], n = msq, k = 0, ldA = 1)
# z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1
# := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1
# := -r1' * uz1 * r1
blas.axpy(x[1], z, n = msq, offsety = msq)
blas.scal(-1.0, z, offset = msq)
__cngrnc(U, z, offsetx = msq)
__cngrnc(W['r'][1], z, trans = 'T', offsetx = msq)
# x[1] := S * x[1] * S
# = usx1
blas.tbmv(S, x[1], n = msq, k = 0, ldA = 1)
# z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0
# = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0
# = r0' * U' * usz0 * U * r0
# = r0' * uz0 * r0
blas.axpy(x1, z, -1.0, n = msq)
blas.scal(-1.0, z, n = msq)
blas.axpy(x[1], z, -1.0, n = msq)
__cngrnc(U, z)
__cngrnc(W['r'][0], z, trans = 'T')
# x[1] := Uti * x[1] * Uti'
# = ux[1]
__cngrnc(Uti, x[1])
return solve
def projected_inverse(L):
"""
Supernodal multifrontal projected inverse. The routine computes the projected inverse
.. math::
Y = P(L^{-T}L^{-1})
where :math:`L` is a Cholesky factor. On exit, the argument :math:`L` contains the
projected inverse :math:`Y`.
:param L: :py:class:`cspmatrix` (factor)
"""
assert isinstance(L, cspmatrix) and L.is_factor is True, "L must be a cspmatrix factor"
n = L.symb.n
snpost = L.symb.snpost
snptr = L.symb.snptr
chptr = L.symb.chptr
chidx = L.symb.chidx
relptr = L.symb.relptr
relidx = L.symb.relidx
blkptr = L.symb.blkptr
blkval = L.blkval
stack = []
for k in reversed(list(snpost)):
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# invert factor of D_{Nk,Nk}
lapack.trtri(blkval, offsetA = blkptr[k], ldA = nj, n = nn)
# zero-out strict upper triangular part of {Nj,Nj} block (just in case!)
for i in range(1,nn): blas.scal(0.0, blkval, offset = blkptr[k] + nj*i, n = i)
# compute inv(D_{Nk,Nk}) (store in 1,1 block of F)
F = matrix(0.0, (nj,nj))
blas.syrk(blkval, F, trans = 'T', offsetA = blkptr[k], ldA = nj, n = nn, k = nn)
# if supernode k is not a root node:
if na > 0:
# copy "update matrix" to 2,2 block of F
Vk = stack.pop()
lapack.lacpy(Vk, F, ldB = nj, offsetB = nn*nj+nn, m = na, n = na, uplo = 'L')
# compute S_{Ak,Nk} = -Vk*L_{Ak,Nk}; store in 2,1 block of F
blas.symm(Vk, blkval, F, m = na, n = nn, offsetB = blkptr[k]+nn,\
ldB = nj, offsetC = nn, ldC = nj, alpha = -1.0)
# compute S_nn = inv(D_{Nk,Nk}) - S_{Ak,Nk}'*L_{Ak,Nk}; store in 1,1 block of F
blas.gemm(F, blkval, F, transA = 'T', m = nn, n = nn, k = na,\
offsetA = nn, alpha = -1.0, beta = 1.0,\
offsetB = blkptr[k]+nn, ldB = nj)
# extract update matrices if supernode k has any children
for ii in range(chptr[k],chptr[k+1]):
i = chidx[ii]
stack.append(frontal_get_update(F, relidx, relptr, i))
# copy S_{Jk,Nk} (i.e., 1,1 and 2,1 blocks of F) to blkval
lapack.lacpy(F, blkval, m = nj, n = nn, offsetB = blkptr[k], ldB = nj, uplo = 'L')
L._is_factor = False
return