def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -W1^2 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -W2 ] [ z[m:] ] [ bz[m:] ]
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (W['di'] .* z is
# returned instead of z).
d1, d2 = W['d'][:m], W['d'][m:]
D = 4*(d1**2 + d2**2)**-1
A = P.T * spdiag(D) * P
lapack.potrf(A)
def f(x, y, z):
x[:n] += P.T * ( mul( div(d2**2 - d1**2, d1**2 + d2**2), x[n:])
+ mul( .5*D, z[:m]-z[m:] ) )
lapack.potrs(A, x)
u = P*x[:n]
x[n:] = div( x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) +
mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2 )
z[:m] = div(u-x[n:]-z[:m], d1)
z[m:] = div(-u-x[n:]-z[m:], d2)
return f
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -W1^2 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -W2 ] [ z[m:] ] [ bz[m:] ]
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (W['di'] .* z is
# returned instead of z).
d1, d2 = W['d'][:m], W['d'][m:]
D = 4 * (d1**2 + d2**2)**-1
A = P.T * spdiag(D) * P
lapack.potrf(A)
def f(x, y, z):
x[:n] += P.T * (mul(div(d2**2 - d1**2, d1**2 + d2**2), x[n:]) +
mul(.5 * D, z[:m] - z[m:]))
lapack.potrs(A, x)
u = P * x[:n]
x[n:] = div(
x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) +
mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2)
z[:m] = div(u - x[n:] - z[:m], d1)
z[m:] = div(-u - x[n:] - z[m:], d2)
return f
def F(W):
"""
Return a function f(x,y,z) that solves
[P , G' W^-1] [ux] [bx]
[G , -W ] [uy] = [bz]
"""
#d = spdiag(matrix(numpy.array(W['d'])))
#dinv= spdiag(matrix(numpy.array(W['di'])))
d = spdiag(W['d'])
dinv = spdiag(W['di'])
#KKT1 = d*( P * d + dinv )
KKT1 = d * P * d + spdiag(matrix(1.0, (2 * dim, 1)))
lapack.potrf(KKT1)
#raw_input('inputpppp')
def f(x, y, z):
uz = -d * (x + P * z)
#uz = matrix(numpy.linalg.solve(KKT1, uz)) # slow version
#lapack.gesv(KKT1,uz) # JZ: gesv have cond issue
lapack.potrs(KKT1, uz)
x[:] = matrix(-z - d * uz)
blas.copy(uz, z)
return f
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(A, Asc)
for k in range(m):
blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m + 1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \
mul(d1, z[:n] + mul(d3, z[:n])) - \
mul(d2, z[n:] - mul(d3, z[n:])) )
x[:n] = div(x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n])
z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:])
return g
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), sqrt(d1+d2) )
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(A, Asc)
for k in range(m):
blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \
mul(d1, z[:n] + mul(d3, z[:n])) - \
mul(d2, z[n:] - mul(d3, z[n:])) )
x[:n] = div( x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] )
z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] )
return g
def get_psd_matrix(p):
tmp = matrix(normal((p)**2),(p,p))/2.0
tmp = tmp + tmp.T
while(1):
try:
lapack.potrf(+tmp)
break
except:
tmp = tmp + .1*eye(p)
return tmp
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
# ds is square root of diagonal of D
ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]),
sqrt(d1+d2) )
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
Asc = A * spdiag(ds**-1)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) +
mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] -
mul(d3, z[n:])) )
x[:n] = div( x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n] -
# (D2-D1)*(D1+D2)^-1 * bx[n:] +
# D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
# D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
# zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] )
z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] )
return g
def Fkkt(W):
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
d1, d2 = W['di'][:n]**2, W['di'][n:]**2
print 'printing: ', W['di']
# ds is square root of diagonal of D
ds = math.sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
Asc = matrix(0.0, (m, n))
# Asc = A*diag(d)^-1/2
Asc = A * spdiag(ds**-1)
# S = I + A * D^-1 * A'
blas.syrk(Asc, S)
S[::m + 1] += 1.0
lapack.potrf(S)
def g(x, y, z):
x[:n] = 0.5 * (x[:n] - mul(d3, x[n:]) + mul(
d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - mul(d3, z[n:])))
x[:n] = div(x[:n], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n] -
# (D2-D1)*(D1+D2)^-1 * bx[n:] +
# D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
# D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
x[:n] = div(x[:n], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
- mul( d3, x[:n] )
# zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
# zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n])
z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:])
return g
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ]
#
# where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di'].
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (di .* z is
# returned instead of z).
# Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and
# d1 = d[:m].^2, d2 = d[m:].^2.
di = W['di']
d1, d2 = di[:m]**2, di[m:]**2
D = div(mul(d1, d2), d1 + d2)
Ds = spdiag(2 * sqrt(D))
base.gemm(Ds, P, Ps)
blas.syrk(Ps, A, trans='T')
lapack.potrf(A)
def f(x, y, z):
# Solve for x[:n]:
#
# A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:]
# + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ).
blas.copy((mul(div(d1 - d2, d1 + d2), x[n:]) +
mul(2 * D, z[:m] - z[m:])), u)
blas.gemv(P, u, x, beta=1.0, trans='T')
lapack.potrs(A, x)
# x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:]
# + (D1-D2)*P*x[:n])
base.gemv(P, x, u)
x[n:] = div(
x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) + mul(d1 - d2, u),
d1 + d2)
# z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m])
# z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:])
z[:m] = mul(di[:m], u - x[n:] - z[:m])
z[m:] = mul(di[m:], -u - x[n:] - z[m:])
return f
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ]
# [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ]
# [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ]
# [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ]
#
# where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di'].
#
# On entry bx, bz are stored in x, z.
# On exit x, z contain the solution, with z scaled (di .* z is
# returned instead of z).
# Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and
# d1 = d[:m].^2, d2 = d[m:].^2.
di = W['di']
d1, d2 = di[:m]**2, di[m:]**2
D = div( mul(d1,d2), d1+d2 )
Ds = spdiag(2 * sqrt(D))
base.gemm(Ds, P, Ps)
blas.syrk(Ps, A, trans = 'T')
lapack.potrf(A)
def f(x, y, z):
# Solve for x[:n]:
#
# A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:]
# + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ).
blas.copy(( mul( div(d1-d2, d1+d2), x[n:]) +
mul( 2*D, z[:m]-z[m:] ) ), u)
blas.gemv(P, u, x, beta = 1.0, trans = 'T')
lapack.potrs(A, x)
# x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:]
# + (D1-D2)*P*x[:n])
base.gemv(P, x, u)
x[n:] = div( x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) +
mul(d1-d2, u), d1+d2 )
# z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m])
# z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:])
z[:m] = mul(di[:m], u-x[n:]-z[:m])
z[m:] = mul(di[m:], -u-x[n:]-z[m:])
return f
def F(W):
"""
Returns a function f(x, y, z) that solves
-diag(z) = bx
-diag(x) - r*r'*z*r*r' = bz
where r = W['r'][0] = W['rti'][0]^{-T}.
"""
rti = W['rti'][0]
# t = rti*rti' as a nonsymmetric matrix.
t = matrix(0.0, (n,n))
blas.gemm(rti, rti, t, transB = 'T')
# Cholesky factorization of tsq = t.*t.
tsq = t**2
lapack.potrf(tsq)
def f(x, y, z):
"""
On entry, x contains bx, y is empty, and z contains bz stored
in column major order.
On exit, they contain the solution, with z scaled
(vec(r'*z*r) is returned instead of z).
We first solve
((rti*rti') .* (rti*rti')) * x = bx - diag(t*bz*t)
and take z = - rti' * (diag(x) + bz) * rti.
"""
# tbst := t * bz * t
tbst = +z
cngrnc(t, tbst)
# x := x - diag(tbst) = bx - diag(rti*rti' * bz * rti*rti')
x -= tbst[::n+1]
# x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bz*t))
lapack.potrs(tsq, x)
# z := z + diag(x) = bz + diag(x)
z[::n+1] += x
# z := -vec(rti' * z * rti)
# = -vec(rti' * (diag(x) + bz) * rti
cngrnc(rti, z, alpha = -1.0)
return f
def Fkkt(x, z, W):
ds = (2.0 * div(1 + x**2, (1 - x**2)**2))**-0.5
Asc = A * spdiag(ds)
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
a = z[0]
def g(x, y, z):
x[:] = mul(x, ds) / a
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
blas.gemv(Asc, v, x, alpha = -1.0, beta = 1.0, trans = 'T')
x[:] = mul(x, ds)
return g
def Fkkt(x, z, W):
ds = (2.0 * div(1 + x**2, (1 - x**2)**2))**-0.5
Asc = A * spdiag(ds)
blas.syrk(Asc, S)
S[::m+1] += 1.0
lapack.potrf(S)
a = z[0]
def g(x, y, z):
x[:] = mul(x, ds) / a
blas.gemv(Asc, x, v)
lapack.potrs(S, v)
blas.gemv(Asc, v, x, alpha = -1.0, beta = 1.0, trans = 'T')
x[:] = mul(x, ds)
return g
def F(x=None, z=None):
if x is None: return 0, matrix(1.0, (n,1))
X = V * spdiag(x) * V.T
L = +X
try: lapack.potrf(L)
except ArithmeticError: return None
f = - 2.0 * (log(L[0,0]) + log(L[1,1]))
W = +V
blas.trsm(L, W)
gradf = matrix(-1.0, (1,2)) * W**2
if z is None: return f, gradf
H = matrix(0.0, (n,n))
blas.syrk(W, H, trans='T')
return f, gradf, z[0] * H**2
def Fkkt(W):
rti = W['rti'][0]
# t = rti*rti' as a nonsymmetric matrix.
t = matrix(0.0, (n, n))
blas.gemm(rti, rti, t, transB='T')
# Cholesky factorization of tsq = t.*t.
tsq = t**2
lapack.potrf(tsq)
def f(x, y, z):
"""
Solve
-diag(z) = bx
-diag(x) - inv(rti*rti') * z * inv(rti*rti') = bs
On entry, x and z contain bx and bs.
On exit, they contain the solution, with z scaled
(inv(rti)'*z*inv(rti) is returned instead of z).
We first solve
((rti*rti') .* (rti*rti')) * x = bx - diag(t*bs*t)
and take z = -rti' * (diag(x) + bs) * rti.
"""
# tbst := t * zs * t = t * bs * t
tbst = matrix(z, (n, n))
cngrnc(t, tbst)
# x := x - diag(tbst) = bx - diag(rti*rti' * bs * rti*rti')
x -= tbst[::n + 1]
# x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bs*t))
lapack.potrs(tsq, x)
# z := z + diag(x) = bs + diag(x)
z[::n + 1] += x
# z := -rti' * z * rti = -rti' * (diag(x) + bs) * rti
cngrnc(rti, z, alpha=-1.0)
return f
def Fkkt(W):
rti = W['rti'][0]
# t = rti*rti' as a nonsymmetric matrix.
t = matrix(0.0, (n,n))
blas.gemm(rti, rti, t, transB = 'T')
# Cholesky factorization of tsq = t.*t.
tsq = t**2
lapack.potrf(tsq)
def f(x, y, z):
"""
Solve
-diag(z) = bx
-diag(x) - inv(rti*rti') * z * inv(rti*rti') = bs
On entry, x and z contain bx and bs.
On exit, they contain the solution, with z scaled
(inv(rti)'*z*inv(rti) is returned instead of z).
We first solve
((rti*rti') .* (rti*rti')) * x = bx - diag(t*bs*t)
and take z = -rti' * (diag(x) + bs) * rti.
"""
# tbst := t * zs * t = t * bs * t
tbst = matrix(z, (n,n))
cngrnc(t, tbst)
# x := x - diag(tbst) = bx - diag(rti*rti' * bs * rti*rti')
x -= tbst[::n+1]
# x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bs*t))
lapack.potrs(tsq, x)
# z := z + diag(x) = bs + diag(x)
z[::n+1] += x
# z := -rti' * z * rti = -rti' * (diag(x) + bs) * rti
cngrnc(rti, z, alpha = -1.0)
return f
def Fkkt(W):
d1, d2 = W['d'][:m], W['d'][m:]
D = 4*(d1**2 + d2**2)**-1
A = P.T * spdiag(D) * P
lapack.potrf(A)
def f(x, y, z):
x[:n] += P.T * ( mul( div(d2**2 - d1**2, d1**2 + d2**2), x[n:])
+ mul( .5*D, z[:m]-z[m:] ) )
lapack.potrs(A, x)
u = P*x[:n]
x[n:] = div( x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) +
mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2 )
z[:m] = div(u-x[n:]-z[:m], d1)
z[m:] = div(-u-x[n:]-z[m:], d2)
return f
def F(W):
"""
Generate a solver for
A'(uz0) = bx[0]
-uz0 - uz1 = bx[1]
A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0
- ux[1] - r1*r1' * uz1 * r1*r1' = bz1.
uz0, uz1, bz0, bz1 are symmetric m x m-matrices.
ux[0], bx[0] are n-vectors.
ux[1], bx[1] are symmetric m x m-matrices.
We first calculate a congruence that diagonalizes r0*r0' and r1*r1':
U' * r0 * r0' * U = I, U' * r1 * r1' * U = S.
We then make a change of variables
usx[0] = ux[0],
usx[1] = U' * ux[1] * U
usz0 = U^-1 * uz0 * U^-T
usz1 = U^-1 * uz1 * U^-T
and define
As() = U' * A() * U'
bsx[1] = U^-1 * bx[1] * U^-T
bsz0 = U' * bz0 * U
bsz1 = U' * bz1 * U.
This gives
As'(usz0) = bx[0]
-usz0 - usz1 = bsx[1]
As(usx[0]) - usx[1] - usz0 = bsz0
-usx[1] - S * usz1 * S = bsz1.
1. Eliminate usz0, usz1 using equations 3 and 4,
usz0 = As(usx[0]) - usx[1] - bsz0
usz1 = -S^-1 * (usx[1] + bsz1) * S^-1.
This gives two equations in usx[0] an usx[1].
As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0)
-As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1
= bsx[1] - bsz0 - S^-1 * bsz1 * S^-1.
2. Eliminate usx[1] using equation 2:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j],
usx[1] = ( S * As(usx[0]) * S ) ./ Gamma
+ ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma.
This gives an equation in usx[0].
As'( As(usx[0]) ./ Gamma )
= bx0 + As'(bsz0) +
As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma )
= bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ).
"""
# Calculate U s.t.
#
# U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s).
# Cholesky factorization r0 * r0' = L * L'
blas.syrk(W['r'][0], L)
lapack.potrf(L)
# SVD L^-1 * r1 = U * diag(s) * V'
blas.copy(W['r'][1], U)
blas.trsm(L, U)
lapack.gesvd(U, s, jobu='O')
# s := s**2
s[:] = s**2
# Uti := U
blas.copy(U, Uti)
# U := L^-T * U
blas.trsm(L, U, transA='T')
# Uti := L * Uti = U^-T
blas.trmm(L, Uti)
# Us := U * diag(s)^-1
blas.copy(U, Us)
for i in range(m):
blas.tbsv(s, Us, n=m, k=0, ldA=1, incx=m, offsetx=i)
# S is m x m with lower triangular entries s[i] * s[j]
# sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j])
# Upper triangular entries are undefined but nonzero.
blas.scal(0.0, S)
blas.syrk(s, S)
Gamma = 1.0 + S
sqrtG = sqrt(Gamma)
# Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n
# = Asi ./ sqrt(Gamma)
blas.copy(A, Asc)
misc.scale(
Asc, # only 'r' part of the dictionary is used
{
'dnl': matrix(0.0, (0, 1)),
'dnli': matrix(0.0, (0, 1)),
'd': matrix(0.0, (0, 1)),
'di': matrix(0.0, (0, 1)),
'v': [],
'beta': [],
'r': [U],
'rti': [U]
})
for i in range(n):
blas.tbsv(sqrtG, Asc, n=msq, k=0, ldA=1, offsetx=i * msq)
# Convert columns of Asc to packed storage
misc.pack2(Asc, {'l': 0, 'q': [], 's': [m]})
# Cholesky factorization of Asc' * Asc.
H = matrix(0.0, (n, n))
blas.syrk(Asc, H, trans='T', k=mpckd)
lapack.potrf(H)
def solve(x, y, z):
"""
1. Solve for usx[0]:
Asc'(Asc(usx[0]))
= bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
= bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S )
./ sqrtG)
where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U,
bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1
2. Solve for usx[1]:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
usx[1]
= ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma
Unscale ux[1] = Uti * usx[1] * Uti'
3. Compute usz0, usz1
r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T
r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T
"""
# z0 := U' * z0 * U
# = bsz0
__cngrnc(U, z, trans='T')
# z1 := Us' * bz1 * Us
# = S^-1 * U' * bz1 * U * S^-1
# = S^-1 * bsz1 * S^-1
__cngrnc(Us, z, trans='T', offsetx=msq)
# x[1] := Uti' * x[1] * Uti
# = bsx[1]
__cngrnc(Uti, x[1], trans='T')
# x[1] := x[1] - z[msq:]
# = bsx[1] - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], alpha=-1.0, offsetx=msq)
# x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG
# = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG
# = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG
# in packed storage
blas.copy(x[1], x1)
blas.tbmv(S, x1, n=msq, k=0, ldA=1)
blas.axpy(z, x1, n=msq)
blas.tbsv(sqrtG, x1, n=msq, k=0, ldA=1)
misc.pack2(x1, {'l': 0, 'q': [], 's': [m]})
# x[0] := x[0] + Asc'*x1
# = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
# = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma )
blas.gemv(Asc, x1, x[0], m=mpckd, trans='T', beta=1.0)
# x[0] := H^-1 * x[0]
# = ux[0]
lapack.potrs(H, x[0])
# x1 = Asc(x[0]) .* sqrtG (unpacked)
# = As(x[0])
blas.gemv(Asc, x[0], tmp, m=mpckd)
misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]})
blas.tbmv(sqrtG, x1, n=msq, k=0, ldA=1)
# usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2
# = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1)
# ./ Gamma
# x[1] := x[1] - z[:msq]
# = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], -1.0, n=msq)
# x[1] := x[1] + x1
# = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(x1, x[1])
# x[1] := x[1] / Gammma
# = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma
# = S^-1 * usx[1] * S^-1
blas.tbsv(Gamma, x[1], n=msq, k=0, ldA=1)
# z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1
# := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1
# := -r1' * uz1 * r1
blas.axpy(x[1], z, n=msq, offsety=msq)
blas.scal(-1.0, z, offset=msq)
__cngrnc(U, z, offsetx=msq)
__cngrnc(W['r'][1], z, trans='T', offsetx=msq)
# x[1] := S * x[1] * S
# = usx1
blas.tbmv(S, x[1], n=msq, k=0, ldA=1)
# z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0
# = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0
# = r0' * U' * usz0 * U * r0
# = r0' * uz0 * r0
blas.axpy(x1, z, -1.0, n=msq)
blas.scal(-1.0, z, n=msq)
blas.axpy(x[1], z, -1.0, n=msq)
__cngrnc(U, z)
__cngrnc(W['r'][0], z, trans='T')
# x[1] := Uti * x[1] * Uti'
# = ux[1]
__cngrnc(Uti, x[1])
return solve
def cholesky(X):
"""
Supernodal multifrontal Cholesky factorization:
.. math::
X = LL^T
where :math:`L` is lower-triangular. On exit, the argument :math:`X`
contains the Cholesky factor :math:`L`.
:param X: :py:class:`cspmatrix`
"""
assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix"
n = X.symb.n
snpost = X.symb.snpost
snptr = X.symb.snptr
chptr = X.symb.chptr
chidx = X.symb.chidx
relptr = X.symb.relptr
relidx = X.symb.relidx
blkptr = X.symb.blkptr
blkval = X.blkval
stack = []
for k in snpost:
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# build frontal matrix
F = matrix(0.0, (nj, nj))
lapack.lacpy(blkval, F, offsetA = blkptr[k], m = nj, n = nn, ldA = nj, uplo = 'L')
# add update matrices from children to frontal matrix
for i in range(chptr[k+1]-1,chptr[k]-1,-1):
Ui = stack.pop()
frontal_add_update(F, Ui, relidx, relptr, chidx[i])
# factor L_{Nk,Nk}
lapack.potrf(F, n = nn, ldA = nj)
# if supernode k is not a root node, compute and push update matrix onto stack
if na > 0:
# compute L_{Ak,Nk} := A_{Ak,Nk}*inv(L_{Nk,Nk}')
blas.trsm(F, F, m = na, n = nn, ldA = nj,
ldB = nj, offsetB = nn, transA = 'T', side = 'R')
# compute Uk = Uk - L_{Ak,Nk}*inv(D_{Nk,Nk})*L_{Ak,Nk}'
if nn == 1:
blas.syr(F, F, n = na, offsetx = nn, \
offsetA = nn*nj+nn, ldA = nj, alpha = -1.0)
else:
blas.syrk(F, F, k = nn, n = na, offsetA = nn, ldA = nj,
offsetC = nn*nj+nn, ldC = nj, alpha = -1.0, beta = 1.0)
# compute L_{Ak,Nk} := L_{Ak,Nk}*inv(L_{Nk,Nk})
blas.trsm(F, F, m = na, n = nn,\
ldA = nj, ldB = nj, offsetB = nn, side = 'R')
# add Uk to stack
Uk = matrix(0.0,(na,na))
lapack.lacpy(F, Uk, m = na, n = na, uplo = 'L', offsetA = nn*nj+nn, ldA = nj)
stack.append(Uk)
# copy the leading Nk columns of frontal matrix to blkval
lapack.lacpy(F, blkval, uplo = "L", offsetB = blkptr[k], m = nj, n = nn, ldB = nj)
X.is_factor = True
return
def Fkkt(W):
"""
Custom solver:
v := alpha * 2*A'*A * u + beta * v
"""
global mmS
mmS = matrix(0.0, (iR, iR))
global vvV
vvV = matrix(0.0, (iR, 1))
# Factor
#
# S = A*D^-1*A' + I
#
# where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2.
mmAsc = matrix(0.0, (iR, iC))
d1, d2 = W["di"][:iC] ** 2, W["di"][iC:] ** 2
# ds is square root of diagonal of D
ds = sqrt(2.0) * div(mul(W["di"][:iC], W["di"][iC:]), sqrt(d1 + d2))
d3 = div(d2 - d1, d1 + d2)
# Asc = A*diag(d)^-1/2
blas.copy(mmTh, mmAsc)
for k in range(iR):
blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k)
# S = I + A * D^-1 * A'
blas.syrk(mmAsc, mmS)
mmS[:: iR + 1] += 1.0
lapack.potrf(mmS)
def g(x, y, z):
x[:iC] = 0.5 * (
x[:iC] - mul(d3, x[iC:]) + mul(d1, z[:iC] + mul(d3, z[:iC])) - mul(d2, z[iC:] - mul(d3, z[iC:]))
)
x[:iC] = div(x[:iC], ds)
# Solve
#
# S * v = 0.5 * A * D^-1 * ( bx[:n]
# - (D2-D1)*(D1+D2)^-1 * bx[n:]
# + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n]
# - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] )
blas.gemv(mmAsc, x, vvV)
lapack.potrs(mmS, vvV)
# x[:n] = D^-1 * ( rhs - A'*v ).
blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans="T")
x[:iC] = div(x[:iC], ds)
# x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] )
# - (D2-D1)*(D1+D2)^-1 * x[:n]
x[iC:] = div(x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1 + d2) - mul(d3, x[:iC])
# z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] )
# z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ).
z[:iC] = mul(W["di"][:iC], x[:iC] - x[iC:] - z[:iC])
z[iC:] = mul(W["di"][iC:], -x[:iC] - x[iC:] - z[iC:])
return g
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 G' ] [ x ] = [ bx ]
# [ G -W'*W ] [ z ] [ bz ].
# First factor
#
# S = G' * W**-1 * W**-T * G
# = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1
#
# where
#
# W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n]
# W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:]
# W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J)
# with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I].
# As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A]
minor = 0
if not helpers.sp_minor_empty():
minor = helpers.sp_minor_top()
beta, v = W['beta'][0], W['v'][0]
As = 2 * v * (v[1:].T * A)
As[1:,:] *= -1.0
As[1:,:] -= A
As /= beta
# S = As'*As + 4 * (W1**2 + W2**2)**-1
S = As.T * As
helpers.sp_add_var("S", S)
d1, d2 = W['d'][:n], W['d'][n:]
d = 4.0 * (d1**2 + d2**2)**-1
S[::n+1] += d
lapack.potrf(S)
helpers.sp_create("00-Fkkt", minor)
def f(x, y, z):
minor = 0
if not helpers.sp_minor_empty():
minor = helpers.sp_minor_top()
else:
global loopf
loopf += 1
minor = loopf
helpers.sp_create("00-f", minor)
# z := - W**-T * z
z[:n] = -div( z[:n], d1 )
z[n:2*n] = -div( z[n:2*n], d2 )
z[2*n:] -= 2.0*v*( v[0]*z[2*n] - blas.dot(v[1:], z[2*n+1:]) )
z[2*n+1:] *= -1.0
z[2*n:] /= beta
# x := x - G' * W**-1 * z
x[:n] -= div(z[:n], d1) - div(z[n:2*n], d2) + As.T * z[-(m+1):]
x[n:] += div(z[:n], d1) + div(z[n:2*n], d2)
helpers.sp_create("15-f", minor)
# Solve for x[:n]:
#
# S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:]
x[:n] -= mul( div(d1**2 - d2**2, d1**2 + d2**2), x[n:])
helpers.sp_create("25-f", minor)
lapack.potrs(S, x)
helpers.sp_create("30-f", minor)
# Solve for x[n:]:
#
# (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n]
x[n:] += mul( d1**-2 - d2**-2, x[:n])
helpers.sp_create("35-f", minor)
x[n:] = div( x[n:], d1**-2 + d2**-2)
helpers.sp_create("40-f", minor)
# z := z + W^-T * G*x
z[:n] += div( x[:n] - x[n:2*n], d1)
helpers.sp_create("44-f", minor)
z[n:2*n] += div( -x[:n] - x[n:2*n], d2)
helpers.sp_create("48-f", minor)
z[2*n:] += As*x[:n]
helpers.sp_create("50-f", minor)
return f
H1[3:, 3:] = 2 * B
return f, Df, z[0] * H0 + sum(z[1:]) * H1
sol = solvers.cp(F)
A = matrix(sol['x'][[0, 1, 1, 2]], (2, 2))
b = sol['x'][3:]
if pylab_installed:
pylab.figure(1, facecolor='w')
pylab.plot(X[:, 0], X[:, 1], 'ko', X[:, 0], X[:, 1], '-k')
# Ellipsoid in the form { x | || L' * (x-c) ||_2 <= 1 }
L = +A
lapack.potrf(L)
c = +b
lapack.potrs(L, c)
# 1000 points on the unit circle
nopts = 1000
angles = matrix([a * 2.0 * pi / nopts for a in range(nopts)], (1, nopts))
circle = matrix(0.0, (2, nopts))
circle[0, :], circle[1, :] = cos(angles), sin(angles)
# ellipse = L^-T * circle + c
blas.trsm(L, circle, transA='T')
ellipse = circle + c[:, nopts * [0]]
ellipse2 = 0.5 * circle + c[:, nopts * [0]]
pylab.plot(ellipse[0, :].T, ellipse[1, :].T, 'k-')
def F(W):
"""
Create a solver for the linear equations
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - Dl^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - r*r' * [ ] * r*r' = bzs
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
where Dl = diag(W['l']), r = W['r'][0].
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ].
1. Compute matrices V1, V2 such that (with T = r*r')
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and S = [ diag(s); 0 ], s a positive q-vector.
2. Factor the mapping X -> X + S * X' * S:
X + S * X' * S = L( L'( X )).
3. Compute scaled mappings: a matrix As with as its columns the
coefficients of the scaled mapping
L^-1( V2' * A() * V1' )
and the matrix Gs = Dl^-1 * G.
4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As.
"""
# 1. Compute V1, V2, s.
r = W['r'][0]
# LQ factorization R[:q, :] = L1 * Q1.
lapack.lacpy(r, Q1, m=q)
lapack.gelqf(Q1, tau1)
lapack.lacpy(Q1, L1, n=q, uplo='L')
lapack.orglq(Q1, tau1)
# LQ factorization R[q:, :] = L2 * Q2.
lapack.lacpy(r, Q2, m=p, offsetA=q)
lapack.gelqf(Q2, tau2)
lapack.lacpy(Q2, L2, n=p, uplo='L')
lapack.orglq(Q2, tau2)
# V2, V1, s are computed from an SVD: if
#
# Q2 * Q1' = U * diag(s) * V',
#
# then V1 = V' * L1^-1 and V2 = L2^-T * U.
# T21 = Q2 * Q1.T
blas.gemm(Q2, Q1, T21, transB='T')
# SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1.
lapack.gesvd(T21, s, jobu='A', jobvt='A', U=V2, Vt=V1)
# # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2
# this will requires lapack.ormlq or lapack.unmlq
# V2 = L2^-T * U
blas.trsm(L2, V2, transA='T')
# V1 = V' * L1^-1
blas.trsm(L1, V1, side='R')
# 2. Factorization X + S * X' * S = L( L'( X )).
#
# The factor L is stored as a diagonal matrix D and a sparse lower
# triangular matrix P, such that
#
# L(X)[:] = D**-1 * (I + P) * X[:]
# L^-1(X)[:] = D * (I - P) * X[:].
# SS is q x q with SS[i,j] = si*sj.
blas.scal(0.0, SS)
blas.syr(s, SS)
# For a p x q matrix X, P*X[:] is Y[:] where
#
# Yij = si * sj * Xji if i < j
# = 0 otherwise.
#
P.V = SS[Itril2]
# For a p x q matrix X, D*X[:] is Y[:] where
#
# Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j
# = Xii / sqrt( 1 + si^2 ) if i = j
# = Xij otherwise.
#
DV[Idiag] = sqrt(1.0 + SS[::q + 1])
DV[Itriu] = sqrt(1.0 - SS[Itril3]**2)
D.V = DV**-1
# 3. Scaled linear mappings
# Ask := V2' * Ask * V1'
blas.scal(0.0, As)
base.axpy(A, As)
for i in xrange(n):
# tmp := V2' * As[i, :]
blas.gemm(V2,
As,
tmp,
transA='T',
m=p,
n=q,
k=p,
ldB=p,
offsetB=i * p * q)
# As[:,i] := tmp * V1'
blas.gemm(tmp,
V1,
As,
transB='T',
m=p,
n=q,
k=q,
ldC=p,
offsetC=i * p * q)
# As := D * (I - P) * As
# = L^-1 * As.
blas.copy(As, As2)
base.gemm(P, As, As2, alpha=-1.0, beta=1.0)
base.gemm(D, As2, As)
# Gs := Dl^-1 * G
blas.scal(0.0, Gs)
base.axpy(G, Gs)
for k in xrange(n):
blas.tbmv(W['di'], Gs, n=m, k=0, ldA=1, offsetx=k * m)
# 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As.
blas.syrk(As, H, trans='T', alpha=2.0)
blas.syrk(Gs, H, trans='T', beta=1.0)
base.axpy(C, H)
lapack.potrf(H)
def f(x, y, z):
"""
Solve
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - T * [ ] * T = bzs.
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ].
Define X = uzs21, Z = T * uzs * T:
C * ux + G' * uzl - 2*A'(X) = bx
[ 0 X' ] [ bX1 0 ]
T * [ ] * T - Z = T * [ ] * T
[ X 0 ] [ 0 bX2 ]
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs
[ -A(ux) -uX2 ] [ Z21 Z22 ]
Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ].
We use the congruence transformation
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and the factorization
X + S * X' * S = L( L'(X) )
to write this as
C * ux + G' * uzl - 2*A'(X) = bx
L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs,
[ -A(ux) -uX2 ] [ Z21 Z22 ]
or
C * ux + Gs' * uuzl - 2*As'(XX) = bx
XX - ZZ21 = bX
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - ZZ21 = bbzs_21
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
if we introduce scaled variables
uuzl = D * uzl
XX = L'(V2^-1 * X * V1^-1)
= L'(V2^-1 * uzs21 * V1^-1)
ZZ21 = L^-1(V2' * Z21 * V1')
and define
bbzs_21 = L^-1(V2' * bzs_21 * V1')
[ bX1 0 ]
bX = L^-1( V2' * (T * [ ] * T)_21 * V1').
[ 0 bX2 ]
Eliminating Z21 gives
C * ux + Gs' * uuzl - 2*As'(XX) = bx
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
and eliminating uuzl and XX gives
H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21)
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22.
In summary, we can use the following algorithm:
1. bXX := bX - bbzs21
[ bX1 0 ]
= L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1')
[ 0 bX2 ]
2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
3. From ux, compute
uuzl = Gs*ux - D^-1 * bzl and
X = V2 * L^-T(-As(ux) + bXX) * V1.
4. Return ux, uuzl,
rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r
and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22.
"""
# Save bzs_11, bzs_22, bzs_21.
lapack.lacpy(z, bz11, uplo='L', m=q, n=q, ldA=p + q, offsetA=m)
lapack.lacpy(z, bz21, m=p, n=q, ldA=p + q, offsetA=m + q)
lapack.lacpy(z,
bz22,
uplo='L',
m=p,
n=p,
ldA=p + q,
offsetA=m + (p + q + 1) * q)
# zl := D^-1 * zl
# = D^-1 * bzl
blas.tbmv(W['di'], z, n=m, k=0, ldA=1)
# zs := r' * [ bX1, 0; 0, bX2 ] * r.
# zs := [ bX1, 0; 0, bX2 ]
blas.scal(0.0, z, offset=m)
lapack.lacpy(x[1], z, uplo='L', m=q, n=q, ldB=p + q, offsetB=m)
lapack.lacpy(x[2],
z,
uplo='L',
m=p,
n=p,
ldB=p + q,
offsetB=m + (p + q + 1) * q)
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc=p + q + 1, offset=m)
# a := tril(zs)*r
blas.copy(r, a)
blas.trmm(z,
a,
side='L',
m=p + q,
n=p + q,
ldA=p + q,
ldB=p + q,
offsetA=m)
# zs := a'*r + r'*a
blas.syr2k(r,
a,
z,
trans='T',
n=p + q,
k=p + q,
ldB=p + q,
ldC=p + q,
offsetC=m)
# bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1')
#
# [ bX1 0 ]
# = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1').
# [ 0 bX2 ]
# a = [ r21 r22 ] * z
# = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r
# = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r
blas.symm(z,
r,
a,
side='R',
m=p,
n=p + q,
ldA=p + q,
ldC=p + q,
offsetB=q)
# bz21 := -bz21 + a * [ r11, r12 ]'
# = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21
blas.gemm(a,
r,
bz21,
transB='T',
m=p,
n=q,
k=p + q,
beta=-1.0,
ldA=p + q,
ldC=p)
# bz21 := V2' * bz21 * V1'
# = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1'
blas.gemm(V2, bz21, tmp, transA='T', m=p, n=q, k=p, ldB=p)
blas.gemm(tmp, V1, bz21, transB='T', m=p, n=q, k=q, ldC=p)
# bz21[:] := D * (I-P) * bz21[:]
# = L^-1 * bz21[:]
# = bXX[:]
blas.copy(bz21, tmp)
base.gemv(P, bz21, tmp, alpha=-1.0, beta=1.0)
base.gemv(D, tmp, bz21)
# Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
# x[0] := x[0] + Gs'*zl + 2*As'(bz21)
# = bx + G' * D^-1 * bzl + 2 * As'(bXX)
blas.gemv(Gs, z, x[0], trans='T', alpha=1.0, beta=1.0)
blas.gemv(As, bz21, x[0], trans='T', alpha=2.0, beta=1.0)
# x[0] := H \ x[0]
# = ux
lapack.potrs(H, x[0])
# uuzl = Gs*ux - D^-1 * bzl
blas.gemv(Gs, x[0], z, alpha=1.0, beta=-1.0)
# bz21 := V2 * L^-T(-As(ux) + bz21) * V1
# = X
blas.gemv(As, x[0], bz21, alpha=-1.0, beta=1.0)
blas.tbsv(DV, bz21, n=p * q, k=0, ldA=1)
blas.copy(bz21, tmp)
base.gemv(P, tmp, bz21, alpha=-1.0, beta=1.0, trans='T')
blas.gemm(V2, bz21, tmp)
blas.gemm(tmp, V1, bz21)
# zs := -zs + r' * [ 0, X'; X, 0 ] * r
# = r' * [ -bX1, X'; X, -bX2 ] * r.
# a := bz21 * [ r11, r12 ]
# = X * [ r11, r12 ]
blas.gemm(bz21, r, a, m=p, n=p + q, k=q, ldA=p, ldC=p + q)
# z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ]
# = rti' * uzs * rti
blas.syr2k(r,
a,
z,
trans='T',
beta=-1.0,
n=p + q,
k=p,
offsetA=q,
offsetC=m,
ldB=p + q,
ldC=p + q)
# uX1 = -Z11 - bzs_11
# = -(r*zs*r')_11 - bzs_11
# uX2 = -Z22 - bzs_22
# = -(r*zs*r')_22 - bzs_22
blas.copy(bz11, x[1])
blas.copy(bz22, x[2])
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc=p + q + 1, offset=m)
# a := r*tril(zs)
blas.copy(r, a)
blas.trmm(z,
a,
side='R',
m=p + q,
n=p + q,
ldA=p + q,
ldB=p + q,
offsetA=m)
# x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]'
# = -bzs_11 - (r*zs*r')_11
blas.syr2k(a, r, x[1], n=q, alpha=-1.0, beta=-1.0)
# x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]'
# = -bzs_22 - (r*zs*r')_22
blas.syr2k(a,
r,
x[2],
n=p,
alpha=-1.0,
beta=-1.0,
offsetA=q,
offsetB=q)
# scale diagonal of zs by 1/2
blas.scal(2.0, z, inc=p + q + 1, offset=m)
return f
if pylab_installed:
pylab.figure(1, facecolor='w', figsize=(6,6))
pylab.plot(V[0,:], V[1,:],'ow', mec='k')
pylab.plot([0], [0], 'k+')
I = [ k for k in range(n) if xd[k] > 1e-5 ]
pylab.plot(V[0,I], V[1,I],'or')
# Enclosing ellipse is {x | x' * (V*diag(xe)*V')^-1 * x = sqrt(2)}
nopts = 1000
angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) )
circle = matrix(0.0, (2,nopts))
circle[0,:], circle[1,:] = cos(angles), sin(angles)
W = V * spdiag(xd) * V.T
lapack.potrf(W)
ellipse = sqrt(2.0) * circle
blas.trmm(W, ellipse)
if pylab_installed:
pylab.plot(ellipse[0,:].T, ellipse[1,:].T, 'k--')
pylab.axis([-5, 5, -5, 5])
pylab.title('D-optimal design (fig. 7.9)')
pylab.axis('off')
# E-design.
#
# maximize w
# subject to w*I <= V*diag(x)*V'
# x >= 0
# sum(x) = 1
def kkt(W):
"""
KKT solver for
Q * ux + uy * 1_m' + mat(uz) = bx
ux * 1_m = by
ux - d.^2 .* mat(uz) = mat(bz).
ux and bx are N x m matrices.
uy and by are N-vectors.
uz and bz are N*m-vectors. mat(uz) is the N x m matrix that
satisfies mat(uz)[:] = uz.
d = mat(W['d']) a positive N x m matrix.
If we eliminate uz from the last equation using
mat(uz) = (ux - mat(bz)) ./ d.^2
we get two equations in ux, uy:
Q * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2
ux * 1_m = by.
From the 1st equation
uxk = -(Q + Dk)^-1 * uy + (Q + Dk)^-1 * (bxk + Dk * bzk)
where uxk is column k of ux, Dk = diag(d[:,k].^-2), and bzk is
column k of mat(bz). Substituting this in the second equation
gives an equation for uy.
1. Solve for uy
sum_k (Q + Dk)^-1 * uy =
sum_k (Q + Dk)^-1 * (bxk + Dk * bzk) - by.
2. Solve for ux (column by column)
Q * ux + ux ./ d.^2 = bx + mat(bz) ./ d.^2 - uy * 1_m'.
3. Solve for uz
mat(uz) = ( ux - mat(bz) ) ./ d.^2.
Return ux, uy, d .* uz.
"""
# D = d.^-2
D = matrix(W['di']**2, (N, m))
blas.scal(0.0, S)
for k in range(m):
# Hk := Q + Dk
blas.copy(Q, H[k])
H[k][::N + 1] += D[:, k]
# Hk := Hk^-1
# = (Q + Dk)^-1
lapack.potrf(H[k])
lapack.potri(H[k])
# S := S + Hk
# = S + (Q + Dk)^-1
blas.axpy(H[k], S)
# Factor S = sum_k (Q + Dk)^-1
lapack.potrf(S)
def f(x, y, z):
# z := mat(z)
# = mat(bz)
z.size = N, m
# x := x + D .* z
# = bx + mat(bz) ./ d.^2
x += mul(D, z)
# y := y - sum_k (Q + Dk)^-1 * X[:,k]
# = by - sum_k (Q + Dk)^-1 * (bxk + Dk * bzk)
for k in range(m):
blas.symv(H[k], x[:, k], y, alpha=-1.0, beta=1.0)
# y := H^-1 * y
# = -uy
lapack.potrs(S, y)
# x[:,k] := H[k] * (x[:,k] + y)
# = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y)
# = ux[:,k]
w = matrix(0.0, (N, 1))
for k in range(m):
# x[:,k] := x[:,k] + y
blas.axpy(y, x, offsety=N * k, n=N)
# w := H[k] * x[:,k]
# = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y)
blas.symv(H[k], x, w, offsetx=N * k)
# x[:,k] := w
# = ux[:,k]
blas.copy(w, x, offsety=N * k)
# y := -y
# = uy
blas.scal(-1.0, y)
# z := (x - z) ./ d
blas.axpy(x, z, -1.0)
blas.tbsv(W['d'], z, n=m * N, k=0, ldA=1)
blas.scal(-1.0, z)
z.size = N * m, 1
return f
def kkt(W):
"""
KKT solver for
X*X' * ux + uy * 1_m' + mat(uz) = bx
ux * 1_m = by
ux - d.^2 .* mat(uz) = mat(bz).
ux and bx are N x m matrices.
uy and by are N-vectors.
uz and bz are N*m-vectors. mat(uz) is the N x m matrix that
satisfies mat(uz)[:] = uz.
d = mat(W['d']) a positive N x m matrix.
If we eliminate uz from the last equation using
mat(uz) = (ux - mat(bz)) ./ d.^2
we get two equations in ux, uy:
X*X' * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2
ux * 1_m = by.
From the 1st equation,
uxk = (X*X' + Dk^-2)^-1 * (-uy + bxk + Dk^-2 * bzk)
= Dk * (I + Xk*Xk')^-1 * Dk * (-uy + bxk + Dk^-2 * bzk)
for k = 1, ..., m, where Dk = diag(d[:,k]), Xk = Dk * X,
uxk is column k of ux, and bzk is column k of mat(bz).
We use the matrix inversion lemma
( I + Xk * Xk' )^-1 = I - Xk * (I + Xk' * Xk)^-1 * Xk'
= I - Xk * Hk^-1 * Xk'
= I - Xk * Lk^-T * Lk^-1 * Xk'
where Hk = I + Xk' * Xk = Lk * Lk' to write this as
uxk = Dk * (I - Xk * Hk^-1 * Xk') * Dk *
(-uy + bxk + Dk^-2 * bzk)
= (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk).
Substituting this in the second equation gives an equation
for uy:
sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 * bzk ),
i.e., with D = (sum_k Dk^2)^1/2, Yk = D^-1 * Dk^2 * X * Lk^-T,
D * ( I - sum_k Yk * Yk' ) * D * uy
= -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
( bxk + Dk^-2 *bzk ).
Another application of the matrix inversion lemma gives
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k ( Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) *
( bxk + Dk^-2 *bzk ) )
with S = I - Y' * Y, Y = [ Y1 ... Ym ].
Summary:
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
d = matrix(W['d'], (N, m))
dsq = matrix(W['d']**2, (N, m))
# Factor the matrices
#
# H[k] = I + Xk' * Xk
# = I + X' * Dk^2 * X.
#
# Dk = diag(d[:,k]).
for k in range(m):
# H[k] = I
blas.scal(0.0, H[k])
H[k][::n + 1] = 1.0
# Xs = Dk * X
# = diag(d[:,k]]) * X
blas.copy(X, Xs)
for j in range(n):
blas.tbmv(d,
Xs,
n=N,
k=0,
ldA=1,
offsetA=k * N,
offsetx=j * N)
# H[k] := H[k] + Xs' * Xs
# = I + Xk' * Xk
blas.syrk(Xs, H[k], trans='T', beta=1.0)
# Factorization H[k] = Lk * Lk'
lapack.potrf(H[k])
###
utime, stime = cputime()
print("Factor Hk's: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# diag(D) = ( sum_k d[:,k]**2 ) ** 1/2
# = ( sum_k Dk^2) ** 1/2.
blas.gemv(dsq, ones, D)
D[:] = sqrt(D)
###
# utime, stime = cputime()
# print("Compute D: utime = %.2f, stime = %.2f" \
# %(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = I - Y'* Y is an m x m block matrix.
# The i,j block of Y' * Y is
#
# Yi' * Yj = Li^-1 * X' * Di^2 * D^-1 * Dj^2 * X * Lj^-T.
#
# We compute only the lower triangular blocks in Y'*Y.
blas.scal(0.0, S)
for i in range(m):
for j in range(i + 1):
# Xs = Di * Dj * D^-1 * X
blas.copy(X, Xs)
blas.copy(d, wN, n=N, offsetx=i * N)
blas.tbmv(d, wN, n=N, k=0, ldA=1, offsetA=j * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
for k in range(n):
blas.tbmv(wN, Xs, n=N, k=0, ldA=1, offsetx=k * N)
# block i, j of S is Xs' * Xs (as nonsymmetric matrix so we
# get the correct multiple after scaling with Li, Lj)
blas.gemm(Xs,
Xs,
S,
transA='T',
ldC=m * n,
offsetC=(j * n) * m * n + i * n)
###
utime, stime = cputime()
print("Form S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
for i in range(m):
# multiply block row i of S on the left with Li^-1
blas.trsm(H[i],
S,
m=n,
n=(i + 1) * n,
ldB=m * n,
offsetB=i * n)
# multiply block column i of S on the right with Li^-T
blas.trsm(H[i],
S,
side='R',
transA='T',
m=(m - i) * n,
n=n,
ldB=m * n,
offsetB=i * n * (m * n + 1))
blas.scal(-1.0, S)
S[::(m * n + 1)] += 1.0
###
utime, stime = cputime()
print("Form S (2): utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
# S = L*L'
lapack.potrf(S)
###
utime, stime = cputime()
print("Factor S: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
utime0, stime0 = cputime()
###
def f(x, y, z):
"""
1. Compute
uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 *
( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2)
* ( bxk + Dk^-2 *bzk ) )
2. For k = 1, ..., m:
uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) *
(-uy + bxk + Dk^-2 * bzk)
3. Solve for uz
d .* uz = ( ux - mat(bz) ) ./ d.
Return ux, uy, d .* uz.
"""
###
utime0, stime0 = cputime()
###
# xk := Dk^2 * xk + zk
# = Dk^2 * bxk + bzk
blas.tbmv(dsq, x, n=N * m, k=0, ldA=1)
blas.axpy(z, x)
# y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk
# = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk
# y := -y + x*ones
blas.gemv(x, ones, y, alpha=1.0, beta=-1.0)
# wnm = X' * x (wnm interpreted as an n x m matrix)
blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n)
# wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix)
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=k * n)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=n * k)
# wN = Dk^2 * wN
blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1)
# y := y - wN
blas.axpy(wN, y, -1.0)
# y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y
#
# Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T.
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# wnm = Y' * y (interpreted as an Nm vector)
# = [ L1^-1 * X' * D1^2 * D^-1 * y;
# L2^-1 * X' * D2^2 * D^-1 * y;
# ...
# Lm^-1 * X' * Dm^2 * D^-1 * y ]
for k in range(m):
# wN = D^-1 * Dk^2 * y
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# wnm[:,k] = X' * wN
blas.gemv(X, wN, wnm, trans='T', offsety=k * n)
# wnm[:,k] = Lk^-1 * wnm[:,k]
blas.trsv(H[k], wnm, offsetx=k * n)
# wnm := S^-1 * wnm (an mn-vector)
lapack.potrs(S, wnm)
# y := y + Y * wnm
# = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T]
# * wnm
for k in range(m):
# wnm[:,k] = Lk^-T * wnm[:,k]
blas.trsv(H[k], wnm, trans='T', offsetx=k * n)
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = D^-1 * Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.tbsv(D, wN, n=N, k=0, ldA=1)
# y += wN
blas.axpy(wN, y)
# y := D^-1 * y
blas.tbsv(D, y, n=N, k=0, ldA=1)
# For k = 1, ..., m:
#
# xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk)
# x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix)
for k in range(m):
blas.copy(y, wN)
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
blas.axpy(wN, x, -1.0, offsety=k * N)
# wnm = X' * x (as an n x m matrix)
blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n)
# wnm[:,k] = Hk^-1 * wnm[:,k]
for k in range(m):
lapack.potrs(H[k], wnm, offsetB=n * k)
for k in range(m):
# wN = X * wnm[:,k]
blas.gemv(X, wnm, wN, offsetx=k * n)
# wN = Dk^2 * wN
blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N)
# x[:,k] := x[:,k] - wN
blas.axpy(wN, x, -1.0, n=N, offsety=k * N)
# z := ( x - z ) ./ d
blas.axpy(x, z, -1.0)
blas.scal(-1.0, z)
blas.tbsv(d, z, n=N * m, k=0, ldA=1)
###
utime, stime = cputime()
print("Solve: utime = %.2f, stime = %.2f" \
%(utime-utime0, stime-stime0))
###
return f
def F(W):
"""
Generate a solver for
A'(uz0) = bx[0]
-uz0 - uz1 = bx[1]
A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0
- ux[1] - r1*r1' * uz1 * r1*r1' = bz1.
uz0, uz1, bz0, bz1 are symmetric m x m-matrices.
ux[0], bx[0] are n-vectors.
ux[1], bx[1] are symmetric m x m-matrices.
We first calculate a congruence that diagonalizes r0*r0' and r1*r1':
U' * r0 * r0' * U = I, U' * r1 * r1' * U = S.
We then make a change of variables
usx[0] = ux[0],
usx[1] = U' * ux[1] * U
usz0 = U^-1 * uz0 * U^-T
usz1 = U^-1 * uz1 * U^-T
and define
As() = U' * A() * U'
bsx[1] = U^-1 * bx[1] * U^-T
bsz0 = U' * bz0 * U
bsz1 = U' * bz1 * U.
This gives
As'(usz0) = bx[0]
-usz0 - usz1 = bsx[1]
As(usx[0]) - usx[1] - usz0 = bsz0
-usx[1] - S * usz1 * S = bsz1.
1. Eliminate usz0, usz1 using equations 3 and 4,
usz0 = As(usx[0]) - usx[1] - bsz0
usz1 = -S^-1 * (usx[1] + bsz1) * S^-1.
This gives two equations in usx[0] an usx[1].
As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0)
-As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1
= bsx[1] - bsz0 - S^-1 * bsz1 * S^-1.
2. Eliminate usx[1] using equation 2:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j],
usx[1] = ( S * As(usx[0]) * S ) ./ Gamma
+ ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma.
This gives an equation in usx[0].
As'( As(usx[0]) ./ Gamma )
= bx0 + As'(bsz0) +
As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma )
= bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ).
"""
# Calculate U s.t.
#
# U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s).
# Cholesky factorization r0 * r0' = L * L'
blas.syrk(W['r'][0], L)
lapack.potrf(L)
# SVD L^-1 * r1 = U * diag(s) * V'
blas.copy(W['r'][1], U)
blas.trsm(L, U)
lapack.gesvd(U, s, jobu = 'O')
# s := s**2
s[:] = s**2
# Uti := U
blas.copy(U, Uti)
# U := L^-T * U
blas.trsm(L, U, transA = 'T')
# Uti := L * Uti = U^-T
blas.trmm(L, Uti)
# Us := U * diag(s)^-1
blas.copy(U, Us)
for i in range(m):
blas.tbsv(s, Us, n = m, k = 0, ldA = 1, incx = m, offsetx = i)
# S is m x m with lower triangular entries s[i] * s[j]
# sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j])
# Upper triangular entries are undefined but nonzero.
blas.scal(0.0, S)
blas.syrk(s, S)
Gamma = 1.0 + S
sqrtG = sqrt(Gamma)
# Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n
# = Asi ./ sqrt(Gamma)
blas.copy(A, Asc)
misc.scale(Asc, # only 'r' part of the dictionary is used
{'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)),
'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)),
'v': [], 'beta': [], 'r': [ U ], 'rti': [ U ]})
for i in range(n):
blas.tbsv(sqrtG, Asc, n = msq, k = 0, ldA = 1, offsetx = i*msq)
# Convert columns of Asc to packed storage
misc.pack2(Asc, {'l': 0, 'q': [], 's': [ m ]})
# Cholesky factorization of Asc' * Asc.
H = matrix(0.0, (n, n))
blas.syrk(Asc, H, trans = 'T', k = mpckd)
lapack.potrf(H)
def solve(x, y, z):
"""
1. Solve for usx[0]:
Asc'(Asc(usx[0]))
= bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
= bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S )
./ sqrtG)
where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U,
bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1
2. Solve for usx[1]:
usx[1] + S * usx[1] * S
= S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1
usx[1]
= ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma
= -bsz0 + (S * As(usx[0]) * S) ./ Gamma
+ (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma
Unscale ux[1] = Uti * usx[1] * Uti'
3. Compute usz0, usz1
r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T
r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T
"""
# z0 := U' * z0 * U
# = bsz0
__cngrnc(U, z, trans = 'T')
# z1 := Us' * bz1 * Us
# = S^-1 * U' * bz1 * U * S^-1
# = S^-1 * bsz1 * S^-1
__cngrnc(Us, z, trans = 'T', offsetx = msq)
# x[1] := Uti' * x[1] * Uti
# = bsx[1]
__cngrnc(Uti, x[1], trans = 'T')
# x[1] := x[1] - z[msq:]
# = bsx[1] - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], alpha = -1.0, offsetx = msq)
# x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG
# = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG
# = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG
# in packed storage
blas.copy(x[1], x1)
blas.tbmv(S, x1, n = msq, k = 0, ldA = 1)
blas.axpy(z, x1, n = msq)
blas.tbsv(sqrtG, x1, n = msq, k = 0, ldA = 1)
misc.pack2(x1, {'l': 0, 'q': [], 's': [m]})
# x[0] := x[0] + Asc'*x1
# = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG)
# = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma )
blas.gemv(Asc, x1, x[0], m = mpckd, trans = 'T', beta = 1.0)
# x[0] := H^-1 * x[0]
# = ux[0]
lapack.potrs(H, x[0])
# x1 = Asc(x[0]) .* sqrtG (unpacked)
# = As(x[0])
blas.gemv(Asc, x[0], tmp, m = mpckd)
misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]})
blas.tbmv(sqrtG, x1, n = msq, k = 0, ldA = 1)
# usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2
# = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1)
# ./ Gamma
# x[1] := x[1] - z[:msq]
# = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(z, x[1], -1.0, n = msq)
# x[1] := x[1] + x1
# = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1
blas.axpy(x1, x[1])
# x[1] := x[1] / Gammma
# = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma
# = S^-1 * usx[1] * S^-1
blas.tbsv(Gamma, x[1], n = msq, k = 0, ldA = 1)
# z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1
# := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1
# := -r1' * uz1 * r1
blas.axpy(x[1], z, n = msq, offsety = msq)
blas.scal(-1.0, z, offset = msq)
__cngrnc(U, z, offsetx = msq)
__cngrnc(W['r'][1], z, trans = 'T', offsetx = msq)
# x[1] := S * x[1] * S
# = usx1
blas.tbmv(S, x[1], n = msq, k = 0, ldA = 1)
# z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0
# = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0
# = r0' * U' * usz0 * U * r0
# = r0' * uz0 * r0
blas.axpy(x1, z, -1.0, n = msq)
blas.scal(-1.0, z, n = msq)
blas.axpy(x[1], z, -1.0, n = msq)
__cngrnc(U, z)
__cngrnc(W['r'][0], z, trans = 'T')
# x[1] := Uti * x[1] * Uti'
# = ux[1]
__cngrnc(Uti, x[1])
return solve
def F(W):
# SVD R[j] = U[j] * diag(sig[j]) * Vt[j]
lapack.gesvd(+W['r'][0], sv, jobu='A', jobvt='A', U=U, Vt=Vt)
# Vt[j] := diag(sig[j])^-1 * Vt[j]
for k in xrange(ns):
blas.tbsv(sv, Vt, n=ns, k=0, ldA=1, offsetx=k * ns)
# Gamma[j] is an ns[j] x ns[j] symmetric matrix
#
# (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2)
# S = sig[j] * sig[j]'
S = matrix(0.0, (ns, ns))
blas.syrk(sv, S)
Gamma = div(S, sqrt(1.0 + rho * S**2))
symmetrize(Gamma, ns)
# As represents the scaled mapping
#
# As(x) = A(u * (Gamma .* x) * u')
# As'(y) = Gamma .* (u' * A'(y) * u)
#
# stored in a similar format as A, except that we use packed
# storage for the columns of As[i][j].
if type(A) is spmatrix:
blas.scal(0.0, As)
try:
As[VecAIndex] = +A['s'][VecAIndex]
except:
As[VecAIndex] = +A[VecAIndex]
else:
blas.copy(A, As)
# As[i][j][:,k] = diag( diag(Gamma[j]))*As[i][j][:,k]
# As[i][j][l,:] = Gamma[j][l,l]*As[i][j][l,:]
for k in xrange(ms):
cngrnc(U, As, trans='T', offsetx=k * (ns2))
blas.tbmv(Gamma, As, n=ns2, k=0, ldA=1, offsetx=k * (ns2))
misc.pack(As, Aspkd, {'l': 0, 'q': [], 's': [ns] * ms})
# H is an m times m block matrix with i, k block
#
# Hik = sum_j As[i,j]' * As[k,j]
#
# of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j]
# are zero for all j
H = matrix(0.0, (ms, ms))
blas.syrk(Aspkd, H, trans='T', beta=1.0, k=ns * (ns + 1) / 2)
lapack.potrf(H)
def solve(x, y, z):
"""
Returns solution of
rho * ux + A'(uy) - r^-T * uz * r^-1 = bx
A(ux) = by
-ux - r * uz * r' = bz.
On entry, x = bx, y = by, z = bz.
On exit, x = ux, y = uy, z = uz.
"""
# bz is a copy of z in the format of x
blas.copy(z, bz)
blas.axpy(bz, x, alpha=rho)
# x := Gamma .* (u' * x * u)
# = Gamma .* (u' * (bx + rho * bz) * u)
cngrnc(U, x, trans='T', offsetx=0)
blas.tbmv(Gamma, x, n=ns2, k=0, ldA=1, offsetx=0)
# y := y - As(x)
# := by - As( Gamma .* u' * (bx + rho * bz) * u)
#blas.copy(x,xp)
#pack_ip(xp,n = ns,m=1,nl=nl)
misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]})
blas.gemv(Aspkd, xp, y, trans = 'T',alpha = -1.0, beta = 1.0, \
m = ns*(ns+1)/2, n = ms,offsetx = 0)
# y := -y - A(bz)
# = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u)
Af(bz, y, alpha=-1.0, beta=-1.0)
# y := H^-1 * y
# = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) )
# = uy
blas.trsv(H, y)
blas.trsv(H, y, trans='T')
# bz = Vt' * vz * Vt
# = uz where
# vz := Gamma .* ( As'(uy) - x )
# = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) )
# = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ).
#blas.copy(x,xp)
#pack_ip(xp,n=ns,m=1,nl=nl)
misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]})
blas.scal(-1.0, xp)
blas.gemv(Aspkd,
y,
xp,
alpha=1.0,
beta=1.0,
m=ns * (ns + 1) / 2,
n=ms,
offsety=0)
# bz[j] is xp unpacked and multiplied with Gamma
misc.unpack(xp, bz, {'l': 0, 'q': [], 's': [ns]})
blas.tbmv(Gamma, bz, n=ns2, k=0, ldA=1, offsetx=0)
# bz = Vt' * bz * Vt
# = uz
cngrnc(Vt, bz, trans='T', offsetx=0)
symmetrize(bz, ns, offset=0)
# x = -bz - r * uz * r'
# z contains r.h.s. bz; copy to x
blas.copy(z, x)
blas.copy(bz, z)
cngrnc(W['r'][0], bz, offsetx=0)
blas.axpy(bz, x)
blas.scal(-1.0, x)
return solve
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 G' ] [ x ] = [ bx ]
# [ G -W'*W ] [ z ] [ bz ].
# First factor
#
# S = G' * W**-1 * W**-T * G
# = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1
#
# where
#
# W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n]
# W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:]
# W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J)
# with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I].
# As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A]
minor = 0
if not helpers.sp_minor_empty():
minor = helpers.sp_minor_top()
beta, v = W['beta'][0], W['v'][0]
As = 2 * v * (v[1:].T * A)
As[1:, :] *= -1.0
As[1:, :] -= A
As /= beta
# S = As'*As + 4 * (W1**2 + W2**2)**-1
S = As.T * As
helpers.sp_add_var("S", S)
d1, d2 = W['d'][:n], W['d'][n:]
d = 4.0 * (d1**2 + d2**2)**-1
S[::n + 1] += d
lapack.potrf(S)
helpers.sp_create("00-Fkkt", minor)
def f(x, y, z):
minor = 0
if not helpers.sp_minor_empty():
minor = helpers.sp_minor_top()
else:
global loopf
loopf += 1
minor = loopf
helpers.sp_create("00-f", minor)
# z := - W**-T * z
z[:n] = -div(z[:n], d1)
z[n:2 * n] = -div(z[n:2 * n], d2)
z[2 * n:] -= 2.0 * v * (v[0] * z[2 * n] -
blas.dot(v[1:], z[2 * n + 1:]))
z[2 * n + 1:] *= -1.0
z[2 * n:] /= beta
# x := x - G' * W**-1 * z
x[:n] -= div(z[:n], d1) - div(z[n:2 * n], d2) + As.T * z[-(m + 1):]
x[n:] += div(z[:n], d1) + div(z[n:2 * n], d2)
helpers.sp_create("15-f", minor)
# Solve for x[:n]:
#
# S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:]
x[:n] -= mul(div(d1**2 - d2**2, d1**2 + d2**2), x[n:])
helpers.sp_create("25-f", minor)
lapack.potrs(S, x)
helpers.sp_create("30-f", minor)
# Solve for x[n:]:
#
# (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n]
x[n:] += mul(d1**-2 - d2**-2, x[:n])
helpers.sp_create("35-f", minor)
x[n:] = div(x[n:], d1**-2 + d2**-2)
helpers.sp_create("40-f", minor)
# z := z + W^-T * G*x
z[:n] += div(x[:n] - x[n:2 * n], d1)
helpers.sp_create("44-f", minor)
z[n:2 * n] += div(-x[:n] - x[n:2 * n], d2)
helpers.sp_create("48-f", minor)
z[2 * n:] += As * x[:n]
helpers.sp_create("50-f", minor)
return f
def custom_kkt(W):
"""
Custom KKT solver for the following conic LP
formulation of the Schur relaxation of the
balance-constrained min/max cut problem
maximize Tr(C,X)
subject to
X_{ii} = 1, i=1...n
sum(X) + x = const
x >= 0, X psd
"""
r = W['rti'][0]
N = r.size[0]
e = matrix(1.0, (N, 1))
# Form and factorize reduced KKT system
H = matrix(0.0, (N + 1, N + 1))
blas.syrk(r, H, n=N, ldC=N + 1)
blas.symv(H, e, H, n=N, ldA=N + 1, offsety=N, incy=N + 1)
H[N, N] = blas.dot(H, e, n=N, offsetx=N, incx=N + 1)
rr = H[:N, :N] # Extract and symmetrize (1,1) block
misc.symm(rr, N) #
q = H[N, :N].T # Extract q = rr*e
H = mul(H, H)
H[N, N] += W['di'][0]**2
lapack.potrf(H)
def fsolve(x, y, z):
"""
Solves the system of equations
[ 0 G'*W^{-1} ] [ ux ] = [ bx ]
[ G -W' ] [ uz ] [ bz ]
"""
# Compute bx := bx + G'*W^{-1}*W^{-T}*bz
v = matrix(0., (N, 1))
for i in range(N):
blas.symv(z, rr, v, ldA=N, offsetA=1, n=N, offsetx=N * i)
x[i] += blas.dot(rr, v, n=N, offsetx=N * i)
blas.symv(z, q, v, ldA=N, offsetA=1, n=N)
x[N] += blas.dot(q, v) + z[0] * W['di'][0]**2
# Solve G'*W^{-1}*W^{-T}*G*ux = bx
lapack.potrs(H, x)
# Compute bz := -W^{-T}*(bz-G*ux)
# z -= G*x
z[1::N + 1] -= x[:-1]
z -= x[-1]
# Apply scaling
z[0] *= -W['di'][0]
blas.scal(0.5, z, n=N, offset=1, inc=N + 1)
tmp = +r
blas.trmm(z, tmp, ldA=N, offsetA=1, n=N, m=N)
blas.syr2k(r,
tmp,
z,
trans='T',
offsetC=1,
ldC=N,
n=N,
k=N,
alpha=-1.0)
return fsolve
H1[3:,2] = -2.0 * c[1] * B[:,1]
H1[3:,3:] = 2*B
return f, Df, z[0]*H0 + sum(z[1:])*H1
sol = solvers.cp(F)
A = matrix( sol['x'][[0, 1, 1, 2]], (2,2))
b = sol['x'][3:]
if pylab_installed:
pylab.figure(1, facecolor='w')
pylab.plot(X[:,0], X[:,1], 'ko', X[:,0], X[:,1], '-k')
# Ellipsoid in the form { x | || L' * (x-c) ||_2 <= 1 }
L = +A
lapack.potrf(L)
c = +b
lapack.potrs(L, c)
# 1000 points on the unit circle
nopts = 1000
angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) )
circle = matrix(0.0, (2,nopts))
circle[0,:], circle[1,:] = cos(angles), sin(angles)
# ellipse = L^-T * circle + c
blas.trsm(L, circle, transA='T')
ellipse = circle + c[:, nopts*[0]]
ellipse2 = 0.5 * circle + c[:, nopts*[0]]
pylab.plot(ellipse[0,:].T, ellipse[1,:].T, 'k-')
Hac = G.T * spdiag((h-G*xac)**-1) * G
if pylab_installed:
pylab.figure(3, facecolor='w')
# polyhedron
for k in range(m):
edge = X[[k,k+1],:] + 0.1 * matrix([1., 0., 0., -1.], (2,2)) * \
(X[2*[k],:] - X[2*[k+1],:])
pylab.plot(edge[:,0], edge[:,1], 'k')
# 1000 points on the unit circle
nopts = 1000
angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) )
circle = matrix(0.0, (2,nopts))
circle[0,:], circle[1,:] = cos(angles), sin(angles)
# ellipse = L^-T * circle + xc where Hac = L*L'
lapack.potrf(Hac)
ellipse = +circle
blas.trsm(Hac, ellipse, transA='T')
ellipse += xac[:, nopts*[0]]
pylab.fill(ellipse[0,:].T, ellipse[1,:].T, facecolor = '#F0F0F0')
pylab.plot([xac[0]], [xac[1]], 'ko')
pylab.title('Analytic center (fig 8.7)')
pylab.axis('equal')
pylab.axis('off')
pylab.show()
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 G' ] [ x ] = [ bx ]
# [ G -W'*W ] [ z ] [ bz ].
# First factor
#
# S = G' * W**-1 * W**-T * G
# = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1
#
# where
#
# W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n]
# W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:]
# W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J)
# with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I].
# As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A]
beta, v = W['beta'][0], W['v'][0]
As = 2 * v * (v[1:].T * A)
As[1:,:] *= -1.0
As[1:,:] -= A
As /= beta
# S = As'*As + 4 * (W1**2 + W2**2)**-1
S = As.T * As
d1, d2 = W['d'][:n], W['d'][n:]
d = 4.0 * (d1**2 + d2**2)**-1
S[::n+1] += d
lapack.potrf(S)
def f(x, y, z):
# z := - W**-T * z
z[:n] = -div( z[:n], d1 )
z[n:2*n] = -div( z[n:2*n], d2 )
z[2*n:] -= 2.0*v*( v[0]*z[2*n] - blas.dot(v[1:], z[2*n+1:]) )
z[2*n+1:] *= -1.0
z[2*n:] /= beta
# x := x - G' * W**-1 * z
x[:n] -= div(z[:n], d1) - div(z[n:2*n], d2) + As.T * z[-(m+1):]
x[n:] += div(z[:n], d1) + div(z[n:2*n], d2)
# Solve for x[:n]:
#
# S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:]
x[:n] -= mul( div(d1**2 - d2**2, d1**2 + d2**2), x[n:])
lapack.potrs(S, x)
# Solve for x[n:]:
#
# (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n]
x[n:] += mul( d1**-2 - d2**-2, x[:n])
x[n:] = div( x[n:], d1**-2 + d2**-2)
# z := z + W^-T * G*x
z[:n] += div( x[:n] - x[n:2*n], d1)
z[n:2*n] += div( -x[:n] - x[n:2*n], d2)
z[2*n:] += As*x[:n]
return f
def F(x=None, z=None):
if x is None:
return m, matrix([1.0, 0.0, 1.0, 0.0, 0.0])
# Factor A as A = L*L'. Compute inverse B = A^-1.
A = matrix([x[0], x[1], x[1], x[2]], (2, 2))
L = +A
try:
lapack.potrf(L)
except:
return None
B = +L
lapack.potri(B)
B[0, 1] = B[1, 0]
# f0 = -log det A
f = matrix(0.0, (m + 1, 1))
f[0] = -2.0 * (log(L[0, 0]) + log(L[1, 1]))
# fk = xk'*A*xk - 2*xk'*b + b*A^-1*b - 1
# = (xk - c)' * A * (xk - c) - 1 where c = A^-1*b
c = x[3:]
lapack.potrs(L, c)
for k in range(m):
f[k + 1] = (X[k, :].T - c).T * A * (X[k, :].T - c) - 1.0
# gradf0 = (-A^-1, 0) = (-B, 0)
Df = matrix(0.0, (m + 1, 5))
Df[0, 0], Df[0, 1], Df[0, 2] = -B[0, 0], -2.0 * B[1, 0], -B[1, 1]
# gradfk = (xk*xk' - A^-1*b*b'*A^-1, 2*(-xk + A^-1*b))
# = (xk*xk' - c*c', 2*(-xk+c))
Df[1:, 0] = X[:m, 0]**2 - c[0]**2
Df[1:, 1] = 2.0 * (mul(X[:m, 0], X[:m, 1]) - c[0] * c[1])
Df[1:, 2] = X[:m, 1]**2 - c[1]**2
Df[1:, 3] = 2.0 * (-X[:m, 0] + c[0])
Df[1:, 4] = 2.0 * (-X[:m, 1] + c[1])
if z is None: return f, Df
# hessf0(Y, y) = (A^-1*Y*A^-1, 0) = (B*YB, 0)
H0 = matrix(0.0, (5, 5))
H0[0, 0] = B[0, 0]**2
H0[1, 0] = 2.0 * B[0, 0] * B[1, 0]
H0[2, 0] = B[1, 0]**2
H0[1, 1] = 2.0 * (B[0, 0] * B[1, 1] + B[1, 0]**2)
H0[2, 1] = 2.0 * B[1, 0] * B[1, 1]
H0[2, 2] = B[1, 1]**2
# hessfi(Y, y)
# = ( A^-1*Y*A^-1*b*b'*A^-1 + A^-1*b*b'*A^-1*Y*A^-1
# - A^-1*y*b'*A^-1 - A^-1*b*y'*A^-1,
# -2*A^-1*Y*A^-1*b + 2*A^-1*y )
# = ( B*Y*c*c' + c*c'*Y*B - B*y*c' - c*y'*B, -2*B*Y*c + 2*B*y )
# = ( B*(Y*c-y)*c' + c*(Y*c-y)'*B, -2*B*(Y*c - y) )
H1 = matrix(0.0, (5, 5))
H1[0, 0] = 2.0 * c[0]**2 * B[0, 0]
H1[1, 0] = 2.0 * (c[0] * c[1] * B[0, 0] + c[0]**2 * B[1, 0])
H1[2, 0] = 2.0 * c[0] * c[1] * B[1, 0]
H1[3:, 0] = -2.0 * c[0] * B[:, 0]
H1[1,1] = 2.0 * c[0]**2 * B[1,1] + 4.0 * c[0]*c[1]*B[1,0] + \
2.0 * c[1]**2 + B[0,0]
H1[2, 1] = 2.0 * (c[1]**2 * B[1, 0] + c[0] * c[1] * B[1, 1])
H1[3:, 1] = -2.0 * B * c[[1, 0]]
H1[2, 2] = 2.0 * c[1]**2 * B[1, 1]
H1[3:, 2] = -2.0 * c[1] * B[:, 1]
H1[3:, 3:] = 2 * B
return f, Df, z[0] * H0 + sum(z[1:]) * H1
def cholesky(A):
""" Cholesky with clean-up """
lapack.potrf(A)
makeLT(A)
def F(W):
"""
Create a solver for the linear equations
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - Dl^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - r*r' * [ ] * r*r' = bzs
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
where Dl = diag(W['l']), r = W['r'][0].
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ].
1. Compute matrices V1, V2 such that (with T = r*r')
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and S = [ diag(s); 0 ], s a positive q-vector.
2. Factor the mapping X -> X + S * X' * S:
X + S * X' * S = L( L'( X )).
3. Compute scaled mappings: a matrix As with as its columns the
coefficients of the scaled mapping
L^-1( V2' * A() * V1' )
and the matrix Gs = Dl^-1 * G.
4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As.
"""
# 1. Compute V1, V2, s.
r = W['r'][0]
# LQ factorization R[:q, :] = L1 * Q1.
lapack.lacpy(r, Q1, m = q)
lapack.gelqf(Q1, tau1)
lapack.lacpy(Q1, L1, n = q, uplo = 'L')
lapack.orglq(Q1, tau1)
# LQ factorization R[q:, :] = L2 * Q2.
lapack.lacpy(r, Q2, m = p, offsetA = q)
lapack.gelqf(Q2, tau2)
lapack.lacpy(Q2, L2, n = p, uplo = 'L')
lapack.orglq(Q2, tau2)
# V2, V1, s are computed from an SVD: if
#
# Q2 * Q1' = U * diag(s) * V',
#
# then V1 = V' * L1^-1 and V2 = L2^-T * U.
# T21 = Q2 * Q1.T
blas.gemm(Q2, Q1, T21, transB = 'T')
# SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1.
lapack.gesvd(T21, s, jobu = 'A', jobvt = 'A', U = V2, Vt = V1)
# # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2
# this will requires lapack.ormlq or lapack.unmlq
# V2 = L2^-T * U
blas.trsm(L2, V2, transA = 'T')
# V1 = V' * L1^-1
blas.trsm(L1, V1, side = 'R')
# 2. Factorization X + S * X' * S = L( L'( X )).
#
# The factor L is stored as a diagonal matrix D and a sparse lower
# triangular matrix P, such that
#
# L(X)[:] = D**-1 * (I + P) * X[:]
# L^-1(X)[:] = D * (I - P) * X[:].
# SS is q x q with SS[i,j] = si*sj.
blas.scal(0.0, SS)
blas.syr(s, SS)
# For a p x q matrix X, P*X[:] is Y[:] where
#
# Yij = si * sj * Xji if i < j
# = 0 otherwise.
#
P.V = SS[Itril2]
# For a p x q matrix X, D*X[:] is Y[:] where
#
# Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j
# = Xii / sqrt( 1 + si^2 ) if i = j
# = Xij otherwise.
#
DV[Idiag] = sqrt(1.0 + SS[::q+1])
DV[Itriu] = sqrt(1.0 - SS[Itril3]**2)
D.V = DV**-1
# 3. Scaled linear mappings
# Ask := V2' * Ask * V1'
blas.scal(0.0, As)
base.axpy(A, As)
for i in xrange(n):
# tmp := V2' * As[i, :]
blas.gemm(V2, As, tmp, transA = 'T', m = p, n = q, k = p,
ldB = p, offsetB = i*p*q)
# As[:,i] := tmp * V1'
blas.gemm(tmp, V1, As, transB = 'T', m = p, n = q, k = q,
ldC = p, offsetC = i*p*q)
# As := D * (I - P) * As
# = L^-1 * As.
blas.copy(As, As2)
base.gemm(P, As, As2, alpha = -1.0, beta = 1.0)
base.gemm(D, As2, As)
# Gs := Dl^-1 * G
blas.scal(0.0, Gs)
base.axpy(G, Gs)
for k in xrange(n):
blas.tbmv(W['di'], Gs, n = m, k = 0, ldA = 1, offsetx = k*m)
# 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As.
blas.syrk(As, H, trans = 'T', alpha = 2.0)
blas.syrk(Gs, H, trans = 'T', beta = 1.0)
base.axpy(C, H)
lapack.potrf(H)
def f(x, y, z):
"""
Solve
C * ux + G' * uzl - 2*A'(uzs21) = bx
-uzs11 = bX1
-uzs22 = bX2
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ uzs11 uzs21' ]
[ ] - T * [ ] * T = bzs.
[ -A(ux) -uX2 ] [ uzs21 uzs22 ]
On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ].
On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ].
Define X = uzs21, Z = T * uzs * T:
C * ux + G' * uzl - 2*A'(X) = bx
[ 0 X' ] [ bX1 0 ]
T * [ ] * T - Z = T * [ ] * T
[ X 0 ] [ 0 bX2 ]
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs
[ -A(ux) -uX2 ] [ Z21 Z22 ]
Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ].
We use the congruence transformation
[ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ]
[ ] [ ] [ ] = [ ]
[ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ]
and the factorization
X + S * X' * S = L( L'(X) )
to write this as
C * ux + G' * uzl - 2*A'(X) = bx
L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX
G * ux - D^2 * uzl = bzl
[ -uX1 -A(ux)' ] [ Z11 Z21' ]
[ ] - [ ] = bzs,
[ -A(ux) -uX2 ] [ Z21 Z22 ]
or
C * ux + Gs' * uuzl - 2*As'(XX) = bx
XX - ZZ21 = bX
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - ZZ21 = bbzs_21
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
if we introduce scaled variables
uuzl = D * uzl
XX = L'(V2^-1 * X * V1^-1)
= L'(V2^-1 * uzs21 * V1^-1)
ZZ21 = L^-1(V2' * Z21 * V1')
and define
bbzs_21 = L^-1(V2' * bzs_21 * V1')
[ bX1 0 ]
bX = L^-1( V2' * (T * [ ] * T)_21 * V1').
[ 0 bX2 ]
Eliminating Z21 gives
C * ux + Gs' * uuzl - 2*As'(XX) = bx
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22
and eliminating uuzl and XX gives
H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21)
Gs * ux - uuzl = D^-1 * bzl
-As(ux) - XX = bbzs_21 - bX
-uX1 - Z11 = bzs_11
-uX2 - Z22 = bzs_22.
In summary, we can use the following algorithm:
1. bXX := bX - bbzs21
[ bX1 0 ]
= L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1')
[ 0 bX2 ]
2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
3. From ux, compute
uuzl = Gs*ux - D^-1 * bzl and
X = V2 * L^-T(-As(ux) + bXX) * V1.
4. Return ux, uuzl,
rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r
and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22.
"""
# Save bzs_11, bzs_22, bzs_21.
lapack.lacpy(z, bz11, uplo = 'L', m = q, n = q, ldA = p+q,
offsetA = m)
lapack.lacpy(z, bz21, m = p, n = q, ldA = p+q, offsetA = m+q)
lapack.lacpy(z, bz22, uplo = 'L', m = p, n = p, ldA = p+q,
offsetA = m + (p+q+1)*q)
# zl := D^-1 * zl
# = D^-1 * bzl
blas.tbmv(W['di'], z, n = m, k = 0, ldA = 1)
# zs := r' * [ bX1, 0; 0, bX2 ] * r.
# zs := [ bX1, 0; 0, bX2 ]
blas.scal(0.0, z, offset = m)
lapack.lacpy(x[1], z, uplo = 'L', m = q, n = q, ldB = p+q,
offsetB = m)
lapack.lacpy(x[2], z, uplo = 'L', m = p, n = p, ldB = p+q,
offsetB = m + (p+q+1)*q)
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc = p+q+1, offset = m)
# a := tril(zs)*r
blas.copy(r, a)
blas.trmm(z, a, side = 'L', m = p+q, n = p+q, ldA = p+q, ldB =
p+q, offsetA = m)
# zs := a'*r + r'*a
blas.syr2k(r, a, z, trans = 'T', n = p+q, k = p+q, ldB = p+q,
ldC = p+q, offsetC = m)
# bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1')
#
# [ bX1 0 ]
# = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1').
# [ 0 bX2 ]
# a = [ r21 r22 ] * z
# = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r
# = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r
blas.symm(z, r, a, side = 'R', m = p, n = p+q, ldA = p+q,
ldC = p+q, offsetB = q)
# bz21 := -bz21 + a * [ r11, r12 ]'
# = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21
blas.gemm(a, r, bz21, transB = 'T', m = p, n = q, k = p+q,
beta = -1.0, ldA = p+q, ldC = p)
# bz21 := V2' * bz21 * V1'
# = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1'
blas.gemm(V2, bz21, tmp, transA = 'T', m = p, n = q, k = p,
ldB = p)
blas.gemm(tmp, V1, bz21, transB = 'T', m = p, n = q, k = q,
ldC = p)
# bz21[:] := D * (I-P) * bz21[:]
# = L^-1 * bz21[:]
# = bXX[:]
blas.copy(bz21, tmp)
base.gemv(P, bz21, tmp, alpha = -1.0, beta = 1.0)
base.gemv(D, tmp, bz21)
# Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX).
# x[0] := x[0] + Gs'*zl + 2*As'(bz21)
# = bx + G' * D^-1 * bzl + 2 * As'(bXX)
blas.gemv(Gs, z, x[0], trans = 'T', alpha = 1.0, beta = 1.0)
blas.gemv(As, bz21, x[0], trans = 'T', alpha = 2.0, beta = 1.0)
# x[0] := H \ x[0]
# = ux
lapack.potrs(H, x[0])
# uuzl = Gs*ux - D^-1 * bzl
blas.gemv(Gs, x[0], z, alpha = 1.0, beta = -1.0)
# bz21 := V2 * L^-T(-As(ux) + bz21) * V1
# = X
blas.gemv(As, x[0], bz21, alpha = -1.0, beta = 1.0)
blas.tbsv(DV, bz21, n = p*q, k = 0, ldA = 1)
blas.copy(bz21, tmp)
base.gemv(P, tmp, bz21, alpha = -1.0, beta = 1.0, trans = 'T')
blas.gemm(V2, bz21, tmp)
blas.gemm(tmp, V1, bz21)
# zs := -zs + r' * [ 0, X'; X, 0 ] * r
# = r' * [ -bX1, X'; X, -bX2 ] * r.
# a := bz21 * [ r11, r12 ]
# = X * [ r11, r12 ]
blas.gemm(bz21, r, a, m = p, n = p+q, k = q, ldA = p, ldC = p+q)
# z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ]
# = rti' * uzs * rti
blas.syr2k(r, a, z, trans = 'T', beta = -1.0, n = p+q, k = p,
offsetA = q, offsetC = m, ldB = p+q, ldC = p+q)
# uX1 = -Z11 - bzs_11
# = -(r*zs*r')_11 - bzs_11
# uX2 = -Z22 - bzs_22
# = -(r*zs*r')_22 - bzs_22
blas.copy(bz11, x[1])
blas.copy(bz22, x[2])
# scale diagonal of zs by 1/2
blas.scal(0.5, z, inc = p+q+1, offset = m)
# a := r*tril(zs)
blas.copy(r, a)
blas.trmm(z, a, side = 'R', m = p+q, n = p+q, ldA = p+q, ldB =
p+q, offsetA = m)
# x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]'
# = -bzs_11 - (r*zs*r')_11
blas.syr2k(a, r, x[1], n = q, alpha = -1.0, beta = -1.0)
# x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]'
# = -bzs_22 - (r*zs*r')_22
blas.syr2k(a, r, x[2], n = p, alpha = -1.0, beta = -1.0,
offsetA = q, offsetB = q)
# scale diagonal of zs by 1/2
blas.scal(2.0, z, inc = p+q+1, offset = m)
return f
def completion(X, factored_updates = True):
"""
Supernodal multifrontal maximum determinant positive definite
matrix completion. The routine computes the Cholesky factor
:math:`L` of the inverse of the maximum determinant positive
definite matrix completion of :math:`X`:, i.e.,
.. math::
P( S^{-1} ) = X
where :math:`S = LL^T`. On exit, the argument `X` contains the
lower-triangular Cholesky factor :math:`L`.
The optional argument `factored_updates` can be used to enable (if
True) or disable (if False) updating of intermediate
factorizations.
:param X: :py:class:`cspmatrix`
:param factored_updates: boolean
"""
assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix"
n = X.symb.n
snpost = X.symb.snpost
snptr = X.symb.snptr
chptr = X.symb.chptr
chidx = X.symb.chidx
relptr = X.symb.relptr
relidx = X.symb.relidx
blkptr = X.symb.blkptr
blkval = X.blkval
stack = []
for k in reversed(list(snpost)):
nn = snptr[k+1]-snptr[k] # |Nk|
na = relptr[k+1]-relptr[k] # |Ak|
nj = na + nn
# allocate F and copy X_{Jk,Nk} to leading columns of F
F = matrix(0.0, (nj,nj))
lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj, m = nj, n = nn, uplo = 'L')
# if supernode k is not a root node:
if na > 0:
# copy Vk to 2,2 block of F
Vk = stack.pop()
lapack.lacpy(Vk, F, offsetB = nn*nj+nn, m = na, n = na, uplo = 'L')
# if supernode k has any children:
for ii in range(chptr[k],chptr[k+1]):
i = chidx[ii]
if factored_updates:
r = relidx[relptr[i]:relptr[i+1]]
stack.append(frontal_get_update_factor(F,r,nn,na))
else:
stack.append(frontal_get_update(F,relidx,relptr,i))
# if supernode k is not a root node:
if na > 0:
if factored_updates:
# In this case we have Vk = Lk'*Lk
trL1 = 'T'
trL2 = 'N'
else:
# factorize Vk
lapack.potrf(F, offsetA = nj*nn+nn, n = na, ldA = nj)
# In this case we have Vk = Lk*Lk'
trL1 = 'N'
trL2 = 'T'
# compute L_{Ak,Nk} and inv(D_{Nk,Nk}) = S_{Nk,Nk} - S_{Ak,Nk}'*L_{Ak,Nk}
lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL1,\
offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn)
blas.syrk(blkval, blkval, n = nn, k = na, trans= 'T', alpha = -1.0, beta = 1.0,
offsetA = blkptr[k]+nn, offsetC = blkptr[k], ldA = nj, ldC = nj)
lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL2,\
offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn)
for i in range(nn):
blas.scal(-1.0, blkval, n = na, offset = blkptr[k] + i*nj + nn)
# factorize inv(D_{Nk,Nk}) as R*R' so that D_{Nk,Nk} = L*L' with L = inv(R)'
lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj,\
ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX!
F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX!
lapack.potrf(F, ldA = nj, n = nn, uplo = 'U') # factorize
F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX!
lapack.lacpy(F, blkval, offsetB = blkptr[k], ldA = nj,\
ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX!
# compute L = inv(R')
lapack.trtri(blkval, offsetA = blkptr[k], ldA = nj, n = nn)
X._is_factor = True
return
def __scale(L, Y, U, adj=False, inv=False, factored_updates=True):
n = L.symb.n
snpost = L.symb.snpost
snptr = L.symb.snptr
chptr = L.symb.chptr
chidx = L.symb.chidx
relptr = L.symb.relptr
relidx = L.symb.relidx
blkptr = L.symb.blkptr
stack = []
for k in reversed(list(snpost)):
nn = snptr[k + 1] - snptr[k] # |Nk|
na = relptr[k + 1] - relptr[k] # |Ak|
nj = na + nn
F = matrix(0.0, (nj, nj))
lapack.lacpy(Y.blkval,
F,
m=nj,
n=nn,
ldA=nj,
offsetA=blkptr[k],
uplo='L')
# if supernode k is not a root node:
if na > 0:
# copy Vk to 2,2 block of F
Vk = stack.pop()
lapack.lacpy(Vk,
F,
ldB=nj,
offsetB=nn * (nj + 1),
m=na,
n=na,
uplo='L')
# if supernode k has any children:
for ii in range(chptr[k], chptr[k + 1]):
i = chidx[ii]
if factored_updates:
r = relidx[relptr[i]:relptr[i + 1]]
stack.append(frontal_get_update_factor(F, r, nn, na))
else:
stack.append(frontal_get_update(F, relidx, relptr, i))
# if supernode k is not a root node:
if na > 0:
if factored_updates:
# In this case we have Vk = Lk'*Lk
if adj is False: trns = 'N'
elif adj is True: trns = 'T'
else:
# factorize Vk
lapack.potrf(F, offsetA=nj * nn + nn, n=na, ldA=nj)
# In this case we have Vk = Lk*Lk'
if adj is False: trns = 'T'
elif adj is True: trns = 'N'
if adj is False: tr = ['T', 'N']
elif adj is True: tr = ['N', 'T']
if inv is False:
for Ut in U:
# symmetrize (1,1) block of Ut_{k} and scale
U11 = matrix(0.0, (nn, nn))
lapack.lacpy(Ut.blkval,
U11,
offsetA=blkptr[k],
m=nn,
n=nn,
ldA=nj,
uplo='L')
U11 += U11.T
U11[::nn + 1] *= 0.5
lapack.lacpy(U11,
Ut.blkval,
offsetB=blkptr[k],
m=nn,
n=nn,
ldB=nj,
uplo='N')
blas.trsm(L.blkval, Ut.blkval, side = 'R', transA = tr[0],\
m = nj, n = nn, offsetA = blkptr[k], ldA = nj,\
offsetB = blkptr[k], ldB = nj)
blas.trsm(L.blkval, Ut.blkval, m = nn, n = nn, transA = tr[1],\
offsetA = blkptr[k], offsetB = blkptr[k],\
ldA = nj, ldB = nj)
# zero-out strict upper triangular part of {Nj,Nj} block
for i in range(1, nn):
blas.scal(0.0, Ut.blkval, offset=blkptr[k] + nj * i, n=i)
if na > 0: blas.trmm(F, Ut.blkval, m = na, n = nn, transA = trns,\
offsetA = nj*nn+nn, ldA = nj,\
offsetB = blkptr[k]+nn, ldB = nj)
else: # inv is True
for Ut in U:
# symmetrize (1,1) block of Ut_{k} and scale
U11 = matrix(0.0, (nn, nn))
lapack.lacpy(Ut.blkval,
U11,
offsetA=blkptr[k],
m=nn,
n=nn,
ldA=nj,
uplo='L')
U11 += U11.T
U11[::nn + 1] *= 0.5
lapack.lacpy(U11,
Ut.blkval,
offsetB=blkptr[k],
m=nn,
n=nn,
ldB=nj,
uplo='N')
blas.trmm(L.blkval, Ut.blkval, side = 'R', transA = tr[0],\
m = nj, n = nn, offsetA = blkptr[k], ldA = nj,\
offsetB = blkptr[k], ldB = nj)
blas.trmm(L.blkval, Ut.blkval, m = nn, n = nn, transA = tr[1],\
offsetA = blkptr[k], offsetB = blkptr[k],\
ldA = nj, ldB = nj)
# zero-out strict upper triangular part of {Nj,Nj} block
for i in range(1, nn):
blas.scal(0.0, Ut.blkval, offset=blkptr[k] + nj * i, n=i)
if na > 0: blas.trsm(F, Ut.blkval, m = na, n = nn, transA = trns,\
offsetA = nj*nn+nn, ldA = nj,\
offsetB = blkptr[k]+nn, ldB = nj)
return
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2*m]**2
dbi = div(1.0,db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2,dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt,mul(W['di'][:m],d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0,(k,1))
base.gemv(E, mul(W['di'][:m],lam), lt, trans = 'T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt,W['di'][:m])
Xs = spmatrix(tmp,range(m),range(m))*X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d,tmp),range(m),range(m))*E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0,(n,n))
Ab[::n+1] = 1.0
base.syrk(Xs,Ab,trans = 'T', beta = 1.0)
Bb = matrix(0.0,(n,k))
Bb = -Xs.T*Es # inefficient!?
D2 = spmatrix(0.0,range(k),range(k))
base.syrk(Es,D2,trans = 'T', partial = True)
d2 = +D2.V
del D2
py = matrix(0.0,(n,1))
for i in range(k):
binvsq = (1.0/W['beta'][i])**2
Ab += binvsq*Pt[i]
dvv = blas.dot(W['v'][i],W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans = 'T', alpha = 1.0, beta = 0.0)
blas.syrk(py, Ab, alpha = 4*binvsq*(dvv+1), beta = 1.0)
Bb[:,i] -= 4*binvsq*W['v'][i][0]*dvv*py
d2[i] += binvsq*(1+4*(W['v'][i][0]**2)*(dvv-1))
d2i = div(1.0,d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam,mul(W['di'][:m],d))
tmp = matrix(0.0,(k,1))
base.gemv(E,mul(W['di'][:m],lam), tmp, trans = 'T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp,tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,M12, alpha = 1.0)
tmp = mul(d,mul(W['di'][:m],lam))
blas.gemv(X,tmp,M12, trans = 'T', alpha = 1.0, beta = 1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt,range(k), range(k))
base.syrk(sBb, Ab, alpha = -1.0, beta = 1.0)
M = matrix([[Ab, M12.T],[M12, a]])
lapack.potrf(M)
def f(x,y,z):
# residuals
rwt = x[:n+k]
rb = x[n+k]
rv = x[n+k+1:n+k+1+m]
iw_rl1 = mul(W['di'][:m],z[:m])
iw_rl2 = mul(W['di'][m:2*m],z[m:2*m])
ri = [z[2*m+i*(n+1):2*m+(i+1)*(n+1)] for i in range(k)]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
qscal(tmp,W['beta'][i],W['v'][i],inv=True)
rbwt[n+i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans = 'T', alpha = -1.0, beta = 1.0)
tmp = mul(W['di'][:m],iw_rl1)
tmp2 = matrix(0.0,(k,1))
base.gemv(E,tmp,tmp2,trans='T')
rbwt[n:] += tmp2
tmp = mul(d,tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X,tmp,rbwt,trans='T', alpha = -1.0, beta = 1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2*m],iw_rl2) - mul(W['di'][:m],iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi,rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans = 'T', alpha = 1.0, beta = 1.0)
rtw = +rbwt[:n]
tmp = mul(d,tmp)
blas.gemv(X, tmp, rtw, trans = 'T', alpha = -1.0, beta = 1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i,rtt)
tmp2 = matrix(0.0,(n,1))
blas.gemv(Bb,tmp,tmp2)
dwdb = matrix([rtw - tmp2,rtb + blas.dot(mul(d2i,lt),rtt)])
lapack.potrs(M,dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0,(k,1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt*dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0,(m,1))
blas.gemv(X,dwdb[:n],dv,alpha = -1.0)
dv = mul(d,dv) - d*dwdb[-1]
base.gemv(E, dt, dv, beta = 1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2,dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = - mul(W['di'][m:2*m],dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0,(n,1))
for i in range(k):
blas.gemv(P[i],dwdb[:n],tmp, alpha = -1.0, beta = 0.0)
tmp1 = matrix([-dt[i],tmp])
blas.axpy(ri[i],tmp1,alpha = -1.0)
qscal(tmp1,W['beta'][i],W['v'][i],inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n+k] = dt
x[n+k] = dwdb[-1]
x[n+k+1:] = dv
z[:m] = wdz1
z[m:2*m] = wdz2
for i in range(k):
z[2*m+i*(n+1):2*m+(i+1)*(n+1)] = wdzi[i]
return f
def softmargin_appr(X,
d,
gamma,
width,
kernel='linear',
sigma=1.0,
degree=1,
theta=1.0,
Q=None):
"""
Solves the approximate 'soft-margin' SVM problem
maximize -(1/2)*z'*Qc*z + d'*z
subject to 0 <= diag(d)*z <= gamma*ones
sum(z) = 0
(with variables z), and its dual problem
minimize (1/2)*y'*Qc^{-1}*y + gamma*sum(v)
subject to diag(d)*(y + b*ones) + v >= 1
v >= 0
(with variables y, v, b).
Qc is the maximum determinant completion of the projection of Q
on a band with bandwidth 2*w+1. Q_ij = K(xi, xj) where K is a kernel
function and xi is the ith row of X (xi' = X[i,:]).
Input arguments.
X is an N x n matrix.
d is an N-vector with elements -1 or 1; d[i] is the label of
row X[i,:].
gamma is a positive parameter.
kernel is a string with values 'linear', 'rfb', 'poly', or 'tanh'.
'linear': k(u,v) = u'*v/sigma.
'rbf': k(u,v) = exp(-||u - v||^2 / (2*sigma)).
'poly': k(u,v) = (u'*v/sigma)**degree.
'tanh': k(u,v) = tanh(u'*v/sigma - theta).
sigma and theta are positive numbers.
degree is a positive integer.
width is a positive integer.
Output.
Returns a dictionary with the keys:
'classifier'
a Python function object that takes an M x n matrix with
test vectors as rows and returns a vector with labels
'completion classifier'
a Python function object that takes an M x n matrix with
test vectors as rows and returns a vector with labels
'z'
a sparse m-vector
'cputime'
a tuple (Ttot, Tqp, Tker) where Ttot is the total
CPU time, Tqp is the CPU time spent solving the QP, and
Tker is the CPU time spent computing the kernel matrix
'iterations'
the number of interior-point iteations
'misclassified'
a tuple (L1, L2) where L1 is a list of indices of
misclassified training vectors from class 1, and L2 is a
list of indices of misclassified training vectors from
class 2
"""
Tstart = cputime()
if verbose: solvers.options['show_progress'] = True
else: solvers.options['show_progress'] = False
N, n = X.size
if Q is None:
Q, a = kernel_matrix(X,
kernel,
sigma=sigma,
degree=degree,
theta=theta,
V='band',
width=width)
else:
if not (Q.size[0] == N and Q.size[1] == N):
raise ValueError("invalid kernel matrix dimensions")
elif not type(Q) is cvxopt.base.spmatrix:
raise ValueError("invalid kernel matrix type")
elif verbose:
print("using precomputed kernel matrix ..")
if kernel == 'rbf':
Ad = spmatrix(0.0, range(N), range(N))
base.syrk(X, V, partial=True)
a = Ad.V
del Ad
Tkernel = cputime(Tstart)
# solve qp
Tqp = cputime()
y, b, v, z, optval, Lc, iters = softmargin_completion(
Q, matrix(d, tc='d'), gamma)
Tqp = cputime(Tqp)
if verbose: print("utime = %f, stime = %f." % Tqp)
# extract nonzero support vectors
nrmz = max(abs(z))
sv = [k for k in range(N) if abs(z[k]) > Tsv * nrmz]
zs = spmatrix(z[sv], sv, [0 for i in range(len(sv))], (len(d), 1))
if verbose: print("%d support vectors." % len(sv))
Xr, zr, Nr = X[sv, :], z[sv], len(sv)
# find misclassified training vectors
err1 = [i for i in range(Q.size[0]) if (v[i] > 1 and d[i] == 1)]
err2 = [i for i in range(Q.size[0]) if (v[i] > 1 and d[i] == -1)]
if verbose:
e1, n1 = len(err1), list(d).count(1)
e2, n2 = len(err2), list(d).count(-1)
print("class 1: %i/%i = %.1f%% misclassified." %
(e1, n1, 100. * e1 / n1))
print("class 2: %i/%i = %.1f%% misclassified." %
(e2, n2, 100. * e2 / n2))
del e1, e2, n1, n2
# create classifier function object
# CLASSIFIER 1 (standard kernel classifier)
if kernel == 'linear':
# w = X'*z / sigma
w = matrix(0.0, (n, 1))
blas.gemv(Xr, zr, w, trans='T', alpha=1.0 / sigma)
def classifier(Y, soft=False):
M = Y.size[0]
x = matrix(b, (M, 1))
blas.gemv(Y, w, x, beta=1.0)
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'rbf':
def classifier(Y, soft=False):
M = Y.size[0]
# K = Y*X' / sigma
K = matrix(0.0, (M, Nr))
blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma)
# c[i] = ||Yi||^2 / sigma
ones = matrix(1.0, (max([M, Nr, n]), 1))
c = Y**2 * ones[:n]
blas.scal(1.0 / sigma, c)
# Kij := Kij - 0.5 * (ci + aj)
# = || yi - xj ||^2 / (2*sigma)
blas.ger(c, ones, K, alpha=-0.5)
blas.ger(ones, a[sv], K, alpha=-0.5)
x = exp(K) * zr + b
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'tanh':
def classifier(Y, soft=False):
M = Y.size[0]
# K = Y*X' / sigma - theta
K = matrix(theta, (M, Nr))
blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma, beta=-1.0)
K = exp(K)
x = div(K - K**-1, K + K**-1) * zr + b
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'poly':
def classifier(Y, soft=False):
M = Y.size[0]
# K = Y*X' / sigma
K = matrix(0.0, (M, Nr))
blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma)
x = K**degree * zr + b
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
else:
pass
# CLASSIFIER 2 (completion kernel classifier)
# TODO: generalize to arbitrary sparsity pattern
L11 = matrix(Q[:width, :width])
lapack.potrf(L11)
if kernel == 'linear':
def classifier2(Y, soft=False):
M = Y.size[0]
W = matrix(0., (width, M))
blas.gemm(X, Y, W, transB='T', alpha=1.0 / sigma, m=width)
lapack.potrs(L11, W)
W = matrix([W, matrix(0., (N - width, M))])
chompack.trsm(Lc, W, trans='N')
chompack.trsm(Lc, W, trans='T')
x = matrix(b, (M, 1))
blas.gemv(W, z, x, trans='T', beta=1.0)
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'poly':
def classifier2(Y, soft=False):
if Y is None: return zs
M = Y.size[0]
W = matrix(0., (width, M))
blas.gemm(X, Y, W, transB='T', alpha=1.0 / sigma, m=width)
W = W**degree
lapack.potrs(L11, W)
W = matrix([W, matrix(0., (N - width, M))])
chompack.trsm(Lc, W, trans='N')
chompack.trsm(Lc, W, trans='T')
x = matrix(b, (M, 1))
blas.gemv(W, z, x, trans='T', beta=1.0)
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'rbf':
def classifier2(Y, soft=False):
M = Y.size[0]
# K = Y*X' / sigma
K = matrix(0.0, (width, M))
blas.gemm(X, Y, K, transB='T', alpha=1.0 / sigma, m=width)
# c[i] = ||Yi||^2 / sigma
ones = matrix(1.0, (max(width, n, M), 1))
c = Y**2 * ones[:n]
blas.scal(1.0 / sigma, c)
# Kij := Kij - 0.5 * (ci + aj)
# = || yi - xj ||^2 / (2*sigma)
blas.ger(ones[:width], c, K, alpha=-0.5)
blas.ger(a[:width], ones[:M], K, alpha=-0.5)
# Kij = exp(Kij)
K = exp(K)
# complete K
lapack.potrs(L11, K)
K = matrix([K, matrix(0., (N - width, M))], (N, M))
chompack.trsm(Lc, K, trans='N')
chompack.trsm(Lc, K, trans='T')
x = matrix(b, (M, 1))
blas.gemv(K, z, x, trans='T', beta=1.0)
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
elif kernel == 'tanh':
def classifier2(Y, soft=False):
M = Y.size[0]
# K = Y*X' / sigma
K = matrix(theta, (width, M))
blas.gemm(X,
Y,
K,
transB='T',
alpha=1.0 / sigma,
beta=-1.0,
m=width)
K = exp(K)
K = div(K - K**-1, K + K**-1)
# complete K
lapack.potrs(L11, K)
K = matrix([K, matrix(0., (N - width, M))], (N, M))
chompack.trsm(Lc, K, trans='N')
chompack.trsm(Lc, K, trans='T')
x = matrix(b, (M, 1))
blas.gemv(K, z, x, trans='T', beta=1.0)
if soft: return x
else: return matrix([2 * (xk > 0.0) - 1 for xk in x])
Ttotal = cputime(Tstart)
return {
'classifier': classifier,
'completion classifier': classifier2,
'cputime': (sum(Ttotal), sum(Tqp), sum(Tkernel)),
'iterations': iters,
'z': zs,
'misclassified': (err1, err2)
}
def computefunc(self,cinfo):
ainv=cinfo
L = +cvxopt.matrix(ainv)
lapack.potrf(L)
f=2.0*np.sum(np.log(np.diag(L)))
return f
def cholesky(A):
""" Cholesky with clean-up """
lapack.potrf(A)
makeLT(A)
def Fkkt(W):
# Returns a function f(x, y, z) that solves
#
# [ 0 G' ] [ x ] = [ bx ]
# [ G -W'*W ] [ z ] [ bz ].
# First factor
#
# S = G' * W**-1 * W**-T * G
# = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1
#
# where
#
# W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n]
# W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:]
# W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J)
# with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I].
# As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A]
beta, v = W['beta'][0], W['v'][0]
As = 2 * v * (v[1:].T * A)
As[1:, :] *= -1.0
As[1:, :] -= A
As /= beta
# S = As'*As + 4 * (W1**2 + W2**2)**-1
S = As.T * As
d1, d2 = W['d'][:n], W['d'][n:]
d = 4.0 * (d1**2 + d2**2)**-1
S[::n + 1] += d
lapack.potrf(S)
def f(x, y, z):
# z := - W**-T * z
z[:n] = -div(z[:n], d1)
z[n:2 * n] = -div(z[n:2 * n], d2)
z[2 * n:] -= 2.0 * v * (v[0] * z[2 * n] -
blas.dot(v[1:], z[2 * n + 1:]))
z[2 * n + 1:] *= -1.0
z[2 * n:] /= beta
# x := x - G' * W**-1 * z
x[:n] -= div(z[:n], d1) - div(z[n:2 * n], d2) + As.T * z[-(m + 1):]
x[n:] += div(z[:n], d1) + div(z[n:2 * n], d2)
# Solve for x[:n]:
#
# S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:]
x[:n] -= mul(div(d1**2 - d2**2, d1**2 + d2**2), x[n:])
lapack.potrs(S, x)
# Solve for x[n:]:
#
# (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n]
x[n:] += mul(d1**-2 - d2**-2, x[:n])
x[n:] = div(x[n:], d1**-2 + d2**-2)
# z := z + W^-T * G*x
z[:n] += div(x[:n] - x[n:2 * n], d1)
z[n:2 * n] += div(-x[:n] - x[n:2 * n], d2)
z[2 * n:] += As * x[:n]
return f
def F(W):
"""
Custom solver for the system
[ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ]
[ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ]
[ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ]
[ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ]
[ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ]
[ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ]
[ | | | . ][ | ] [ | ]
[ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ]
where
It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ]
[ 0 0 ] [ -Pi 0 ]
dwt = [ dw ] rwt = [ rw ]
[ dt ] [ rt ].
"""
# scalings and 'intermediate' vectors
# db = inv(Wl1)^2 + inv(Wl2)^2
db = W['di'][:m]**2 + W['di'][m:2 * m]**2
dbi = div(1.0, db)
# dt = I - inv(Wl1)*Dbi*inv(Wl1)
dt = 1.0 - mul(W['di'][:m]**2, dbi)
dtsqrt = sqrt(dt)
# lam = Dt*inv(Wl1)*d
lam = mul(dt, mul(W['di'][:m], d))
# lt = E'*inv(Wl1)*lam
lt = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), lt, trans='T')
# Xs = sqrt(Dt)*inv(Wl1)*X
tmp = mul(dtsqrt, W['di'][:m])
Xs = spmatrix(tmp, range(m), range(m)) * X
# Es = D*sqrt(Dt)*inv(Wl1)*E
Es = spmatrix(mul(d, tmp), range(m), range(m)) * E
# form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs
# and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es
# and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1))
Ab = matrix(0.0, (n, n))
Ab[::n + 1] = 1.0
base.syrk(Xs, Ab, trans='T', beta=1.0)
Bb = matrix(0.0, (n, k))
Bb = -Xs.T * Es # inefficient!?
D2 = spmatrix(0.0, range(k), range(k))
base.syrk(Es, D2, trans='T', partial=True)
d2 = +D2.V
del D2
py = matrix(0.0, (n, 1))
for i in range(k):
binvsq = (1.0 / W['beta'][i])**2
Ab += binvsq * Pt[i]
dvv = blas.dot(W['v'][i], W['v'][i])
blas.gemv(P[i], W['v'][i][1:], py, trans='T', alpha=1.0, beta=0.0)
blas.syrk(py, Ab, alpha=4 * binvsq * (dvv + 1), beta=1.0)
Bb[:, i] -= 4 * binvsq * W['v'][i][0] * dvv * py
d2[i] += binvsq * (1 + 4 * (W['v'][i][0]**2) * (dvv - 1))
d2i = div(1.0, d2)
d2isqrt = sqrt(d2i)
# compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam
alpha = blas.dot(lam, mul(W['di'][:m], d))
tmp = matrix(0.0, (k, 1))
base.gemv(E, mul(W['di'][:m], lam), tmp, trans='T')
tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam
a = alpha - blas.dot(tmp, tmp)
# compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam
tmp = mul(tmp, d2isqrt)
M12 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, M12, alpha=1.0)
tmp = mul(d, mul(W['di'][:m], lam))
blas.gemv(X, tmp, M12, trans='T', alpha=1.0, beta=1.0)
# form and factor M
sBb = Bb * spmatrix(d2isqrt, range(k), range(k))
base.syrk(sBb, Ab, alpha=-1.0, beta=1.0)
M = matrix([[Ab, M12.T], [M12, a]])
lapack.potrf(M)
def f(x, y, z):
# residuals
rwt = x[:n + k]
rb = x[n + k]
rv = x[n + k + 1:n + k + 1 + m]
iw_rl1 = mul(W['di'][:m], z[:m])
iw_rl2 = mul(W['di'][m:2 * m], z[m:2 * m])
ri = [
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)]
for i in range(k)
]
# compute 'derived' residuals
# rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1
rbwt = +rwt
for i in range(k):
tmp = +ri[i]
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
qscal(tmp, W['beta'][i], W['v'][i], inv=True)
rbwt[n + i] -= tmp[0]
blas.gemv(P[i], tmp[1:], rbwt, trans='T', alpha=-1.0, beta=1.0)
tmp = mul(W['di'][:m], iw_rl1)
tmp2 = matrix(0.0, (k, 1))
base.gemv(E, tmp, tmp2, trans='T')
rbwt[n:] += tmp2
tmp = mul(d, tmp) # tmp = D*inv(Wl1)^2*rl1
blas.gemv(X, tmp, rbwt, trans='T', alpha=-1.0, beta=1.0)
# rbb = rb - d'*inv(Wl1)^2*rl1
rbb = rb - sum(tmp)
# rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1
rbv = rv - mul(W['di'][m:2 * m], iw_rl2) - mul(W['di'][:m], iw_rl1)
# [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv
tmp = mul(W['di'][:m]**2, mul(dbi, rbv))
rtt = +rbwt[n:]
base.gemv(E, tmp, rtt, trans='T', alpha=1.0, beta=1.0)
rtw = +rbwt[:n]
tmp = mul(d, tmp)
blas.gemv(X, tmp, rtw, trans='T', alpha=-1.0, beta=1.0)
# rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv
rtb = rbb - sum(tmp)
# solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt]
tmp = mul(d2i, rtt)
tmp2 = matrix(0.0, (n, 1))
blas.gemv(Bb, tmp, tmp2)
dwdb = matrix([rtw - tmp2, rtb + blas.dot(mul(d2i, lt), rtt)])
lapack.potrs(M, dwdb)
# compute dt = inv(D2)*(rtt - Bb'*dw + lt*db)
tmp2 = matrix(0.0, (k, 1))
blas.gemv(Bb, dwdb[:n], tmp2, trans='T')
dt = mul(d2i, rtt - tmp2 + lt * dwdb[-1])
# compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db))
dv = matrix(0.0, (m, 1))
blas.gemv(X, dwdb[:n], dv, alpha=-1.0)
dv = mul(d, dv) - d * dwdb[-1]
base.gemv(E, dt, dv, beta=1.0)
tmp = +dv # tmp = E*dt - D*X*dw - d*db
dv = mul(dbi, rbv + mul(W['di'][:m]**2, dv))
# compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1)
wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1
# compute wdz2 = - inv(Wl2)*(dv + rl2)
wdz2 = -mul(W['di'][m:2 * m], dv) - iw_rl2
# compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri)
wdzi = []
tmp = matrix(0.0, (n, 1))
for i in range(k):
blas.gemv(P[i], dwdb[:n], tmp, alpha=-1.0, beta=0.0)
tmp1 = matrix([-dt[i], tmp])
blas.axpy(ri[i], tmp1, alpha=-1.0)
qscal(tmp1, W['beta'][i], W['v'][i], inv=True)
wdzi.append(tmp1)
# solution
x[:n] = dwdb[:n]
x[n:n + k] = dt
x[n + k] = dwdb[-1]
x[n + k + 1:] = dv
z[:m] = wdz1
z[m:2 * m] = wdz2
for i in range(k):
z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)] = wdzi[i]
return f
def psdcompletion(A, reordered = True, **kwargs):
"""
Maximum determinant positive semidefinite matrix completion. The
routine takes a cspmatrix :math:`A` and returns the maximum determinant
positive semidefinite matrix completion :math:`X` as a dense matrix, i.e.,
.. math::
P( X ) = A
:param A: :py:class:`cspmatrix`
:param reordered: boolean
"""
assert isinstance(A, cspmatrix) and A.is_factor is False, "A must be a cspmatrix"
tol = kwargs.get('tol',1e-15)
X = matrix(A.spmatrix(reordered = True, symmetric = True))
symb = A.symb
n = symb.n
snptr = symb.snptr
sncolptr = symb.sncolptr
snrowidx = symb.snrowidx
# visit supernodes in reverse (descending) order
for k in range(symb.Nsn-1,-1,-1):
nn = snptr[k+1]-snptr[k]
beta = snrowidx[sncolptr[k]:sncolptr[k+1]]
nj = len(beta)
if nj-nn == 0: continue
alpha = beta[nn:]
nu = beta[:nn]
eta = matrix([matrix(range(beta[kk]+1,beta[kk+1])) for kk in range(nj-1)] + [matrix(range(beta[-1]+1,n))])
try:
# Try Cholesky factorization first
Xaa = X[alpha,alpha]
lapack.potrf(Xaa)
Xan = X[alpha,nu]
lapack.trtrs(Xaa, Xan, trans = 'N')
XeaT = X[eta,alpha].T
lapack.trtrs(Xaa, XeaT, trans = 'N')
# Compute update
tmp = XeaT.T*Xan
except:
# If Cholesky fact. fails, switch to EVD: Xaa = Z*diag(w)*Z.T
Xaa = X[alpha,alpha]
w = matrix(0.0,(Xaa.size[0],1))
Z = matrix(0.0,Xaa.size)
lapack.syevr(Xaa, w, jobz='V', range='A', uplo='L', Z=Z)
# Pseudo-inverse: Xp = pinv(Xaa)
lambda_max = max(w)
Xp = Z*spmatrix([1.0/wi if wi > lambda_max*tol else 0.0 for wi in w],range(len(w)),range(len(w)))*Z.T
# Compute update
tmp = X[eta,alpha]*Xp*X[alpha,nu]
X[eta,nu] = tmp
X[nu,eta] = tmp.T
if reordered:
return X
else:
return X[symb.ip,symb.ip]
def F(x=None, z=None):
if x is None:
return m, matrix([ 1.0, 0.0, 1.0, 0.0, 0.0 ])
# Factor A as A = L*L'. Compute inverse B = A^-1.
A = matrix( [x[0], x[1], x[1], x[2]], (2,2))
L = +A
try: lapack.potrf(L)
except: return None
B = +L
lapack.potri(B)
B[0,1] = B[1,0]
# f0 = -log det A
f = matrix(0.0, (m+1,1))
f[0] = -2.0 * (log(L[0,0]) + log(L[1,1]))
# fk = xk'*A*xk - 2*xk'*b + b*A^-1*b - 1
# = (xk - c)' * A * (xk - c) - 1 where c = A^-1*b
c = x[3:]
lapack.potrs(L, c)
for k in range(m):
f[k+1] = (X[k,:].T - c).T * A * (X[k,:].T - c) - 1.0
# gradf0 = (-A^-1, 0) = (-B, 0)
Df = matrix(0.0, (m+1,5))
Df[0,0], Df[0,1], Df[0,2] = -B[0,0], -2.0*B[1,0], -B[1,1]
# gradfk = (xk*xk' - A^-1*b*b'*A^-1, 2*(-xk + A^-1*b))
# = (xk*xk' - c*c', 2*(-xk+c))
Df[1:,0] = X[:m,0]**2 - c[0]**2
Df[1:,1] = 2.0 * (mul(X[:m,0], X[:m,1]) - c[0]*c[1])
Df[1:,2] = X[:m,1]**2 - c[1]**2
Df[1:,3] = 2.0 * (-X[:m,0] + c[0])
Df[1:,4] = 2.0 * (-X[:m,1] + c[1])
if z is None: return f, Df
# hessf0(Y, y) = (A^-1*Y*A^-1, 0) = (B*YB, 0)
H0 = matrix(0.0, (5,5))
H0[0,0] = B[0,0]**2
H0[1,0] = 2.0 * B[0,0] * B[1,0]
H0[2,0] = B[1,0]**2
H0[1,1] = 2.0 * ( B[0,0] * B[1,1] + B[1,0]**2 )
H0[2,1] = 2.0 * B[1,0] * B[1,1]
H0[2,2] = B[1,1]**2
# hessfi(Y, y)
# = ( A^-1*Y*A^-1*b*b'*A^-1 + A^-1*b*b'*A^-1*Y*A^-1
# - A^-1*y*b'*A^-1 - A^-1*b*y'*A^-1,
# -2*A^-1*Y*A^-1*b + 2*A^-1*y )
# = ( B*Y*c*c' + c*c'*Y*B - B*y*c' - c*y'*B, -2*B*Y*c + 2*B*y )
# = ( B*(Y*c-y)*c' + c*(Y*c-y)'*B, -2*B*(Y*c - y) )
H1 = matrix(0.0, (5,5))
H1[0,0] = 2.0 * c[0]**2 * B[0,0]
H1[1,0] = 2.0 * ( c[0] * c[1] * B[0,0] + c[0]**2 * B[1,0] )
H1[2,0] = 2.0 * c[0] * c[1] * B[1,0]
H1[3:,0] = -2.0 * c[0] * B[:,0]
H1[1,1] = 2.0 * c[0]**2 * B[1,1] + 4.0 * c[0]*c[1]*B[1,0] + \
2.0 * c[1]**2 + B[0,0]
H1[2,1] = 2.0 * (c[1]**2 * B[1,0] + c[0]*c[1]*B[1,1])
H1[3:,1] = -2.0 * B * c[[1,0]]
H1[2,2] = 2.0 * c[1]**2 * B[1,1]
H1[3:,2] = -2.0 * c[1] * B[:,1]
H1[3:,3:] = 2*B
return f, Df, z[0]*H0 + sum(z[1:])*H1
def factor(W, H=None, Df=None):
if F['firstcall']:
if type(G) is matrix:
F['Gs'] = matrix(0.0, G.size)
else:
F['Gs'] = spmatrix(0.0, G.I, G.J, G.size)
if mnl:
if type(Df) is matrix:
F['Dfs'] = matrix(0.0, Df.size)
else:
F['Dfs'] = spmatrix(0.0, Df.I, Df.J, Df.size)
if (mnl and type(Df) is matrix) or type(G) is matrix or \
type(H) is matrix:
F['S'] = matrix(0.0, (n, n))
F['K'] = matrix(0.0, (p, p))
else:
F['S'] = spmatrix([], [], [], (n, n), 'd')
F['Sf'] = None
if type(A) is matrix:
F['K'] = matrix(0.0, (p, p))
else:
F['K'] = spmatrix([], [], [], (p, p), 'd')
# Dfs = Wnl^{-1} * Df
if mnl:
base.gemm(spmatrix(W['dnli'], list(range(mnl)),
list(range(mnl))), Df, F['Dfs'], partial=True)
# Gs = Wl^{-1} * G.
base.gemm(spmatrix(W['di'], list(range(ml)), list(range(ml))),
G, F['Gs'], partial=True)
if F['firstcall']:
base.syrk(F['Gs'], F['S'], trans='T')
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0)
if H is not None:
F['S'] += H
try:
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
F['Sf'] = cholmod.symbolic(F['S'])
cholmod.numeric(F['S'], F['Sf'])
except ArithmeticError:
F['singular'] = True
if type(A) is matrix and type(F['S']) is spmatrix:
F['S'] = matrix(0.0, (n, n))
base.syrk(F['Gs'], F['S'], trans='T')
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0)
base.syrk(A, F['S'], trans='T', beta=1.0)
if H is not None:
F['S'] += H
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
F['Sf'] = cholmod.symbolic(F['S'])
cholmod.numeric(F['S'], F['Sf'])
F['firstcall'] = False
else:
base.syrk(F['Gs'], F['S'], trans='T', partial=True)
if mnl:
base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0,
partial=True)
if H is not None:
F['S'] += H
if F['singular']:
base.syrk(A, F['S'], trans='T', beta=1.0, partial=True)
if type(F['S']) is matrix:
lapack.potrf(F['S'])
else:
cholmod.numeric(F['S'], F['Sf'])
if type(F['S']) is matrix:
# Asct := L^{-1}*A'. Factor K = Asct'*Asct.
if type(A) is matrix:
Asct = A.T
else:
Asct = matrix(A.T)
blas.trsm(F['S'], Asct)
blas.syrk(Asct, F['K'], trans='T')
lapack.potrf(F['K'])
else:
# Asct := L^{-1}*P*A'. Factor K = Asct'*Asct.
if type(A) is matrix:
Asct = A.T
cholmod.solve(F['Sf'], Asct, sys=7)
cholmod.solve(F['Sf'], Asct, sys=4)
blas.syrk(Asct, F['K'], trans='T')
lapack.potrf(F['K'])
else:
Asct = cholmod.spsolve(F['Sf'], A.T, sys=7)
Asct = cholmod.spsolve(F['Sf'], Asct, sys=4)
base.syrk(Asct, F['K'], trans='T')
Kf = cholmod.symbolic(F['K'])
cholmod.numeric(F['K'], Kf)
def solve(x, y, z):
# Solve
#
# [ H A' GG'*W^{-1} ] [ ux ] [ bx ]
# [ A 0 0 ] * [ uy ] = [ by ]
# [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
#
# and return ux, uy, W*uz.
#
# If not F['singular']:
#
# K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz ) - by
# S*ux = bx + GG'*W^{-1}*W^{-T}*bz - A'*uy
# W*uz = W^{-T} * ( GG*ux - bz ).
#
# If F['singular']:
#
# K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz + A'*by )
# - by
# S*ux = bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y.
# W*uz = W^{-T} * ( GG*ux - bz ).
# z := W^{-1} * z = W^{-1} * bz
scale(z, W, trans='T', inverse='I')
# If not F['singular']:
# x := L^{-1} * P * (x + GGs'*z)
# = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz)
#
# If F['singular']:
# x := L^{-1} * P * (x + GGs'*z + A'*y))
# = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz + A'*y)
if mnl:
base.gemv(F['Dfs'], z, x, trans='T', beta=1.0)
base.gemv(F['Gs'], z, x, offsetx=mnl, trans='T',
beta=1.0)
if F['singular']:
base.gemv(A, y, x, trans='T', beta=1.0)
if type(F['S']) is matrix:
blas.trsv(F['S'], x)
else:
cholmod.solve(F['Sf'], x, sys=7)
cholmod.solve(F['Sf'], x, sys=4)
# y := K^{-1} * (Asc*x - y)
# = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz) - by)
# (if not F['singular'])
# = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz +
# A'*by) - by)
# (if F['singular']).
base.gemv(Asct, x, y, trans='T', beta=-1.0)
if type(F['K']) is matrix:
lapack.potrs(F['K'], y)
else:
cholmod.solve(Kf, y)
# x := P' * L^{-T} * (x - Asc'*y)
# = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz - A'*y)
# (if not F['singular'])
# = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y)
# (if F['singular'])
base.gemv(Asct, y, x, alpha=-1.0, beta=1.0)
if type(F['S']) is matrix:
blas.trsv(F['S'], x, trans='T')
else:
cholmod.solve(F['Sf'], x, sys=5)
cholmod.solve(F['Sf'], x, sys=8)
# W*z := GGs*x - z = W^{-T} * (GG*x - bz)
if mnl:
base.gemv(F['Dfs'], x, z, beta=-1.0)
base.gemv(F['Gs'], x, z, beta=-1.0, offsety=mnl)
return solve
