def test_hessian_fu(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
Y = L.copy()
cp.projected_inverse(Y)
U = cp.cspmatrix(self.symb) + 0.1*self.A
cp.hessian(L, Y, U, adj = False, inv = False, factored_updates = True)
cp.hessian(L, Y, U, adj = True, inv = False, factored_updates = True)
cp.hessian(L, Y, U, adj = True, inv = True, factored_updates = True)
cp.hessian(L, Y, U, adj = False, inv = True, factored_updates = True)
diff = list((0.1*self.A-U.spmatrix(reordered=False,symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_hessian_fu(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
Y = L.copy()
cp.projected_inverse(Y)
U = cp.cspmatrix(self.symb) + 0.1 * self.A
cp.hessian(L, Y, U, adj=False, inv=False, factored_updates=True)
cp.hessian(L, Y, U, adj=True, inv=False, factored_updates=True)
cp.hessian(L, Y, U, adj=True, inv=True, factored_updates=True)
cp.hessian(L, Y, U, adj=False, inv=True, factored_updates=True)
diff = list(
(0.1 * self.A - U.spmatrix(reordered=False, symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_projected_inverse(self):
Y = cp.cspmatrix(self.symb) + self.A
Am = Y.spmatrix(symmetric=True, reordered=False)
cp.cholesky(Y)
cp.projected_inverse(Y)
Ym = Y.spmatrix(symmetric=True, reordered=False)
self.assertAlmostEqual((Ym.V.T * (Am.V))[0], self.symb.n)
def test_llt(self):
A = cp.cspmatrix(self.symb) + self.A
cp.cholesky(A)
cp.llt(A)
Am = A.spmatrix(reordered=False)
diff = list((Am - self.A).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_pfcholesky(self):
U = matrix(range(1,2*self.symb.n+1),(self.symb.n,2),tc='d')/self.symb.n
alpha = matrix([1.2,-0.01])
D = matrix(0.0,(2,2))
D[::3] = alpha
random.seed(1)
V = matrix([random.random() for i in range(self.symb.n*3)],(self.symb.n,3))
# PF Cholesky from spmatrix
Lpf = cp.pfcholesky(self.A,U,alpha,p=amd.order)
Vt = +V
Lpf.trmm(Vt,trans='T')
Lpf.trmm(Vt,trans='N')
diff = list( (Vt - (cp.symmetrize(self.A) + U*D*U.T)*V)[:] )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
Lpf.trsm(Vt,trans='N')
Lpf.trsm(Vt,trans='T')
diff = list( (Vt-V)[:] )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
# PF Cholesky from cspmatrix factor
L = cp.cspmatrix(self.symb) + self.A
Lpf = cp.pfcholesky(L,U,alpha)
Vt = +V
Lpf.trmm(Vt,trans='T')
Lpf.trmm(Vt,trans='N')
diff = list( (Vt - (cp.symmetrize(self.A) + U*D*U.T)*V)[:] )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
Lpf.trsm(Vt,trans='N')
Lpf.trsm(Vt,trans='T')
diff = list( (Vt-V)[:] )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_llt(self):
A = cp.cspmatrix(self.symb) + self.A
cp.cholesky(A)
cp.llt(A)
Am = A.spmatrix(reordered=False)
diff = list( (Am - self.A).V )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_projected_inverse(self):
Y = cp.cspmatrix(self.symb) + self.A
Am = Y.spmatrix(symmetric=True,reordered=False)
cp.cholesky(Y)
cp.projected_inverse(Y)
Ym = Y.spmatrix(symmetric=True,reordered=False)
self.assertAlmostEqual((Ym.V.T*(Am.V))[0], self.symb.n)
def test_cspmatrix(self):
Ac = cp.cspmatrix(self.symb) + self.A
self.assertTrue(Ac.is_factor is False)
self.assertTrue(Ac.size[0] == Ac.size[1])
self.assertTrue(Ac.size[0] == 17)
diff = list((self.A - Ac.spmatrix(reordered=False,symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
diff = list(cp.symmetrize(self.A) - Ac.spmatrix(reordered=False,symmetric=True))
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
diff = list(cp.symmetrize(self.A)[self.symb.p,self.symb.p] - Ac.spmatrix(reordered=True,symmetric=True))
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
diff = list(cp.tril(cp.symmetrize(self.A)[self.symb.p,self.symb.p]) - Ac.spmatrix(reordered=True,symmetric=False))
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
diff = list((2*self.A - (2*Ac).spmatrix(reordered=False,symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
diff = list((2*self.A - (Ac+Ac).spmatrix(reordered=False,symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
Ac2 = Ac.copy()
Ac2 += Ac
diff = list((2*self.A - Ac2.spmatrix(reordered=False,symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
self.assertAlmostEqualLists(list(Ac.diag(reordered=False)), list(self.A[::18]))
self.assertAlmostEqualLists(list(Ac.diag(reordered=True)), list(self.A[self.symb.p,self.symb.p][::18]))
def mk_rand(V, cone='posdef', seed=0):
"""
Generates random matrix U with sparsity pattern V.
- U is positive definite if cone is 'posdef'.
- U is completable if cone is 'completable'.
"""
if not (cone == 'posdef' or cone == 'completable'):
raise ValueError, "cone must be 'posdef' (default) or 'completable' "
from cvxopt import amd
setseed(seed)
n = V.size[0]
U = +V
U.V *= 0.0
for i in xrange(n):
u = normal(n, 1) / sqrt(n)
base.syrk(u, U, beta=1.0, partial=True)
# test if U is in cone: if not, add multiple of identity
t = 0.1
Ut = +U
p = amd.order(Ut)
# Vc, NF = chompack.embed(U,p)
symb = chompack.symbolic(U, p)
Vc = chompack.cspmatrix(symb) + U
while True:
# Uc = chompack.project(Vc,Ut)
Uc = chompack.cspmatrix(symb) + Ut
try:
if cone == 'posdef':
# positive definite?
# Y = chompack.cholesky(Uc)
Y = Uc.copy()
chompack.cholesky(Y)
elif cone == 'completable':
# completable?
# Y = chompack.completion(Uc)
Y = Uc.copy()
chompack.completion(Y)
# Success: Ut is in cone
U = +Ut
break
except:
Ut = U + spmatrix(t, xrange(n), xrange(n), (n, n))
t *= 2.0
return U
def mk_rand(V,cone='posdef',seed=0):
"""
Generates random matrix U with sparsity pattern V.
- U is positive definite if cone is 'posdef'.
- U is completable if cone is 'completable'.
"""
if not (cone=='posdef' or cone=='completable'):
raise ValueError("cone must be 'posdef' (default) or 'completable' ")
from cvxopt import amd
setseed(seed)
n = V.size[0]
U = +V
U.V *= 0.0
for i in range(n):
u = normal(n,1)/sqrt(n)
base.syrk(u,U,beta=1.0,partial=True)
# test if U is in cone: if not, add multiple of identity
t = 0.1; Ut = +U
p = amd.order(Ut)
# Vc, NF = chompack.embed(U,p)
symb = chompack.symbolic(U,p)
Vc = chompack.cspmatrix(symb) + U
while True:
# Uc = chompack.project(Vc,Ut)
Uc = chompack.cspmatrix(symb) + Ut
try:
if cone=='posdef':
# positive definite?
# Y = chompack.cholesky(Uc)
Y = Uc.copy()
chompack.cholesky(Y)
elif cone=='completable':
# completable?
# Y = chompack.completion(Uc)
Y = Uc.copy()
chompack.completion(Y)
# Success: Ut is in cone
U = +Ut
break
except:
Ut = U + spmatrix(t,range(n),range(n),(n,n))
t*=2.0
return U
def test_completion_fu(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
L2 = L.copy()
cp.projected_inverse(L2)
cp.completion(L2, factored_updates = True)
diff = list((L.spmatrix()-L2.spmatrix()).V)
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_completion_fu(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
L2 = L.copy()
cp.projected_inverse(L2)
cp.completion(L2, factored_updates=True)
diff = list((L.spmatrix() - L2.spmatrix()).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_mrcompletion(self):
Ac = cp.cspmatrix(self.symb) + self.A
Y = cp.mrcompletion(Ac)
C = Y * Y.T
Ap = cp.cspmatrix(Ac.symb)
Ap.add_projection(C, beta=0.0, reordered=True)
diff = list((Ac.spmatrix() - Ap.spmatrix()).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
U = normal(17, 2)
cp.syr2(Ac, U[:, 0], U[:, 0], alpha=0.5, beta=0.0)
cp.syr2(Ac, U[:, 1], U[:, 1], alpha=0.5, beta=1.0)
Y = cp.mrcompletion(Ac)
Ap.add_projection(Y * Y.T, beta=0.0, reordered=True)
diff = list((Ac.spmatrix() - Ap.spmatrix()).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
Ap.add_projection(U * U.T, beta=0.0, reordered=False)
diff = list((Ac.spmatrix() - Ap.spmatrix()).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_trmm(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
B = cp.eye(self.symb.n)
cp.trmm(L, B)
diff = list((L.spmatrix(reordered=True) - B[self.symb.p,self.symb.p])[:])
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
B = cp.eye(self.symb.n)
cp.trmm(L, B, trans = 'T')
diff = list((L.spmatrix(reordered=True).T - B[self.symb.p,self.symb.p])[:])
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_trmm(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
B = cp.eye(self.symb.n)
cp.trmm(L, B)
diff = list(
(L.spmatrix(reordered=True) - B[self.symb.p, self.symb.p])[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
B = cp.eye(self.symb.n)
cp.trmm(L, B, trans='T')
diff = list(
(L.spmatrix(reordered=True).T - B[self.symb.p, self.symb.p])[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_trsm(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
B = cp.eye(self.symb.n)
Bt = matrix(B)[self.symb.p, :]
Lt = matrix(L.spmatrix(reordered=True, symmetric=False))
cp.trsm(L, B)
blas.trsm(Lt, Bt)
diff = list((B - Bt[self.symb.ip, :])[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
B = cp.eye(self.symb.n)
Bt = matrix(B)[self.symb.p, :]
Lt = matrix(L.spmatrix(reordered=True, symmetric=False))
cp.trsm(L, B, trans='T')
blas.trsm(Lt, Bt, transA='T')
diff = list(B - Bt[self.symb.ip, :])[:]
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def test_cspmatrix(self):
Ac = cp.cspmatrix(self.symb) + self.A
self.assertTrue(Ac.is_factor is False)
self.assertTrue(Ac.size[0] == Ac.size[1])
self.assertTrue(Ac.size[0] == 17)
diff = list((self.A - Ac.spmatrix(reordered=False, symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
diff = list(
cp.symmetrize(self.A) -
Ac.spmatrix(reordered=False, symmetric=True))
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
diff = list(
cp.symmetrize(self.A)[self.symb.p, self.symb.p] -
Ac.spmatrix(reordered=True, symmetric=True))
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
diff = list(
cp.tril(cp.symmetrize(self.A)[self.symb.p, self.symb.p]) -
Ac.spmatrix(reordered=True, symmetric=False))
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
diff = list((2 * self.A -
(2 * Ac).spmatrix(reordered=False, symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
diff = list((2 * self.A -
(Ac + Ac).spmatrix(reordered=False, symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
Ac2 = Ac.copy()
Ac2 += Ac
diff = list(
(2 * self.A - Ac2.spmatrix(reordered=False, symmetric=False)).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
self.assertAlmostEqualLists(list(Ac.diag(reordered=False)),
list(self.A[::18]))
self.assertAlmostEqualLists(
list(Ac.diag(reordered=True)),
list(self.A[self.symb.p, self.symb.p][::18]))
def test_trsm(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
B = cp.eye(self.symb.n)
Bt = matrix(B)[self.symb.p,:]
Lt = matrix(L.spmatrix(reordered=True,symmetric=False))
cp.trsm(L, B)
blas.trsm(Lt,Bt)
diff = list((B-Bt[self.symb.ip,:])[:])
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
B = cp.eye(self.symb.n)
Bt = matrix(B)[self.symb.p,:]
Lt = matrix(L.spmatrix(reordered=True,symmetric=False))
cp.trsm(L, B, trans = 'T')
blas.trsm(Lt,Bt,transA='T')
diff = list(B-Bt[self.symb.ip,:])[:]
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def completion(X):
"""
Returns maximum-determinant positive definite completion of X
if it exists, and otherwise an exception is raised.
"""
from cvxopt.amd import order
n = X.size[0]
Xt = chompack.tril(X)
p = order(Xt)
# Xc,N = chompack.embed(Xt,p)
# L = chompack.completion(Xc)
symb = chompack.symbolic(Xt, p)
L = chompack.cspmatrix(symb) + Xt
chompack.completion(L)
Xt = matrix(0., (n, n))
Xt[::n + 1] = 1.
# chompack.solve(L, Xt, mode=0)
# chompack.solve(L, Xt, mode=1)
chompack.trsm(L, Xt)
chompack.trsm(L, Xt, trans='T')
return Xt
def completion(X):
"""
Returns maximum-determinant positive definite completion of X
if it exists, and otherwise an exception is raised.
"""
from cvxopt.amd import order
n = X.size[0]
Xt = chompack.tril(X)
p = order(Xt)
# Xc,N = chompack.embed(Xt,p)
# L = chompack.completion(Xc)
symb = chompack.symbolic(Xt,p)
L = chompack.cspmatrix(symb) + Xt
chompack.completion(L)
Xt = matrix(0.,(n,n))
Xt[::n+1] = 1.
# chompack.solve(L, Xt, mode=0)
# chompack.solve(L, Xt, mode=1)
chompack.trsm(L, Xt)
chompack.trsm(L, Xt, trans = 'T')
return Xt
def test_pfcholesky(self):
U = matrix(range(1, 2 * self.symb.n + 1),
(self.symb.n, 2), tc='d') / self.symb.n
alpha = matrix([1.2, -0.01])
D = matrix(0.0, (2, 2))
D[::3] = alpha
random.seed(1)
V = matrix([random.random() for i in range(self.symb.n * 3)],
(self.symb.n, 3))
# PF Cholesky from spmatrix
Lpf = cp.pfcholesky(self.A, U, alpha, p=amd.order)
Vt = +V
Lpf.trmm(Vt, trans='T')
Lpf.trmm(Vt, trans='N')
diff = list((Vt - (cp.symmetrize(self.A) + U * D * U.T) * V)[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
Lpf.trsm(Vt, trans='N')
Lpf.trsm(Vt, trans='T')
diff = list((Vt - V)[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
# PF Cholesky from cspmatrix factor
L = cp.cspmatrix(self.symb) + self.A
Lpf = cp.pfcholesky(L, U, alpha)
Vt = +V
Lpf.trmm(Vt, trans='T')
Lpf.trmm(Vt, trans='N')
diff = list((Vt - (cp.symmetrize(self.A) + U * D * U.T) * V)[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
Lpf.trsm(Vt, trans='N')
Lpf.trsm(Vt, trans='T')
diff = list((Vt - V)[:])
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def softmargin_completion(Q, d, gamma):
"""
Solves the QP
minimize (1/2)*y'*Qc^{-1}*y + gamma*sum(v)
subject to diag(d)*(y + b*ones) + v >= 1
v >= 0
(with variables y, b, v) and its dual, the 'soft-margin' SVM problem,
maximize -(1/2)*z'*Qc*z + d'*z
subject to 0 <= diag(d)*z <= gamma*ones
sum(z) = 0
(with variables z).
Qc is the max determinant completion of Q.
Input arguments.
Q is a sparse N x N sparse matrix with chordal sparsity pattern
and a positive definite completion
d is an N-vector of labels -1 or 1.
gamma is a positive parameter.
F is the chompack pattern corresponding to Q. If F is None, the
pattern is computed.
Output.
z, y, b, v, optval, L, iters
"""
if verbose: solvers.options['show_progress'] = True
else: solvers.options['show_progress'] = False
N = Q.size[0]
p = chompack.maxcardsearch(Q)
symb = chompack.symbolic(Q, p)
Qc = chompack.cspmatrix(symb) + Q
# Qinv is the inverse of the max. determinant p.d. completion of Q
Lc = Qc.copy()
chompack.completion(Lc)
Qinv = Lc.copy()
chompack.llt(Qinv)
Qinv = Qinv.spmatrix(reordered=False)
Qinv = chompack.symmetrize(Qinv)
def P(u, v, alpha=1.0, beta=0.0):
"""
v := alpha * [ Qc^-1, 0, 0; 0, 0, 0; 0, 0, 0 ] * u + beta * v
"""
v *= beta
base.symv(Qinv, u, v, alpha=alpha, beta=1.0)
def G(u, v, alpha=1.0, beta=0.0, trans='N'):
"""
If trans is 'N':
v := alpha * [ -diag(d), -d, -I; 0, 0, -I ] * u + beta * v.
If trans is 'T':
v := alpha * [ -diag(d), 0; -d', 0; -I, -I ] * u + beta * v.
"""
v *= beta
if trans is 'N':
v[:N] -= alpha * (base.mul(d, u[:N] + u[N]) + u[-N:])
v[-N:] -= alpha * u[-N:]
else:
v[:N] -= alpha * base.mul(d, u[:N])
v[N] -= alpha * blas.dot(d, u, n=N)
v[-N:] -= alpha * (u[:N] + u[N:])
K = spmatrix(0.0, Qinv.I, Qinv.J)
dy1, dy2 = matrix(0.0, (N, 1)), matrix(0.0, (N, 1))
def Fkkt(W):
"""
Custom KKT solver for
[ Qinv 0 0 -D 0 ] [ ux_y ] [ bx_y ]
[ 0 0 0 -d' 0 ] [ ux_b ] [ bx_b ]
[ 0 0 0 -I -I ] [ ux_v ] = [ bx_v ]
[ -D -d -I -D1 0 ] [ uz_z ] [ bz_z ]
[ 0 0 -I 0 -D2 ] [ uz_w ] [ bz_w ]
with D1 = diag(d1), D2 = diag(d2), d1 = W['d'][:N]**2,
d2 = W['d'][N:])**2.
"""
d1, d2 = W['d'][:N]**2, W['d'][N:]**2
d3, d4 = (d1 + d2)**-1, (d1**-1 + d2**-1)**-1
# Factor the chordal matrix K = Qinv + (D_1+D_2)^-1.
K.V = Qinv.V
K[::N + 1] = K[::N + 1] + d3
L = chompack.cspmatrix(symb) + K
chompack.cholesky(L)
# Solve (Qinv + (D1+D2)^-1) * dy2 = (D1 + D2)^{-1} * 1
blas.copy(d3, dy2)
chompack.trsm(L, dy2, trans='N')
chompack.trsm(L, dy2, trans='T')
def g(x, y, z):
# Solve
#
# [ K d3 ] [ ux_y ]
# [ ] [ ] =
# [ d3' 1'*d3 ] [ ux_b ]
#
# [ bx_y ] [ D ]
# [ ] - [ ] * D3 * (D2 * bx_v + bx_z - bx_w).
# [ bx_b ] [ d' ]
x[:N] -= mul(d, mul(d3, mul(d2, x[-N:]) + z[:N] - z[-N:]))
x[N] -= blas.dot(d, mul(d3, mul(d2, x[-N:]) + z[:N] - z[-N:]))
# Solve dy1 := K^-1 * x[:N]
blas.copy(x, dy1, n=N)
chompack.trsm(L, dy1, trans='N')
chompack.trsm(L, dy1, trans='T')
# Find ux_y = dy1 - ux_b * dy2 s.t
#
# d3' * ( dy1 - ux_b * dy2 + ux_b ) = x[N]
#
# i.e. x[N] := ( x[N] - d3'* dy1 ) / ( d3'* ( 1 - dy2 ) ).
x[N] = ( x[N] - blas.dot(d3, dy1) ) / \
( blas.asum(d3) - blas.dot(d3, dy2) )
x[:N] = dy1 - x[N] * dy2
# ux_v = D4 * ( bx_v - D1^-1 (bz_z + D * (ux_y + ux_b))
# - D2^-1 * bz_w )
x[-N:] = mul(
d4, x[-N:] - div(z[:N] + mul(d, x[:N] + x[N]), d1) -
div(z[N:], d2))
# uz_z = - D1^-1 * ( bx_z - D * ( ux_y + ux_b ) - ux_v )
# uz_w = - D2^-1 * ( bx_w - uz_w )
z[:N] += base.mul(d, x[:N] + x[N]) + x[-N:]
z[-N:] += x[-N:]
blas.scal(-1.0, z)
# Return W['di'] * uz
blas.tbmv(W['di'], z, n=2 * N, k=0, ldA=1)
return g
q = matrix(0.0, (2 * N + 1, 1))
if weights is 'proportional':
dlist = list(d)
C1 = 0.5 * N * gamma / dlist.count(1)
C2 = 0.5 * N * gamma / dlist.count(-1)
gvec = matrix([C1 if w == 1 else C2 for w in dlist], (N, 1))
del dlist
q[-N:] = gvec
elif weights is 'equal':
q[-N:] = gamma
h = matrix(0.0, (2 * N, 1))
h[:N] = -1.0
sol = solvers.coneqp(P, q, G, h, kktsolver=Fkkt)
u = matrix(0.0, (N, 1))
y, b, v = sol['x'][:N], sol['x'][N], sol['x'][N + 1:]
z = mul(d, sol['z'][:N])
base.symv(Qinv, y, u)
optval = 0.5 * blas.dot(y, u) + gamma * sum(v)
return y, b, v, z, optval, Lc, sol['iterations']
def solve_phase1(self,kktsolver='chol',MM = 1e5):
"""
Solves primal Phase I problem using the feasible
start solver.
Returns primal feasible X.
"""
from cvxopt import cholmod, amd
k = 1e-3
# compute Schur complement matrix
Id = [i*(self.n+1) for i in range(self.n)]
As = self._A[:,1:]
As[Id,:] /= sqrt(2.0)
M = spmatrix([],[],[],(self.m,self.m))
base.syrk(As,M,trans='T')
u = +self.b
# compute least-norm solution
F = cholmod.symbolic(M)
cholmod.numeric(M,F)
cholmod.solve(F,u)
x = 0.5*self._A[:,1:]*u
X0 = spmatrix(x[self.V[:].I],self.V.I,self.V.J,(self.n,self.n))
# test feasibility
p = amd.order(self.V)
#Xc,Nf = chompack.embed(X0,p)
#E = chompack.project(Xc,spmatrix(1.0,range(self.n),range(self.n)))
symb = chompack.symbolic(self.V,p)
Xc = chompack.cspmatrix(symb) + X0
try:
# L = chompack.completion(Xc)
L = Xc.copy()
chompack.completion(L)
# least-norm solution is feasible
return X0,None
except:
pass
# create Phase I SDP object
trA = matrix(0.0,(self.m+1,1))
e = matrix(1.0,(self.n,1))
Aa = self._A[Id,1:]
base.gemv(Aa,e,trA,trans='T')
trA[-1] = MM
P1 = SDP()
P1._A = misc.phase1_sdp(self._A,trA)
P1._b = matrix([self.b-k*trA[:self.m],MM])
P1._agg_sparsity()
# find feasible starting point for Phase I problem
tMIN = 0.0
tMAX = 1.0
while True:
t = (tMIN+tMAX)/2.0
#Xt = chompack.copy(Xc)
#chompack.axpy(E,Xt,t)
Xt = Xc.copy() + spmatrix(t,list(range(self.n)),list(range(self.n)))
try:
# L = chompack.completion(Xt)
L = Xt.copy()
chompack.completion(L)
tMAX = t
if tMAX - tMIN < 1e-1:
break
except:
tMAX *= 2.0
tMIN = t
tt = t + 1.0
U = X0 + spmatrix(tt,list(range(self.n,)),list(range(self.n)))
trU = sum(U[:][Id])
Z0 = spdiag([U,spmatrix([tt+k,MM-trU],[0,1],[0,1],(2,2))])
sol = P1.solve_feas(primalstart = {'x':Z0}, kktsolver = kktsolver)
s = sol['x'][-2,-2] - k
if s > 0:
return None,P1
else:
sol.pop('y')
sol.pop('s')
X0 = sol.pop('x')[:self.n,:self.n]\
- spmatrix(s,list(range(self.n)),list(range(self.n)))
return X0,sol
def test_cholesky(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
Lm = L.spmatrix(reordered=True)
diff = list( (cp.tril(cp.perm(Lm*Lm.T,self.symb.ip)) - self.A).V )
self.assertAlmostEqualLists(diff, len(diff)*[0.0])
def test_dot(self):
A = cp.cspmatrix(self.symb) + self.A
B = cp.cspmatrix(self.symb) - 2*self.A
self.assertAlmostEqual(2.0*blas.dot(A.blkval, B.blkval) - blas.dot(A.diag(),B.diag()), cp.dot(A,B))
print("Solving SDP with CVXOPT..")
prob = (c, G, matrix(h), dims)
sol = solvers.conelp(*prob)
Z1 = matrix(sol['z'], (n,n))
# convert SDP and solve
prob2, blocks_to_sparse, symbs = cp.convert_conelp(*prob)
print("Solving converted SDP (no merging)..")
sol2 = solvers.conelp(*prob2)
# convert block-diagonal solution to spmatrix
blki,I,J,bn = blocks_to_sparse[0]
Z2 = spmatrix(sol2['z'][blki],I,J)
# compute completion
symb = cp.symbolic(Z2, p=cp.maxcardsearch)
Z2c = cp.psdcompletion(cp.cspmatrix(symb)+Z2, reordered=False)
prob3, blocks_to_sparse, symbs = cp.convert_conelp(*prob, coupling = 'full', merge_function = cp.merge_size_fill(5,5))
print("Solving converted SDP (with merging)..")
sol3 = solvers.conelp(*prob3)
# convert block-diagonal solution to spmatrix
blki,I,J,bn = blocks_to_sparse[0]
Z3 = spmatrix(sol3['z'][blki],I,J)
# compute completion
symb = cp.symbolic(Z3, p=cp.maxcardsearch)
Z3c = cp.psdcompletion(cp.cspmatrix(symb)+Z3, reordered=False)
def test_dot(self):
A = cp.cspmatrix(self.symb) + self.A
B = cp.cspmatrix(self.symb) - 2 * self.A
self.assertAlmostEqual(
2.0 * blas.dot(A.blkval, B.blkval) - blas.dot(A.diag(), B.diag()),
cp.dot(A, B))
# We will use (greedy) clique merging in this example:
fmerge = merge_size_fill(16,4)
# Compute symbolic factorization with AMD ordering and clique merging; only lower triangular part of As is accessed
print("Computing symbolic factorization..")
p = amd.order
symb = symbolic(As, p = p, merge_function = fmerge)
print("Order of matrix : %i" % (symb.n))
print("Number of nonzeros : %i" % (symb.nnz))
print("Number of supernodes : %i" % (symb.Nsn))
print("Largest supernode : %i" % (max([symb.snptr[k+1]-symb.snptr[k] for k in range(symb.Nsn)])))
print("Largest clique : %i\n" % (symb.clique_number))
A = cspmatrix(symb) # create new cspmatrix from symbolic factorization
A += As # add spmatrix 'As' to cspmatrix 'A'; this ignores the upper triangular entries in As
print("Computing Cholesky factorization..")
L = A.copy() # make a copy of A
cholesky(L) # compute Cholesky factorization; overwrites L
print("Computing Cholesky product..")
At = L.copy() # make a copy of L
llt(At) # compute Cholesky product; overwrites At
print("Computing projected inverse..")
Y = L.copy() # make a copy of L
projected_inverse(Y) # compute projected inverse; overwrites Y
print("Computing completion..")
def test_cholesky(self):
L = cp.cspmatrix(self.symb) + self.A
cp.cholesky(L)
Lm = L.spmatrix(reordered=True)
diff = list((cp.tril(cp.perm(Lm * Lm.T, self.symb.ip)) - self.A).V)
self.assertAlmostEqualLists(diff, len(diff) * [0.0])
def kernel_matrix(X,
kernel,
sigma=1.0,
theta=1.0,
degree=1,
V=None,
width=None):
"""
Computes the kernel matrix or a partial kernel matrix.
Input arguments.
X is an N x n matrix.
kernel is a string with values 'linear', 'rfb', 'poly', or 'tanh'.
'linear': k(u,v) = u'*v/sigma.
'rbf': k(u,v) = exp(-||u - v||^2 / (2*sigma)).
'poly': k(u,v) = (u'*v/sigma)**degree.
'tanh': k(u,v) = tanh(u'*v/sigma - theta). kernel is a
sigma and theta are positive numbers.
degree is a positive integer.
V is an N x N sparse matrix (default is None).
width is a positive integer (default is None).
Output.
Q, an N x N matrix or sparse matrix.
If V is a sparse matrix, a partial kernel matrix with the sparsity
pattern V is returned.
If width is specified and V = 'band', a partial kernel matrix
with band sparsity is returned (width is the half-bandwidth).
a, an N x 1 matrix with the products <xi,xi>/sigma.
"""
N, n = X.size
#### dense (full) kernel matrix
if V is None:
if verbose: print("building kernel matrix ..")
# Qij = xi'*xj / sigma
Q = matrix(0.0, (N, N))
blas.syrk(X, Q, alpha=1.0 / sigma)
a = Q[::N + 1] # ai = ||xi||**2 / sigma
if kernel == 'linear':
pass
elif kernel == 'rbf':
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
ones = matrix(1.0, (N, 1))
blas.syr2(a, ones, Q, alpha=-0.5)
Q = exp(Q)
elif kernel == 'tanh':
Q = exp(Q - theta)
Q = div(Q - Q**-1, Q + Q**-1)
elif kernel == 'poly':
Q = Q**degree
else:
raise ValueError('invalid kernel type')
#### general sparse partial kernel matrix
elif type(V) is cvxopt.base.spmatrix:
if verbose: print("building projected kernel matrix ...")
Q = +V
base.syrk(X, Q, partial=True, alpha=1.0 / sigma)
# ai = ||xi||**2 / sigma
a = matrix(Q[::N + 1], (N, 1))
if kernel == 'linear':
pass
elif kernel == 'rbf':
ones = matrix(1.0, (N, 1))
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
p = chompack.maxcardsearch(V)
symb = chompack.symbolic(Q, p)
Qc = chompack.cspmatrix(symb) + Q
chompack.syr2(Qc, a, ones, alpha=-0.5)
Q = Qc.spmatrix(reordered=False)
Q.V = exp(Q.V)
elif kernel == 'tanh':
v = +Q.V
v = exp(v - theta)
v = div(v - v**-1, v + v**-1)
Q.V = v
elif kernel == 'poly':
Q.V = Q.V**degree
else:
raise ValueError('invalid kernel type')
#### banded partial kernel matrix
elif V == 'band' and width is not None:
# Lower triangular part of band matrix with bandwidth 2*w+1.
if verbose: print("building projected kernel matrix ...")
I = [i for k in range(N) for i in range(k, min(width + k + 1, N))]
J = [k for k in range(N) for i in range(min(width + 1, N - k))]
V = matrix(0.0, (len(I), 1))
oy = 0
for k in range(N): # V[:,k] = Xtrain[k:k+w, :] * Xtrain[k,:].T
m = min(width + 1, N - k)
blas.gemv(X,
X,
V,
m=m,
ldA=N,
incx=N,
offsetA=k,
offsetx=k,
offsety=oy)
oy += m
blas.scal(1.0 / sigma, V)
# ai = ||xi||**2 / sigma
a = matrix(V[[i for i in range(len(I)) if I[i] == J[i]]], (N, 1))
if kernel == 'linear':
Q = spmatrix(V, I, J, (N, N))
elif kernel == 'rbf':
Q = spmatrix(V, I, J, (N, N))
ones = matrix(1.0, (N, 1))
# Qij := Qij - 0.5 * ( ai + aj )
# = -||xi - xj||^2 / (2*sigma)
symb = chompack.symbolic(Q)
Qc = chompack.cspmatrix(symb) + Q
chompack.syr2(Qc, a, ones, alpha=-0.5)
Q = Qc.spmatrix(reordered=False)
Q.V = exp(Q.V)
elif kernel == 'tanh':
V = exp(V - theta)
V = div(V - V**-1, V + V**-1)
Q = spmatrix(V, I, J, (N, N))
elif kernel == 'poly':
Q = spmatrix(V**degree, I, J, (N, N))
else:
raise ValueError('invalid kernel type')
else:
raise TypeError('invalid type V')
return Q, a
def Fkkt(W):
"""
Custom KKT solver for
[ Qinv 0 0 -D 0 ] [ ux_y ] [ bx_y ]
[ 0 0 0 -d' 0 ] [ ux_b ] [ bx_b ]
[ 0 0 0 -I -I ] [ ux_v ] = [ bx_v ]
[ -D -d -I -D1 0 ] [ uz_z ] [ bz_z ]
[ 0 0 -I 0 -D2 ] [ uz_w ] [ bz_w ]
with D1 = diag(d1), D2 = diag(d2), d1 = W['d'][:N]**2,
d2 = W['d'][N:])**2.
"""
d1, d2 = W['d'][:N]**2, W['d'][N:]**2
d3, d4 = (d1 + d2)**-1, (d1**-1 + d2**-1)**-1
# Factor the chordal matrix K = Qinv + (D_1+D_2)^-1.
K.V = Qinv.V
K[::N + 1] = K[::N + 1] + d3
L = chompack.cspmatrix(symb) + K
chompack.cholesky(L)
# Solve (Qinv + (D1+D2)^-1) * dy2 = (D1 + D2)^{-1} * 1
blas.copy(d3, dy2)
chompack.trsm(L, dy2, trans='N')
chompack.trsm(L, dy2, trans='T')
def g(x, y, z):
# Solve
#
# [ K d3 ] [ ux_y ]
# [ ] [ ] =
# [ d3' 1'*d3 ] [ ux_b ]
#
# [ bx_y ] [ D ]
# [ ] - [ ] * D3 * (D2 * bx_v + bx_z - bx_w).
# [ bx_b ] [ d' ]
x[:N] -= mul(d, mul(d3, mul(d2, x[-N:]) + z[:N] - z[-N:]))
x[N] -= blas.dot(d, mul(d3, mul(d2, x[-N:]) + z[:N] - z[-N:]))
# Solve dy1 := K^-1 * x[:N]
blas.copy(x, dy1, n=N)
chompack.trsm(L, dy1, trans='N')
chompack.trsm(L, dy1, trans='T')
# Find ux_y = dy1 - ux_b * dy2 s.t
#
# d3' * ( dy1 - ux_b * dy2 + ux_b ) = x[N]
#
# i.e. x[N] := ( x[N] - d3'* dy1 ) / ( d3'* ( 1 - dy2 ) ).
x[N] = ( x[N] - blas.dot(d3, dy1) ) / \
( blas.asum(d3) - blas.dot(d3, dy2) )
x[:N] = dy1 - x[N] * dy2
# ux_v = D4 * ( bx_v - D1^-1 (bz_z + D * (ux_y + ux_b))
# - D2^-1 * bz_w )
x[-N:] = mul(
d4, x[-N:] - div(z[:N] + mul(d, x[:N] + x[N]), d1) -
div(z[N:], d2))
# uz_z = - D1^-1 * ( bx_z - D * ( ux_y + ux_b ) - ux_v )
# uz_w = - D2^-1 * ( bx_w - uz_w )
z[:N] += base.mul(d, x[:N] + x[N]) + x[-N:]
z[-N:] += x[-N:]
blas.scal(-1.0, z)
# Return W['di'] * uz
blas.tbmv(W['di'], z, n=2 * N, k=0, ldA=1)
return g
def solve_phase1(self, kktsolver='chol', MM=1e5):
"""
Solves primal Phase I problem using the feasible
start solver.
Returns primal feasible X.
"""
from cvxopt import cholmod, amd
k = 1e-3
# compute Schur complement matrix
Id = [i * (self.n + 1) for i in range(self.n)]
As = self._A[:, 1:]
As[Id, :] /= sqrt(2.0)
M = spmatrix([], [], [], (self.m, self.m))
base.syrk(As, M, trans='T')
u = +self.b
# compute least-norm solution
F = cholmod.symbolic(M)
cholmod.numeric(M, F)
cholmod.solve(F, u)
x = 0.5 * self._A[:, 1:] * u
X0 = spmatrix(x[self.V[:].I], self.V.I, self.V.J, (self.n, self.n))
# test feasibility
p = amd.order(self.V)
#Xc,Nf = chompack.embed(X0,p)
#E = chompack.project(Xc,spmatrix(1.0,range(self.n),range(self.n)))
symb = chompack.symbolic(self.V, p)
Xc = chompack.cspmatrix(symb) + X0
try:
# L = chompack.completion(Xc)
L = Xc.copy()
chompack.completion(L)
# least-norm solution is feasible
return X0
except:
pass
# create Phase I SDP object
trA = matrix(0.0, (self.m + 1, 1))
e = matrix(1.0, (self.n, 1))
Aa = self._A[Id, 1:]
base.gemv(Aa, e, trA, trans='T')
trA[-1] = MM
P1 = SDP()
P1._A = misc.phase1_sdp(self._A, trA)
P1._b = matrix([self.b - k * trA[:self.m], MM])
P1._agg_sparsity()
# find feasible starting point for Phase I problem
tMIN = 0.0
tMAX = 1.0
while True:
t = (tMIN + tMAX) / 2.0
#Xt = chompack.copy(Xc)
#chompack.axpy(E,Xt,t)
Xt = Xc.copy() + spmatrix(t, range(self.n), range(self.n))
try:
# L = chompack.completion(Xt)
L = Xt.copy()
chompack.completion(L)
tMAX = t
if tMAX - tMIN < 1e-1:
break
except:
tMAX *= 2.0
tMIN = t
tt = t + 1.0
U = X0 + spmatrix(tt, range(self.n, ), range(self.n))
trU = sum(U[:][Id])
Z0 = spdiag([U, spmatrix([tt + k, MM - trU], [0, 1], [0, 1], (2, 2))])
sol = P1.solve_feas(primalstart={'x': Z0}, kktsolver=kktsolver)
s = sol['x'][-2, -2] - k
if s > 0:
return None, P1
else:
sol.pop('y')
sol.pop('s')
X0 = sol.pop('x')[:self.n,:self.n]\
- spmatrix(s,range(self.n),range(self.n))
return X0, sol
