def callback(self, xk=None, every=10):
'Callback function after each iteration of minres'
self.iter += 1
# Access current iteration from lin_solver
xy = numpy_to_cvxopt_matrix(xk)
# Set x[I], y, and czl czl(if nonempty)
nI = len(self.PDAS.I)
ny = self.PDAS.QP.numeq
ncL = len(self.PDAS.cAL)
ncU = len(self.PDAS.cAU)
self.PDAS.x[self.PDAS.realI + self.PDAS.F] = xy[:nI]
self.PDAS.y = xy[nI:]
if self.PDAS.czl.size[0] > 0:
self.PDAS.czl[self.PDAS.cAL] = xy[nI + ny:nI + ny + ncL]
self.PDAS.czu[self.PDAS.cAU] = xy[nI + ny + ncL:]
# Set residuals, and inv_norm estimate
self.PDAS.CG_r = self.Lhs * xy - self.rhs
# If iter not modulo of 'every', do nothing
if self.iter % every != 0:
return
if self.correct_inv:
self.inv_norm = min(
1.0e+8,
max(nrm2(xy) / nrm2(self.rhs + self.PDAS.CG_r), self.inv_norm))
# Compute tilde v
Qinvv = self.PDAS.CG_r[len(self.PDAS.realI):]
rI = self.PDAS.CG_r[:len(self.PDAS.realI)]
lapack.sytrs(self.PDAS.Q, self.PDAS.ipiv, Qinvv)
viration = nrm2(rI - self.SI.T * Qinvv) * self.PDAS.inv_norm * matrix(
1.0, (len(self.PDAS.realI), 1))
self.err_lb = -viration
self.err_ub = viration
# Update other variables
self.PDAS._back_substitute()
# Caveat z_A shifted
self.PDAS.zu[
self.PDAS.AU] = self.PDAS.zu[self.PDAS.AU] + self.SA.T * Qinvv
self.PDAS.identify_violation_inexact_c(self.err_lb, self.err_ub,
self.SaQinvSi)
frame = inspect.currentframe().f_back
# self.iter = frame.f_locals['itn']
# If condition satisfied, terminate the linear equation solve
if self.PDAS.ask('conditioner') is True:
set_in_frame(inspect.currentframe().f_back, 'istop', 9)
def solve(x, y, z):
# Solve
#
# [ H A' GG'*W^{-1} ] [ ux ] [ bx ]
# [ A 0 0 ] * [ uy [ = [ by ]
# [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
#
# and return ux, uy, W*uz.
#
# On entry, x, y, z contain bx, by, bz. On exit, they contain
# the solution ux, uy, W*uz.
blas.copy(x, u)
blas.copy(y, u, offsety=n)
scale(z, W, trans='T', inverse='I')
pack(z, u, dims, mnl, offsety=n + p)
lapack.sytrs(K, ipiv, u)
blas.copy(u, x, n=n)
blas.copy(u, y, offsetx=n, n=p)
unpack(u, z, dims, mnl, offsetx=n + p)
def solve(x, y, z):
# Solve
#
# [ H A' GG'*W^{-1} ] [ ux ] [ bx ]
# [ A 0 0 ] * [ uy [ = [ by ]
# [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
#
# and return ux, uy, W*uz.
#
# On entry, x, y, z contain bx, by, bz. On exit, they contain
# the solution ux, uy, W*uz.
blas.copy(x, u)
blas.copy(y, u, offsety=n)
scale(z, W, trans='T', inverse='I')
pack(z, u, dims, mnl, offsety=n + p)
lapack.sytrs(K, ipiv, u)
blas.copy(u, x, n=n)
blas.copy(u, y, offsetx=n, n=p)
unpack(u, z, dims, mnl, offsetx=n + p)
def solve(x, y, z):
# Solve
#
# [ H A' GG'*W^{-1} ] [ ux ] [ bx ]
# [ A 0 0 ] * [ uy [ = [ by ]
# [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
#
# and return ux, uy, W*uz.
#
# On entry, x, y, z contain bx, by, bz. On exit, they contain
# the solution ux, uy, W*uz.
blas.scal(0.0, sltn)
blas.copy(x, u)
blas.copy(y, u, offsety = n)
scale(z, W, trans = 'T', inverse = 'I')
pack(z, u, dims, mnl, offsety = n + p)
blas.copy(u, r)
# Iterative refinement algorithm:
# Init: sltn = 0, r_0 = [bx; by; W^{-T}*bz]
# 1. u_k = Ktilde^-1 * r_k
# 2. sltn += u_k
# 3. r_k+1 = r - K*sltn
# Repeat until exceed MAX_ITER iterations or ||r|| <= ERROR_BOUND
iteration = 0
resid_norm = 1
while iteration <= MAX_ITER and resid_norm > ERROR_BOUND:
lapack.sytrs(Ktilde, ipiv, u)
blas.axpy(u, sltn, alpha = 1.0)
blas.copy(r, u)
blas.symv(K, sltn, u, alpha = -1.0, beta = 1.0)
resid_norm = math.sqrt(blas.dot(u, u))
iteration += 1
blas.copy(sltn, x, n = n)
blas.copy(sltn, y, offsetx = n, n = p)
unpack(sltn, z, dims, mnl, offsetx = n + p)
def kkt_solver(x, y, z):
if (TEST_KKT):
x0 = matrix(0., x.size)
y0 = matrix(0., y.size)
z0 = matrix(0., z.size)
x0[:] = x[:]
y0[:] = y[:]
z0[:] = z[:]
# Get default solver solutions.
xp = matrix(0., x.size)
yp = matrix(0., y.size)
zp = matrix(0., z.size)
xp[:] = x[:]
yp[:] = y[:]
zp[:] = z[:]
default_solver(xp, yp, zp)
offset = K * (K + 1) / 2
for i in xrange(K):
symmetrize_matrix(zp, K + 1, offset)
offset += (K + 1) * (K + 1)
# pab = x[:K*(K+1)/2] # p_{ab} 1<=a<=b<=K
# pis = x[K*(K+1)/2:] # \pi_i 1<=i<=K
# z_{ab} := d_{ab}^{-1} z_{ab}
# \mat{z}_i = r_i^{-1} \mat{z}_i r_i^{-t}
misc.scale(z, W, trans='T', inverse='I')
l = z[:]
# l_{ab} := d_{ab}^{-2} z_{ab}
# \mat{z}_i := r_i^{-t}r_i^{-1} \mat{z}_i r_i^{-t} r_i^{-1}
misc.scale(l, W, trans='N', inverse='I')
# The RHS of equations
#
# d_{ab}^{-2}n_{ab} + vec(V_{ab})^t . vec( \sum_i R_i* F R_i*)
# + \sum_i vec(V_{ab})^t . vec( g_i g_i^t) u_i + y
# = -d_{ab}^{-2} l_{ab} + ( p_{ab} - vec(V_{ab})^t . vec(\sum_i L_i*)
#
###
# Lsum := \sum_i L_i
moffset = K * (K + 1) / 2
Lsum = np.sum(np.array(l[moffset:]).reshape((K, (K + 1) * (K + 1))),
axis=0)
Lsum = matrix(Lsum, (K + 1, K + 1))
Ls = Lsum[:K, :K]
x[:K * (K + 1) / 2] -= l[:K * (K + 1) / 2]
dL = matrix(0., (K * (K + 1) / 2, 1))
ab = 0
for a in xrange(K):
dL[ab] = Ls[a, a]
ab += 1
for b in xrange(a + 1, K):
dL[ab] = Ls[a, a] + Ls[b, b] - 2 * Ls[b, a]
ab += 1
x[:K * (K + 1) / 2] -= cvxopt.mul(si2ab, dL)
# The RHS of equations
# g_i^t F g_i + R_{i,K+1,K+1}^2 u_i = pi - L_{i,K+1,K+1}
x[K * (K + 1) / 2:] -= l[K * (K + 1) / 2 + (K + 1) * (K + 1) -
1::(K + 1) * (K + 1)]
# x := B^{-1} Cv
lapack.sytrs(Bm, ipiv, x)
# lapack.potrs( Bm, x)
# y := (oz'.B^{-1}.Cv[:-1] - y)/(oz'.B^{-1}.oz)
y[0] = (blas.dotu(oz, x) - y[0]) / blas.dotu(oz, iB1)
# x := B^{-1} Cv - B^{-1}.oz y
blas.axpy(iB1, x, -y[0])
# Solve for -n_{ab} - d_{ab}^2 z_{ab} = l_{ab}
# We need to return scaled d*z.
# z := d_{ab} d_{ab}^{-2}(n_{ab} + l_{ab})
# = d_{ab}^{-1}n_{ab} + d_{ab}^{-1}l_{ab}
z[:K * (K + 1) / 2] += cvxopt.mul(dis, x[:K * (K + 1) / 2])
z[:K * (K + 1) / 2] *= -1.
# Solve for \mat{z}_i = -R_i (\mat{l}_i + diag(F, u_i)) R_i
# = -L_i - R_i diag(F, u_i) R_i
# We return
# r_i^t \mat{z}_i r_i = -r_i^{-1} (\mat{l}_i + diag(F, u_i)) r_i^{-t}
ui = x[-K:]
nab = tri2symm(x, K)
F = Fisher_matrix(si2, nab)
offset = K * (K + 1) / 2
for i in xrange(K):
start, end = i * (K + 1) * (K + 1), (i + 1) * (K + 1) * (K + 1)
Fu = matrix(0.0, (K + 1, K + 1))
Fu[:K, :K] = F
Fu[K, K] = ui[i]
Fu = matrix(Fu, ((K + 1) * (K + 1), 1))
# Fu := -r_i^{-1} diag( F, u_i) r_i^{-t}
cngrnc(rtis[i], Fu, K + 1, alpha=-1.)
# Fu := -r_i^{-1} (\mat{l}_i + diag( F, u_i )) r_i^{-t}
blas.axpy(z[offset + start:offset + end], Fu, alpha=-1.)
z[offset + start:offset + end] = Fu
if (TEST_KKT):
offset = K * (K + 1) / 2
for i in xrange(K):
symmetrize_matrix(z, K + 1, offset)
offset += (K + 1) * (K + 1)
dz = np.max(np.abs(z - zp))
dx = np.max(np.abs(x - xp))
dy = np.max(np.abs(y - yp))
tol = 1e-5
if dx > tol:
print 'dx='
print dx
print x
print xp
if dy > tol:
print 'dy='
print dy
print y
print yp
if dz > tol:
print 'dz='
print dz
print z
print zp
if dx > tol or dy > tol or dz > tol:
for i, (r, rti) in enumerate(zip(ris, rtis)):
print 'r[%d]=' % i
print r
print 'rti[%d]=' % i
print rti
print 'rti.T*r='
print rti.T * r
for i, d in enumerate(ds):
print 'd[%d]=%g' % (i, d)
print 'x0, y0, z0='
print x0
print y0
print z0
print Bm0
def Aopt_KKT_solver(si2, W):
'''
Construct a solver that solves the KKT equations associated with the cone
programming for A-optimal:
/ 0 At Gt \ / x \ / p \
| A 0 0 | | y | = | q |
\ G 0 -Wt W / \ z / \ s /
Args:
si2: symmetric KxK matrix, si2[i,j] = 1/s_{ij}^2
'''
K = si2.size[0]
ds = W['d']
dis = W['di'] # dis[i] := 1./ds[i]
rtis = W['rti']
ris = W['r']
d2s = ds**2
di2s = dis**2
# R_i = r_i^{-t}r_i^{-1}
Ris = [matrix(0.0, (K + 1, K + 1)) for i in xrange(K)]
for i in xrange(K):
blas.gemm(rtis[i], rtis[i], Ris[i], transB='T')
ddR2 = matrix(0., (K * (K + 1) / 2, K * (K + 1) / 2))
sumdR2_C(Ris, ddR2, K)
# upper triangular representation si2ab[(a,b)] := si2[a,b]
si2ab = matrix(0., (K * (K + 1) / 2, 1))
p = 0
for i in xrange(K):
si2ab[p:p + (K - i)] = si2[i:, i]
p += (K - i)
si2q = matrix(0., (K * (K + 1) / 2, K * (K + 1) / 2))
blas.syr(si2ab, si2q)
sRVR = cvxopt.mul(si2q, ddR2)
# We first solve for K(K+1)/2 n_{ab}, K u_i, 1 y
nvars = K * (K + 1) / 2 + K # + 1 We solve y by elimination of n and u.
Bm = matrix(0.0, (nvars, nvars))
# The LHS matrix of equations
#
# d_{ab}^{-2} n_{ab} + vec(V_{ab})^t . vec( \sum_i R_i* F R_i*)
# + \sum_i vec(V_{ab})^t . vec( g_i g_i^t) u_i + y
# = -d_{ab}^{-2}l_{ab} + ( p_{ab} - vec(V_{ab})^t . vec(\sum_i L_i*)
#
# Coefficients for n_{ab}
Bm[:K * (K + 1) / 2, :K * (K + 1) / 2] = cvxopt.mul(si2q, ddR2)
row = 0
for a in xrange(K):
for b in xrange(a, K):
Bm[row, row] += di2s[row] # d_{ab}^{-2} n_{ab}
row += 1
assert (K * (K + 1) / 2 == row)
# Coefficients for u_i
# The LHS of equations
# g_i^t F g_i + R_{i,K+1,K+1}^2 u_i = pi - L_{i,K+1,K+1}
dg = matrix(0., (K, K * (K + 1) / 2))
g = matrix(0., (K, K))
for i in xrange(K):
g[i, :] = Ris[i][K, :K]
# dg[:,(a,b)] = g[a] - g[b] if a!=b else g[a]
pairwise_diff(g, dg, K)
dg2 = dg**2
# dg2 := s[(a,b)]^{-2} dg[(a,b)]^2
for i in xrange(K):
dg2[i, :] = cvxopt.mul(si2ab.T, dg2[i, :])
Bm[K * (K + 1) / 2:K * (K + 1) / 2 + K, :-K] = dg2
# Diagonal coefficients for u_i.
uoffset = K * (K + 1) / 2
for i in xrange(K):
RiKK = Ris[i][K, K]
Bm[uoffset + i, uoffset + i] = RiKK**2
# Compare with the default KKT solver.
TEST_KKT = False
if (TEST_KKT):
Bm0 = matrix(0., Bm.size)
blas.copy(Bm, Bm0)
G, h, A = Aopt_GhA(si2)
dims = dict(l=K * (K + 1) / 2, q=[], s=[K + 1] * K)
default_solver = misc.kkt_ldl(G, dims, A)(W)
ipiv = matrix(0, Bm.size)
lapack.sytrf(Bm, ipiv)
# TODO: IS THIS A POSITIVE DEFINITE MATRIX?
# lapack.potrf( Bm)
# oz := (1, ..., 1, 0, ..., 0)' with K*(K+1)/2 ones and K zeros
oz = matrix(0., (Bm.size[0], 1))
oz[:K * (K + 1) / 2] = 1.
# iB1 := B^{-1} oz
iB1 = matrix(oz[:], oz.size)
lapack.sytrs(Bm, ipiv, iB1)
# lapack.potrs( Bm, iB1)
#######
#
# The solver
#
#######
def kkt_solver(x, y, z):
if (TEST_KKT):
x0 = matrix(0., x.size)
y0 = matrix(0., y.size)
z0 = matrix(0., z.size)
x0[:] = x[:]
y0[:] = y[:]
z0[:] = z[:]
# Get default solver solutions.
xp = matrix(0., x.size)
yp = matrix(0., y.size)
zp = matrix(0., z.size)
xp[:] = x[:]
yp[:] = y[:]
zp[:] = z[:]
default_solver(xp, yp, zp)
offset = K * (K + 1) / 2
for i in xrange(K):
symmetrize_matrix(zp, K + 1, offset)
offset += (K + 1) * (K + 1)
# pab = x[:K*(K+1)/2] # p_{ab} 1<=a<=b<=K
# pis = x[K*(K+1)/2:] # \pi_i 1<=i<=K
# z_{ab} := d_{ab}^{-1} z_{ab}
# \mat{z}_i = r_i^{-1} \mat{z}_i r_i^{-t}
misc.scale(z, W, trans='T', inverse='I')
l = z[:]
# l_{ab} := d_{ab}^{-2} z_{ab}
# \mat{z}_i := r_i^{-t}r_i^{-1} \mat{z}_i r_i^{-t} r_i^{-1}
misc.scale(l, W, trans='N', inverse='I')
# The RHS of equations
#
# d_{ab}^{-2}n_{ab} + vec(V_{ab})^t . vec( \sum_i R_i* F R_i*)
# + \sum_i vec(V_{ab})^t . vec( g_i g_i^t) u_i + y
# = -d_{ab}^{-2} l_{ab} + ( p_{ab} - vec(V_{ab})^t . vec(\sum_i L_i*)
#
###
# Lsum := \sum_i L_i
moffset = K * (K + 1) / 2
Lsum = np.sum(np.array(l[moffset:]).reshape((K, (K + 1) * (K + 1))),
axis=0)
Lsum = matrix(Lsum, (K + 1, K + 1))
Ls = Lsum[:K, :K]
x[:K * (K + 1) / 2] -= l[:K * (K + 1) / 2]
dL = matrix(0., (K * (K + 1) / 2, 1))
ab = 0
for a in xrange(K):
dL[ab] = Ls[a, a]
ab += 1
for b in xrange(a + 1, K):
dL[ab] = Ls[a, a] + Ls[b, b] - 2 * Ls[b, a]
ab += 1
x[:K * (K + 1) / 2] -= cvxopt.mul(si2ab, dL)
# The RHS of equations
# g_i^t F g_i + R_{i,K+1,K+1}^2 u_i = pi - L_{i,K+1,K+1}
x[K * (K + 1) / 2:] -= l[K * (K + 1) / 2 + (K + 1) * (K + 1) -
1::(K + 1) * (K + 1)]
# x := B^{-1} Cv
lapack.sytrs(Bm, ipiv, x)
# lapack.potrs( Bm, x)
# y := (oz'.B^{-1}.Cv[:-1] - y)/(oz'.B^{-1}.oz)
y[0] = (blas.dotu(oz, x) - y[0]) / blas.dotu(oz, iB1)
# x := B^{-1} Cv - B^{-1}.oz y
blas.axpy(iB1, x, -y[0])
# Solve for -n_{ab} - d_{ab}^2 z_{ab} = l_{ab}
# We need to return scaled d*z.
# z := d_{ab} d_{ab}^{-2}(n_{ab} + l_{ab})
# = d_{ab}^{-1}n_{ab} + d_{ab}^{-1}l_{ab}
z[:K * (K + 1) / 2] += cvxopt.mul(dis, x[:K * (K + 1) / 2])
z[:K * (K + 1) / 2] *= -1.
# Solve for \mat{z}_i = -R_i (\mat{l}_i + diag(F, u_i)) R_i
# = -L_i - R_i diag(F, u_i) R_i
# We return
# r_i^t \mat{z}_i r_i = -r_i^{-1} (\mat{l}_i + diag(F, u_i)) r_i^{-t}
ui = x[-K:]
nab = tri2symm(x, K)
F = Fisher_matrix(si2, nab)
offset = K * (K + 1) / 2
for i in xrange(K):
start, end = i * (K + 1) * (K + 1), (i + 1) * (K + 1) * (K + 1)
Fu = matrix(0.0, (K + 1, K + 1))
Fu[:K, :K] = F
Fu[K, K] = ui[i]
Fu = matrix(Fu, ((K + 1) * (K + 1), 1))
# Fu := -r_i^{-1} diag( F, u_i) r_i^{-t}
cngrnc(rtis[i], Fu, K + 1, alpha=-1.)
# Fu := -r_i^{-1} (\mat{l}_i + diag( F, u_i )) r_i^{-t}
blas.axpy(z[offset + start:offset + end], Fu, alpha=-1.)
z[offset + start:offset + end] = Fu
if (TEST_KKT):
offset = K * (K + 1) / 2
for i in xrange(K):
symmetrize_matrix(z, K + 1, offset)
offset += (K + 1) * (K + 1)
dz = np.max(np.abs(z - zp))
dx = np.max(np.abs(x - xp))
dy = np.max(np.abs(y - yp))
tol = 1e-5
if dx > tol:
print 'dx='
print dx
print x
print xp
if dy > tol:
print 'dy='
print dy
print y
print yp
if dz > tol:
print 'dz='
print dz
print z
print zp
if dx > tol or dy > tol or dz > tol:
for i, (r, rti) in enumerate(zip(ris, rtis)):
print 'r[%d]=' % i
print r
print 'rti[%d]=' % i
print rti
print 'rti.T*r='
print rti.T * r
for i, d in enumerate(ds):
print 'd[%d]=%g' % (i, d)
print 'x0, y0, z0='
print x0
print y0
print z0
print Bm0
###
# END of kkt_solver.
###
return kkt_solver
