def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 # ds is square root of diagonal of D ds = sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(A, Asc) for k in range(m): blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m + 1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \ mul(d1, z[:n] + mul(d3, z[:n])) - \ mul(d2, z[n:] - mul(d3, z[n:])) ) x[:n] = div(x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n]) z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:]) return g
def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 # ds is square root of diagonal of D ds = sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), sqrt(d1+d2) ) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(A, Asc) for k in range(m): blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m+1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \ mul(d1, z[:n] + mul(d3, z[:n])) - \ mul(d2, z[n:] - mul(d3, z[n:])) ) x[:n] = div( x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] ) z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] ) return g
def f(x, y, z): # z := mat(z) # = mat(bz) z.size = N, m # x := x + D .* z # = bx + mat(bz) ./ d.^2 x += mul(D, z) # y := y - sum_k (Q + Dk)^-1 * X[:,k] # = by - sum_k (Q + Dk)^-1 * (bxk + Dk * bzk) for k in range(m): blas.symv(H[k], x[:, k], y, alpha=-1.0, beta=1.0) # y := H^-1 * y # = -uy lapack.potrs(S, y) # x[:,k] := H[k] * (x[:,k] + y) # = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y) # = ux[:,k] w = matrix(0.0, (N, 1)) for k in range(m): # x[:,k] := x[:,k] + y blas.axpy(y, x, offsety=N * k, n=N) # w := H[k] * x[:,k] # = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y) blas.symv(H[k], x, w, offsetx=N * k) # x[:,k] := w # = ux[:,k] blas.copy(w, x, offsety=N * k) # y := -y # = uy blas.scal(-1.0, y) # z := (x - z) ./ d blas.axpy(x, z, -1.0) blas.tbsv(W['d'], z, n=m * N, k=0, ldA=1) blas.scal(-1.0, z) z.size = N * m, 1
def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans='T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans='T', offsetx=msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans='T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha=-1.0, offsetx=msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n=msq, k=0, ldA=1) blas.axpy(z, x1, n=msq) blas.tbsv(sqrtG, x1, n=msq, k=0, ldA=1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m=mpckd, trans='T', beta=1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m=mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n=msq, k=0, ldA=1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n=msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n=msq, k=0, ldA=1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n=msq, offsety=msq) blas.scal(-1.0, z, offset=msq) __cngrnc(U, z, offsetx=msq) __cngrnc(W['r'][1], z, trans='T', offsetx=msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n=msq, k=0, ldA=1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n=msq) blas.scal(-1.0, z, n=msq) blas.axpy(x[1], z, -1.0, n=msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans='T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1])
def F(W): """ Generate a solver for A'(uz0) = bx[0] -uz0 - uz1 = bx[1] A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0 - ux[1] - r1*r1' * uz1 * r1*r1' = bz1. uz0, uz1, bz0, bz1 are symmetric m x m-matrices. ux[0], bx[0] are n-vectors. ux[1], bx[1] are symmetric m x m-matrices. We first calculate a congruence that diagonalizes r0*r0' and r1*r1': U' * r0 * r0' * U = I, U' * r1 * r1' * U = S. We then make a change of variables usx[0] = ux[0], usx[1] = U' * ux[1] * U usz0 = U^-1 * uz0 * U^-T usz1 = U^-1 * uz1 * U^-T and define As() = U' * A() * U' bsx[1] = U^-1 * bx[1] * U^-T bsz0 = U' * bz0 * U bsz1 = U' * bz1 * U. This gives As'(usz0) = bx[0] -usz0 - usz1 = bsx[1] As(usx[0]) - usx[1] - usz0 = bsz0 -usx[1] - S * usz1 * S = bsz1. 1. Eliminate usz0, usz1 using equations 3 and 4, usz0 = As(usx[0]) - usx[1] - bsz0 usz1 = -S^-1 * (usx[1] + bsz1) * S^-1. This gives two equations in usx[0] an usx[1]. As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0) -As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1 = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1. 2. Eliminate usx[1] using equation 2: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j], usx[1] = ( S * As(usx[0]) * S ) ./ Gamma + ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma. This gives an equation in usx[0]. As'( As(usx[0]) ./ Gamma ) = bx0 + As'(bsz0) + As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma ) = bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ). """ # Calculate U s.t. # # U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s). # Cholesky factorization r0 * r0' = L * L' blas.syrk(W['r'][0], L) lapack.potrf(L) # SVD L^-1 * r1 = U * diag(s) * V' blas.copy(W['r'][1], U) blas.trsm(L, U) lapack.gesvd(U, s, jobu='O') # s := s**2 s[:] = s**2 # Uti := U blas.copy(U, Uti) # U := L^-T * U blas.trsm(L, U, transA='T') # Uti := L * Uti = U^-T blas.trmm(L, Uti) # Us := U * diag(s)^-1 blas.copy(U, Us) for i in range(m): blas.tbsv(s, Us, n=m, k=0, ldA=1, incx=m, offsetx=i) # S is m x m with lower triangular entries s[i] * s[j] # sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j]) # Upper triangular entries are undefined but nonzero. blas.scal(0.0, S) blas.syrk(s, S) Gamma = 1.0 + S sqrtG = sqrt(Gamma) # Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n # = Asi ./ sqrt(Gamma) blas.copy(A, Asc) misc.scale( Asc, # only 'r' part of the dictionary is used { 'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)), 'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)), 'v': [], 'beta': [], 'r': [U], 'rti': [U] }) for i in range(n): blas.tbsv(sqrtG, Asc, n=msq, k=0, ldA=1, offsetx=i * msq) # Convert columns of Asc to packed storage misc.pack2(Asc, {'l': 0, 'q': [], 's': [m]}) # Cholesky factorization of Asc' * Asc. H = matrix(0.0, (n, n)) blas.syrk(Asc, H, trans='T', k=mpckd) lapack.potrf(H) def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans='T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans='T', offsetx=msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans='T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha=-1.0, offsetx=msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n=msq, k=0, ldA=1) blas.axpy(z, x1, n=msq) blas.tbsv(sqrtG, x1, n=msq, k=0, ldA=1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m=mpckd, trans='T', beta=1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m=mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n=msq, k=0, ldA=1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n=msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n=msq, k=0, ldA=1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n=msq, offsety=msq) blas.scal(-1.0, z, offset=msq) __cngrnc(U, z, offsetx=msq) __cngrnc(W['r'][1], z, trans='T', offsetx=msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n=msq, k=0, ldA=1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n=msq) blas.scal(-1.0, z, n=msq) blas.axpy(x[1], z, -1.0, n=msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans='T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1]) return solve
def f(x, y, z): """ Solve C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - T * [ ] * T = bzs. [ -A(ux) -uX2 ] [ uzs21 uzs22 ] On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ]. Define X = uzs21, Z = T * uzs * T: C * ux + G' * uzl - 2*A'(X) = bx [ 0 X' ] [ bX1 0 ] T * [ ] * T - Z = T * [ ] * T [ X 0 ] [ 0 bX2 ] G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs [ -A(ux) -uX2 ] [ Z21 Z22 ] Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ]. We use the congruence transformation [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and the factorization X + S * X' * S = L( L'(X) ) to write this as C * ux + G' * uzl - 2*A'(X) = bx L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs, [ -A(ux) -uX2 ] [ Z21 Z22 ] or C * ux + Gs' * uuzl - 2*As'(XX) = bx XX - ZZ21 = bX Gs * ux - uuzl = D^-1 * bzl -As(ux) - ZZ21 = bbzs_21 -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 if we introduce scaled variables uuzl = D * uzl XX = L'(V2^-1 * X * V1^-1) = L'(V2^-1 * uzs21 * V1^-1) ZZ21 = L^-1(V2' * Z21 * V1') and define bbzs_21 = L^-1(V2' * bzs_21 * V1') [ bX1 0 ] bX = L^-1( V2' * (T * [ ] * T)_21 * V1'). [ 0 bX2 ] Eliminating Z21 gives C * ux + Gs' * uuzl - 2*As'(XX) = bx Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 and eliminating uuzl and XX gives H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21) Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22. In summary, we can use the following algorithm: 1. bXX := bX - bbzs21 [ bX1 0 ] = L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1') [ 0 bX2 ] 2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). 3. From ux, compute uuzl = Gs*ux - D^-1 * bzl and X = V2 * L^-T(-As(ux) + bXX) * V1. 4. Return ux, uuzl, rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22. """ # Save bzs_11, bzs_22, bzs_21. lapack.lacpy(z, bz11, uplo = 'L', m = q, n = q, ldA = p+q, offsetA = m) lapack.lacpy(z, bz21, m = p, n = q, ldA = p+q, offsetA = m+q) lapack.lacpy(z, bz22, uplo = 'L', m = p, n = p, ldA = p+q, offsetA = m + (p+q+1)*q) # zl := D^-1 * zl # = D^-1 * bzl blas.tbmv(W['di'], z, n = m, k = 0, ldA = 1) # zs := r' * [ bX1, 0; 0, bX2 ] * r. # zs := [ bX1, 0; 0, bX2 ] blas.scal(0.0, z, offset = m) lapack.lacpy(x[1], z, uplo = 'L', m = q, n = q, ldB = p+q, offsetB = m) lapack.lacpy(x[2], z, uplo = 'L', m = p, n = p, ldB = p+q, offsetB = m + (p+q+1)*q) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc = p+q+1, offset = m) # a := tril(zs)*r blas.copy(r, a) blas.trmm(z, a, side = 'L', m = p+q, n = p+q, ldA = p+q, ldB = p+q, offsetA = m) # zs := a'*r + r'*a blas.syr2k(r, a, z, trans = 'T', n = p+q, k = p+q, ldB = p+q, ldC = p+q, offsetC = m) # bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1') # # [ bX1 0 ] # = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1'). # [ 0 bX2 ] # a = [ r21 r22 ] * z # = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r # = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r blas.symm(z, r, a, side = 'R', m = p, n = p+q, ldA = p+q, ldC = p+q, offsetB = q) # bz21 := -bz21 + a * [ r11, r12 ]' # = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21 blas.gemm(a, r, bz21, transB = 'T', m = p, n = q, k = p+q, beta = -1.0, ldA = p+q, ldC = p) # bz21 := V2' * bz21 * V1' # = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1' blas.gemm(V2, bz21, tmp, transA = 'T', m = p, n = q, k = p, ldB = p) blas.gemm(tmp, V1, bz21, transB = 'T', m = p, n = q, k = q, ldC = p) # bz21[:] := D * (I-P) * bz21[:] # = L^-1 * bz21[:] # = bXX[:] blas.copy(bz21, tmp) base.gemv(P, bz21, tmp, alpha = -1.0, beta = 1.0) base.gemv(D, tmp, bz21) # Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). # x[0] := x[0] + Gs'*zl + 2*As'(bz21) # = bx + G' * D^-1 * bzl + 2 * As'(bXX) blas.gemv(Gs, z, x[0], trans = 'T', alpha = 1.0, beta = 1.0) blas.gemv(As, bz21, x[0], trans = 'T', alpha = 2.0, beta = 1.0) # x[0] := H \ x[0] # = ux lapack.potrs(H, x[0]) # uuzl = Gs*ux - D^-1 * bzl blas.gemv(Gs, x[0], z, alpha = 1.0, beta = -1.0) # bz21 := V2 * L^-T(-As(ux) + bz21) * V1 # = X blas.gemv(As, x[0], bz21, alpha = -1.0, beta = 1.0) blas.tbsv(DV, bz21, n = p*q, k = 0, ldA = 1) blas.copy(bz21, tmp) base.gemv(P, tmp, bz21, alpha = -1.0, beta = 1.0, trans = 'T') blas.gemm(V2, bz21, tmp) blas.gemm(tmp, V1, bz21) # zs := -zs + r' * [ 0, X'; X, 0 ] * r # = r' * [ -bX1, X'; X, -bX2 ] * r. # a := bz21 * [ r11, r12 ] # = X * [ r11, r12 ] blas.gemm(bz21, r, a, m = p, n = p+q, k = q, ldA = p, ldC = p+q) # z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ] # = rti' * uzs * rti blas.syr2k(r, a, z, trans = 'T', beta = -1.0, n = p+q, k = p, offsetA = q, offsetC = m, ldB = p+q, ldC = p+q) # uX1 = -Z11 - bzs_11 # = -(r*zs*r')_11 - bzs_11 # uX2 = -Z22 - bzs_22 # = -(r*zs*r')_22 - bzs_22 blas.copy(bz11, x[1]) blas.copy(bz22, x[2]) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc = p+q+1, offset = m) # a := r*tril(zs) blas.copy(r, a) blas.trmm(z, a, side = 'R', m = p+q, n = p+q, ldA = p+q, ldB = p+q, offsetA = m) # x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]' # = -bzs_11 - (r*zs*r')_11 blas.syr2k(a, r, x[1], n = q, alpha = -1.0, beta = -1.0) # x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]' # = -bzs_22 - (r*zs*r')_22 blas.syr2k(a, r, x[2], n = p, alpha = -1.0, beta = -1.0, offsetA = q, offsetB = q) # scale diagonal of zs by 1/2 blas.scal(2.0, z, inc = p+q+1, offset = m)
def f(x, y, z): """ Solve C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - T * [ ] * T = bzs. [ -A(ux) -uX2 ] [ uzs21 uzs22 ] On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ]. Define X = uzs21, Z = T * uzs * T: C * ux + G' * uzl - 2*A'(X) = bx [ 0 X' ] [ bX1 0 ] T * [ ] * T - Z = T * [ ] * T [ X 0 ] [ 0 bX2 ] G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs [ -A(ux) -uX2 ] [ Z21 Z22 ] Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ]. We use the congruence transformation [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and the factorization X + S * X' * S = L( L'(X) ) to write this as C * ux + G' * uzl - 2*A'(X) = bx L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs, [ -A(ux) -uX2 ] [ Z21 Z22 ] or C * ux + Gs' * uuzl - 2*As'(XX) = bx XX - ZZ21 = bX Gs * ux - uuzl = D^-1 * bzl -As(ux) - ZZ21 = bbzs_21 -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 if we introduce scaled variables uuzl = D * uzl XX = L'(V2^-1 * X * V1^-1) = L'(V2^-1 * uzs21 * V1^-1) ZZ21 = L^-1(V2' * Z21 * V1') and define bbzs_21 = L^-1(V2' * bzs_21 * V1') [ bX1 0 ] bX = L^-1( V2' * (T * [ ] * T)_21 * V1'). [ 0 bX2 ] Eliminating Z21 gives C * ux + Gs' * uuzl - 2*As'(XX) = bx Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 and eliminating uuzl and XX gives H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21) Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22. In summary, we can use the following algorithm: 1. bXX := bX - bbzs21 [ bX1 0 ] = L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1') [ 0 bX2 ] 2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). 3. From ux, compute uuzl = Gs*ux - D^-1 * bzl and X = V2 * L^-T(-As(ux) + bXX) * V1. 4. Return ux, uuzl, rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22. """ # Save bzs_11, bzs_22, bzs_21. lapack.lacpy(z, bz11, uplo='L', m=q, n=q, ldA=p + q, offsetA=m) lapack.lacpy(z, bz21, m=p, n=q, ldA=p + q, offsetA=m + q) lapack.lacpy(z, bz22, uplo='L', m=p, n=p, ldA=p + q, offsetA=m + (p + q + 1) * q) # zl := D^-1 * zl # = D^-1 * bzl blas.tbmv(W['di'], z, n=m, k=0, ldA=1) # zs := r' * [ bX1, 0; 0, bX2 ] * r. # zs := [ bX1, 0; 0, bX2 ] blas.scal(0.0, z, offset=m) lapack.lacpy(x[1], z, uplo='L', m=q, n=q, ldB=p + q, offsetB=m) lapack.lacpy(x[2], z, uplo='L', m=p, n=p, ldB=p + q, offsetB=m + (p + q + 1) * q) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc=p + q + 1, offset=m) # a := tril(zs)*r blas.copy(r, a) blas.trmm(z, a, side='L', m=p + q, n=p + q, ldA=p + q, ldB=p + q, offsetA=m) # zs := a'*r + r'*a blas.syr2k(r, a, z, trans='T', n=p + q, k=p + q, ldB=p + q, ldC=p + q, offsetC=m) # bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1') # # [ bX1 0 ] # = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1'). # [ 0 bX2 ] # a = [ r21 r22 ] * z # = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r # = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r blas.symm(z, r, a, side='R', m=p, n=p + q, ldA=p + q, ldC=p + q, offsetB=q) # bz21 := -bz21 + a * [ r11, r12 ]' # = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21 blas.gemm(a, r, bz21, transB='T', m=p, n=q, k=p + q, beta=-1.0, ldA=p + q, ldC=p) # bz21 := V2' * bz21 * V1' # = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1' blas.gemm(V2, bz21, tmp, transA='T', m=p, n=q, k=p, ldB=p) blas.gemm(tmp, V1, bz21, transB='T', m=p, n=q, k=q, ldC=p) # bz21[:] := D * (I-P) * bz21[:] # = L^-1 * bz21[:] # = bXX[:] blas.copy(bz21, tmp) base.gemv(P, bz21, tmp, alpha=-1.0, beta=1.0) base.gemv(D, tmp, bz21) # Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). # x[0] := x[0] + Gs'*zl + 2*As'(bz21) # = bx + G' * D^-1 * bzl + 2 * As'(bXX) blas.gemv(Gs, z, x[0], trans='T', alpha=1.0, beta=1.0) blas.gemv(As, bz21, x[0], trans='T', alpha=2.0, beta=1.0) # x[0] := H \ x[0] # = ux lapack.potrs(H, x[0]) # uuzl = Gs*ux - D^-1 * bzl blas.gemv(Gs, x[0], z, alpha=1.0, beta=-1.0) # bz21 := V2 * L^-T(-As(ux) + bz21) * V1 # = X blas.gemv(As, x[0], bz21, alpha=-1.0, beta=1.0) blas.tbsv(DV, bz21, n=p * q, k=0, ldA=1) blas.copy(bz21, tmp) base.gemv(P, tmp, bz21, alpha=-1.0, beta=1.0, trans='T') blas.gemm(V2, bz21, tmp) blas.gemm(tmp, V1, bz21) # zs := -zs + r' * [ 0, X'; X, 0 ] * r # = r' * [ -bX1, X'; X, -bX2 ] * r. # a := bz21 * [ r11, r12 ] # = X * [ r11, r12 ] blas.gemm(bz21, r, a, m=p, n=p + q, k=q, ldA=p, ldC=p + q) # z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ] # = rti' * uzs * rti blas.syr2k(r, a, z, trans='T', beta=-1.0, n=p + q, k=p, offsetA=q, offsetC=m, ldB=p + q, ldC=p + q) # uX1 = -Z11 - bzs_11 # = -(r*zs*r')_11 - bzs_11 # uX2 = -Z22 - bzs_22 # = -(r*zs*r')_22 - bzs_22 blas.copy(bz11, x[1]) blas.copy(bz22, x[2]) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc=p + q + 1, offset=m) # a := r*tril(zs) blas.copy(r, a) blas.trmm(z, a, side='R', m=p + q, n=p + q, ldA=p + q, ldB=p + q, offsetA=m) # x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]' # = -bzs_11 - (r*zs*r')_11 blas.syr2k(a, r, x[1], n=q, alpha=-1.0, beta=-1.0) # x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]' # = -bzs_22 - (r*zs*r')_22 blas.syr2k(a, r, x[2], n=p, alpha=-1.0, beta=-1.0, offsetA=q, offsetB=q) # scale diagonal of zs by 1/2 blas.scal(2.0, z, inc=p + q + 1, offset=m)
def qp(P, q, G=None, h=None, A=None, b=None): """ Solves a pair of primal and dual convex quadratic cone programs minimize (1/2)*x'*P*x + q'*x subject to G*x + s = h A*x = b s >= 0 maximize -(1/2)*(q + G'*z + A'*y)' * pinv(P) * (q + G'*z + A'*y) - h'*z - b'*y subject to q + G'*z + A'*y in range(P) z >= 0. The inequalities are with respect to a cone C defined as the Cartesian product of N + M + 1 cones: C = C_0 x C_1 x .... x C_N x C_{N+1} x ... x C_{N+M}. The first cone C_0 is the nonnegative orthant of dimension ml. The next N cones are 2nd order cones of dimension mq[0], ..., mq[N-1]. The second order cone of dimension m is defined as { (u0, u1) in R x R^{m-1} | u0 >= ||u1||_2 }. The next M cones are positive semidefinite cones of order ms[0], ..., ms[M-1] >= 0. Input arguments (basic usage). P is a dense or sparse 'd' matrix of size (n,n) with the lower triangular part of the Hessian of the objective stored in the lower triangle. Must be positive semidefinite. q is a dense 'd' matrix of size (n,1). dims is a dictionary with the dimensions of the components of C. It has three fields. - dims['l'] = ml, the dimension of the nonnegative orthant C_0. (ml >= 0.) - dims['q'] = mq = [ mq[0], mq[1], ..., mq[N-1] ], a list of N integers with the dimensions of the second order cones C_1, ..., C_N. (N >= 0 and mq[k] >= 1.) - dims['s'] = ms = [ ms[0], ms[1], ..., ms[M-1] ], a list of M integers with the orders of the semidefinite cones C_{N+1}, ..., C_{N+M}. (M >= 0 and ms[k] >= 0.) The default value of dims = {'l': G.size[0], 'q': [], 's': []}. G is a dense or sparse 'd' matrix of size (K,n), where K = ml + mq[0] + ... + mq[N-1] + ms[0]**2 + ... + ms[M-1]**2. Each column of G describes a vector v = ( v_0, v_1, ..., v_N, vec(v_{N+1}), ..., vec(v_{N+M}) ) in V = R^ml x R^mq[0] x ... x R^mq[N-1] x S^ms[0] x ... x S^ms[M-1] stored as a column vector [ v_0; v_1; ...; v_N; vec(v_{N+1}); ...; vec(v_{N+M}) ]. Here, if u is a symmetric matrix of order m, then vec(u) is the matrix u stored in column major order as a vector of length m**2. We use BLAS unpacked 'L' storage, i.e., the entries in vec(u) corresponding to the strictly upper triangular entries of u are not referenced. h is a dense 'd' matrix of size (K,1), representing a vector in V, in the same format as the columns of G. A is a dense or sparse 'd' matrix of size (p,n). The default value is a sparse 'd' matrix of size (0,n). b is a dense 'd' matrix of size (p,1). The default value is a dense 'd' matrix of size (0,1). It is assumed that rank(A) = p and rank([P; A; G]) = n. The other arguments are normally not needed. They make it possible to exploit certain types of structure, as described below. Output arguments. Returns a dictionary with keys 'status', 'x', 's', 'z', 'y', 'primal objective', 'dual objective', 'gap', 'relative gap', 'primal infeasibility', 'dual infeasibility', 'primal slack', 'dual slack', 'iterations'. The 'status' field has values 'optimal' or 'unknown'. 'iterations' is the number of iterations taken. If the status is 'optimal', 'x', 's', 'y', 'z' are an approximate solution of the primal and dual optimality conditions G*x + s = h, A*x = b P*x + G'*z + A'*y + q = 0 s >= 0, z >= 0 s'*z = 0. If the status is 'unknown', 'x', 'y', 's', 'z' are the last iterates before termination. These satisfy s > 0 and z > 0, but are not necessarily feasible. The values of the other fields are defined as follows. - 'primal objective': the primal objective (1/2)*x'*P*x + q'*x. - 'dual objective': the dual objective L(x,y,z) = (1/2)*x'*P*x + q'*x + z'*(G*x - h) + y'*(A*x-b). - 'gap': the duality gap s'*z. - 'relative gap': the relative gap, defined as gap / -primal objective if the primal objective is negative, gap / dual objective if the dual objective is positive, and None otherwise. - 'primal infeasibility': the residual in the primal constraints, defined as the maximum of the residual in the inequalities || G*x + s + h || / max(1, ||h||) and the residual in the equalities || A*x - b || / max(1, ||b||). - 'dual infeasibility': the residual in the dual constraints, defined as || P*x + G'*z + A'*y + q || / max(1, ||q||). - 'primal slack': the smallest primal slack, sup {t | s >= t*e }, where e = ( e_0, e_1, ..., e_N, e_{N+1}, ..., e_{M+N} ) is the identity vector in C. e_0 is an ml-vector of ones, e_k, k = 1,..., N, is the unit vector (1,0,...,0) of length mq[k], and e_k = vec(I) where I is the identity matrix of order ms[k]. - 'dual slack': the smallest dual slack, sup {t | z >= t*e }. If the exit status is 'optimal', then the primal and dual infeasibilities are guaranteed to be less than Termination with status 'unknown' indicates that the algorithm failed to find a solution that satisfies the specified tolerances. In some cases, the returned solution may be fairly accurate. If the primal and dual infeasibilities, the gap, and the relative gap are small, then x, y, s, z are close to optimal. Advanced usage. Three mechanisms are provided to express problem structure. 1. The user can provide a customized routine for solving linear equations (`KKT systems') [ P A' G' ] [ ux ] [ bx ] [ A 0 0 ] [ uy ] = [ by ]. [ G 0 -W'*W ] [ uz ] [ bz ] W is a scaling matrix, a block diagonal mapping W*u = ( W0*u_0, ..., W_{N+M}*u_{N+M} ) defined as follows. - For the 'l' block (W_0): W_0 = diag(d), with d a positive vector of length ml. - For the 'q' blocks (W_{k+1}, k = 0, ..., N-1): W_{k+1} = beta_k * ( 2 * v_k * v_k' - J ) where beta_k is a positive scalar, v_k is a vector in R^mq[k] with v_k[0] > 0 and v_k'*J*v_k = 1, and J = [1, 0; 0, -I]. - For the 's' blocks (W_{k+N}, k = 0, ..., M-1): W_k * u = vec(r_k' * mat(u) * r_k) where r_k is a nonsingular matrix of order ms[k], and mat(x) is the inverse of the vec operation. The optional argument kktsolver is a Python function that will be called as g = kktsolver(W). W is a dictionary that contains the parameters of the scaling: - W['d'] is a positive 'd' matrix of size (ml,1). - W['di'] is a positive 'd' matrix with the elementwise inverse of W['d']. - W['beta'] is a list [ beta_0, ..., beta_{N-1} ] - W['v'] is a list [ v_0, ..., v_{N-1} ] - W['r'] is a list [ r_0, ..., r_{M-1} ] - W['rti'] is a list [ rti_0, ..., rti_{M-1} ], with rti_k the inverse of the transpose of r_k. The call g = kktsolver(W) should return a function g that solves the KKT system by g(x, y, z). On entry, x, y, z contain the righthand side bx, by, bz. On exit, they contain the solution, with uz scaled, the argument z contains W*uz. In other words, on exit x, y, z are the solution of [ P A' G'*W^{-1} ] [ ux ] [ bx ] [ A 0 0 ] [ uy ] = [ by ]. [ G 0 -W' ] [ uz ] [ bz ] 2. The linear operators P*u, G*u and A*u can be specified by providing Python functions instead of matrices. This can only be done in combination with 1. above, i.e., it requires the kktsolver argument. If P is a function, the call P(u, v, alpha, beta) should evaluate the matrix-vectors product v := alpha * P * u + beta * v. The arguments u and v are required. The other arguments have default values alpha = 1.0, beta = 0.0. If G is a function, the call G(u, v, alpha, beta, trans) should evaluate the matrix-vector products v := alpha * G * u + beta * v if trans is 'N' v := alpha * G' * u + beta * v if trans is 'T'. The arguments u and v are required. The other arguments have default values alpha = 1.0, beta = 0.0, trans = 'N'. If A is a function, the call A(u, v, alpha, beta, trans) should evaluate the matrix-vectors products v := alpha * A * u + beta * v if trans is 'N' v := alpha * A' * u + beta * v if trans is 'T'. The arguments u and v are required. The other arguments have default values alpha = 1.0, beta = 0.0, trans = 'N'. If X is the vector space of primal variables x, then: If this option is used, the argument q must be in the same format as x, the argument P must be a Python function, the arguments A and G must be Python functions or None, and the argument kktsolver is required. If Y is the vector space of primal variables y: If this option is used, the argument b must be in the same format as y, the argument A must be a Python function or None, and the argument kktsolver is required. """ from cvxopt import base, blas, misc from cvxopt.base import matrix, spmatrix dims = None kktsolver = 'chol2' # Argument error checking depends on level of customization. customkkt = not isinstance(kktsolver, str) matrixP = isinstance(P, (matrix, spmatrix)) matrixG = isinstance(G, (matrix, spmatrix)) matrixA = isinstance(A, (matrix, spmatrix)) if (not matrixP or (not matrixG and G is not None) or (not matrixA and A is not None)) and not customkkt: raise ValueError("use of function valued P, G, A requires a " "user-provided kktsolver") if False and (matrixA or not customkkt): raise ValueError("use of non vector type for y requires " "function valued A and user-provided kktsolver") if (not isinstance(q, matrix) or q.typecode != 'd' or q.size[1] != 1): raise TypeError("'q' must be a 'd' matrix with one column") if matrixP: if P.typecode != 'd' or P.size != (q.size[0], q.size[0]): raise TypeError("'P' must be a 'd' matrix of size (%d, %d)" % (q.size[0], q.size[0])) def fP(x, y, alpha=1.0, beta=0.0): base.symv(P, x, y, alpha=alpha, beta=beta) else: fP = P if h is None: h = matrix(0.0, (0, 1)) if not isinstance(h, matrix) or h.typecode != 'd' or h.size[1] != 1: raise TypeError("'h' must be a 'd' matrix with one column") if not dims: dims = {'l': h.size[0], 'q': [], 's': []} if not isinstance(dims['l'], (int, long)) or dims['l'] < 0: raise TypeError("'dims['l']' must be a nonnegative integer") if [k for k in dims['q'] if not isinstance(k, (int, long)) or k < 1]: raise TypeError("'dims['q']' must be a list of positive integers") if [k for k in dims['s'] if not isinstance(k, (int, long)) or k < 0]: raise TypeError("'dims['s']' must be a list of nonnegative " "integers") if dims['q'] or dims['s']: refinement = 1 else: refinement = 0 cdim = dims['l'] + sum(dims['q']) + sum([k ** 2 for k in dims['s']]) if h.size[0] != cdim: raise TypeError("'h' must be a 'd' matrix of size (%d,1)" % cdim) # Data for kth 'q' constraint are found in rows indq[k]:indq[k+1] of G. indq = [dims['l']] for k in dims['q']: indq = indq + [indq[-1] + k] # Data for kth 's' constraint are found in rows inds[k]:inds[k+1] of G. inds = [indq[-1]] for k in dims['s']: inds = inds + [inds[-1] + k ** 2] if G is None: G = spmatrix([], [], [], (0, q.size[0])) matrixG = True if matrixG: if G.typecode != 'd' or G.size != (cdim, q.size[0]): raise TypeError("'G' must be a 'd' matrix of size (%d, %d)" % (cdim, q.size[0])) def fG(x, y, trans='N', alpha=1.0, beta=0.0): misc.sgemv(G, x, y, dims, trans=trans, alpha=alpha, beta=beta) else: fG = G if A is None: A = spmatrix([], [], [], (0, q.size[0])) matrixA = True if matrixA: if A.typecode != 'd' or A.size[1] != q.size[0]: raise TypeError("'A' must be a 'd' matrix with %d columns" % q.size[0]) def fA(x, y, trans='N', alpha=1.0, beta=0.0): base.gemv(A, x, y, trans=trans, alpha=alpha, beta=beta) else: fA = A if b is None: b = matrix(0.0, (0, 1)) if not isinstance(b, matrix) or b.typecode != 'd' or b.size[1] != 1: raise TypeError("'b' must be a 'd' matrix with one column") if matrixA and b.size[0] != A.size[0]: raise TypeError("'b' must have length %d" % A.size[0]) ws3, wz3 = matrix(0.0, (cdim, 1)), matrix(0.0, (cdim, 1)) def res(ux, uy, uz, us, vx, vy, vz, vs, W, lmbda): # Evaluates residual in Newton equations: # # [ vx ] [ vx ] [ 0 ] [ P A' G' ] [ ux ] # [ vy ] := [ vy ] - [ 0 ] - [ A 0 0 ] * [ uy ] # [ vz ] [ vz ] [ W'*us ] [ G 0 0 ] [ W^{-1}*uz ] # # vs := vs - lmbda o (uz + us). # vx := vx - P*ux - A'*uy - G'*W^{-1}*uz fP(ux, vx, alpha=-1.0, beta=1.0) fA(uy, vx, alpha=-1.0, beta=1.0, trans='T') blas.copy(uz, wz3) misc.scale(wz3, W, inverse='I') fG(wz3, vx, alpha=-1.0, beta=1.0, trans='T') # vy := vy - A*ux fA(ux, vy, alpha=-1.0, beta=1.0) # vz := vz - G*ux - W'*us fG(ux, vz, alpha=-1.0, beta=1.0) blas.copy(us, ws3) misc.scale(ws3, W, trans='T') blas.axpy(ws3, vz, alpha=-1.0) # vs := vs - lmbda o (uz + us) blas.copy(us, ws3) blas.axpy(uz, ws3) misc.sprod(ws3, lmbda, dims, diag='D') blas.axpy(ws3, vs, alpha=-1.0) # kktsolver(W) returns a routine for solving # # [ P A' G'*W^{-1} ] [ ux ] [ bx ] # [ A 0 0 ] [ uy ] = [ by ]. # [ G 0 -W' ] [ uz ] [ bz ] factor = kkt_chol2(G, dims, A) def kktsolver(W): return factor(W, P) resx0 = max(1.0, math.sqrt(np.dot(q.T, q))) resy0 = max(1.0, math.sqrt(np.dot(b.T, b))) resz0 = max(1.0, misc.snrm2(h, dims)) if cdim == 0: return solve_only_equalities_qp(kktsolver, fP, fA, resx0, resy0, dims) x, y = matrix(q), matrix(b) s, z = matrix(0.0, (cdim, 1)), matrix(0.0, (cdim, 1)) # Factor # # [ P A' G' ] # [ A 0 0 ]. # [ G 0 -I ] W = {} W['d'] = matrix(1.0, (dims['l'], 1)) W['di'] = matrix(1.0, (dims['l'], 1)) W['v'] = [matrix(0.0, (m, 1)) for m in dims['q']] W['beta'] = len(dims['q']) * [1.0] for v in W['v']: v[0] = 1.0 W['r'] = [matrix(0.0, (m, m)) for m in dims['s']] W['rti'] = [matrix(0.0, (m, m)) for m in dims['s']] for r in W['r']: r[::r.size[0] + 1] = 1.0 for rti in W['rti']: rti[::rti.size[0] + 1] = 1.0 try: f = kktsolver(W) except ArithmeticError: raise ValueError("Rank(A) < p or Rank([P; A; G]) < n") # Solve # # [ P A' G' ] [ x ] [ -q ] # [ A 0 0 ] * [ y ] = [ b ]. # [ G 0 -I ] [ z ] [ h ] x = matrix(np.copy(q)) x *= -1.0 y = matrix(np.copy(b)) z = matrix(np.copy(h)) try: f(x, y, z) except ArithmeticError: raise ValueError("Rank(A) < p or Rank([P; G; A]) < n") s = matrix(np.copy(z)) blas.scal(-1.0, s) nrms = misc.snrm2(s, dims) ts = misc.max_step(s, dims) if ts >= -1e-8 * max(nrms, 1.0): a = 1.0 + ts s[:dims['l']] += a s[indq[:-1]] += a ind = dims['l'] + sum(dims['q']) for m in dims['s']: s[ind: ind + m * m: m + 1] += a ind += m ** 2 nrmz = misc.snrm2(z, dims) tz = misc.max_step(z, dims) if tz >= -1e-8 * max(nrmz, 1.0): a = 1.0 + tz z[:dims['l']] += a z[indq[:-1]] += a ind = dims['l'] + sum(dims['q']) for m in dims['s']: z[ind: ind + m * m: m + 1] += a ind += m ** 2 rx, ry, rz = matrix(q), matrix(b), matrix(0.0, (cdim, 1)) dx, dy = matrix(x), matrix(y) dz, ds = matrix(0.0, (cdim, 1)), matrix(0.0, (cdim, 1)) lmbda = matrix(0.0, (dims['l'] + sum(dims['q']) + sum(dims['s']), 1)) lmbdasq = matrix(0.0, (dims['l'] + sum(dims['q']) + sum(dims['s']), 1)) sigs = matrix(0.0, (sum(dims['s']), 1)) sigz = matrix(0.0, (sum(dims['s']), 1)) if show_progress: print("% 10s% 12s% 10s% 8s% 7s" % ("pcost", "dcost", "gap", "pres", "dres")) gap = misc.sdot(s, z, dims) for iters in range(MAXITERS + 1): # f0 = (1/2)*x'*P*x + q'*x + r and rx = P*x + q + A'*y + G'*z. rx = matrix(np.copy(q)) fP(x, rx, beta=1.0) f0 = 0.5 * (np.dot(x.T, rx) + np.dot(x.T, q)) fA(y, rx, beta=1.0, trans='T') fG(z, rx, beta=1.0, trans='T') resx = math.sqrt(np.dot(rx.T, rx)) # ry = A*x - b ry = matrix(np.copy(b)) fA(x, ry, alpha=1.0, beta=-1.0) resy = math.sqrt(np.dot(ry.T, ry)) # rz = s + G*x - h rz = matrix(np.copy(s)) blas.axpy(h, rz, alpha=-1.0) fG(x, rz, beta=1.0) resz = misc.snrm2(rz, dims) # Statistics for stopping criteria. # pcost = (1/2)*x'*P*x + q'*x # dcost = (1/2)*x'*P*x + q'*x + y'*(A*x-b) + z'*(G*x-h) # = (1/2)*x'*P*x + q'*x + y'*(A*x-b) + z'*(G*x-h+s) - z'*s # = (1/2)*x'*P*x + q'*x + y'*ry + z'*rz - gap pcost = f0 dcost = f0 + np.dot(y.T, ry) + misc.sdot(z, rz, dims) - gap if pcost < 0.0: relgap = gap / -pcost elif dcost > 0.0: relgap = gap / dcost else: relgap = None pres = max(resy / resy0, resz / resz0) dres = resx / resx0 if show_progress: print("%2d: % 8.4e % 8.4e % 4.0e% 7.0e% 7.0e" % (iters, pcost, dcost, gap, pres, dres)) if (pres <= FEASTOL and dres <= FEASTOL and (gap <= ABSTOL or (relgap is not None and relgap <= RELTOL))) or \ iters == MAXITERS: ind = dims['l'] + sum(dims['q']) for m in dims['s']: misc.symm(s, m, ind) misc.symm(z, m, ind) ind += m ** 2 ts = misc.max_step(s, dims) tz = misc.max_step(z, dims) if iters == MAXITERS: if show_progress: print("Terminated (maximum number of iterations " "reached).") status = 'unknown' else: if show_progress: print("Optimal solution found.") status = 'optimal' return {'x': x, 'y': y, 's': s, 'z': z, 'status': status, 'gap': gap, 'relative gap': relgap, 'primal objective': pcost, 'dual objective': dcost, 'primal infeasibility': pres, 'dual infeasibility': dres, 'primal slack': -ts, 'dual slack': -tz, 'iterations': iters} # Compute initial scaling W and scaled iterates: # # W * z = W^{-T} * s = lambda. # # lmbdasq = lambda o lambda. if iters == 0: W = misc.compute_scaling(s, z, lmbda, dims) misc.ssqr(lmbdasq, lmbda, dims) # f3(x, y, z) solves # # [ P A' G' ] [ ux ] [ bx ] # [ A 0 0 ] [ uy ] = [ by ]. # [ G 0 -W'*W ] [ W^{-1}*uz ] [ bz ] # # On entry, x, y, z containg bx, by, bz. # On exit, they contain ux, uy, uz. try: f3 = kktsolver(W) except ArithmeticError: if iters == 0: raise ValueError("Rank(A) < p or Rank([P; A; G]) < n") else: ind = dims['l'] + sum(dims['q']) for m in dims['s']: misc.symm(s, m, ind) misc.symm(z, m, ind) ind += m ** 2 ts = misc.max_step(s, dims) tz = misc.max_step(z, dims) if show_progress: print("Terminated (singular KKT matrix).") return {'x': x, 'y': y, 's': s, 'z': z, 'status': 'unknown', 'gap': gap, 'relative gap': relgap, 'primal objective': pcost, 'dual objective': dcost, 'primal infeasibility': pres, 'dual infeasibility': dres, 'primal slack': -ts, 'dual slack': -tz, 'iterations': iters} # f4_no_ir(x, y, z, s) solves # # [ 0 ] [ P A' G' ] [ ux ] [ bx ] # [ 0 ] + [ A 0 0 ] * [ uy ] = [ by ] # [ W'*us ] [ G 0 0 ] [ W^{-1}*uz ] [ bz ] # # lmbda o (uz + us) = bs. # # On entry, x, y, z, s contain bx, by, bz, bs. # On exit, they contain ux, uy, uz, us. def f4_no_ir(x, y, z, s): # Solve # # [ P A' G' ] [ ux ] [ bx ] # [ A 0 0 ] [ uy ] = [ by ] # [ G 0 -W'*W ] [ W^{-1}*uz ] [ bz - W'*(lmbda o\ bs) ] # # us = lmbda o\ bs - uz. # # On entry, x, y, z, s contains bx, by, bz, bs. # On exit they contain x, y, z, s. # s := lmbda o\ s # = lmbda o\ bs misc.sinv(s, lmbda, dims) # z := z - W'*s # = bz - W'*(lambda o\ bs) ws3 = matrix(np.copy(s)) misc.scale(ws3, W, trans='T') blas.axpy(ws3, z, alpha=-1.0) # Solve for ux, uy, uz f3(x, y, z) # s := s - z # = lambda o\ bs - uz. blas.axpy(z, s, alpha=-1.0) # f4(x, y, z, s) solves the same system as f4_no_ir, but applies # iterative refinement. if iters == 0: if refinement: wx, wy = matrix(q), matrix(b) wz, ws = matrix(0.0, (cdim, 1)), matrix(0.0, (cdim, 1)) if refinement: wx2, wy2 = matrix(q), matrix(b) wz2, ws2 = matrix(0.0, (cdim, 1)), matrix(0.0, (cdim, 1)) def f4(x, y, z, s): if refinement: wx = matrix(np.copy(x)) wy = matrix(np.copy(y)) wz = matrix(np.copy(z)) ws = matrix(np.copy(s)) f4_no_ir(x, y, z, s) for i in range(refinement): wx2 = matrix(np.copy(wx)) wy2 = matrix(np.copy(wy)) wz2 = matrix(np.copy(wz)) ws2 = matrix(np.copy(ws)) res(x, y, z, s, wx2, wy2, wz2, ws2, W, lmbda) f4_no_ir(wx2, wy2, wz2, ws2) y += wx2 y += wy2 blas.axpy(wz2, z) blas.axpy(ws2, s) mu = gap / (dims['l'] + len(dims['q']) + sum(dims['s'])) sigma, eta = 0.0, 0.0 for i in [0, 1]: # Solve # # [ 0 ] [ P A' G' ] [ dx ] # [ 0 ] + [ A 0 0 ] * [ dy ] = -(1 - eta) * r # [ W'*ds ] [ G 0 0 ] [ W^{-1}*dz ] # # lmbda o (dz + ds) = -lmbda o lmbda + sigma*mu*e (i=0) # lmbda o (dz + ds) = -lmbda o lmbda - dsa o dza # + sigma*mu*e (i=1) where dsa, dza # are the solution for i=0. # ds = -lmbdasq + sigma * mu * e (if i is 0) # = -lmbdasq - dsa o dza + sigma * mu * e (if i is 1), # where ds, dz are solution for i is 0. blas.scal(0.0, ds) if i == 1: blas.axpy(ws3, ds, alpha=-1.0) blas.axpy(lmbdasq, ds, n=dims['l'] + sum(dims['q']), alpha=-1.0) ds[:dims['l']] += sigma * mu ind = dims['l'] for m in dims['q']: ds[ind] += sigma * mu ind += m ind2 = ind for m in dims['s']: blas.axpy(lmbdasq, ds, n=m, offsetx=ind2, offsety=ind, incy=m + 1, alpha=-1.0) ds[ind: ind + m * m: m + 1] += sigma * mu ind += m * m ind2 += m # (dx, dy, dz) := -(1 - eta) * (rx, ry, rz) dx *= 0.0 dx += (-1.0 + eta) * rx dy *= 0.0 dy += (-1.0 + eta) * ry blas.scal(0.0, dz) blas.axpy(rz, dz, alpha=-1.0 + eta) try: f4(dx, dy, dz, ds) except ArithmeticError: if iters == 0: raise ValueError("Rank(A) < p or Rank([P; A; G]) < n") else: ind = dims['l'] + sum(dims['q']) for m in dims['s']: misc.symm(s, m, ind) misc.symm(z, m, ind) ind += m ** 2 ts = misc.max_step(s, dims) tz = misc.max_step(z, dims) if show_progress: print("Terminated (singular KKT matrix).") return {'x': x, 'y': y, 's': s, 'z': z, 'status': 'unknown', 'gap': gap, 'relative gap': relgap, 'primal objective': pcost, 'dual objective': dcost, 'primal infeasibility': pres, 'dual infeasibility': dres, 'primal slack': -ts, 'dual slack': -tz, 'iterations': iters} dsdz = misc.sdot(ds, dz, dims) # Save ds o dz for Mehrotra correction if i == 0: ws3 = matrix(np.copy(ds)) misc.sprod(ws3, dz, dims) # Maximum steps to boundary. # # If i is 1, also compute eigenvalue decomposition of the # 's' blocks in ds,dz. The eigenvectors Qs, Qz are stored in # dsk, dzk. The eigenvalues are stored in sigs, sigz. misc.scale2(lmbda, ds, dims) misc.scale2(lmbda, dz, dims) if i == 0: ts = misc.max_step(ds, dims) tz = misc.max_step(dz, dims) else: ts = misc.max_step(ds, dims, sigma=sigs) tz = misc.max_step(dz, dims, sigma=sigz) t = max([0.0, ts, tz]) if t == 0: step = 1.0 else: if i == 0: step = min(1.0, 1.0 / t) else: step = min(1.0, STEP / t) if i == 0: sigma = min(1.0, max(0.0, 1.0 - step + dsdz / gap * step ** 2)) ** EXPON eta = 0.0 x += step * dx y += step * dy # We will now replace the 'l' and 'q' blocks of ds and dz with # the updated iterates in the current scaling. # We also replace the 's' blocks of ds and dz with the factors # Ls, Lz in a factorization Ls*Ls', Lz*Lz' of the updated variables # in the current scaling. # ds := e + step*ds for nonlinear, 'l' and 'q' blocks. # dz := e + step*dz for nonlinear, 'l' and 'q' blocks. blas.scal(step, ds, n=dims['l'] + sum(dims['q'])) blas.scal(step, dz, n=dims['l'] + sum(dims['q'])) ind = dims['l'] ds[:ind] += 1.0 dz[:ind] += 1.0 for m in dims['q']: ds[ind] += 1.0 dz[ind] += 1.0 ind += m # ds := H(lambda)^{-1/2} * ds and dz := H(lambda)^{-1/2} * dz. # # This replaced the 'l' and 'q' components of ds and dz with the # updated iterates in the current scaling. # The 's' components of ds and dz are replaced with # # diag(lmbda_k)^{1/2} * Qs * diag(lmbda_k)^{1/2} # diag(lmbda_k)^{1/2} * Qz * diag(lmbda_k)^{1/2} # misc.scale2(lmbda, ds, dims, inverse='I') misc.scale2(lmbda, dz, dims, inverse='I') # sigs := ( e + step*sigs ) ./ lambda for 's' blocks. # sigz := ( e + step*sigz ) ./ lmabda for 's' blocks. blas.scal(step, sigs) blas.scal(step, sigz) sigs += 1.0 sigz += 1.0 blas.tbsv(lmbda, sigs, n=sum(dims['s']), k=0, ldA=1, offsetA=dims['l'] + sum(dims['q'])) blas.tbsv(lmbda, sigz, n=sum(dims['s']), k=0, ldA=1, offsetA=dims['l'] + sum(dims['q'])) # dsk := Ls = dsk * sqrt(sigs). # dzk := Lz = dzk * sqrt(sigz). ind2, ind3 = dims['l'] + sum(dims['q']), 0 for k in range(len(dims['s'])): m = dims['s'][k] for i in range(m): blas.scal(math.sqrt(sigs[ind3 + i]), ds, offset=ind2 + m * i, n=m) blas.scal(math.sqrt(sigz[ind3 + i]), dz, offset=ind2 + m * i, n=m) ind2 += m * m ind3 += m # Update lambda and scaling. misc.update_scaling(W, lmbda, ds, dz) # Unscale s, z (unscaled variables are used only to compute # feasibility residuals). blas.copy(lmbda, s, n=dims['l'] + sum(dims['q'])) ind = dims['l'] + sum(dims['q']) ind2 = ind for m in dims['s']: blas.scal(0.0, s, offset=ind2) blas.copy(lmbda, s, offsetx=ind, offsety=ind2, n=m, incy=m + 1) ind += m ind2 += m * m misc.scale(s, W, trans='T') blas.copy(lmbda, z, n=dims['l'] + sum(dims['q'])) ind = dims['l'] + sum(dims['q']) ind2 = ind for m in dims['s']: blas.scal(0.0, z, offset=ind2) blas.copy(lmbda, z, offsetx=ind, offsety=ind2, n=m, incy=m + 1) ind += m ind2 += m * m misc.scale(z, W, inverse='I') gap = blas.dot(lmbda, lmbda)
def f(x, y, z): """ 1. Compute uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ) ) 2. For k = 1, ..., m: uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk) 3. Solve for uz d .* uz = ( ux - mat(bz) ) ./ d. Return ux, uy, d .* uz. """ ### utime0, stime0 = cputime() ### # xk := Dk^2 * xk + zk # = Dk^2 * bxk + bzk blas.tbmv(dsq, x, n=N * m, k=0, ldA=1) blas.axpy(z, x) # y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk # = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk # y := -y + x*ones blas.gemv(x, ones, y, alpha=1.0, beta=-1.0) # wnm = X' * x (wnm interpreted as an n x m matrix) blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n) # wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix) for k in range(m): lapack.potrs(H[k], wnm, offsetB=k * n) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=n * k) # wN = Dk^2 * wN blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1) # y := y - wN blas.axpy(wN, y, -1.0) # y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y # # Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T. # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # wnm = Y' * y (interpreted as an Nm vector) # = [ L1^-1 * X' * D1^2 * D^-1 * y; # L2^-1 * X' * D2^2 * D^-1 * y; # ... # Lm^-1 * X' * Dm^2 * D^-1 * y ] for k in range(m): # wN = D^-1 * Dk^2 * y blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # wnm[:,k] = X' * wN blas.gemv(X, wN, wnm, trans='T', offsety=k * n) # wnm[:,k] = Lk^-1 * wnm[:,k] blas.trsv(H[k], wnm, offsetx=k * n) # wnm := S^-1 * wnm (an mn-vector) lapack.potrs(S, wnm) # y := y + Y * wnm # = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T] # * wnm for k in range(m): # wnm[:,k] = Lk^-T * wnm[:,k] blas.trsv(H[k], wnm, trans='T', offsetx=k * n) # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = D^-1 * Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # y += wN blas.axpy(wN, y) # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # For k = 1, ..., m: # # xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk) # x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix) for k in range(m): blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.axpy(wN, x, -1.0, offsety=k * N) # wnm = X' * x (as an n x m matrix) blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n) # wnm[:,k] = Hk^-1 * wnm[:,k] for k in range(m): lapack.potrs(H[k], wnm, offsetB=n * k) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) # x[:,k] := x[:,k] - wN blas.axpy(wN, x, -1.0, n=N, offsety=k * N) # z := ( x - z ) ./ d blas.axpy(x, z, -1.0) blas.scal(-1.0, z) blas.tbsv(d, z, n=N * m, k=0, ldA=1) ### utime, stime = cputime() print("Solve: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0))
def kkt(W): """ KKT solver for X*X' * ux + uy * 1_m' + mat(uz) = bx ux * 1_m = by ux - d.^2 .* mat(uz) = mat(bz). ux and bx are N x m matrices. uy and by are N-vectors. uz and bz are N*m-vectors. mat(uz) is the N x m matrix that satisfies mat(uz)[:] = uz. d = mat(W['d']) a positive N x m matrix. If we eliminate uz from the last equation using mat(uz) = (ux - mat(bz)) ./ d.^2 we get two equations in ux, uy: X*X' * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2 ux * 1_m = by. From the 1st equation, uxk = (X*X' + Dk^-2)^-1 * (-uy + bxk + Dk^-2 * bzk) = Dk * (I + Xk*Xk')^-1 * Dk * (-uy + bxk + Dk^-2 * bzk) for k = 1, ..., m, where Dk = diag(d[:,k]), Xk = Dk * X, uxk is column k of ux, and bzk is column k of mat(bz). We use the matrix inversion lemma ( I + Xk * Xk' )^-1 = I - Xk * (I + Xk' * Xk)^-1 * Xk' = I - Xk * Hk^-1 * Xk' = I - Xk * Lk^-T * Lk^-1 * Xk' where Hk = I + Xk' * Xk = Lk * Lk' to write this as uxk = Dk * (I - Xk * Hk^-1 * Xk') * Dk * (-uy + bxk + Dk^-2 * bzk) = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk). Substituting this in the second equation gives an equation for uy: sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * uy = -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 * bzk ), i.e., with D = (sum_k Dk^2)^1/2, Yk = D^-1 * Dk^2 * X * Lk^-T, D * ( I - sum_k Yk * Yk' ) * D * uy = -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ). Another application of the matrix inversion lemma gives uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k ( Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * ( bxk + Dk^-2 *bzk ) ) with S = I - Y' * Y, Y = [ Y1 ... Ym ]. Summary: 1. Compute uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ) ) 2. For k = 1, ..., m: uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk) 3. Solve for uz d .* uz = ( ux - mat(bz) ) ./ d. Return ux, uy, d .* uz. """ ### utime0, stime0 = cputime() ### d = matrix(W['d'], (N, m)) dsq = matrix(W['d']**2, (N, m)) # Factor the matrices # # H[k] = I + Xk' * Xk # = I + X' * Dk^2 * X. # # Dk = diag(d[:,k]). for k in range(m): # H[k] = I blas.scal(0.0, H[k]) H[k][::n + 1] = 1.0 # Xs = Dk * X # = diag(d[:,k]]) * X blas.copy(X, Xs) for j in range(n): blas.tbmv(d, Xs, n=N, k=0, ldA=1, offsetA=k * N, offsetx=j * N) # H[k] := H[k] + Xs' * Xs # = I + Xk' * Xk blas.syrk(Xs, H[k], trans='T', beta=1.0) # Factorization H[k] = Lk * Lk' lapack.potrf(H[k]) ### utime, stime = cputime() print("Factor Hk's: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # diag(D) = ( sum_k d[:,k]**2 ) ** 1/2 # = ( sum_k Dk^2) ** 1/2. blas.gemv(dsq, ones, D) D[:] = sqrt(D) ### # utime, stime = cputime() # print("Compute D: utime = %.2f, stime = %.2f" \ # %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # S = I - Y'* Y is an m x m block matrix. # The i,j block of Y' * Y is # # Yi' * Yj = Li^-1 * X' * Di^2 * D^-1 * Dj^2 * X * Lj^-T. # # We compute only the lower triangular blocks in Y'*Y. blas.scal(0.0, S) for i in range(m): for j in range(i + 1): # Xs = Di * Dj * D^-1 * X blas.copy(X, Xs) blas.copy(d, wN, n=N, offsetx=i * N) blas.tbmv(d, wN, n=N, k=0, ldA=1, offsetA=j * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) for k in range(n): blas.tbmv(wN, Xs, n=N, k=0, ldA=1, offsetx=k * N) # block i, j of S is Xs' * Xs (as nonsymmetric matrix so we # get the correct multiple after scaling with Li, Lj) blas.gemm(Xs, Xs, S, transA='T', ldC=m * n, offsetC=(j * n) * m * n + i * n) ### utime, stime = cputime() print("Form S: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### for i in range(m): # multiply block row i of S on the left with Li^-1 blas.trsm(H[i], S, m=n, n=(i + 1) * n, ldB=m * n, offsetB=i * n) # multiply block column i of S on the right with Li^-T blas.trsm(H[i], S, side='R', transA='T', m=(m - i) * n, n=n, ldB=m * n, offsetB=i * n * (m * n + 1)) blas.scal(-1.0, S) S[::(m * n + 1)] += 1.0 ### utime, stime = cputime() print("Form S (2): utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # S = L*L' lapack.potrf(S) ### utime, stime = cputime() print("Factor S: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### def f(x, y, z): """ 1. Compute uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ) ) 2. For k = 1, ..., m: uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk) 3. Solve for uz d .* uz = ( ux - mat(bz) ) ./ d. Return ux, uy, d .* uz. """ ### utime0, stime0 = cputime() ### # xk := Dk^2 * xk + zk # = Dk^2 * bxk + bzk blas.tbmv(dsq, x, n=N * m, k=0, ldA=1) blas.axpy(z, x) # y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk # = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk # y := -y + x*ones blas.gemv(x, ones, y, alpha=1.0, beta=-1.0) # wnm = X' * x (wnm interpreted as an n x m matrix) blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n) # wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix) for k in range(m): lapack.potrs(H[k], wnm, offsetB=k * n) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=n * k) # wN = Dk^2 * wN blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1) # y := y - wN blas.axpy(wN, y, -1.0) # y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y # # Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T. # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # wnm = Y' * y (interpreted as an Nm vector) # = [ L1^-1 * X' * D1^2 * D^-1 * y; # L2^-1 * X' * D2^2 * D^-1 * y; # ... # Lm^-1 * X' * Dm^2 * D^-1 * y ] for k in range(m): # wN = D^-1 * Dk^2 * y blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # wnm[:,k] = X' * wN blas.gemv(X, wN, wnm, trans='T', offsety=k * n) # wnm[:,k] = Lk^-1 * wnm[:,k] blas.trsv(H[k], wnm, offsetx=k * n) # wnm := S^-1 * wnm (an mn-vector) lapack.potrs(S, wnm) # y := y + Y * wnm # = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T] # * wnm for k in range(m): # wnm[:,k] = Lk^-T * wnm[:,k] blas.trsv(H[k], wnm, trans='T', offsetx=k * n) # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = D^-1 * Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # y += wN blas.axpy(wN, y) # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # For k = 1, ..., m: # # xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk) # x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix) for k in range(m): blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.axpy(wN, x, -1.0, offsety=k * N) # wnm = X' * x (as an n x m matrix) blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n) # wnm[:,k] = Hk^-1 * wnm[:,k] for k in range(m): lapack.potrs(H[k], wnm, offsetB=n * k) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) # x[:,k] := x[:,k] - wN blas.axpy(wN, x, -1.0, n=N, offsety=k * N) # z := ( x - z ) ./ d blas.axpy(x, z, -1.0) blas.scal(-1.0, z) blas.tbsv(d, z, n=N * m, k=0, ldA=1) ### utime, stime = cputime() print("Solve: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) ### return f
def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans = 'T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans = 'T', offsetx = msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans = 'T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha = -1.0, offsetx = msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n = msq, k = 0, ldA = 1) blas.axpy(z, x1, n = msq) blas.tbsv(sqrtG, x1, n = msq, k = 0, ldA = 1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m = mpckd, trans = 'T', beta = 1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m = mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n = msq, k = 0, ldA = 1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n = msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n = msq, k = 0, ldA = 1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n = msq, offsety = msq) blas.scal(-1.0, z, offset = msq) __cngrnc(U, z, offsetx = msq) __cngrnc(W['r'][1], z, trans = 'T', offsetx = msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n = msq, k = 0, ldA = 1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n = msq) blas.scal(-1.0, z, n = msq) blas.axpy(x[1], z, -1.0, n = msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans = 'T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1])
def F(W): """ Generate a solver for A'(uz0) = bx[0] -uz0 - uz1 = bx[1] A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0 - ux[1] - r1*r1' * uz1 * r1*r1' = bz1. uz0, uz1, bz0, bz1 are symmetric m x m-matrices. ux[0], bx[0] are n-vectors. ux[1], bx[1] are symmetric m x m-matrices. We first calculate a congruence that diagonalizes r0*r0' and r1*r1': U' * r0 * r0' * U = I, U' * r1 * r1' * U = S. We then make a change of variables usx[0] = ux[0], usx[1] = U' * ux[1] * U usz0 = U^-1 * uz0 * U^-T usz1 = U^-1 * uz1 * U^-T and define As() = U' * A() * U' bsx[1] = U^-1 * bx[1] * U^-T bsz0 = U' * bz0 * U bsz1 = U' * bz1 * U. This gives As'(usz0) = bx[0] -usz0 - usz1 = bsx[1] As(usx[0]) - usx[1] - usz0 = bsz0 -usx[1] - S * usz1 * S = bsz1. 1. Eliminate usz0, usz1 using equations 3 and 4, usz0 = As(usx[0]) - usx[1] - bsz0 usz1 = -S^-1 * (usx[1] + bsz1) * S^-1. This gives two equations in usx[0] an usx[1]. As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0) -As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1 = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1. 2. Eliminate usx[1] using equation 2: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j], usx[1] = ( S * As(usx[0]) * S ) ./ Gamma + ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma. This gives an equation in usx[0]. As'( As(usx[0]) ./ Gamma ) = bx0 + As'(bsz0) + As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma ) = bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ). """ # Calculate U s.t. # # U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s). # Cholesky factorization r0 * r0' = L * L' blas.syrk(W['r'][0], L) lapack.potrf(L) # SVD L^-1 * r1 = U * diag(s) * V' blas.copy(W['r'][1], U) blas.trsm(L, U) lapack.gesvd(U, s, jobu = 'O') # s := s**2 s[:] = s**2 # Uti := U blas.copy(U, Uti) # U := L^-T * U blas.trsm(L, U, transA = 'T') # Uti := L * Uti = U^-T blas.trmm(L, Uti) # Us := U * diag(s)^-1 blas.copy(U, Us) for i in range(m): blas.tbsv(s, Us, n = m, k = 0, ldA = 1, incx = m, offsetx = i) # S is m x m with lower triangular entries s[i] * s[j] # sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j]) # Upper triangular entries are undefined but nonzero. blas.scal(0.0, S) blas.syrk(s, S) Gamma = 1.0 + S sqrtG = sqrt(Gamma) # Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n # = Asi ./ sqrt(Gamma) blas.copy(A, Asc) misc.scale(Asc, # only 'r' part of the dictionary is used {'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)), 'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)), 'v': [], 'beta': [], 'r': [ U ], 'rti': [ U ]}) for i in range(n): blas.tbsv(sqrtG, Asc, n = msq, k = 0, ldA = 1, offsetx = i*msq) # Convert columns of Asc to packed storage misc.pack2(Asc, {'l': 0, 'q': [], 's': [ m ]}) # Cholesky factorization of Asc' * Asc. H = matrix(0.0, (n, n)) blas.syrk(Asc, H, trans = 'T', k = mpckd) lapack.potrf(H) def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans = 'T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans = 'T', offsetx = msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans = 'T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha = -1.0, offsetx = msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n = msq, k = 0, ldA = 1) blas.axpy(z, x1, n = msq) blas.tbsv(sqrtG, x1, n = msq, k = 0, ldA = 1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m = mpckd, trans = 'T', beta = 1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m = mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n = msq, k = 0, ldA = 1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n = msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n = msq, k = 0, ldA = 1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n = msq, offsety = msq) blas.scal(-1.0, z, offset = msq) __cngrnc(U, z, offsetx = msq) __cngrnc(W['r'][1], z, trans = 'T', offsetx = msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n = msq, k = 0, ldA = 1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n = msq) blas.scal(-1.0, z, n = msq) blas.axpy(x[1], z, -1.0, n = msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans = 'T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1]) return solve
def F(W): # SVD R[j] = U[j] * diag(sig[j]) * Vt[j] lapack.gesvd(+W['r'][0], sv, jobu='A', jobvt='A', U=U, Vt=Vt) # Vt[j] := diag(sig[j])^-1 * Vt[j] for k in xrange(ns): blas.tbsv(sv, Vt, n=ns, k=0, ldA=1, offsetx=k * ns) # Gamma[j] is an ns[j] x ns[j] symmetric matrix # # (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2) # S = sig[j] * sig[j]' S = matrix(0.0, (ns, ns)) blas.syrk(sv, S) Gamma = div(S, sqrt(1.0 + rho * S**2)) symmetrize(Gamma, ns) # As represents the scaled mapping # # As(x) = A(u * (Gamma .* x) * u') # As'(y) = Gamma .* (u' * A'(y) * u) # # stored in a similar format as A, except that we use packed # storage for the columns of As[i][j]. if type(A) is spmatrix: blas.scal(0.0, As) try: As[VecAIndex] = +A['s'][VecAIndex] except: As[VecAIndex] = +A[VecAIndex] else: blas.copy(A, As) # As[i][j][:,k] = diag( diag(Gamma[j]))*As[i][j][:,k] # As[i][j][l,:] = Gamma[j][l,l]*As[i][j][l,:] for k in xrange(ms): cngrnc(U, As, trans='T', offsetx=k * (ns2)) blas.tbmv(Gamma, As, n=ns2, k=0, ldA=1, offsetx=k * (ns2)) misc.pack(As, Aspkd, {'l': 0, 'q': [], 's': [ns] * ms}) # H is an m times m block matrix with i, k block # # Hik = sum_j As[i,j]' * As[k,j] # # of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j] # are zero for all j H = matrix(0.0, (ms, ms)) blas.syrk(Aspkd, H, trans='T', beta=1.0, k=ns * (ns + 1) / 2) lapack.potrf(H) def solve(x, y, z): """ Returns solution of rho * ux + A'(uy) - r^-T * uz * r^-1 = bx A(ux) = by -ux - r * uz * r' = bz. On entry, x = bx, y = by, z = bz. On exit, x = ux, y = uy, z = uz. """ # bz is a copy of z in the format of x blas.copy(z, bz) blas.axpy(bz, x, alpha=rho) # x := Gamma .* (u' * x * u) # = Gamma .* (u' * (bx + rho * bz) * u) cngrnc(U, x, trans='T', offsetx=0) blas.tbmv(Gamma, x, n=ns2, k=0, ldA=1, offsetx=0) # y := y - As(x) # := by - As( Gamma .* u' * (bx + rho * bz) * u) #blas.copy(x,xp) #pack_ip(xp,n = ns,m=1,nl=nl) misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]}) blas.gemv(Aspkd, xp, y, trans = 'T',alpha = -1.0, beta = 1.0, \ m = ns*(ns+1)/2, n = ms,offsetx = 0) # y := -y - A(bz) # = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u) Af(bz, y, alpha=-1.0, beta=-1.0) # y := H^-1 * y # = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) ) # = uy blas.trsv(H, y) blas.trsv(H, y, trans='T') # bz = Vt' * vz * Vt # = uz where # vz := Gamma .* ( As'(uy) - x ) # = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) ) # = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ). #blas.copy(x,xp) #pack_ip(xp,n=ns,m=1,nl=nl) misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]}) blas.scal(-1.0, xp) blas.gemv(Aspkd, y, xp, alpha=1.0, beta=1.0, m=ns * (ns + 1) / 2, n=ms, offsety=0) # bz[j] is xp unpacked and multiplied with Gamma misc.unpack(xp, bz, {'l': 0, 'q': [], 's': [ns]}) blas.tbmv(Gamma, bz, n=ns2, k=0, ldA=1, offsetx=0) # bz = Vt' * bz * Vt # = uz cngrnc(Vt, bz, trans='T', offsetx=0) symmetrize(bz, ns, offset=0) # x = -bz - r * uz * r' # z contains r.h.s. bz; copy to x blas.copy(z, x) blas.copy(bz, z) cngrnc(W['r'][0], bz, offsetx=0) blas.axpy(bz, x) blas.scal(-1.0, x) return solve
def F(W): for j in xrange(N): # SVD R[j] = U[j] * diag(sig[j]) * Vt[j] lapack.gesvd(+W['r'][j], sv[j], jobu='A', jobvt='A', U=U[j], Vt=Vt[j]) # Vt[j] := diag(sig[j])^-1 * Vt[j] for k in xrange(ns[j]): blas.tbsv(sv[j], Vt[j], n=ns[j], k=0, ldA=1, offsetx=k * ns[j]) # Gamma[j] is an ns[j] x ns[j] symmetric matrix # # (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2) # S = sig[j] * sig[j]' S = matrix(0.0, (ns[j], ns[j])) blas.syrk(sv[j], S) Gamma[j][:] = div(S, sqrt(1.0 + rho * S**2))[:] symmetrize(Gamma[j], ns[j]) # As represents the scaled mapping # # As(x) = A(u * (Gamma .* x) * u') # As'(y) = Gamma .* (u' * A'(y) * u) # # stored in a similar format as A, except that we use packed # storage for the columns of As[i][j]. for i in xrange(M): if (type(A[i][j]) is matrix) or (type(A[i][j]) is spmatrix): # As[i][j][:,k] = vec( # (U[j]' * mat( A[i][j][:,k] ) * U[j]) .* Gamma[j]) copy(A[i][j], As[i][j]) As[i][j] = matrix(As[i][j]) for k in xrange(ms[i]): cngrnc(U[j], As[i][j], trans='T', offsetx=k * (ns[j]**2), n=ns[j]) blas.tbmv(Gamma[j], As[i][j], n=ns[j]**2, k=0, ldA=1, offsetx=k * (ns[j]**2)) # pack As[i][j] in place #pack_ip(As[i][j], ns[j]) for k in xrange(As[i][j].size[1]): tmp = +As[i][j][:, k] misc.pack2(tmp, {'l': 0, 'q': [], 's': [ns[j]]}) As[i][j][:, k] = tmp else: As[i][j] = 0.0 # H is an m times m block matrix with i, k block # # Hik = sum_j As[i,j]' * As[k,j] # # of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j] # are zero for all j H = spmatrix([], [], [], (sum(ms), sum(ms))) rowid = 0 for i in xrange(M): colid = 0 for k in xrange(i + 1): sparse_block = True Hik = matrix(0.0, (ms[i], ms[k])) for j in xrange(N): if (type(As[i][j]) is matrix) and \ (type(As[k][j]) is matrix): sparse_block = False # Hik += As[i,j]' * As[k,j] if i == k: blas.syrk(As[i][j], Hik, trans='T', beta=1.0, k=ns[j] * (ns[j] + 1) / 2, ldA=ns[j]**2) else: blas.gemm(As[i][j], As[k][j], Hik, transA='T', beta=1.0, k=ns[j] * (ns[j] + 1) / 2, ldA=ns[j]**2, ldB=ns[j]**2) if not (sparse_block): H[rowid:rowid+ms[i], colid:colid+ms[k]] \ = sparse(Hik) colid += ms[k] rowid += ms[i] HF = cholmod.symbolic(H) cholmod.numeric(H, HF) def solve(x, y, z): """ Returns solution of rho * ux + A'(uy) - r^-T * uz * r^-1 = bx A(ux) = by -ux - r * uz * r' = bz. On entry, x = bx, y = by, z = bz. On exit, x = ux, y = uy, z = uz. """ # bz is a copy of z in the format of x blas.copy(z, bz) # x := x + rho * bz # = bx + rho * bz blas.axpy(bz, x, alpha=rho) # x := Gamma .* (u' * x * u) # = Gamma .* (u' * (bx + rho * bz) * u) offsetj = 0 for j in xrange(N): cngrnc(U[j], x, trans='T', offsetx=offsetj, n=ns[j]) blas.tbmv(Gamma[j], x, n=ns[j]**2, k=0, ldA=1, offsetx=offsetj) offsetj += ns[j]**2 # y := y - As(x) # := by - As( Gamma .* u' * (bx + rho * bz) * u) blas.copy(x, xp) offsetj = 0 for j in xrange(N): misc.pack2(xp, {'l': offsetj, 'q': [], 's': [ns[j]]}) offsetj += ns[j]**2 offseti = 0 for i in xrange(M): offsetj = 0 for j in xrange(N): if type(As[i][j]) is matrix: blas.gemv(As[i][j], xp, y, trans='T', alpha=-1.0, beta=1.0, m=ns[j] * (ns[j] + 1) / 2, n=ms[i], ldA=ns[j]**2, offsetx=offsetj, offsety=offseti) offsetj += ns[j]**2 offseti += ms[i] # y := -y - A(bz) # = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u) Af(bz, y, alpha=-1.0, beta=-1.0) # y := H^-1 * y # = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) ) # = uy cholmod.solve(HF, y) # bz = Vt' * vz * Vt # = uz where # vz := Gamma .* ( As'(uy) - x ) # = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) ) # = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ). blas.copy(x, xp) offsetj = 0 for j in xrange(N): # xp is -x[j] = -Gamma .* (u' * (bx + rho*bz) * u) # in packed storage misc.pack2(xp, {'l': offsetj, 'q': [], 's': [ns[j]]}) offsetj += ns[j]**2 blas.scal(-1.0, xp) offsetj = 0 for j in xrange(N): # xp += As'(uy) offseti = 0 for i in xrange(M): if type(As[i][j]) is matrix: blas.gemv(As[i][j], y, xp, alpha = 1.0, beta = 1.0, m = ns[j]*(ns[j]+1)/2, \ n = ms[i],ldA = ns[j]**2, \ offsetx = offseti, offsety = offsetj) offseti += ms[i] # bz[j] is xp unpacked and multiplied with Gamma #unpack(xp, bz[j], ns[j]) misc.unpack(xp, bz, { 'l': 0, 'q': [], 's': [ns[j]] }, offsetx=offsetj, offsety=offsetj) blas.tbmv(Gamma[j], bz, n=ns[j]**2, k=0, ldA=1, offsetx=offsetj) # bz = Vt' * bz * Vt # = uz cngrnc(Vt[j], bz, trans='T', offsetx=offsetj, n=ns[j]) symmetrize(bz, ns[j], offset=offsetj) offsetj += ns[j]**2 # x = -bz - r * uz * r' blas.copy(z, x) blas.copy(bz, z) offsetj = 0 for j in xrange(N): cngrnc(+W['r'][j], bz, offsetx=offsetj, n=ns[j]) offsetj += ns[j]**2 blas.axpy(bz, x) blas.scal(-1.0, x) return solve
def Fkkt(W): """ Custom solver: v := alpha * 2*A'*A * u + beta * v """ global mmS mmS = matrix(0.0, (iR, iR)) global vvV vvV = matrix(0.0, (iR, 1)) # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. mmAsc = matrix(0.0, (iR, iC)) d1, d2 = W["di"][:iC] ** 2, W["di"][iC:] ** 2 # ds is square root of diagonal of D ds = sqrt(2.0) * div(mul(W["di"][:iC], W["di"][iC:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(mmTh, mmAsc) for k in range(iR): blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(mmAsc, mmS) mmS[:: iR + 1] += 1.0 lapack.potrf(mmS) def g(x, y, z): x[:iC] = 0.5 * ( x[:iC] - mul(d3, x[iC:]) + mul(d1, z[:iC] + mul(d3, z[:iC])) - mul(d2, z[iC:] - mul(d3, z[iC:])) ) x[:iC] = div(x[:iC], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(mmAsc, x, vvV) lapack.potrs(mmS, vvV) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans="T") x[:iC] = div(x[:iC], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[iC:] = div(x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1 + d2) - mul(d3, x[:iC]) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:iC] = mul(W["di"][:iC], x[:iC] - x[iC:] - z[:iC]) z[iC:] = mul(W["di"][iC:], -x[:iC] - x[iC:] - z[iC:]) return g
def Fkkt(W): """ Custom solver: v := alpha * 2*A'*A * u + beta * v """ global mmS mmS = matrix(0.0, (iR, iR)) global vvV vvV = matrix(0.0, (iR, 1)) # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. mmAsc = matrix(0.0, (iR, iC)) d1, d2 = W['di'][:iC]**2, W['di'][iC:]**2 # ds is square root of diagonal of D ds = sqrt(2.0) * div(mul(W['di'][:iC], W['di'][iC:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(mmTh, mmAsc) for k in range(iR): blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(mmAsc, mmS) mmS[::iR + 1] += 1.0 lapack.potrf(mmS) def g(x, y, z): x[:iC] = 0.5 * ( x[:iC] - mul(d3, x[iC:]) + \ mul(d1, z[:iC] + mul(d3, z[:iC])) - \ mul(d2, z[iC:] - mul(d3, z[iC:])) ) x[:iC] = div(x[:iC], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(mmAsc, x, vvV) lapack.potrs(mmS, vvV) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans='T') x[:iC] = div(x[:iC], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[iC:] = div( x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1+d2 )\ - mul( d3, x[:iC] ) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:iC] = mul(W['di'][:iC], x[:iC] - x[iC:] - z[:iC]) z[iC:] = mul(W['di'][iC:], -x[:iC] - x[iC:] - z[iC:]) return g