def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ] # [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ] # [ P -I -W1^2 0 ] [ z[:m] ] = [ bz[:m] ] # [-P -I 0 -W2 ] [ z[m:] ] [ bz[m:] ] # # On entry bx, bz are stored in x, z. # On exit x, z contain the solution, with z scaled (W['di'] .* z is # returned instead of z). d1, d2 = W['d'][:m], W['d'][m:] D = 4*(d1**2 + d2**2)**-1 A = P.T * spdiag(D) * P lapack.potrf(A) def f(x, y, z): x[:n] += P.T * ( mul( div(d2**2 - d1**2, d1**2 + d2**2), x[n:]) + mul( .5*D, z[:m]-z[m:] ) ) lapack.potrs(A, x) u = P*x[:n] x[n:] = div( x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) + mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2 ) z[:m] = div(u-x[n:]-z[:m], d1) z[m:] = div(-u-x[n:]-z[m:], d2) return f
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ] # [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ] # [ P -I -W1^2 0 ] [ z[:m] ] = [ bz[:m] ] # [-P -I 0 -W2 ] [ z[m:] ] [ bz[m:] ] # # On entry bx, bz are stored in x, z. # On exit x, z contain the solution, with z scaled (W['di'] .* z is # returned instead of z). d1, d2 = W['d'][:m], W['d'][m:] D = 4 * (d1**2 + d2**2)**-1 A = P.T * spdiag(D) * P lapack.potrf(A) def f(x, y, z): x[:n] += P.T * (mul(div(d2**2 - d1**2, d1**2 + d2**2), x[n:]) + mul(.5 * D, z[:m] - z[m:])) lapack.potrs(A, x) u = P * x[:n] x[n:] = div( x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) + mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2) z[:m] = div(u - x[n:] - z[:m], d1) z[m:] = div(-u - x[n:] - z[m:], d2) return f
def F(W): """ Return a function f(x,y,z) that solves [P , G' W^-1] [ux] [bx] [G , -W ] [uy] = [bz] """ #d = spdiag(matrix(numpy.array(W['d']))) #dinv= spdiag(matrix(numpy.array(W['di']))) d = spdiag(W['d']) dinv = spdiag(W['di']) #KKT1 = d*( P * d + dinv ) KKT1 = d * P * d + spdiag(matrix(1.0, (2 * dim, 1))) lapack.potrf(KKT1) #raw_input('inputpppp') def f(x, y, z): uz = -d * (x + P * z) #uz = matrix(numpy.linalg.solve(KKT1, uz)) # slow version #lapack.gesv(KKT1,uz) # JZ: gesv have cond issue lapack.potrs(KKT1, uz) x[:] = matrix(-z - d * uz) blas.copy(uz, z) return f
def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 # ds is square root of diagonal of D ds = sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(A, Asc) for k in range(m): blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m + 1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \ mul(d1, z[:n] + mul(d3, z[:n])) - \ mul(d2, z[n:] - mul(d3, z[n:])) ) x[:n] = div(x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n]) z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:]) return g
def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 # ds is square root of diagonal of D ds = sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), sqrt(d1+d2) ) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(A, Asc) for k in range(m): blas.tbsv(ds, Asc, n=n, k=0, ldA=1, incx=m, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m+1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \ mul(d1, z[:n] + mul(d3, z[:n])) - \ mul(d2, z[n:] - mul(d3, z[n:])) ) x[:n] = div( x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] ) z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] ) return g
def get_psd_matrix(p): tmp = matrix(normal((p)**2),(p,p))/2.0 tmp = tmp + tmp.T while(1): try: lapack.potrf(+tmp) break except: tmp = tmp + .1*eye(p) return tmp
def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 # ds is square root of diagonal of D ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), sqrt(d1+d2) ) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 Asc = A * spdiag(ds**-1) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m+1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - mul(d3, z[n:])) ) x[:n] = div( x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] - # (D2-D1)*(D1+D2)^-1 * bx[n:] + # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] ) # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ). z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] ) z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] ) return g
def Fkkt(W): # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2. d1, d2 = W['di'][:n]**2, W['di'][n:]**2 print 'printing: ', W['di'] # ds is square root of diagonal of D ds = math.sqrt(2.0) * div(mul(W['di'][:n], W['di'][n:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) Asc = matrix(0.0, (m, n)) # Asc = A*diag(d)^-1/2 Asc = A * spdiag(ds**-1) # S = I + A * D^-1 * A' blas.syrk(Asc, S) S[::m + 1] += 1.0 lapack.potrf(S) def g(x, y, z): x[:n] = 0.5 * (x[:n] - mul(d3, x[n:]) + mul( d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - mul(d3, z[n:]))) x[:n] = div(x[:n], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] - # (D2-D1)*(D1+D2)^-1 * bx[n:] + # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] ) blas.gemv(Asc, x, v) lapack.potrs(S, v) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T') x[:n] = div(x[:n], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\ - mul( d3, x[:n] ) # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] ) # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ). z[:n] = mul(W['di'][:n], x[:n] - x[n:] - z[:n]) z[n:] = mul(W['di'][n:], -x[:n] - x[n:] - z[n:]) return g
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ] # [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ] # [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ] # [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ] # # where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di']. # # On entry bx, bz are stored in x, z. # On exit x, z contain the solution, with z scaled (di .* z is # returned instead of z). # Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and # d1 = d[:m].^2, d2 = d[m:].^2. di = W['di'] d1, d2 = di[:m]**2, di[m:]**2 D = div(mul(d1, d2), d1 + d2) Ds = spdiag(2 * sqrt(D)) base.gemm(Ds, P, Ps) blas.syrk(Ps, A, trans='T') lapack.potrf(A) def f(x, y, z): # Solve for x[:n]: # # A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:] # + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ). blas.copy((mul(div(d1 - d2, d1 + d2), x[n:]) + mul(2 * D, z[:m] - z[m:])), u) blas.gemv(P, u, x, beta=1.0, trans='T') lapack.potrs(A, x) # x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:] # + (D1-D2)*P*x[:n]) base.gemv(P, x, u) x[n:] = div( x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) + mul(d1 - d2, u), d1 + d2) # z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m]) # z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:]) z[:m] = mul(di[:m], u - x[n:] - z[:m]) z[m:] = mul(di[m:], -u - x[n:] - z[m:]) return f
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 0 P' -P' ] [ x[:n] ] [ bx[:n] ] # [ 0 0 -I -I ] [ x[n:] ] [ bx[n:] ] # [ P -I -D1^{-1} 0 ] [ z[:m] ] = [ bz[:m] ] # [-P -I 0 -D2^{-1} ] [ z[m:] ] [ bz[m:] ] # # where D1 = diag(di[:m])^2, D2 = diag(di[m:])^2 and di = W['di']. # # On entry bx, bz are stored in x, z. # On exit x, z contain the solution, with z scaled (di .* z is # returned instead of z). # Factor A = 4*P'*D*P where D = d1.*d2 ./(d1+d2) and # d1 = d[:m].^2, d2 = d[m:].^2. di = W['di'] d1, d2 = di[:m]**2, di[m:]**2 D = div( mul(d1,d2), d1+d2 ) Ds = spdiag(2 * sqrt(D)) base.gemm(Ds, P, Ps) blas.syrk(Ps, A, trans = 'T') lapack.potrf(A) def f(x, y, z): # Solve for x[:n]: # # A*x[:n] = bx[:n] + P' * ( ((D1-D2)*(D1+D2)^{-1})*bx[n:] # + (2*D1*D2*(D1+D2)^{-1}) * (bz[:m] - bz[m:]) ). blas.copy(( mul( div(d1-d2, d1+d2), x[n:]) + mul( 2*D, z[:m]-z[m:] ) ), u) blas.gemv(P, u, x, beta = 1.0, trans = 'T') lapack.potrs(A, x) # x[n:] := (D1+D2)^{-1} * (bx[n:] - D1*bz[:m] - D2*bz[m:] # + (D1-D2)*P*x[:n]) base.gemv(P, x, u) x[n:] = div( x[n:] - mul(d1, z[:m]) - mul(d2, z[m:]) + mul(d1-d2, u), d1+d2 ) # z[:m] := d1[:m] .* ( P*x[:n] - x[n:] - bz[:m]) # z[m:] := d2[m:] .* (-P*x[:n] - x[n:] - bz[m:]) z[:m] = mul(di[:m], u-x[n:]-z[:m]) z[m:] = mul(di[m:], -u-x[n:]-z[m:]) return f
def F(W): """ Returns a function f(x, y, z) that solves -diag(z) = bx -diag(x) - r*r'*z*r*r' = bz where r = W['r'][0] = W['rti'][0]^{-T}. """ rti = W['rti'][0] # t = rti*rti' as a nonsymmetric matrix. t = matrix(0.0, (n,n)) blas.gemm(rti, rti, t, transB = 'T') # Cholesky factorization of tsq = t.*t. tsq = t**2 lapack.potrf(tsq) def f(x, y, z): """ On entry, x contains bx, y is empty, and z contains bz stored in column major order. On exit, they contain the solution, with z scaled (vec(r'*z*r) is returned instead of z). We first solve ((rti*rti') .* (rti*rti')) * x = bx - diag(t*bz*t) and take z = - rti' * (diag(x) + bz) * rti. """ # tbst := t * bz * t tbst = +z cngrnc(t, tbst) # x := x - diag(tbst) = bx - diag(rti*rti' * bz * rti*rti') x -= tbst[::n+1] # x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bz*t)) lapack.potrs(tsq, x) # z := z + diag(x) = bz + diag(x) z[::n+1] += x # z := -vec(rti' * z * rti) # = -vec(rti' * (diag(x) + bz) * rti cngrnc(rti, z, alpha = -1.0) return f
def Fkkt(x, z, W): ds = (2.0 * div(1 + x**2, (1 - x**2)**2))**-0.5 Asc = A * spdiag(ds) blas.syrk(Asc, S) S[::m+1] += 1.0 lapack.potrf(S) a = z[0] def g(x, y, z): x[:] = mul(x, ds) / a blas.gemv(Asc, x, v) lapack.potrs(S, v) blas.gemv(Asc, v, x, alpha = -1.0, beta = 1.0, trans = 'T') x[:] = mul(x, ds) return g
def F(x=None, z=None): if x is None: return 0, matrix(1.0, (n,1)) X = V * spdiag(x) * V.T L = +X try: lapack.potrf(L) except ArithmeticError: return None f = - 2.0 * (log(L[0,0]) + log(L[1,1])) W = +V blas.trsm(L, W) gradf = matrix(-1.0, (1,2)) * W**2 if z is None: return f, gradf H = matrix(0.0, (n,n)) blas.syrk(W, H, trans='T') return f, gradf, z[0] * H**2
def Fkkt(W): rti = W['rti'][0] # t = rti*rti' as a nonsymmetric matrix. t = matrix(0.0, (n, n)) blas.gemm(rti, rti, t, transB='T') # Cholesky factorization of tsq = t.*t. tsq = t**2 lapack.potrf(tsq) def f(x, y, z): """ Solve -diag(z) = bx -diag(x) - inv(rti*rti') * z * inv(rti*rti') = bs On entry, x and z contain bx and bs. On exit, they contain the solution, with z scaled (inv(rti)'*z*inv(rti) is returned instead of z). We first solve ((rti*rti') .* (rti*rti')) * x = bx - diag(t*bs*t) and take z = -rti' * (diag(x) + bs) * rti. """ # tbst := t * zs * t = t * bs * t tbst = matrix(z, (n, n)) cngrnc(t, tbst) # x := x - diag(tbst) = bx - diag(rti*rti' * bs * rti*rti') x -= tbst[::n + 1] # x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bs*t)) lapack.potrs(tsq, x) # z := z + diag(x) = bs + diag(x) z[::n + 1] += x # z := -rti' * z * rti = -rti' * (diag(x) + bs) * rti cngrnc(rti, z, alpha=-1.0) return f
def Fkkt(W): rti = W['rti'][0] # t = rti*rti' as a nonsymmetric matrix. t = matrix(0.0, (n,n)) blas.gemm(rti, rti, t, transB = 'T') # Cholesky factorization of tsq = t.*t. tsq = t**2 lapack.potrf(tsq) def f(x, y, z): """ Solve -diag(z) = bx -diag(x) - inv(rti*rti') * z * inv(rti*rti') = bs On entry, x and z contain bx and bs. On exit, they contain the solution, with z scaled (inv(rti)'*z*inv(rti) is returned instead of z). We first solve ((rti*rti') .* (rti*rti')) * x = bx - diag(t*bs*t) and take z = -rti' * (diag(x) + bs) * rti. """ # tbst := t * zs * t = t * bs * t tbst = matrix(z, (n,n)) cngrnc(t, tbst) # x := x - diag(tbst) = bx - diag(rti*rti' * bs * rti*rti') x -= tbst[::n+1] # x := (t.*t)^{-1} * x = (t.*t)^{-1} * (bx - diag(t*bs*t)) lapack.potrs(tsq, x) # z := z + diag(x) = bs + diag(x) z[::n+1] += x # z := -rti' * z * rti = -rti' * (diag(x) + bs) * rti cngrnc(rti, z, alpha = -1.0) return f
def Fkkt(W): d1, d2 = W['d'][:m], W['d'][m:] D = 4*(d1**2 + d2**2)**-1 A = P.T * spdiag(D) * P lapack.potrf(A) def f(x, y, z): x[:n] += P.T * ( mul( div(d2**2 - d1**2, d1**2 + d2**2), x[n:]) + mul( .5*D, z[:m]-z[m:] ) ) lapack.potrs(A, x) u = P*x[:n] x[n:] = div( x[n:] - div(z[:m], d1**2) - div(z[m:], d2**2) + mul(d1**-2 - d2**-2, u), d1**-2 + d2**-2 ) z[:m] = div(u-x[n:]-z[:m], d1) z[m:] = div(-u-x[n:]-z[m:], d2) return f
def F(W): """ Generate a solver for A'(uz0) = bx[0] -uz0 - uz1 = bx[1] A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0 - ux[1] - r1*r1' * uz1 * r1*r1' = bz1. uz0, uz1, bz0, bz1 are symmetric m x m-matrices. ux[0], bx[0] are n-vectors. ux[1], bx[1] are symmetric m x m-matrices. We first calculate a congruence that diagonalizes r0*r0' and r1*r1': U' * r0 * r0' * U = I, U' * r1 * r1' * U = S. We then make a change of variables usx[0] = ux[0], usx[1] = U' * ux[1] * U usz0 = U^-1 * uz0 * U^-T usz1 = U^-1 * uz1 * U^-T and define As() = U' * A() * U' bsx[1] = U^-1 * bx[1] * U^-T bsz0 = U' * bz0 * U bsz1 = U' * bz1 * U. This gives As'(usz0) = bx[0] -usz0 - usz1 = bsx[1] As(usx[0]) - usx[1] - usz0 = bsz0 -usx[1] - S * usz1 * S = bsz1. 1. Eliminate usz0, usz1 using equations 3 and 4, usz0 = As(usx[0]) - usx[1] - bsz0 usz1 = -S^-1 * (usx[1] + bsz1) * S^-1. This gives two equations in usx[0] an usx[1]. As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0) -As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1 = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1. 2. Eliminate usx[1] using equation 2: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j], usx[1] = ( S * As(usx[0]) * S ) ./ Gamma + ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma. This gives an equation in usx[0]. As'( As(usx[0]) ./ Gamma ) = bx0 + As'(bsz0) + As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma ) = bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ). """ # Calculate U s.t. # # U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s). # Cholesky factorization r0 * r0' = L * L' blas.syrk(W['r'][0], L) lapack.potrf(L) # SVD L^-1 * r1 = U * diag(s) * V' blas.copy(W['r'][1], U) blas.trsm(L, U) lapack.gesvd(U, s, jobu='O') # s := s**2 s[:] = s**2 # Uti := U blas.copy(U, Uti) # U := L^-T * U blas.trsm(L, U, transA='T') # Uti := L * Uti = U^-T blas.trmm(L, Uti) # Us := U * diag(s)^-1 blas.copy(U, Us) for i in range(m): blas.tbsv(s, Us, n=m, k=0, ldA=1, incx=m, offsetx=i) # S is m x m with lower triangular entries s[i] * s[j] # sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j]) # Upper triangular entries are undefined but nonzero. blas.scal(0.0, S) blas.syrk(s, S) Gamma = 1.0 + S sqrtG = sqrt(Gamma) # Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n # = Asi ./ sqrt(Gamma) blas.copy(A, Asc) misc.scale( Asc, # only 'r' part of the dictionary is used { 'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)), 'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)), 'v': [], 'beta': [], 'r': [U], 'rti': [U] }) for i in range(n): blas.tbsv(sqrtG, Asc, n=msq, k=0, ldA=1, offsetx=i * msq) # Convert columns of Asc to packed storage misc.pack2(Asc, {'l': 0, 'q': [], 's': [m]}) # Cholesky factorization of Asc' * Asc. H = matrix(0.0, (n, n)) blas.syrk(Asc, H, trans='T', k=mpckd) lapack.potrf(H) def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans='T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans='T', offsetx=msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans='T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha=-1.0, offsetx=msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n=msq, k=0, ldA=1) blas.axpy(z, x1, n=msq) blas.tbsv(sqrtG, x1, n=msq, k=0, ldA=1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m=mpckd, trans='T', beta=1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m=mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n=msq, k=0, ldA=1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n=msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n=msq, k=0, ldA=1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n=msq, offsety=msq) blas.scal(-1.0, z, offset=msq) __cngrnc(U, z, offsetx=msq) __cngrnc(W['r'][1], z, trans='T', offsetx=msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n=msq, k=0, ldA=1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n=msq) blas.scal(-1.0, z, n=msq) blas.axpy(x[1], z, -1.0, n=msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans='T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1]) return solve
def cholesky(X): """ Supernodal multifrontal Cholesky factorization: .. math:: X = LL^T where :math:`L` is lower-triangular. On exit, the argument :math:`X` contains the Cholesky factor :math:`L`. :param X: :py:class:`cspmatrix` """ assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix" n = X.symb.n snpost = X.symb.snpost snptr = X.symb.snptr chptr = X.symb.chptr chidx = X.symb.chidx relptr = X.symb.relptr relidx = X.symb.relidx blkptr = X.symb.blkptr blkval = X.blkval stack = [] for k in snpost: nn = snptr[k+1]-snptr[k] # |Nk| na = relptr[k+1]-relptr[k] # |Ak| nj = na + nn # build frontal matrix F = matrix(0.0, (nj, nj)) lapack.lacpy(blkval, F, offsetA = blkptr[k], m = nj, n = nn, ldA = nj, uplo = 'L') # add update matrices from children to frontal matrix for i in range(chptr[k+1]-1,chptr[k]-1,-1): Ui = stack.pop() frontal_add_update(F, Ui, relidx, relptr, chidx[i]) # factor L_{Nk,Nk} lapack.potrf(F, n = nn, ldA = nj) # if supernode k is not a root node, compute and push update matrix onto stack if na > 0: # compute L_{Ak,Nk} := A_{Ak,Nk}*inv(L_{Nk,Nk}') blas.trsm(F, F, m = na, n = nn, ldA = nj, ldB = nj, offsetB = nn, transA = 'T', side = 'R') # compute Uk = Uk - L_{Ak,Nk}*inv(D_{Nk,Nk})*L_{Ak,Nk}' if nn == 1: blas.syr(F, F, n = na, offsetx = nn, \ offsetA = nn*nj+nn, ldA = nj, alpha = -1.0) else: blas.syrk(F, F, k = nn, n = na, offsetA = nn, ldA = nj, offsetC = nn*nj+nn, ldC = nj, alpha = -1.0, beta = 1.0) # compute L_{Ak,Nk} := L_{Ak,Nk}*inv(L_{Nk,Nk}) blas.trsm(F, F, m = na, n = nn,\ ldA = nj, ldB = nj, offsetB = nn, side = 'R') # add Uk to stack Uk = matrix(0.0,(na,na)) lapack.lacpy(F, Uk, m = na, n = na, uplo = 'L', offsetA = nn*nj+nn, ldA = nj) stack.append(Uk) # copy the leading Nk columns of frontal matrix to blkval lapack.lacpy(F, blkval, uplo = "L", offsetB = blkptr[k], m = nj, n = nn, ldB = nj) X.is_factor = True return
def Fkkt(W): """ Custom solver: v := alpha * 2*A'*A * u + beta * v """ global mmS mmS = matrix(0.0, (iR, iR)) global vvV vvV = matrix(0.0, (iR, 1)) # Factor # # S = A*D^-1*A' + I # # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**2, D2 = d[n:]**2. mmAsc = matrix(0.0, (iR, iC)) d1, d2 = W["di"][:iC] ** 2, W["di"][iC:] ** 2 # ds is square root of diagonal of D ds = sqrt(2.0) * div(mul(W["di"][:iC], W["di"][iC:]), sqrt(d1 + d2)) d3 = div(d2 - d1, d1 + d2) # Asc = A*diag(d)^-1/2 blas.copy(mmTh, mmAsc) for k in range(iR): blas.tbsv(ds, mmAsc, n=iC, k=0, ldA=1, incx=iR, offsetx=k) # S = I + A * D^-1 * A' blas.syrk(mmAsc, mmS) mmS[:: iR + 1] += 1.0 lapack.potrf(mmS) def g(x, y, z): x[:iC] = 0.5 * ( x[:iC] - mul(d3, x[iC:]) + mul(d1, z[:iC] + mul(d3, z[:iC])) - mul(d2, z[iC:] - mul(d3, z[iC:])) ) x[:iC] = div(x[:iC], ds) # Solve # # S * v = 0.5 * A * D^-1 * ( bx[:n] # - (D2-D1)*(D1+D2)^-1 * bx[n:] # + D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bz[:n] # - D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bz[n:] ) blas.gemv(mmAsc, x, vvV) lapack.potrs(mmS, vvV) # x[:n] = D^-1 * ( rhs - A'*v ). blas.gemv(mmAsc, vvV, x, alpha=-1.0, beta=1.0, trans="T") x[:iC] = div(x[:iC], ds) # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bz[:n] - D2*bz[n:] ) # - (D2-D1)*(D1+D2)^-1 * x[:n] x[iC:] = div(x[iC:] - mul(d1, z[:iC]) - mul(d2, z[iC:]), d1 + d2) - mul(d3, x[:iC]) # z[:n] = D1^1/2 * ( x[:n] - x[n:] - bz[:n] ) # z[n:] = D2^1/2 * ( -x[:n] - x[n:] - bz[n:] ). z[:iC] = mul(W["di"][:iC], x[:iC] - x[iC:] - z[:iC]) z[iC:] = mul(W["di"][iC:], -x[:iC] - x[iC:] - z[iC:]) return g
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 G' ] [ x ] = [ bx ] # [ G -W'*W ] [ z ] [ bz ]. # First factor # # S = G' * W**-1 * W**-T * G # = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1 # # where # # W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n] # W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:] # W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J) # with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I]. # As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A] minor = 0 if not helpers.sp_minor_empty(): minor = helpers.sp_minor_top() beta, v = W['beta'][0], W['v'][0] As = 2 * v * (v[1:].T * A) As[1:,:] *= -1.0 As[1:,:] -= A As /= beta # S = As'*As + 4 * (W1**2 + W2**2)**-1 S = As.T * As helpers.sp_add_var("S", S) d1, d2 = W['d'][:n], W['d'][n:] d = 4.0 * (d1**2 + d2**2)**-1 S[::n+1] += d lapack.potrf(S) helpers.sp_create("00-Fkkt", minor) def f(x, y, z): minor = 0 if not helpers.sp_minor_empty(): minor = helpers.sp_minor_top() else: global loopf loopf += 1 minor = loopf helpers.sp_create("00-f", minor) # z := - W**-T * z z[:n] = -div( z[:n], d1 ) z[n:2*n] = -div( z[n:2*n], d2 ) z[2*n:] -= 2.0*v*( v[0]*z[2*n] - blas.dot(v[1:], z[2*n+1:]) ) z[2*n+1:] *= -1.0 z[2*n:] /= beta # x := x - G' * W**-1 * z x[:n] -= div(z[:n], d1) - div(z[n:2*n], d2) + As.T * z[-(m+1):] x[n:] += div(z[:n], d1) + div(z[n:2*n], d2) helpers.sp_create("15-f", minor) # Solve for x[:n]: # # S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:] x[:n] -= mul( div(d1**2 - d2**2, d1**2 + d2**2), x[n:]) helpers.sp_create("25-f", minor) lapack.potrs(S, x) helpers.sp_create("30-f", minor) # Solve for x[n:]: # # (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n] x[n:] += mul( d1**-2 - d2**-2, x[:n]) helpers.sp_create("35-f", minor) x[n:] = div( x[n:], d1**-2 + d2**-2) helpers.sp_create("40-f", minor) # z := z + W^-T * G*x z[:n] += div( x[:n] - x[n:2*n], d1) helpers.sp_create("44-f", minor) z[n:2*n] += div( -x[:n] - x[n:2*n], d2) helpers.sp_create("48-f", minor) z[2*n:] += As*x[:n] helpers.sp_create("50-f", minor) return f
H1[3:, 3:] = 2 * B return f, Df, z[0] * H0 + sum(z[1:]) * H1 sol = solvers.cp(F) A = matrix(sol['x'][[0, 1, 1, 2]], (2, 2)) b = sol['x'][3:] if pylab_installed: pylab.figure(1, facecolor='w') pylab.plot(X[:, 0], X[:, 1], 'ko', X[:, 0], X[:, 1], '-k') # Ellipsoid in the form { x | || L' * (x-c) ||_2 <= 1 } L = +A lapack.potrf(L) c = +b lapack.potrs(L, c) # 1000 points on the unit circle nopts = 1000 angles = matrix([a * 2.0 * pi / nopts for a in range(nopts)], (1, nopts)) circle = matrix(0.0, (2, nopts)) circle[0, :], circle[1, :] = cos(angles), sin(angles) # ellipse = L^-T * circle + c blas.trsm(L, circle, transA='T') ellipse = circle + c[:, nopts * [0]] ellipse2 = 0.5 * circle + c[:, nopts * [0]] pylab.plot(ellipse[0, :].T, ellipse[1, :].T, 'k-')
def F(W): """ Create a solver for the linear equations C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - Dl^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - r*r' * [ ] * r*r' = bzs [ -A(ux) -uX2 ] [ uzs21 uzs22 ] where Dl = diag(W['l']), r = W['r'][0]. On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ]. 1. Compute matrices V1, V2 such that (with T = r*r') [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and S = [ diag(s); 0 ], s a positive q-vector. 2. Factor the mapping X -> X + S * X' * S: X + S * X' * S = L( L'( X )). 3. Compute scaled mappings: a matrix As with as its columns the coefficients of the scaled mapping L^-1( V2' * A() * V1' ) and the matrix Gs = Dl^-1 * G. 4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As. """ # 1. Compute V1, V2, s. r = W['r'][0] # LQ factorization R[:q, :] = L1 * Q1. lapack.lacpy(r, Q1, m=q) lapack.gelqf(Q1, tau1) lapack.lacpy(Q1, L1, n=q, uplo='L') lapack.orglq(Q1, tau1) # LQ factorization R[q:, :] = L2 * Q2. lapack.lacpy(r, Q2, m=p, offsetA=q) lapack.gelqf(Q2, tau2) lapack.lacpy(Q2, L2, n=p, uplo='L') lapack.orglq(Q2, tau2) # V2, V1, s are computed from an SVD: if # # Q2 * Q1' = U * diag(s) * V', # # then V1 = V' * L1^-1 and V2 = L2^-T * U. # T21 = Q2 * Q1.T blas.gemm(Q2, Q1, T21, transB='T') # SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1. lapack.gesvd(T21, s, jobu='A', jobvt='A', U=V2, Vt=V1) # # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2 # this will requires lapack.ormlq or lapack.unmlq # V2 = L2^-T * U blas.trsm(L2, V2, transA='T') # V1 = V' * L1^-1 blas.trsm(L1, V1, side='R') # 2. Factorization X + S * X' * S = L( L'( X )). # # The factor L is stored as a diagonal matrix D and a sparse lower # triangular matrix P, such that # # L(X)[:] = D**-1 * (I + P) * X[:] # L^-1(X)[:] = D * (I - P) * X[:]. # SS is q x q with SS[i,j] = si*sj. blas.scal(0.0, SS) blas.syr(s, SS) # For a p x q matrix X, P*X[:] is Y[:] where # # Yij = si * sj * Xji if i < j # = 0 otherwise. # P.V = SS[Itril2] # For a p x q matrix X, D*X[:] is Y[:] where # # Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j # = Xii / sqrt( 1 + si^2 ) if i = j # = Xij otherwise. # DV[Idiag] = sqrt(1.0 + SS[::q + 1]) DV[Itriu] = sqrt(1.0 - SS[Itril3]**2) D.V = DV**-1 # 3. Scaled linear mappings # Ask := V2' * Ask * V1' blas.scal(0.0, As) base.axpy(A, As) for i in xrange(n): # tmp := V2' * As[i, :] blas.gemm(V2, As, tmp, transA='T', m=p, n=q, k=p, ldB=p, offsetB=i * p * q) # As[:,i] := tmp * V1' blas.gemm(tmp, V1, As, transB='T', m=p, n=q, k=q, ldC=p, offsetC=i * p * q) # As := D * (I - P) * As # = L^-1 * As. blas.copy(As, As2) base.gemm(P, As, As2, alpha=-1.0, beta=1.0) base.gemm(D, As2, As) # Gs := Dl^-1 * G blas.scal(0.0, Gs) base.axpy(G, Gs) for k in xrange(n): blas.tbmv(W['di'], Gs, n=m, k=0, ldA=1, offsetx=k * m) # 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As. blas.syrk(As, H, trans='T', alpha=2.0) blas.syrk(Gs, H, trans='T', beta=1.0) base.axpy(C, H) lapack.potrf(H) def f(x, y, z): """ Solve C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - T * [ ] * T = bzs. [ -A(ux) -uX2 ] [ uzs21 uzs22 ] On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ]. Define X = uzs21, Z = T * uzs * T: C * ux + G' * uzl - 2*A'(X) = bx [ 0 X' ] [ bX1 0 ] T * [ ] * T - Z = T * [ ] * T [ X 0 ] [ 0 bX2 ] G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs [ -A(ux) -uX2 ] [ Z21 Z22 ] Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ]. We use the congruence transformation [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and the factorization X + S * X' * S = L( L'(X) ) to write this as C * ux + G' * uzl - 2*A'(X) = bx L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs, [ -A(ux) -uX2 ] [ Z21 Z22 ] or C * ux + Gs' * uuzl - 2*As'(XX) = bx XX - ZZ21 = bX Gs * ux - uuzl = D^-1 * bzl -As(ux) - ZZ21 = bbzs_21 -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 if we introduce scaled variables uuzl = D * uzl XX = L'(V2^-1 * X * V1^-1) = L'(V2^-1 * uzs21 * V1^-1) ZZ21 = L^-1(V2' * Z21 * V1') and define bbzs_21 = L^-1(V2' * bzs_21 * V1') [ bX1 0 ] bX = L^-1( V2' * (T * [ ] * T)_21 * V1'). [ 0 bX2 ] Eliminating Z21 gives C * ux + Gs' * uuzl - 2*As'(XX) = bx Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 and eliminating uuzl and XX gives H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21) Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22. In summary, we can use the following algorithm: 1. bXX := bX - bbzs21 [ bX1 0 ] = L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1') [ 0 bX2 ] 2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). 3. From ux, compute uuzl = Gs*ux - D^-1 * bzl and X = V2 * L^-T(-As(ux) + bXX) * V1. 4. Return ux, uuzl, rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22. """ # Save bzs_11, bzs_22, bzs_21. lapack.lacpy(z, bz11, uplo='L', m=q, n=q, ldA=p + q, offsetA=m) lapack.lacpy(z, bz21, m=p, n=q, ldA=p + q, offsetA=m + q) lapack.lacpy(z, bz22, uplo='L', m=p, n=p, ldA=p + q, offsetA=m + (p + q + 1) * q) # zl := D^-1 * zl # = D^-1 * bzl blas.tbmv(W['di'], z, n=m, k=0, ldA=1) # zs := r' * [ bX1, 0; 0, bX2 ] * r. # zs := [ bX1, 0; 0, bX2 ] blas.scal(0.0, z, offset=m) lapack.lacpy(x[1], z, uplo='L', m=q, n=q, ldB=p + q, offsetB=m) lapack.lacpy(x[2], z, uplo='L', m=p, n=p, ldB=p + q, offsetB=m + (p + q + 1) * q) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc=p + q + 1, offset=m) # a := tril(zs)*r blas.copy(r, a) blas.trmm(z, a, side='L', m=p + q, n=p + q, ldA=p + q, ldB=p + q, offsetA=m) # zs := a'*r + r'*a blas.syr2k(r, a, z, trans='T', n=p + q, k=p + q, ldB=p + q, ldC=p + q, offsetC=m) # bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1') # # [ bX1 0 ] # = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1'). # [ 0 bX2 ] # a = [ r21 r22 ] * z # = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r # = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r blas.symm(z, r, a, side='R', m=p, n=p + q, ldA=p + q, ldC=p + q, offsetB=q) # bz21 := -bz21 + a * [ r11, r12 ]' # = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21 blas.gemm(a, r, bz21, transB='T', m=p, n=q, k=p + q, beta=-1.0, ldA=p + q, ldC=p) # bz21 := V2' * bz21 * V1' # = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1' blas.gemm(V2, bz21, tmp, transA='T', m=p, n=q, k=p, ldB=p) blas.gemm(tmp, V1, bz21, transB='T', m=p, n=q, k=q, ldC=p) # bz21[:] := D * (I-P) * bz21[:] # = L^-1 * bz21[:] # = bXX[:] blas.copy(bz21, tmp) base.gemv(P, bz21, tmp, alpha=-1.0, beta=1.0) base.gemv(D, tmp, bz21) # Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). # x[0] := x[0] + Gs'*zl + 2*As'(bz21) # = bx + G' * D^-1 * bzl + 2 * As'(bXX) blas.gemv(Gs, z, x[0], trans='T', alpha=1.0, beta=1.0) blas.gemv(As, bz21, x[0], trans='T', alpha=2.0, beta=1.0) # x[0] := H \ x[0] # = ux lapack.potrs(H, x[0]) # uuzl = Gs*ux - D^-1 * bzl blas.gemv(Gs, x[0], z, alpha=1.0, beta=-1.0) # bz21 := V2 * L^-T(-As(ux) + bz21) * V1 # = X blas.gemv(As, x[0], bz21, alpha=-1.0, beta=1.0) blas.tbsv(DV, bz21, n=p * q, k=0, ldA=1) blas.copy(bz21, tmp) base.gemv(P, tmp, bz21, alpha=-1.0, beta=1.0, trans='T') blas.gemm(V2, bz21, tmp) blas.gemm(tmp, V1, bz21) # zs := -zs + r' * [ 0, X'; X, 0 ] * r # = r' * [ -bX1, X'; X, -bX2 ] * r. # a := bz21 * [ r11, r12 ] # = X * [ r11, r12 ] blas.gemm(bz21, r, a, m=p, n=p + q, k=q, ldA=p, ldC=p + q) # z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ] # = rti' * uzs * rti blas.syr2k(r, a, z, trans='T', beta=-1.0, n=p + q, k=p, offsetA=q, offsetC=m, ldB=p + q, ldC=p + q) # uX1 = -Z11 - bzs_11 # = -(r*zs*r')_11 - bzs_11 # uX2 = -Z22 - bzs_22 # = -(r*zs*r')_22 - bzs_22 blas.copy(bz11, x[1]) blas.copy(bz22, x[2]) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc=p + q + 1, offset=m) # a := r*tril(zs) blas.copy(r, a) blas.trmm(z, a, side='R', m=p + q, n=p + q, ldA=p + q, ldB=p + q, offsetA=m) # x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]' # = -bzs_11 - (r*zs*r')_11 blas.syr2k(a, r, x[1], n=q, alpha=-1.0, beta=-1.0) # x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]' # = -bzs_22 - (r*zs*r')_22 blas.syr2k(a, r, x[2], n=p, alpha=-1.0, beta=-1.0, offsetA=q, offsetB=q) # scale diagonal of zs by 1/2 blas.scal(2.0, z, inc=p + q + 1, offset=m) return f
if pylab_installed: pylab.figure(1, facecolor='w', figsize=(6,6)) pylab.plot(V[0,:], V[1,:],'ow', mec='k') pylab.plot([0], [0], 'k+') I = [ k for k in range(n) if xd[k] > 1e-5 ] pylab.plot(V[0,I], V[1,I],'or') # Enclosing ellipse is {x | x' * (V*diag(xe)*V')^-1 * x = sqrt(2)} nopts = 1000 angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) ) circle = matrix(0.0, (2,nopts)) circle[0,:], circle[1,:] = cos(angles), sin(angles) W = V * spdiag(xd) * V.T lapack.potrf(W) ellipse = sqrt(2.0) * circle blas.trmm(W, ellipse) if pylab_installed: pylab.plot(ellipse[0,:].T, ellipse[1,:].T, 'k--') pylab.axis([-5, 5, -5, 5]) pylab.title('D-optimal design (fig. 7.9)') pylab.axis('off') # E-design. # # maximize w # subject to w*I <= V*diag(x)*V' # x >= 0 # sum(x) = 1
def kkt(W): """ KKT solver for Q * ux + uy * 1_m' + mat(uz) = bx ux * 1_m = by ux - d.^2 .* mat(uz) = mat(bz). ux and bx are N x m matrices. uy and by are N-vectors. uz and bz are N*m-vectors. mat(uz) is the N x m matrix that satisfies mat(uz)[:] = uz. d = mat(W['d']) a positive N x m matrix. If we eliminate uz from the last equation using mat(uz) = (ux - mat(bz)) ./ d.^2 we get two equations in ux, uy: Q * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2 ux * 1_m = by. From the 1st equation uxk = -(Q + Dk)^-1 * uy + (Q + Dk)^-1 * (bxk + Dk * bzk) where uxk is column k of ux, Dk = diag(d[:,k].^-2), and bzk is column k of mat(bz). Substituting this in the second equation gives an equation for uy. 1. Solve for uy sum_k (Q + Dk)^-1 * uy = sum_k (Q + Dk)^-1 * (bxk + Dk * bzk) - by. 2. Solve for ux (column by column) Q * ux + ux ./ d.^2 = bx + mat(bz) ./ d.^2 - uy * 1_m'. 3. Solve for uz mat(uz) = ( ux - mat(bz) ) ./ d.^2. Return ux, uy, d .* uz. """ # D = d.^-2 D = matrix(W['di']**2, (N, m)) blas.scal(0.0, S) for k in range(m): # Hk := Q + Dk blas.copy(Q, H[k]) H[k][::N + 1] += D[:, k] # Hk := Hk^-1 # = (Q + Dk)^-1 lapack.potrf(H[k]) lapack.potri(H[k]) # S := S + Hk # = S + (Q + Dk)^-1 blas.axpy(H[k], S) # Factor S = sum_k (Q + Dk)^-1 lapack.potrf(S) def f(x, y, z): # z := mat(z) # = mat(bz) z.size = N, m # x := x + D .* z # = bx + mat(bz) ./ d.^2 x += mul(D, z) # y := y - sum_k (Q + Dk)^-1 * X[:,k] # = by - sum_k (Q + Dk)^-1 * (bxk + Dk * bzk) for k in range(m): blas.symv(H[k], x[:, k], y, alpha=-1.0, beta=1.0) # y := H^-1 * y # = -uy lapack.potrs(S, y) # x[:,k] := H[k] * (x[:,k] + y) # = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y) # = ux[:,k] w = matrix(0.0, (N, 1)) for k in range(m): # x[:,k] := x[:,k] + y blas.axpy(y, x, offsety=N * k, n=N) # w := H[k] * x[:,k] # = (Q + Dk)^-1 * (bxk + bzk ./ d.^2 + y) blas.symv(H[k], x, w, offsetx=N * k) # x[:,k] := w # = ux[:,k] blas.copy(w, x, offsety=N * k) # y := -y # = uy blas.scal(-1.0, y) # z := (x - z) ./ d blas.axpy(x, z, -1.0) blas.tbsv(W['d'], z, n=m * N, k=0, ldA=1) blas.scal(-1.0, z) z.size = N * m, 1 return f
def kkt(W): """ KKT solver for X*X' * ux + uy * 1_m' + mat(uz) = bx ux * 1_m = by ux - d.^2 .* mat(uz) = mat(bz). ux and bx are N x m matrices. uy and by are N-vectors. uz and bz are N*m-vectors. mat(uz) is the N x m matrix that satisfies mat(uz)[:] = uz. d = mat(W['d']) a positive N x m matrix. If we eliminate uz from the last equation using mat(uz) = (ux - mat(bz)) ./ d.^2 we get two equations in ux, uy: X*X' * ux + ux ./ d.^2 + uy * 1_m' = bx + mat(bz) ./ d.^2 ux * 1_m = by. From the 1st equation, uxk = (X*X' + Dk^-2)^-1 * (-uy + bxk + Dk^-2 * bzk) = Dk * (I + Xk*Xk')^-1 * Dk * (-uy + bxk + Dk^-2 * bzk) for k = 1, ..., m, where Dk = diag(d[:,k]), Xk = Dk * X, uxk is column k of ux, and bzk is column k of mat(bz). We use the matrix inversion lemma ( I + Xk * Xk' )^-1 = I - Xk * (I + Xk' * Xk)^-1 * Xk' = I - Xk * Hk^-1 * Xk' = I - Xk * Lk^-T * Lk^-1 * Xk' where Hk = I + Xk' * Xk = Lk * Lk' to write this as uxk = Dk * (I - Xk * Hk^-1 * Xk') * Dk * (-uy + bxk + Dk^-2 * bzk) = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk). Substituting this in the second equation gives an equation for uy: sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * uy = -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 * bzk ), i.e., with D = (sum_k Dk^2)^1/2, Yk = D^-1 * Dk^2 * X * Lk^-T, D * ( I - sum_k Yk * Yk' ) * D * uy = -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ). Another application of the matrix inversion lemma gives uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k ( Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2 ) * ( bxk + Dk^-2 *bzk ) ) with S = I - Y' * Y, Y = [ Y1 ... Ym ]. Summary: 1. Compute uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ) ) 2. For k = 1, ..., m: uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk) 3. Solve for uz d .* uz = ( ux - mat(bz) ) ./ d. Return ux, uy, d .* uz. """ ### utime0, stime0 = cputime() ### d = matrix(W['d'], (N, m)) dsq = matrix(W['d']**2, (N, m)) # Factor the matrices # # H[k] = I + Xk' * Xk # = I + X' * Dk^2 * X. # # Dk = diag(d[:,k]). for k in range(m): # H[k] = I blas.scal(0.0, H[k]) H[k][::n + 1] = 1.0 # Xs = Dk * X # = diag(d[:,k]]) * X blas.copy(X, Xs) for j in range(n): blas.tbmv(d, Xs, n=N, k=0, ldA=1, offsetA=k * N, offsetx=j * N) # H[k] := H[k] + Xs' * Xs # = I + Xk' * Xk blas.syrk(Xs, H[k], trans='T', beta=1.0) # Factorization H[k] = Lk * Lk' lapack.potrf(H[k]) ### utime, stime = cputime() print("Factor Hk's: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # diag(D) = ( sum_k d[:,k]**2 ) ** 1/2 # = ( sum_k Dk^2) ** 1/2. blas.gemv(dsq, ones, D) D[:] = sqrt(D) ### # utime, stime = cputime() # print("Compute D: utime = %.2f, stime = %.2f" \ # %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # S = I - Y'* Y is an m x m block matrix. # The i,j block of Y' * Y is # # Yi' * Yj = Li^-1 * X' * Di^2 * D^-1 * Dj^2 * X * Lj^-T. # # We compute only the lower triangular blocks in Y'*Y. blas.scal(0.0, S) for i in range(m): for j in range(i + 1): # Xs = Di * Dj * D^-1 * X blas.copy(X, Xs) blas.copy(d, wN, n=N, offsetx=i * N) blas.tbmv(d, wN, n=N, k=0, ldA=1, offsetA=j * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) for k in range(n): blas.tbmv(wN, Xs, n=N, k=0, ldA=1, offsetx=k * N) # block i, j of S is Xs' * Xs (as nonsymmetric matrix so we # get the correct multiple after scaling with Li, Lj) blas.gemm(Xs, Xs, S, transA='T', ldC=m * n, offsetC=(j * n) * m * n + i * n) ### utime, stime = cputime() print("Form S: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### for i in range(m): # multiply block row i of S on the left with Li^-1 blas.trsm(H[i], S, m=n, n=(i + 1) * n, ldB=m * n, offsetB=i * n) # multiply block column i of S on the right with Li^-T blas.trsm(H[i], S, side='R', transA='T', m=(m - i) * n, n=n, ldB=m * n, offsetB=i * n * (m * n + 1)) blas.scal(-1.0, S) S[::(m * n + 1)] += 1.0 ### utime, stime = cputime() print("Form S (2): utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### # S = L*L' lapack.potrf(S) ### utime, stime = cputime() print("Factor S: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) utime0, stime0 = cputime() ### def f(x, y, z): """ 1. Compute uy = D^-1 * (I + Y * S^-1 * Y') * D^-1 * ( -by + sum_k (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * ( bxk + Dk^-2 *bzk ) ) 2. For k = 1, ..., m: uxk = (Dk^2 - Dk^2 * X * Hk^-1 * X' * Dk^2) * (-uy + bxk + Dk^-2 * bzk) 3. Solve for uz d .* uz = ( ux - mat(bz) ) ./ d. Return ux, uy, d .* uz. """ ### utime0, stime0 = cputime() ### # xk := Dk^2 * xk + zk # = Dk^2 * bxk + bzk blas.tbmv(dsq, x, n=N * m, k=0, ldA=1) blas.axpy(z, x) # y := -y + sum_k ( I - Dk^2 * X * Hk^-1 * X' ) * xk # = -y + x*ones - sum_k Dk^2 * X * Hk^-1 * X' * xk # y := -y + x*ones blas.gemv(x, ones, y, alpha=1.0, beta=-1.0) # wnm = X' * x (wnm interpreted as an n x m matrix) blas.gemm(X, x, wnm, m=n, k=N, n=m, transA='T', ldB=N, ldC=n) # wnm[:,k] = Hk \ wnm[:,k] (for wnm as an n x m matrix) for k in range(m): lapack.potrs(H[k], wnm, offsetB=k * n) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=n * k) # wN = Dk^2 * wN blas.tbmv(dsq[:, k], wN, n=N, k=0, ldA=1) # y := y - wN blas.axpy(wN, y, -1.0) # y = D^-1 * (I + Y * S^-1 * Y') * D^-1 * y # # Y = [Y1 ... Ym ], Yk = D^-1 * Dk^2 * X * Lk^-T. # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # wnm = Y' * y (interpreted as an Nm vector) # = [ L1^-1 * X' * D1^2 * D^-1 * y; # L2^-1 * X' * D2^2 * D^-1 * y; # ... # Lm^-1 * X' * Dm^2 * D^-1 * y ] for k in range(m): # wN = D^-1 * Dk^2 * y blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # wnm[:,k] = X' * wN blas.gemv(X, wN, wnm, trans='T', offsety=k * n) # wnm[:,k] = Lk^-1 * wnm[:,k] blas.trsv(H[k], wnm, offsetx=k * n) # wnm := S^-1 * wnm (an mn-vector) lapack.potrs(S, wnm) # y := y + Y * wnm # = y + D^-1 * [ D1^2 * X * L1^-T ... D2^k * X * Lk^-T] # * wnm for k in range(m): # wnm[:,k] = Lk^-T * wnm[:,k] blas.trsv(H[k], wnm, trans='T', offsetx=k * n) # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = D^-1 * Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.tbsv(D, wN, n=N, k=0, ldA=1) # y += wN blas.axpy(wN, y) # y := D^-1 * y blas.tbsv(D, y, n=N, k=0, ldA=1) # For k = 1, ..., m: # # xk = (I - Dk^2 * X * Hk^-1 * X') * (-Dk^2 * y + xk) # x = x - [ D1^2 * y ... Dm^2 * y] (as an N x m matrix) for k in range(m): blas.copy(y, wN) blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) blas.axpy(wN, x, -1.0, offsety=k * N) # wnm = X' * x (as an n x m matrix) blas.gemm(X, x, wnm, transA='T', m=n, n=m, k=N, ldB=N, ldC=n) # wnm[:,k] = Hk^-1 * wnm[:,k] for k in range(m): lapack.potrs(H[k], wnm, offsetB=n * k) for k in range(m): # wN = X * wnm[:,k] blas.gemv(X, wnm, wN, offsetx=k * n) # wN = Dk^2 * wN blas.tbmv(dsq, wN, n=N, k=0, ldA=1, offsetA=k * N) # x[:,k] := x[:,k] - wN blas.axpy(wN, x, -1.0, n=N, offsety=k * N) # z := ( x - z ) ./ d blas.axpy(x, z, -1.0) blas.scal(-1.0, z) blas.tbsv(d, z, n=N * m, k=0, ldA=1) ### utime, stime = cputime() print("Solve: utime = %.2f, stime = %.2f" \ %(utime-utime0, stime-stime0)) ### return f
def F(W): """ Generate a solver for A'(uz0) = bx[0] -uz0 - uz1 = bx[1] A(ux[0]) - ux[1] - r0*r0' * uz0 * r0*r0' = bz0 - ux[1] - r1*r1' * uz1 * r1*r1' = bz1. uz0, uz1, bz0, bz1 are symmetric m x m-matrices. ux[0], bx[0] are n-vectors. ux[1], bx[1] are symmetric m x m-matrices. We first calculate a congruence that diagonalizes r0*r0' and r1*r1': U' * r0 * r0' * U = I, U' * r1 * r1' * U = S. We then make a change of variables usx[0] = ux[0], usx[1] = U' * ux[1] * U usz0 = U^-1 * uz0 * U^-T usz1 = U^-1 * uz1 * U^-T and define As() = U' * A() * U' bsx[1] = U^-1 * bx[1] * U^-T bsz0 = U' * bz0 * U bsz1 = U' * bz1 * U. This gives As'(usz0) = bx[0] -usz0 - usz1 = bsx[1] As(usx[0]) - usx[1] - usz0 = bsz0 -usx[1] - S * usz1 * S = bsz1. 1. Eliminate usz0, usz1 using equations 3 and 4, usz0 = As(usx[0]) - usx[1] - bsz0 usz1 = -S^-1 * (usx[1] + bsz1) * S^-1. This gives two equations in usx[0] an usx[1]. As'(As(usx[0]) - usx[1]) = bx[0] + As'(bsz0) -As(usx[0]) + usx[1] + S^-1 * usx[1] * S^-1 = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1. 2. Eliminate usx[1] using equation 2: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 i.e., with Gamma[i,j] = 1.0 + S[i,i] * S[j,j], usx[1] = ( S * As(usx[0]) * S ) ./ Gamma + ( S * ( bsx[1] - bsz0 ) * S - bsz1 ) ./ Gamma. This gives an equation in usx[0]. As'( As(usx[0]) ./ Gamma ) = bx0 + As'(bsz0) + As'( (S * ( bsx[1] - bsz0 ) * S - bsz1) ./ Gamma ) = bx0 + As'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ Gamma ). """ # Calculate U s.t. # # U' * r0*r0' * U = I, U' * r1*r1' * U = diag(s). # Cholesky factorization r0 * r0' = L * L' blas.syrk(W['r'][0], L) lapack.potrf(L) # SVD L^-1 * r1 = U * diag(s) * V' blas.copy(W['r'][1], U) blas.trsm(L, U) lapack.gesvd(U, s, jobu = 'O') # s := s**2 s[:] = s**2 # Uti := U blas.copy(U, Uti) # U := L^-T * U blas.trsm(L, U, transA = 'T') # Uti := L * Uti = U^-T blas.trmm(L, Uti) # Us := U * diag(s)^-1 blas.copy(U, Us) for i in range(m): blas.tbsv(s, Us, n = m, k = 0, ldA = 1, incx = m, offsetx = i) # S is m x m with lower triangular entries s[i] * s[j] # sqrtG is m x m with lower triangular entries sqrt(1.0 + s[i]*s[j]) # Upper triangular entries are undefined but nonzero. blas.scal(0.0, S) blas.syrk(s, S) Gamma = 1.0 + S sqrtG = sqrt(Gamma) # Asc[i] = (U' * Ai * * U ) ./ sqrtG, for i = 1, ..., n # = Asi ./ sqrt(Gamma) blas.copy(A, Asc) misc.scale(Asc, # only 'r' part of the dictionary is used {'dnl': matrix(0.0, (0, 1)), 'dnli': matrix(0.0, (0, 1)), 'd': matrix(0.0, (0, 1)), 'di': matrix(0.0, (0, 1)), 'v': [], 'beta': [], 'r': [ U ], 'rti': [ U ]}) for i in range(n): blas.tbsv(sqrtG, Asc, n = msq, k = 0, ldA = 1, offsetx = i*msq) # Convert columns of Asc to packed storage misc.pack2(Asc, {'l': 0, 'q': [], 's': [ m ]}) # Cholesky factorization of Asc' * Asc. H = matrix(0.0, (n, n)) blas.syrk(Asc, H, trans = 'T', k = mpckd) lapack.potrf(H) def solve(x, y, z): """ 1. Solve for usx[0]: Asc'(Asc(usx[0])) = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) = bx0 + Asc'( ( bsz0 + S * ( bsx[1] - bssz1) S ) ./ sqrtG) where bsx[1] = U^-1 * bx[1] * U^-T, bsz0 = U' * bz0 * U, bsz1 = U' * bz1 * U, bssz1 = S^-1 * bsz1 * S^-1 2. Solve for usx[1]: usx[1] + S * usx[1] * S = S * ( As(usx[0]) + bsx[1] - bsz0 ) * S - bsz1 usx[1] = ( S * (As(usx[0]) + bsx[1] - bsz0) * S - bsz1) ./ Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 - bsz1 + S * bsx[1] * S ) . / Gamma = -bsz0 + (S * As(usx[0]) * S) ./ Gamma + (bsz0 + S * ( bsx[1] - bssz1 ) * S ) . / Gamma Unscale ux[1] = Uti * usx[1] * Uti' 3. Compute usz0, usz1 r0' * uz0 * r0 = r0^-1 * ( A(ux[0]) - ux[1] - bz0 ) * r0^-T r1' * uz1 * r1 = r1^-1 * ( -ux[1] - bz1 ) * r1^-T """ # z0 := U' * z0 * U # = bsz0 __cngrnc(U, z, trans = 'T') # z1 := Us' * bz1 * Us # = S^-1 * U' * bz1 * U * S^-1 # = S^-1 * bsz1 * S^-1 __cngrnc(Us, z, trans = 'T', offsetx = msq) # x[1] := Uti' * x[1] * Uti # = bsx[1] __cngrnc(Uti, x[1], trans = 'T') # x[1] := x[1] - z[msq:] # = bsx[1] - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], alpha = -1.0, offsetx = msq) # x1 = (S * x[1] * S + z[:msq] ) ./ sqrtG # = (S * ( bsx[1] - S^-1 * bsz1 * S^-1) * S + bsz0 ) ./ sqrtG # = (S * bsx[1] * S - bsz1 + bsz0 ) ./ sqrtG # in packed storage blas.copy(x[1], x1) blas.tbmv(S, x1, n = msq, k = 0, ldA = 1) blas.axpy(z, x1, n = msq) blas.tbsv(sqrtG, x1, n = msq, k = 0, ldA = 1) misc.pack2(x1, {'l': 0, 'q': [], 's': [m]}) # x[0] := x[0] + Asc'*x1 # = bx0 + Asc'( ( bsz0 - bsz1 + S * bsx[1] * S ) ./ sqrtG) # = bx0 + As'( ( bz0 - bz1 + S * bx[1] * S ) ./ Gamma ) blas.gemv(Asc, x1, x[0], m = mpckd, trans = 'T', beta = 1.0) # x[0] := H^-1 * x[0] # = ux[0] lapack.potrs(H, x[0]) # x1 = Asc(x[0]) .* sqrtG (unpacked) # = As(x[0]) blas.gemv(Asc, x[0], tmp, m = mpckd) misc.unpack(tmp, x1, {'l': 0, 'q': [], 's': [m]}) blas.tbmv(sqrtG, x1, n = msq, k = 0, ldA = 1) # usx[1] = (x1 + (x[1] - z[:msq])) ./ sqrtG**2 # = (As(ux[0]) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1) # ./ Gamma # x[1] := x[1] - z[:msq] # = bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(z, x[1], -1.0, n = msq) # x[1] := x[1] + x1 # = As(ux) + bsx[1] - bsz0 - S^-1 * bsz1 * S^-1 blas.axpy(x1, x[1]) # x[1] := x[1] / Gammma # = (As(ux) + bsx[1] - bsz0 + S^-1 * bsz1 * S^-1 ) / Gamma # = S^-1 * usx[1] * S^-1 blas.tbsv(Gamma, x[1], n = msq, k = 0, ldA = 1) # z[msq:] := r1' * U * (-z[msq:] - x[1]) * U * r1 # := -r1' * U * S^-1 * (bsz1 + ux[1]) * S^-1 * U * r1 # := -r1' * uz1 * r1 blas.axpy(x[1], z, n = msq, offsety = msq) blas.scal(-1.0, z, offset = msq) __cngrnc(U, z, offsetx = msq) __cngrnc(W['r'][1], z, trans = 'T', offsetx = msq) # x[1] := S * x[1] * S # = usx1 blas.tbmv(S, x[1], n = msq, k = 0, ldA = 1) # z[:msq] = r0' * U' * ( x1 - x[1] - z[:msq] ) * U * r0 # = r0' * U' * ( As(ux) - usx1 - bsz0 ) * U * r0 # = r0' * U' * usz0 * U * r0 # = r0' * uz0 * r0 blas.axpy(x1, z, -1.0, n = msq) blas.scal(-1.0, z, n = msq) blas.axpy(x[1], z, -1.0, n = msq) __cngrnc(U, z) __cngrnc(W['r'][0], z, trans = 'T') # x[1] := Uti * x[1] * Uti' # = ux[1] __cngrnc(Uti, x[1]) return solve
def F(W): # SVD R[j] = U[j] * diag(sig[j]) * Vt[j] lapack.gesvd(+W['r'][0], sv, jobu='A', jobvt='A', U=U, Vt=Vt) # Vt[j] := diag(sig[j])^-1 * Vt[j] for k in xrange(ns): blas.tbsv(sv, Vt, n=ns, k=0, ldA=1, offsetx=k * ns) # Gamma[j] is an ns[j] x ns[j] symmetric matrix # # (sig[j] * sig[j]') ./ sqrt(1 + rho * (sig[j] * sig[j]').^2) # S = sig[j] * sig[j]' S = matrix(0.0, (ns, ns)) blas.syrk(sv, S) Gamma = div(S, sqrt(1.0 + rho * S**2)) symmetrize(Gamma, ns) # As represents the scaled mapping # # As(x) = A(u * (Gamma .* x) * u') # As'(y) = Gamma .* (u' * A'(y) * u) # # stored in a similar format as A, except that we use packed # storage for the columns of As[i][j]. if type(A) is spmatrix: blas.scal(0.0, As) try: As[VecAIndex] = +A['s'][VecAIndex] except: As[VecAIndex] = +A[VecAIndex] else: blas.copy(A, As) # As[i][j][:,k] = diag( diag(Gamma[j]))*As[i][j][:,k] # As[i][j][l,:] = Gamma[j][l,l]*As[i][j][l,:] for k in xrange(ms): cngrnc(U, As, trans='T', offsetx=k * (ns2)) blas.tbmv(Gamma, As, n=ns2, k=0, ldA=1, offsetx=k * (ns2)) misc.pack(As, Aspkd, {'l': 0, 'q': [], 's': [ns] * ms}) # H is an m times m block matrix with i, k block # # Hik = sum_j As[i,j]' * As[k,j] # # of size ms[i] x ms[k]. Hik = 0 if As[i,j] or As[k,j] # are zero for all j H = matrix(0.0, (ms, ms)) blas.syrk(Aspkd, H, trans='T', beta=1.0, k=ns * (ns + 1) / 2) lapack.potrf(H) def solve(x, y, z): """ Returns solution of rho * ux + A'(uy) - r^-T * uz * r^-1 = bx A(ux) = by -ux - r * uz * r' = bz. On entry, x = bx, y = by, z = bz. On exit, x = ux, y = uy, z = uz. """ # bz is a copy of z in the format of x blas.copy(z, bz) blas.axpy(bz, x, alpha=rho) # x := Gamma .* (u' * x * u) # = Gamma .* (u' * (bx + rho * bz) * u) cngrnc(U, x, trans='T', offsetx=0) blas.tbmv(Gamma, x, n=ns2, k=0, ldA=1, offsetx=0) # y := y - As(x) # := by - As( Gamma .* u' * (bx + rho * bz) * u) #blas.copy(x,xp) #pack_ip(xp,n = ns,m=1,nl=nl) misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]}) blas.gemv(Aspkd, xp, y, trans = 'T',alpha = -1.0, beta = 1.0, \ m = ns*(ns+1)/2, n = ms,offsetx = 0) # y := -y - A(bz) # = -by - A(bz) + As(Gamma .* (u' * (bx + rho * bz) * u) Af(bz, y, alpha=-1.0, beta=-1.0) # y := H^-1 * y # = H^-1 ( -by - A(bz) + As(Gamma.* u'*(bx + rho*bz)*u) ) # = uy blas.trsv(H, y) blas.trsv(H, y, trans='T') # bz = Vt' * vz * Vt # = uz where # vz := Gamma .* ( As'(uy) - x ) # = Gamma .* ( As'(uy) - Gamma .* (u'*(bx + rho *bz)*u) ) # = Gamma.^2 .* ( u' * (A'(uy) - bx - rho * bz) * u ). #blas.copy(x,xp) #pack_ip(xp,n=ns,m=1,nl=nl) misc.pack(x, xp, {'l': 0, 'q': [], 's': [ns]}) blas.scal(-1.0, xp) blas.gemv(Aspkd, y, xp, alpha=1.0, beta=1.0, m=ns * (ns + 1) / 2, n=ms, offsety=0) # bz[j] is xp unpacked and multiplied with Gamma misc.unpack(xp, bz, {'l': 0, 'q': [], 's': [ns]}) blas.tbmv(Gamma, bz, n=ns2, k=0, ldA=1, offsetx=0) # bz = Vt' * bz * Vt # = uz cngrnc(Vt, bz, trans='T', offsetx=0) symmetrize(bz, ns, offset=0) # x = -bz - r * uz * r' # z contains r.h.s. bz; copy to x blas.copy(z, x) blas.copy(bz, z) cngrnc(W['r'][0], bz, offsetx=0) blas.axpy(bz, x) blas.scal(-1.0, x) return solve
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 G' ] [ x ] = [ bx ] # [ G -W'*W ] [ z ] [ bz ]. # First factor # # S = G' * W**-1 * W**-T * G # = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1 # # where # # W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n] # W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:] # W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J) # with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I]. # As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A] minor = 0 if not helpers.sp_minor_empty(): minor = helpers.sp_minor_top() beta, v = W['beta'][0], W['v'][0] As = 2 * v * (v[1:].T * A) As[1:, :] *= -1.0 As[1:, :] -= A As /= beta # S = As'*As + 4 * (W1**2 + W2**2)**-1 S = As.T * As helpers.sp_add_var("S", S) d1, d2 = W['d'][:n], W['d'][n:] d = 4.0 * (d1**2 + d2**2)**-1 S[::n + 1] += d lapack.potrf(S) helpers.sp_create("00-Fkkt", minor) def f(x, y, z): minor = 0 if not helpers.sp_minor_empty(): minor = helpers.sp_minor_top() else: global loopf loopf += 1 minor = loopf helpers.sp_create("00-f", minor) # z := - W**-T * z z[:n] = -div(z[:n], d1) z[n:2 * n] = -div(z[n:2 * n], d2) z[2 * n:] -= 2.0 * v * (v[0] * z[2 * n] - blas.dot(v[1:], z[2 * n + 1:])) z[2 * n + 1:] *= -1.0 z[2 * n:] /= beta # x := x - G' * W**-1 * z x[:n] -= div(z[:n], d1) - div(z[n:2 * n], d2) + As.T * z[-(m + 1):] x[n:] += div(z[:n], d1) + div(z[n:2 * n], d2) helpers.sp_create("15-f", minor) # Solve for x[:n]: # # S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:] x[:n] -= mul(div(d1**2 - d2**2, d1**2 + d2**2), x[n:]) helpers.sp_create("25-f", minor) lapack.potrs(S, x) helpers.sp_create("30-f", minor) # Solve for x[n:]: # # (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n] x[n:] += mul(d1**-2 - d2**-2, x[:n]) helpers.sp_create("35-f", minor) x[n:] = div(x[n:], d1**-2 + d2**-2) helpers.sp_create("40-f", minor) # z := z + W^-T * G*x z[:n] += div(x[:n] - x[n:2 * n], d1) helpers.sp_create("44-f", minor) z[n:2 * n] += div(-x[:n] - x[n:2 * n], d2) helpers.sp_create("48-f", minor) z[2 * n:] += As * x[:n] helpers.sp_create("50-f", minor) return f
def custom_kkt(W): """ Custom KKT solver for the following conic LP formulation of the Schur relaxation of the balance-constrained min/max cut problem maximize Tr(C,X) subject to X_{ii} = 1, i=1...n sum(X) + x = const x >= 0, X psd """ r = W['rti'][0] N = r.size[0] e = matrix(1.0, (N, 1)) # Form and factorize reduced KKT system H = matrix(0.0, (N + 1, N + 1)) blas.syrk(r, H, n=N, ldC=N + 1) blas.symv(H, e, H, n=N, ldA=N + 1, offsety=N, incy=N + 1) H[N, N] = blas.dot(H, e, n=N, offsetx=N, incx=N + 1) rr = H[:N, :N] # Extract and symmetrize (1,1) block misc.symm(rr, N) # q = H[N, :N].T # Extract q = rr*e H = mul(H, H) H[N, N] += W['di'][0]**2 lapack.potrf(H) def fsolve(x, y, z): """ Solves the system of equations [ 0 G'*W^{-1} ] [ ux ] = [ bx ] [ G -W' ] [ uz ] [ bz ] """ # Compute bx := bx + G'*W^{-1}*W^{-T}*bz v = matrix(0., (N, 1)) for i in range(N): blas.symv(z, rr, v, ldA=N, offsetA=1, n=N, offsetx=N * i) x[i] += blas.dot(rr, v, n=N, offsetx=N * i) blas.symv(z, q, v, ldA=N, offsetA=1, n=N) x[N] += blas.dot(q, v) + z[0] * W['di'][0]**2 # Solve G'*W^{-1}*W^{-T}*G*ux = bx lapack.potrs(H, x) # Compute bz := -W^{-T}*(bz-G*ux) # z -= G*x z[1::N + 1] -= x[:-1] z -= x[-1] # Apply scaling z[0] *= -W['di'][0] blas.scal(0.5, z, n=N, offset=1, inc=N + 1) tmp = +r blas.trmm(z, tmp, ldA=N, offsetA=1, n=N, m=N) blas.syr2k(r, tmp, z, trans='T', offsetC=1, ldC=N, n=N, k=N, alpha=-1.0) return fsolve
H1[3:,2] = -2.0 * c[1] * B[:,1] H1[3:,3:] = 2*B return f, Df, z[0]*H0 + sum(z[1:])*H1 sol = solvers.cp(F) A = matrix( sol['x'][[0, 1, 1, 2]], (2,2)) b = sol['x'][3:] if pylab_installed: pylab.figure(1, facecolor='w') pylab.plot(X[:,0], X[:,1], 'ko', X[:,0], X[:,1], '-k') # Ellipsoid in the form { x | || L' * (x-c) ||_2 <= 1 } L = +A lapack.potrf(L) c = +b lapack.potrs(L, c) # 1000 points on the unit circle nopts = 1000 angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) ) circle = matrix(0.0, (2,nopts)) circle[0,:], circle[1,:] = cos(angles), sin(angles) # ellipse = L^-T * circle + c blas.trsm(L, circle, transA='T') ellipse = circle + c[:, nopts*[0]] ellipse2 = 0.5 * circle + c[:, nopts*[0]] pylab.plot(ellipse[0,:].T, ellipse[1,:].T, 'k-')
Hac = G.T * spdiag((h-G*xac)**-1) * G if pylab_installed: pylab.figure(3, facecolor='w') # polyhedron for k in range(m): edge = X[[k,k+1],:] + 0.1 * matrix([1., 0., 0., -1.], (2,2)) * \ (X[2*[k],:] - X[2*[k+1],:]) pylab.plot(edge[:,0], edge[:,1], 'k') # 1000 points on the unit circle nopts = 1000 angles = matrix( [ a*2.0*pi/nopts for a in range(nopts) ], (1,nopts) ) circle = matrix(0.0, (2,nopts)) circle[0,:], circle[1,:] = cos(angles), sin(angles) # ellipse = L^-T * circle + xc where Hac = L*L' lapack.potrf(Hac) ellipse = +circle blas.trsm(Hac, ellipse, transA='T') ellipse += xac[:, nopts*[0]] pylab.fill(ellipse[0,:].T, ellipse[1,:].T, facecolor = '#F0F0F0') pylab.plot([xac[0]], [xac[1]], 'ko') pylab.title('Analytic center (fig 8.7)') pylab.axis('equal') pylab.axis('off') pylab.show()
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 G' ] [ x ] = [ bx ] # [ G -W'*W ] [ z ] [ bz ]. # First factor # # S = G' * W**-1 * W**-T * G # = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1 # # where # # W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n] # W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:] # W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J) # with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I]. # As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A] beta, v = W['beta'][0], W['v'][0] As = 2 * v * (v[1:].T * A) As[1:,:] *= -1.0 As[1:,:] -= A As /= beta # S = As'*As + 4 * (W1**2 + W2**2)**-1 S = As.T * As d1, d2 = W['d'][:n], W['d'][n:] d = 4.0 * (d1**2 + d2**2)**-1 S[::n+1] += d lapack.potrf(S) def f(x, y, z): # z := - W**-T * z z[:n] = -div( z[:n], d1 ) z[n:2*n] = -div( z[n:2*n], d2 ) z[2*n:] -= 2.0*v*( v[0]*z[2*n] - blas.dot(v[1:], z[2*n+1:]) ) z[2*n+1:] *= -1.0 z[2*n:] /= beta # x := x - G' * W**-1 * z x[:n] -= div(z[:n], d1) - div(z[n:2*n], d2) + As.T * z[-(m+1):] x[n:] += div(z[:n], d1) + div(z[n:2*n], d2) # Solve for x[:n]: # # S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:] x[:n] -= mul( div(d1**2 - d2**2, d1**2 + d2**2), x[n:]) lapack.potrs(S, x) # Solve for x[n:]: # # (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n] x[n:] += mul( d1**-2 - d2**-2, x[:n]) x[n:] = div( x[n:], d1**-2 + d2**-2) # z := z + W^-T * G*x z[:n] += div( x[:n] - x[n:2*n], d1) z[n:2*n] += div( -x[:n] - x[n:2*n], d2) z[2*n:] += As*x[:n] return f
def F(x=None, z=None): if x is None: return m, matrix([1.0, 0.0, 1.0, 0.0, 0.0]) # Factor A as A = L*L'. Compute inverse B = A^-1. A = matrix([x[0], x[1], x[1], x[2]], (2, 2)) L = +A try: lapack.potrf(L) except: return None B = +L lapack.potri(B) B[0, 1] = B[1, 0] # f0 = -log det A f = matrix(0.0, (m + 1, 1)) f[0] = -2.0 * (log(L[0, 0]) + log(L[1, 1])) # fk = xk'*A*xk - 2*xk'*b + b*A^-1*b - 1 # = (xk - c)' * A * (xk - c) - 1 where c = A^-1*b c = x[3:] lapack.potrs(L, c) for k in range(m): f[k + 1] = (X[k, :].T - c).T * A * (X[k, :].T - c) - 1.0 # gradf0 = (-A^-1, 0) = (-B, 0) Df = matrix(0.0, (m + 1, 5)) Df[0, 0], Df[0, 1], Df[0, 2] = -B[0, 0], -2.0 * B[1, 0], -B[1, 1] # gradfk = (xk*xk' - A^-1*b*b'*A^-1, 2*(-xk + A^-1*b)) # = (xk*xk' - c*c', 2*(-xk+c)) Df[1:, 0] = X[:m, 0]**2 - c[0]**2 Df[1:, 1] = 2.0 * (mul(X[:m, 0], X[:m, 1]) - c[0] * c[1]) Df[1:, 2] = X[:m, 1]**2 - c[1]**2 Df[1:, 3] = 2.0 * (-X[:m, 0] + c[0]) Df[1:, 4] = 2.0 * (-X[:m, 1] + c[1]) if z is None: return f, Df # hessf0(Y, y) = (A^-1*Y*A^-1, 0) = (B*YB, 0) H0 = matrix(0.0, (5, 5)) H0[0, 0] = B[0, 0]**2 H0[1, 0] = 2.0 * B[0, 0] * B[1, 0] H0[2, 0] = B[1, 0]**2 H0[1, 1] = 2.0 * (B[0, 0] * B[1, 1] + B[1, 0]**2) H0[2, 1] = 2.0 * B[1, 0] * B[1, 1] H0[2, 2] = B[1, 1]**2 # hessfi(Y, y) # = ( A^-1*Y*A^-1*b*b'*A^-1 + A^-1*b*b'*A^-1*Y*A^-1 # - A^-1*y*b'*A^-1 - A^-1*b*y'*A^-1, # -2*A^-1*Y*A^-1*b + 2*A^-1*y ) # = ( B*Y*c*c' + c*c'*Y*B - B*y*c' - c*y'*B, -2*B*Y*c + 2*B*y ) # = ( B*(Y*c-y)*c' + c*(Y*c-y)'*B, -2*B*(Y*c - y) ) H1 = matrix(0.0, (5, 5)) H1[0, 0] = 2.0 * c[0]**2 * B[0, 0] H1[1, 0] = 2.0 * (c[0] * c[1] * B[0, 0] + c[0]**2 * B[1, 0]) H1[2, 0] = 2.0 * c[0] * c[1] * B[1, 0] H1[3:, 0] = -2.0 * c[0] * B[:, 0] H1[1,1] = 2.0 * c[0]**2 * B[1,1] + 4.0 * c[0]*c[1]*B[1,0] + \ 2.0 * c[1]**2 + B[0,0] H1[2, 1] = 2.0 * (c[1]**2 * B[1, 0] + c[0] * c[1] * B[1, 1]) H1[3:, 1] = -2.0 * B * c[[1, 0]] H1[2, 2] = 2.0 * c[1]**2 * B[1, 1] H1[3:, 2] = -2.0 * c[1] * B[:, 1] H1[3:, 3:] = 2 * B return f, Df, z[0] * H0 + sum(z[1:]) * H1
def cholesky(A): """ Cholesky with clean-up """ lapack.potrf(A) makeLT(A)
def F(W): """ Create a solver for the linear equations C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - Dl^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - r*r' * [ ] * r*r' = bzs [ -A(ux) -uX2 ] [ uzs21 uzs22 ] where Dl = diag(W['l']), r = W['r'][0]. On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ Dl*uzl; (r'*uzs*r)[:] ]. 1. Compute matrices V1, V2 such that (with T = r*r') [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and S = [ diag(s); 0 ], s a positive q-vector. 2. Factor the mapping X -> X + S * X' * S: X + S * X' * S = L( L'( X )). 3. Compute scaled mappings: a matrix As with as its columns the coefficients of the scaled mapping L^-1( V2' * A() * V1' ) and the matrix Gs = Dl^-1 * G. 4. Cholesky factorization of H = C + Gs'*Gs + 2*As'*As. """ # 1. Compute V1, V2, s. r = W['r'][0] # LQ factorization R[:q, :] = L1 * Q1. lapack.lacpy(r, Q1, m = q) lapack.gelqf(Q1, tau1) lapack.lacpy(Q1, L1, n = q, uplo = 'L') lapack.orglq(Q1, tau1) # LQ factorization R[q:, :] = L2 * Q2. lapack.lacpy(r, Q2, m = p, offsetA = q) lapack.gelqf(Q2, tau2) lapack.lacpy(Q2, L2, n = p, uplo = 'L') lapack.orglq(Q2, tau2) # V2, V1, s are computed from an SVD: if # # Q2 * Q1' = U * diag(s) * V', # # then V1 = V' * L1^-1 and V2 = L2^-T * U. # T21 = Q2 * Q1.T blas.gemm(Q2, Q1, T21, transB = 'T') # SVD T21 = U * diag(s) * V'. Store U in V2 and V' in V1. lapack.gesvd(T21, s, jobu = 'A', jobvt = 'A', U = V2, Vt = V1) # # Q2 := Q2 * Q1' without extracting Q1; store T21 in Q2 # this will requires lapack.ormlq or lapack.unmlq # V2 = L2^-T * U blas.trsm(L2, V2, transA = 'T') # V1 = V' * L1^-1 blas.trsm(L1, V1, side = 'R') # 2. Factorization X + S * X' * S = L( L'( X )). # # The factor L is stored as a diagonal matrix D and a sparse lower # triangular matrix P, such that # # L(X)[:] = D**-1 * (I + P) * X[:] # L^-1(X)[:] = D * (I - P) * X[:]. # SS is q x q with SS[i,j] = si*sj. blas.scal(0.0, SS) blas.syr(s, SS) # For a p x q matrix X, P*X[:] is Y[:] where # # Yij = si * sj * Xji if i < j # = 0 otherwise. # P.V = SS[Itril2] # For a p x q matrix X, D*X[:] is Y[:] where # # Yij = Xij / sqrt( 1 - si^2 * sj^2 ) if i < j # = Xii / sqrt( 1 + si^2 ) if i = j # = Xij otherwise. # DV[Idiag] = sqrt(1.0 + SS[::q+1]) DV[Itriu] = sqrt(1.0 - SS[Itril3]**2) D.V = DV**-1 # 3. Scaled linear mappings # Ask := V2' * Ask * V1' blas.scal(0.0, As) base.axpy(A, As) for i in xrange(n): # tmp := V2' * As[i, :] blas.gemm(V2, As, tmp, transA = 'T', m = p, n = q, k = p, ldB = p, offsetB = i*p*q) # As[:,i] := tmp * V1' blas.gemm(tmp, V1, As, transB = 'T', m = p, n = q, k = q, ldC = p, offsetC = i*p*q) # As := D * (I - P) * As # = L^-1 * As. blas.copy(As, As2) base.gemm(P, As, As2, alpha = -1.0, beta = 1.0) base.gemm(D, As2, As) # Gs := Dl^-1 * G blas.scal(0.0, Gs) base.axpy(G, Gs) for k in xrange(n): blas.tbmv(W['di'], Gs, n = m, k = 0, ldA = 1, offsetx = k*m) # 4. Cholesky factorization of H = C + Gs' * Gs + 2 * As' * As. blas.syrk(As, H, trans = 'T', alpha = 2.0) blas.syrk(Gs, H, trans = 'T', beta = 1.0) base.axpy(C, H) lapack.potrf(H) def f(x, y, z): """ Solve C * ux + G' * uzl - 2*A'(uzs21) = bx -uzs11 = bX1 -uzs22 = bX2 G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ uzs11 uzs21' ] [ ] - T * [ ] * T = bzs. [ -A(ux) -uX2 ] [ uzs21 uzs22 ] On entry, x = (bx, bX1, bX2) and z = [ bzl; bzs[:] ]. On exit, x = (ux, uX1, uX2) and z = [ D*uzl; (r'*uzs*r)[:] ]. Define X = uzs21, Z = T * uzs * T: C * ux + G' * uzl - 2*A'(X) = bx [ 0 X' ] [ bX1 0 ] T * [ ] * T - Z = T * [ ] * T [ X 0 ] [ 0 bX2 ] G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs [ -A(ux) -uX2 ] [ Z21 Z22 ] Return x = (ux, uX1, uX2), z = [ D*uzl; (rti'*Z*rti)[:] ]. We use the congruence transformation [ V1 0 ] [ T11 T21' ] [ V1' 0 ] [ I S' ] [ ] [ ] [ ] = [ ] [ 0 V2' ] [ T21 T22 ] [ 0 V2 ] [ S I ] and the factorization X + S * X' * S = L( L'(X) ) to write this as C * ux + G' * uzl - 2*A'(X) = bx L'(V2^-1 * X * V1^-1) - L^-1(V2' * Z21 * V1') = bX G * ux - D^2 * uzl = bzl [ -uX1 -A(ux)' ] [ Z11 Z21' ] [ ] - [ ] = bzs, [ -A(ux) -uX2 ] [ Z21 Z22 ] or C * ux + Gs' * uuzl - 2*As'(XX) = bx XX - ZZ21 = bX Gs * ux - uuzl = D^-1 * bzl -As(ux) - ZZ21 = bbzs_21 -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 if we introduce scaled variables uuzl = D * uzl XX = L'(V2^-1 * X * V1^-1) = L'(V2^-1 * uzs21 * V1^-1) ZZ21 = L^-1(V2' * Z21 * V1') and define bbzs_21 = L^-1(V2' * bzs_21 * V1') [ bX1 0 ] bX = L^-1( V2' * (T * [ ] * T)_21 * V1'). [ 0 bX2 ] Eliminating Z21 gives C * ux + Gs' * uuzl - 2*As'(XX) = bx Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22 and eliminating uuzl and XX gives H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bX - bbzs_21) Gs * ux - uuzl = D^-1 * bzl -As(ux) - XX = bbzs_21 - bX -uX1 - Z11 = bzs_11 -uX2 - Z22 = bzs_22. In summary, we can use the following algorithm: 1. bXX := bX - bbzs21 [ bX1 0 ] = L^-1( V2' * ((T * [ ] * T)_21 - bzs_21) * V1') [ 0 bX2 ] 2. Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). 3. From ux, compute uuzl = Gs*ux - D^-1 * bzl and X = V2 * L^-T(-As(ux) + bXX) * V1. 4. Return ux, uuzl, rti' * Z * rti = r' * [ -bX1, X'; X, -bX2 ] * r and uX1 = -Z11 - bzs_11, uX2 = -Z22 - bzs_22. """ # Save bzs_11, bzs_22, bzs_21. lapack.lacpy(z, bz11, uplo = 'L', m = q, n = q, ldA = p+q, offsetA = m) lapack.lacpy(z, bz21, m = p, n = q, ldA = p+q, offsetA = m+q) lapack.lacpy(z, bz22, uplo = 'L', m = p, n = p, ldA = p+q, offsetA = m + (p+q+1)*q) # zl := D^-1 * zl # = D^-1 * bzl blas.tbmv(W['di'], z, n = m, k = 0, ldA = 1) # zs := r' * [ bX1, 0; 0, bX2 ] * r. # zs := [ bX1, 0; 0, bX2 ] blas.scal(0.0, z, offset = m) lapack.lacpy(x[1], z, uplo = 'L', m = q, n = q, ldB = p+q, offsetB = m) lapack.lacpy(x[2], z, uplo = 'L', m = p, n = p, ldB = p+q, offsetB = m + (p+q+1)*q) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc = p+q+1, offset = m) # a := tril(zs)*r blas.copy(r, a) blas.trmm(z, a, side = 'L', m = p+q, n = p+q, ldA = p+q, ldB = p+q, offsetA = m) # zs := a'*r + r'*a blas.syr2k(r, a, z, trans = 'T', n = p+q, k = p+q, ldB = p+q, ldC = p+q, offsetC = m) # bz21 := L^-1( V2' * ((r * zs * r')_21 - bz21) * V1') # # [ bX1 0 ] # = L^-1( V2' * ((T * [ ] * T)_21 - bz21) * V1'). # [ 0 bX2 ] # a = [ r21 r22 ] * z # = [ r21 r22 ] * r' * [ bX1, 0; 0, bX2 ] * r # = [ T21 T22 ] * [ bX1, 0; 0, bX2 ] * r blas.symm(z, r, a, side = 'R', m = p, n = p+q, ldA = p+q, ldC = p+q, offsetB = q) # bz21 := -bz21 + a * [ r11, r12 ]' # = -bz21 + (T * [ bX1, 0; 0, bX2 ] * T)_21 blas.gemm(a, r, bz21, transB = 'T', m = p, n = q, k = p+q, beta = -1.0, ldA = p+q, ldC = p) # bz21 := V2' * bz21 * V1' # = V2' * (-bz21 + (T*[bX1, 0; 0, bX2]*T)_21) * V1' blas.gemm(V2, bz21, tmp, transA = 'T', m = p, n = q, k = p, ldB = p) blas.gemm(tmp, V1, bz21, transB = 'T', m = p, n = q, k = q, ldC = p) # bz21[:] := D * (I-P) * bz21[:] # = L^-1 * bz21[:] # = bXX[:] blas.copy(bz21, tmp) base.gemv(P, bz21, tmp, alpha = -1.0, beta = 1.0) base.gemv(D, tmp, bz21) # Solve H * ux = bx + Gs' * D^-1 * bzl + 2*As'(bXX). # x[0] := x[0] + Gs'*zl + 2*As'(bz21) # = bx + G' * D^-1 * bzl + 2 * As'(bXX) blas.gemv(Gs, z, x[0], trans = 'T', alpha = 1.0, beta = 1.0) blas.gemv(As, bz21, x[0], trans = 'T', alpha = 2.0, beta = 1.0) # x[0] := H \ x[0] # = ux lapack.potrs(H, x[0]) # uuzl = Gs*ux - D^-1 * bzl blas.gemv(Gs, x[0], z, alpha = 1.0, beta = -1.0) # bz21 := V2 * L^-T(-As(ux) + bz21) * V1 # = X blas.gemv(As, x[0], bz21, alpha = -1.0, beta = 1.0) blas.tbsv(DV, bz21, n = p*q, k = 0, ldA = 1) blas.copy(bz21, tmp) base.gemv(P, tmp, bz21, alpha = -1.0, beta = 1.0, trans = 'T') blas.gemm(V2, bz21, tmp) blas.gemm(tmp, V1, bz21) # zs := -zs + r' * [ 0, X'; X, 0 ] * r # = r' * [ -bX1, X'; X, -bX2 ] * r. # a := bz21 * [ r11, r12 ] # = X * [ r11, r12 ] blas.gemm(bz21, r, a, m = p, n = p+q, k = q, ldA = p, ldC = p+q) # z := -z + [ r21, r22 ]' * a + a' * [ r21, r22 ] # = rti' * uzs * rti blas.syr2k(r, a, z, trans = 'T', beta = -1.0, n = p+q, k = p, offsetA = q, offsetC = m, ldB = p+q, ldC = p+q) # uX1 = -Z11 - bzs_11 # = -(r*zs*r')_11 - bzs_11 # uX2 = -Z22 - bzs_22 # = -(r*zs*r')_22 - bzs_22 blas.copy(bz11, x[1]) blas.copy(bz22, x[2]) # scale diagonal of zs by 1/2 blas.scal(0.5, z, inc = p+q+1, offset = m) # a := r*tril(zs) blas.copy(r, a) blas.trmm(z, a, side = 'R', m = p+q, n = p+q, ldA = p+q, ldB = p+q, offsetA = m) # x[1] := -x[1] - a[:q,:] * r[:q, :]' - r[:q,:] * a[:q,:]' # = -bzs_11 - (r*zs*r')_11 blas.syr2k(a, r, x[1], n = q, alpha = -1.0, beta = -1.0) # x[2] := -x[2] - a[q:,:] * r[q:, :]' - r[q:,:] * a[q:,:]' # = -bzs_22 - (r*zs*r')_22 blas.syr2k(a, r, x[2], n = p, alpha = -1.0, beta = -1.0, offsetA = q, offsetB = q) # scale diagonal of zs by 1/2 blas.scal(2.0, z, inc = p+q+1, offset = m) return f
def completion(X, factored_updates = True): """ Supernodal multifrontal maximum determinant positive definite matrix completion. The routine computes the Cholesky factor :math:`L` of the inverse of the maximum determinant positive definite matrix completion of :math:`X`:, i.e., .. math:: P( S^{-1} ) = X where :math:`S = LL^T`. On exit, the argument `X` contains the lower-triangular Cholesky factor :math:`L`. The optional argument `factored_updates` can be used to enable (if True) or disable (if False) updating of intermediate factorizations. :param X: :py:class:`cspmatrix` :param factored_updates: boolean """ assert isinstance(X, cspmatrix) and X.is_factor is False, "X must be a cspmatrix" n = X.symb.n snpost = X.symb.snpost snptr = X.symb.snptr chptr = X.symb.chptr chidx = X.symb.chidx relptr = X.symb.relptr relidx = X.symb.relidx blkptr = X.symb.blkptr blkval = X.blkval stack = [] for k in reversed(list(snpost)): nn = snptr[k+1]-snptr[k] # |Nk| na = relptr[k+1]-relptr[k] # |Ak| nj = na + nn # allocate F and copy X_{Jk,Nk} to leading columns of F F = matrix(0.0, (nj,nj)) lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj, m = nj, n = nn, uplo = 'L') # if supernode k is not a root node: if na > 0: # copy Vk to 2,2 block of F Vk = stack.pop() lapack.lacpy(Vk, F, offsetB = nn*nj+nn, m = na, n = na, uplo = 'L') # if supernode k has any children: for ii in range(chptr[k],chptr[k+1]): i = chidx[ii] if factored_updates: r = relidx[relptr[i]:relptr[i+1]] stack.append(frontal_get_update_factor(F,r,nn,na)) else: stack.append(frontal_get_update(F,relidx,relptr,i)) # if supernode k is not a root node: if na > 0: if factored_updates: # In this case we have Vk = Lk'*Lk trL1 = 'T' trL2 = 'N' else: # factorize Vk lapack.potrf(F, offsetA = nj*nn+nn, n = na, ldA = nj) # In this case we have Vk = Lk*Lk' trL1 = 'N' trL2 = 'T' # compute L_{Ak,Nk} and inv(D_{Nk,Nk}) = S_{Nk,Nk} - S_{Ak,Nk}'*L_{Ak,Nk} lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL1,\ offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn) blas.syrk(blkval, blkval, n = nn, k = na, trans= 'T', alpha = -1.0, beta = 1.0, offsetA = blkptr[k]+nn, offsetC = blkptr[k], ldA = nj, ldC = nj) lapack.trtrs(F, blkval, offsetA = nj*nn+nn, trans = trL2,\ offsetB = blkptr[k]+nn, ldB = nj, n = na, nrhs = nn) for i in range(nn): blas.scal(-1.0, blkval, n = na, offset = blkptr[k] + i*nj + nn) # factorize inv(D_{Nk,Nk}) as R*R' so that D_{Nk,Nk} = L*L' with L = inv(R)' lapack.lacpy(blkval, F, offsetA = blkptr[k], ldA = nj,\ ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX! F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX! lapack.potrf(F, ldA = nj, n = nn, uplo = 'U') # factorize F[:nn,:nn] = matrix(F[:nn,:nn][::-1],(nn,nn)) # reverse -- FIX! lapack.lacpy(F, blkval, offsetB = blkptr[k], ldA = nj,\ ldB = nj, m = nn, n = nn, uplo = 'L') # copy -- FIX! # compute L = inv(R') lapack.trtri(blkval, offsetA = blkptr[k], ldA = nj, n = nn) X._is_factor = True return
def __scale(L, Y, U, adj=False, inv=False, factored_updates=True): n = L.symb.n snpost = L.symb.snpost snptr = L.symb.snptr chptr = L.symb.chptr chidx = L.symb.chidx relptr = L.symb.relptr relidx = L.symb.relidx blkptr = L.symb.blkptr stack = [] for k in reversed(list(snpost)): nn = snptr[k + 1] - snptr[k] # |Nk| na = relptr[k + 1] - relptr[k] # |Ak| nj = na + nn F = matrix(0.0, (nj, nj)) lapack.lacpy(Y.blkval, F, m=nj, n=nn, ldA=nj, offsetA=blkptr[k], uplo='L') # if supernode k is not a root node: if na > 0: # copy Vk to 2,2 block of F Vk = stack.pop() lapack.lacpy(Vk, F, ldB=nj, offsetB=nn * (nj + 1), m=na, n=na, uplo='L') # if supernode k has any children: for ii in range(chptr[k], chptr[k + 1]): i = chidx[ii] if factored_updates: r = relidx[relptr[i]:relptr[i + 1]] stack.append(frontal_get_update_factor(F, r, nn, na)) else: stack.append(frontal_get_update(F, relidx, relptr, i)) # if supernode k is not a root node: if na > 0: if factored_updates: # In this case we have Vk = Lk'*Lk if adj is False: trns = 'N' elif adj is True: trns = 'T' else: # factorize Vk lapack.potrf(F, offsetA=nj * nn + nn, n=na, ldA=nj) # In this case we have Vk = Lk*Lk' if adj is False: trns = 'T' elif adj is True: trns = 'N' if adj is False: tr = ['T', 'N'] elif adj is True: tr = ['N', 'T'] if inv is False: for Ut in U: # symmetrize (1,1) block of Ut_{k} and scale U11 = matrix(0.0, (nn, nn)) lapack.lacpy(Ut.blkval, U11, offsetA=blkptr[k], m=nn, n=nn, ldA=nj, uplo='L') U11 += U11.T U11[::nn + 1] *= 0.5 lapack.lacpy(U11, Ut.blkval, offsetB=blkptr[k], m=nn, n=nn, ldB=nj, uplo='N') blas.trsm(L.blkval, Ut.blkval, side = 'R', transA = tr[0],\ m = nj, n = nn, offsetA = blkptr[k], ldA = nj,\ offsetB = blkptr[k], ldB = nj) blas.trsm(L.blkval, Ut.blkval, m = nn, n = nn, transA = tr[1],\ offsetA = blkptr[k], offsetB = blkptr[k],\ ldA = nj, ldB = nj) # zero-out strict upper triangular part of {Nj,Nj} block for i in range(1, nn): blas.scal(0.0, Ut.blkval, offset=blkptr[k] + nj * i, n=i) if na > 0: blas.trmm(F, Ut.blkval, m = na, n = nn, transA = trns,\ offsetA = nj*nn+nn, ldA = nj,\ offsetB = blkptr[k]+nn, ldB = nj) else: # inv is True for Ut in U: # symmetrize (1,1) block of Ut_{k} and scale U11 = matrix(0.0, (nn, nn)) lapack.lacpy(Ut.blkval, U11, offsetA=blkptr[k], m=nn, n=nn, ldA=nj, uplo='L') U11 += U11.T U11[::nn + 1] *= 0.5 lapack.lacpy(U11, Ut.blkval, offsetB=blkptr[k], m=nn, n=nn, ldB=nj, uplo='N') blas.trmm(L.blkval, Ut.blkval, side = 'R', transA = tr[0],\ m = nj, n = nn, offsetA = blkptr[k], ldA = nj,\ offsetB = blkptr[k], ldB = nj) blas.trmm(L.blkval, Ut.blkval, m = nn, n = nn, transA = tr[1],\ offsetA = blkptr[k], offsetB = blkptr[k],\ ldA = nj, ldB = nj) # zero-out strict upper triangular part of {Nj,Nj} block for i in range(1, nn): blas.scal(0.0, Ut.blkval, offset=blkptr[k] + nj * i, n=i) if na > 0: blas.trsm(F, Ut.blkval, m = na, n = nn, transA = trns,\ offsetA = nj*nn+nn, ldA = nj,\ offsetB = blkptr[k]+nn, ldB = nj) return
def F(W): """ Custom solver for the system [ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ] [ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ] [ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ] [ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ] [ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ] [ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ] [ | | | . ][ | ] [ | ] [ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ] where It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ] [ 0 0 ] [ -Pi 0 ] dwt = [ dw ] rwt = [ rw ] [ dt ] [ rt ]. """ # scalings and 'intermediate' vectors # db = inv(Wl1)^2 + inv(Wl2)^2 db = W['di'][:m]**2 + W['di'][m:2*m]**2 dbi = div(1.0,db) # dt = I - inv(Wl1)*Dbi*inv(Wl1) dt = 1.0 - mul(W['di'][:m]**2,dbi) dtsqrt = sqrt(dt) # lam = Dt*inv(Wl1)*d lam = mul(dt,mul(W['di'][:m],d)) # lt = E'*inv(Wl1)*lam lt = matrix(0.0,(k,1)) base.gemv(E, mul(W['di'][:m],lam), lt, trans = 'T') # Xs = sqrt(Dt)*inv(Wl1)*X tmp = mul(dtsqrt,W['di'][:m]) Xs = spmatrix(tmp,range(m),range(m))*X # Es = D*sqrt(Dt)*inv(Wl1)*E Es = spmatrix(mul(d,tmp),range(m),range(m))*E # form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs # and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es # and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1)) Ab = matrix(0.0,(n,n)) Ab[::n+1] = 1.0 base.syrk(Xs,Ab,trans = 'T', beta = 1.0) Bb = matrix(0.0,(n,k)) Bb = -Xs.T*Es # inefficient!? D2 = spmatrix(0.0,range(k),range(k)) base.syrk(Es,D2,trans = 'T', partial = True) d2 = +D2.V del D2 py = matrix(0.0,(n,1)) for i in range(k): binvsq = (1.0/W['beta'][i])**2 Ab += binvsq*Pt[i] dvv = blas.dot(W['v'][i],W['v'][i]) blas.gemv(P[i], W['v'][i][1:], py, trans = 'T', alpha = 1.0, beta = 0.0) blas.syrk(py, Ab, alpha = 4*binvsq*(dvv+1), beta = 1.0) Bb[:,i] -= 4*binvsq*W['v'][i][0]*dvv*py d2[i] += binvsq*(1+4*(W['v'][i][0]**2)*(dvv-1)) d2i = div(1.0,d2) d2isqrt = sqrt(d2i) # compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam alpha = blas.dot(lam,mul(W['di'][:m],d)) tmp = matrix(0.0,(k,1)) base.gemv(E,mul(W['di'][:m],lam), tmp, trans = 'T') tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam a = alpha - blas.dot(tmp,tmp) # compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam tmp = mul(tmp, d2isqrt) M12 = matrix(0.0,(n,1)) blas.gemv(Bb,tmp,M12, alpha = 1.0) tmp = mul(d,mul(W['di'][:m],lam)) blas.gemv(X,tmp,M12, trans = 'T', alpha = 1.0, beta = 1.0) # form and factor M sBb = Bb * spmatrix(d2isqrt,range(k), range(k)) base.syrk(sBb, Ab, alpha = -1.0, beta = 1.0) M = matrix([[Ab, M12.T],[M12, a]]) lapack.potrf(M) def f(x,y,z): # residuals rwt = x[:n+k] rb = x[n+k] rv = x[n+k+1:n+k+1+m] iw_rl1 = mul(W['di'][:m],z[:m]) iw_rl2 = mul(W['di'][m:2*m],z[m:2*m]) ri = [z[2*m+i*(n+1):2*m+(i+1)*(n+1)] for i in range(k)] # compute 'derived' residuals # rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1 rbwt = +rwt for i in range(k): tmp = +ri[i] qscal(tmp,W['beta'][i],W['v'][i],inv=True) qscal(tmp,W['beta'][i],W['v'][i],inv=True) rbwt[n+i] -= tmp[0] blas.gemv(P[i], tmp[1:], rbwt, trans = 'T', alpha = -1.0, beta = 1.0) tmp = mul(W['di'][:m],iw_rl1) tmp2 = matrix(0.0,(k,1)) base.gemv(E,tmp,tmp2,trans='T') rbwt[n:] += tmp2 tmp = mul(d,tmp) # tmp = D*inv(Wl1)^2*rl1 blas.gemv(X,tmp,rbwt,trans='T', alpha = -1.0, beta = 1.0) # rbb = rb - d'*inv(Wl1)^2*rl1 rbb = rb - sum(tmp) # rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1 rbv = rv - mul(W['di'][m:2*m],iw_rl2) - mul(W['di'][:m],iw_rl1) # [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv tmp = mul(W['di'][:m]**2, mul(dbi,rbv)) rtt = +rbwt[n:] base.gemv(E, tmp, rtt, trans = 'T', alpha = 1.0, beta = 1.0) rtw = +rbwt[:n] tmp = mul(d,tmp) blas.gemv(X, tmp, rtw, trans = 'T', alpha = -1.0, beta = 1.0) # rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv rtb = rbb - sum(tmp) # solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt] tmp = mul(d2i,rtt) tmp2 = matrix(0.0,(n,1)) blas.gemv(Bb,tmp,tmp2) dwdb = matrix([rtw - tmp2,rtb + blas.dot(mul(d2i,lt),rtt)]) lapack.potrs(M,dwdb) # compute dt = inv(D2)*(rtt - Bb'*dw + lt*db) tmp2 = matrix(0.0,(k,1)) blas.gemv(Bb, dwdb[:n], tmp2, trans='T') dt = mul(d2i, rtt - tmp2 + lt*dwdb[-1]) # compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db)) dv = matrix(0.0,(m,1)) blas.gemv(X,dwdb[:n],dv,alpha = -1.0) dv = mul(d,dv) - d*dwdb[-1] base.gemv(E, dt, dv, beta = 1.0) tmp = +dv # tmp = E*dt - D*X*dw - d*db dv = mul(dbi, rbv + mul(W['di'][:m]**2,dv)) # compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1) wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1 # compute wdz2 = - inv(Wl2)*(dv + rl2) wdz2 = - mul(W['di'][m:2*m],dv) - iw_rl2 # compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri) wdzi = [] tmp = matrix(0.0,(n,1)) for i in range(k): blas.gemv(P[i],dwdb[:n],tmp, alpha = -1.0, beta = 0.0) tmp1 = matrix([-dt[i],tmp]) blas.axpy(ri[i],tmp1,alpha = -1.0) qscal(tmp1,W['beta'][i],W['v'][i],inv=True) wdzi.append(tmp1) # solution x[:n] = dwdb[:n] x[n:n+k] = dt x[n+k] = dwdb[-1] x[n+k+1:] = dv z[:m] = wdz1 z[m:2*m] = wdz2 for i in range(k): z[2*m+i*(n+1):2*m+(i+1)*(n+1)] = wdzi[i] return f
def softmargin_appr(X, d, gamma, width, kernel='linear', sigma=1.0, degree=1, theta=1.0, Q=None): """ Solves the approximate 'soft-margin' SVM problem maximize -(1/2)*z'*Qc*z + d'*z subject to 0 <= diag(d)*z <= gamma*ones sum(z) = 0 (with variables z), and its dual problem minimize (1/2)*y'*Qc^{-1}*y + gamma*sum(v) subject to diag(d)*(y + b*ones) + v >= 1 v >= 0 (with variables y, v, b). Qc is the maximum determinant completion of the projection of Q on a band with bandwidth 2*w+1. Q_ij = K(xi, xj) where K is a kernel function and xi is the ith row of X (xi' = X[i,:]). Input arguments. X is an N x n matrix. d is an N-vector with elements -1 or 1; d[i] is the label of row X[i,:]. gamma is a positive parameter. kernel is a string with values 'linear', 'rfb', 'poly', or 'tanh'. 'linear': k(u,v) = u'*v/sigma. 'rbf': k(u,v) = exp(-||u - v||^2 / (2*sigma)). 'poly': k(u,v) = (u'*v/sigma)**degree. 'tanh': k(u,v) = tanh(u'*v/sigma - theta). sigma and theta are positive numbers. degree is a positive integer. width is a positive integer. Output. Returns a dictionary with the keys: 'classifier' a Python function object that takes an M x n matrix with test vectors as rows and returns a vector with labels 'completion classifier' a Python function object that takes an M x n matrix with test vectors as rows and returns a vector with labels 'z' a sparse m-vector 'cputime' a tuple (Ttot, Tqp, Tker) where Ttot is the total CPU time, Tqp is the CPU time spent solving the QP, and Tker is the CPU time spent computing the kernel matrix 'iterations' the number of interior-point iteations 'misclassified' a tuple (L1, L2) where L1 is a list of indices of misclassified training vectors from class 1, and L2 is a list of indices of misclassified training vectors from class 2 """ Tstart = cputime() if verbose: solvers.options['show_progress'] = True else: solvers.options['show_progress'] = False N, n = X.size if Q is None: Q, a = kernel_matrix(X, kernel, sigma=sigma, degree=degree, theta=theta, V='band', width=width) else: if not (Q.size[0] == N and Q.size[1] == N): raise ValueError("invalid kernel matrix dimensions") elif not type(Q) is cvxopt.base.spmatrix: raise ValueError("invalid kernel matrix type") elif verbose: print("using precomputed kernel matrix ..") if kernel == 'rbf': Ad = spmatrix(0.0, range(N), range(N)) base.syrk(X, V, partial=True) a = Ad.V del Ad Tkernel = cputime(Tstart) # solve qp Tqp = cputime() y, b, v, z, optval, Lc, iters = softmargin_completion( Q, matrix(d, tc='d'), gamma) Tqp = cputime(Tqp) if verbose: print("utime = %f, stime = %f." % Tqp) # extract nonzero support vectors nrmz = max(abs(z)) sv = [k for k in range(N) if abs(z[k]) > Tsv * nrmz] zs = spmatrix(z[sv], sv, [0 for i in range(len(sv))], (len(d), 1)) if verbose: print("%d support vectors." % len(sv)) Xr, zr, Nr = X[sv, :], z[sv], len(sv) # find misclassified training vectors err1 = [i for i in range(Q.size[0]) if (v[i] > 1 and d[i] == 1)] err2 = [i for i in range(Q.size[0]) if (v[i] > 1 and d[i] == -1)] if verbose: e1, n1 = len(err1), list(d).count(1) e2, n2 = len(err2), list(d).count(-1) print("class 1: %i/%i = %.1f%% misclassified." % (e1, n1, 100. * e1 / n1)) print("class 2: %i/%i = %.1f%% misclassified." % (e2, n2, 100. * e2 / n2)) del e1, e2, n1, n2 # create classifier function object # CLASSIFIER 1 (standard kernel classifier) if kernel == 'linear': # w = X'*z / sigma w = matrix(0.0, (n, 1)) blas.gemv(Xr, zr, w, trans='T', alpha=1.0 / sigma) def classifier(Y, soft=False): M = Y.size[0] x = matrix(b, (M, 1)) blas.gemv(Y, w, x, beta=1.0) if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'rbf': def classifier(Y, soft=False): M = Y.size[0] # K = Y*X' / sigma K = matrix(0.0, (M, Nr)) blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma) # c[i] = ||Yi||^2 / sigma ones = matrix(1.0, (max([M, Nr, n]), 1)) c = Y**2 * ones[:n] blas.scal(1.0 / sigma, c) # Kij := Kij - 0.5 * (ci + aj) # = || yi - xj ||^2 / (2*sigma) blas.ger(c, ones, K, alpha=-0.5) blas.ger(ones, a[sv], K, alpha=-0.5) x = exp(K) * zr + b if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'tanh': def classifier(Y, soft=False): M = Y.size[0] # K = Y*X' / sigma - theta K = matrix(theta, (M, Nr)) blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma, beta=-1.0) K = exp(K) x = div(K - K**-1, K + K**-1) * zr + b if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'poly': def classifier(Y, soft=False): M = Y.size[0] # K = Y*X' / sigma K = matrix(0.0, (M, Nr)) blas.gemm(Y, Xr, K, transB='T', alpha=1.0 / sigma) x = K**degree * zr + b if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) else: pass # CLASSIFIER 2 (completion kernel classifier) # TODO: generalize to arbitrary sparsity pattern L11 = matrix(Q[:width, :width]) lapack.potrf(L11) if kernel == 'linear': def classifier2(Y, soft=False): M = Y.size[0] W = matrix(0., (width, M)) blas.gemm(X, Y, W, transB='T', alpha=1.0 / sigma, m=width) lapack.potrs(L11, W) W = matrix([W, matrix(0., (N - width, M))]) chompack.trsm(Lc, W, trans='N') chompack.trsm(Lc, W, trans='T') x = matrix(b, (M, 1)) blas.gemv(W, z, x, trans='T', beta=1.0) if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'poly': def classifier2(Y, soft=False): if Y is None: return zs M = Y.size[0] W = matrix(0., (width, M)) blas.gemm(X, Y, W, transB='T', alpha=1.0 / sigma, m=width) W = W**degree lapack.potrs(L11, W) W = matrix([W, matrix(0., (N - width, M))]) chompack.trsm(Lc, W, trans='N') chompack.trsm(Lc, W, trans='T') x = matrix(b, (M, 1)) blas.gemv(W, z, x, trans='T', beta=1.0) if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'rbf': def classifier2(Y, soft=False): M = Y.size[0] # K = Y*X' / sigma K = matrix(0.0, (width, M)) blas.gemm(X, Y, K, transB='T', alpha=1.0 / sigma, m=width) # c[i] = ||Yi||^2 / sigma ones = matrix(1.0, (max(width, n, M), 1)) c = Y**2 * ones[:n] blas.scal(1.0 / sigma, c) # Kij := Kij - 0.5 * (ci + aj) # = || yi - xj ||^2 / (2*sigma) blas.ger(ones[:width], c, K, alpha=-0.5) blas.ger(a[:width], ones[:M], K, alpha=-0.5) # Kij = exp(Kij) K = exp(K) # complete K lapack.potrs(L11, K) K = matrix([K, matrix(0., (N - width, M))], (N, M)) chompack.trsm(Lc, K, trans='N') chompack.trsm(Lc, K, trans='T') x = matrix(b, (M, 1)) blas.gemv(K, z, x, trans='T', beta=1.0) if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) elif kernel == 'tanh': def classifier2(Y, soft=False): M = Y.size[0] # K = Y*X' / sigma K = matrix(theta, (width, M)) blas.gemm(X, Y, K, transB='T', alpha=1.0 / sigma, beta=-1.0, m=width) K = exp(K) K = div(K - K**-1, K + K**-1) # complete K lapack.potrs(L11, K) K = matrix([K, matrix(0., (N - width, M))], (N, M)) chompack.trsm(Lc, K, trans='N') chompack.trsm(Lc, K, trans='T') x = matrix(b, (M, 1)) blas.gemv(K, z, x, trans='T', beta=1.0) if soft: return x else: return matrix([2 * (xk > 0.0) - 1 for xk in x]) Ttotal = cputime(Tstart) return { 'classifier': classifier, 'completion classifier': classifier2, 'cputime': (sum(Ttotal), sum(Tqp), sum(Tkernel)), 'iterations': iters, 'z': zs, 'misclassified': (err1, err2) }
def computefunc(self,cinfo): ainv=cinfo L = +cvxopt.matrix(ainv) lapack.potrf(L) f=2.0*np.sum(np.log(np.diag(L))) return f
def Fkkt(W): # Returns a function f(x, y, z) that solves # # [ 0 G' ] [ x ] = [ bx ] # [ G -W'*W ] [ z ] [ bz ]. # First factor # # S = G' * W**-1 * W**-T * G # = [0; -A]' * W3^-2 * [0; -A] + 4 * (W1**2 + W2**2)**-1 # # where # # W1 = diag(d1) with d1 = W['d'][:n] = 1 ./ W['di'][:n] # W2 = diag(d2) with d2 = W['d'][n:] = 1 ./ W['di'][n:] # W3 = beta * (2*v*v' - J), W3^-1 = 1/beta * (2*J*v*v'*J - J) # with beta = W['beta'][0], v = W['v'][0], J = [1, 0; 0, -I]. # As = W3^-1 * [ 0 ; -A ] = 1/beta * ( 2*J*v * v' - I ) * [0; A] beta, v = W['beta'][0], W['v'][0] As = 2 * v * (v[1:].T * A) As[1:, :] *= -1.0 As[1:, :] -= A As /= beta # S = As'*As + 4 * (W1**2 + W2**2)**-1 S = As.T * As d1, d2 = W['d'][:n], W['d'][n:] d = 4.0 * (d1**2 + d2**2)**-1 S[::n + 1] += d lapack.potrf(S) def f(x, y, z): # z := - W**-T * z z[:n] = -div(z[:n], d1) z[n:2 * n] = -div(z[n:2 * n], d2) z[2 * n:] -= 2.0 * v * (v[0] * z[2 * n] - blas.dot(v[1:], z[2 * n + 1:])) z[2 * n + 1:] *= -1.0 z[2 * n:] /= beta # x := x - G' * W**-1 * z x[:n] -= div(z[:n], d1) - div(z[n:2 * n], d2) + As.T * z[-(m + 1):] x[n:] += div(z[:n], d1) + div(z[n:2 * n], d2) # Solve for x[:n]: # # S*x[:n] = x[:n] - (W1**2 - W2**2)(W1**2 + W2**2)^-1 * x[n:] x[:n] -= mul(div(d1**2 - d2**2, d1**2 + d2**2), x[n:]) lapack.potrs(S, x) # Solve for x[n:]: # # (d1**-2 + d2**-2) * x[n:] = x[n:] + (d1**-2 - d2**-2)*x[:n] x[n:] += mul(d1**-2 - d2**-2, x[:n]) x[n:] = div(x[n:], d1**-2 + d2**-2) # z := z + W^-T * G*x z[:n] += div(x[:n] - x[n:2 * n], d1) z[n:2 * n] += div(-x[:n] - x[n:2 * n], d2) z[2 * n:] += As * x[:n] return f
def F(W): """ Custom solver for the system [ It 0 0 Xt' 0 At1' ... Atk' ][ dwt ] [ rwt ] [ 0 0 0 -d' 0 0 ... 0 ][ db ] [ rb ] [ 0 0 0 -I -I 0 ... 0 ][ dv ] [ rv ] [ Xt -d -I -Wl1^-2 ][ dzl1 ] [ rl1 ] [ 0 0 -I -Wl2^-2 ][ dzl2 ] = [ rl2 ] [ At1 0 0 -W1^-2 ][ dz1 ] [ r1 ] [ | | | . ][ | ] [ | ] [ Atk 0 0 -Wk^-2 ][ dzk ] [ rk ] where It = [ I 0 ] Xt = [ -D*X E ] Ati = [ 0 -e_i' ] [ 0 0 ] [ -Pi 0 ] dwt = [ dw ] rwt = [ rw ] [ dt ] [ rt ]. """ # scalings and 'intermediate' vectors # db = inv(Wl1)^2 + inv(Wl2)^2 db = W['di'][:m]**2 + W['di'][m:2 * m]**2 dbi = div(1.0, db) # dt = I - inv(Wl1)*Dbi*inv(Wl1) dt = 1.0 - mul(W['di'][:m]**2, dbi) dtsqrt = sqrt(dt) # lam = Dt*inv(Wl1)*d lam = mul(dt, mul(W['di'][:m], d)) # lt = E'*inv(Wl1)*lam lt = matrix(0.0, (k, 1)) base.gemv(E, mul(W['di'][:m], lam), lt, trans='T') # Xs = sqrt(Dt)*inv(Wl1)*X tmp = mul(dtsqrt, W['di'][:m]) Xs = spmatrix(tmp, range(m), range(m)) * X # Es = D*sqrt(Dt)*inv(Wl1)*E Es = spmatrix(mul(d, tmp), range(m), range(m)) * E # form Ab = I + sum((1/bi)^2*(Pi'*Pi + 4*(v'*v + 1)*Pi'*y*y'*Pi)) + Xs'*Xs # and Bb = -sum((1/bi)^2*(4*ui*v'*v*Pi'*y*ei')) - Xs'*Es # and D2 = Es'*Es + sum((1/bi)^2*(1+4*ui^2*(v'*v - 1)) Ab = matrix(0.0, (n, n)) Ab[::n + 1] = 1.0 base.syrk(Xs, Ab, trans='T', beta=1.0) Bb = matrix(0.0, (n, k)) Bb = -Xs.T * Es # inefficient!? D2 = spmatrix(0.0, range(k), range(k)) base.syrk(Es, D2, trans='T', partial=True) d2 = +D2.V del D2 py = matrix(0.0, (n, 1)) for i in range(k): binvsq = (1.0 / W['beta'][i])**2 Ab += binvsq * Pt[i] dvv = blas.dot(W['v'][i], W['v'][i]) blas.gemv(P[i], W['v'][i][1:], py, trans='T', alpha=1.0, beta=0.0) blas.syrk(py, Ab, alpha=4 * binvsq * (dvv + 1), beta=1.0) Bb[:, i] -= 4 * binvsq * W['v'][i][0] * dvv * py d2[i] += binvsq * (1 + 4 * (W['v'][i][0]**2) * (dvv - 1)) d2i = div(1.0, d2) d2isqrt = sqrt(d2i) # compute a = alpha - lam'*inv(Wl1)*E*inv(D2)*E'*inv(Wl1)*lam alpha = blas.dot(lam, mul(W['di'][:m], d)) tmp = matrix(0.0, (k, 1)) base.gemv(E, mul(W['di'][:m], lam), tmp, trans='T') tmp = mul(tmp, d2isqrt) #tmp = inv(D2)^(1/2)*E'*inv(Wl1)*lam a = alpha - blas.dot(tmp, tmp) # compute M12 = X'*D*inv(Wl1)*lam + Bb*inv(D2)*E'*inv(Wl1)*lam tmp = mul(tmp, d2isqrt) M12 = matrix(0.0, (n, 1)) blas.gemv(Bb, tmp, M12, alpha=1.0) tmp = mul(d, mul(W['di'][:m], lam)) blas.gemv(X, tmp, M12, trans='T', alpha=1.0, beta=1.0) # form and factor M sBb = Bb * spmatrix(d2isqrt, range(k), range(k)) base.syrk(sBb, Ab, alpha=-1.0, beta=1.0) M = matrix([[Ab, M12.T], [M12, a]]) lapack.potrf(M) def f(x, y, z): # residuals rwt = x[:n + k] rb = x[n + k] rv = x[n + k + 1:n + k + 1 + m] iw_rl1 = mul(W['di'][:m], z[:m]) iw_rl2 = mul(W['di'][m:2 * m], z[m:2 * m]) ri = [ z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)] for i in range(k) ] # compute 'derived' residuals # rbwt = rwt + sum(Ai'*inv(Wi)^2*ri) + [-X'*D; E']*inv(Wl1)^2*rl1 rbwt = +rwt for i in range(k): tmp = +ri[i] qscal(tmp, W['beta'][i], W['v'][i], inv=True) qscal(tmp, W['beta'][i], W['v'][i], inv=True) rbwt[n + i] -= tmp[0] blas.gemv(P[i], tmp[1:], rbwt, trans='T', alpha=-1.0, beta=1.0) tmp = mul(W['di'][:m], iw_rl1) tmp2 = matrix(0.0, (k, 1)) base.gemv(E, tmp, tmp2, trans='T') rbwt[n:] += tmp2 tmp = mul(d, tmp) # tmp = D*inv(Wl1)^2*rl1 blas.gemv(X, tmp, rbwt, trans='T', alpha=-1.0, beta=1.0) # rbb = rb - d'*inv(Wl1)^2*rl1 rbb = rb - sum(tmp) # rbv = rv - inv(Wl2)*rl2 - inv(Wl1)^2*rl1 rbv = rv - mul(W['di'][m:2 * m], iw_rl2) - mul(W['di'][:m], iw_rl1) # [rtw;rtt] = rbwt + [-X'*D; E']*inv(Wl1)^2*inv(Db)*rbv tmp = mul(W['di'][:m]**2, mul(dbi, rbv)) rtt = +rbwt[n:] base.gemv(E, tmp, rtt, trans='T', alpha=1.0, beta=1.0) rtw = +rbwt[:n] tmp = mul(d, tmp) blas.gemv(X, tmp, rtw, trans='T', alpha=-1.0, beta=1.0) # rtb = rbb - d'*inv(Wl1)^2*inv(Db)*rbv rtb = rbb - sum(tmp) # solve M*[dw;db] = [rtw - Bb*inv(D2)*rtt; rtb + lt'*inv(D2)*rtt] tmp = mul(d2i, rtt) tmp2 = matrix(0.0, (n, 1)) blas.gemv(Bb, tmp, tmp2) dwdb = matrix([rtw - tmp2, rtb + blas.dot(mul(d2i, lt), rtt)]) lapack.potrs(M, dwdb) # compute dt = inv(D2)*(rtt - Bb'*dw + lt*db) tmp2 = matrix(0.0, (k, 1)) blas.gemv(Bb, dwdb[:n], tmp2, trans='T') dt = mul(d2i, rtt - tmp2 + lt * dwdb[-1]) # compute dv = inv(Db)*(rbv + inv(Wl1)^2*(E*dt - D*X*dw - d*db)) dv = matrix(0.0, (m, 1)) blas.gemv(X, dwdb[:n], dv, alpha=-1.0) dv = mul(d, dv) - d * dwdb[-1] base.gemv(E, dt, dv, beta=1.0) tmp = +dv # tmp = E*dt - D*X*dw - d*db dv = mul(dbi, rbv + mul(W['di'][:m]**2, dv)) # compute wdz1 = inv(Wl1)*(E*dt - D*X*dw - d*db - dv - rl1) wdz1 = mul(W['di'][:m], tmp - dv) - iw_rl1 # compute wdz2 = - inv(Wl2)*(dv + rl2) wdz2 = -mul(W['di'][m:2 * m], dv) - iw_rl2 # compute wdzi = inv(Wi)*([-ei'*dt; -Pi*dw] - ri) wdzi = [] tmp = matrix(0.0, (n, 1)) for i in range(k): blas.gemv(P[i], dwdb[:n], tmp, alpha=-1.0, beta=0.0) tmp1 = matrix([-dt[i], tmp]) blas.axpy(ri[i], tmp1, alpha=-1.0) qscal(tmp1, W['beta'][i], W['v'][i], inv=True) wdzi.append(tmp1) # solution x[:n] = dwdb[:n] x[n:n + k] = dt x[n + k] = dwdb[-1] x[n + k + 1:] = dv z[:m] = wdz1 z[m:2 * m] = wdz2 for i in range(k): z[2 * m + i * (n + 1):2 * m + (i + 1) * (n + 1)] = wdzi[i] return f
def psdcompletion(A, reordered = True, **kwargs): """ Maximum determinant positive semidefinite matrix completion. The routine takes a cspmatrix :math:`A` and returns the maximum determinant positive semidefinite matrix completion :math:`X` as a dense matrix, i.e., .. math:: P( X ) = A :param A: :py:class:`cspmatrix` :param reordered: boolean """ assert isinstance(A, cspmatrix) and A.is_factor is False, "A must be a cspmatrix" tol = kwargs.get('tol',1e-15) X = matrix(A.spmatrix(reordered = True, symmetric = True)) symb = A.symb n = symb.n snptr = symb.snptr sncolptr = symb.sncolptr snrowidx = symb.snrowidx # visit supernodes in reverse (descending) order for k in range(symb.Nsn-1,-1,-1): nn = snptr[k+1]-snptr[k] beta = snrowidx[sncolptr[k]:sncolptr[k+1]] nj = len(beta) if nj-nn == 0: continue alpha = beta[nn:] nu = beta[:nn] eta = matrix([matrix(range(beta[kk]+1,beta[kk+1])) for kk in range(nj-1)] + [matrix(range(beta[-1]+1,n))]) try: # Try Cholesky factorization first Xaa = X[alpha,alpha] lapack.potrf(Xaa) Xan = X[alpha,nu] lapack.trtrs(Xaa, Xan, trans = 'N') XeaT = X[eta,alpha].T lapack.trtrs(Xaa, XeaT, trans = 'N') # Compute update tmp = XeaT.T*Xan except: # If Cholesky fact. fails, switch to EVD: Xaa = Z*diag(w)*Z.T Xaa = X[alpha,alpha] w = matrix(0.0,(Xaa.size[0],1)) Z = matrix(0.0,Xaa.size) lapack.syevr(Xaa, w, jobz='V', range='A', uplo='L', Z=Z) # Pseudo-inverse: Xp = pinv(Xaa) lambda_max = max(w) Xp = Z*spmatrix([1.0/wi if wi > lambda_max*tol else 0.0 for wi in w],range(len(w)),range(len(w)))*Z.T # Compute update tmp = X[eta,alpha]*Xp*X[alpha,nu] X[eta,nu] = tmp X[nu,eta] = tmp.T if reordered: return X else: return X[symb.ip,symb.ip]
def F(x=None, z=None): if x is None: return m, matrix([ 1.0, 0.0, 1.0, 0.0, 0.0 ]) # Factor A as A = L*L'. Compute inverse B = A^-1. A = matrix( [x[0], x[1], x[1], x[2]], (2,2)) L = +A try: lapack.potrf(L) except: return None B = +L lapack.potri(B) B[0,1] = B[1,0] # f0 = -log det A f = matrix(0.0, (m+1,1)) f[0] = -2.0 * (log(L[0,0]) + log(L[1,1])) # fk = xk'*A*xk - 2*xk'*b + b*A^-1*b - 1 # = (xk - c)' * A * (xk - c) - 1 where c = A^-1*b c = x[3:] lapack.potrs(L, c) for k in range(m): f[k+1] = (X[k,:].T - c).T * A * (X[k,:].T - c) - 1.0 # gradf0 = (-A^-1, 0) = (-B, 0) Df = matrix(0.0, (m+1,5)) Df[0,0], Df[0,1], Df[0,2] = -B[0,0], -2.0*B[1,0], -B[1,1] # gradfk = (xk*xk' - A^-1*b*b'*A^-1, 2*(-xk + A^-1*b)) # = (xk*xk' - c*c', 2*(-xk+c)) Df[1:,0] = X[:m,0]**2 - c[0]**2 Df[1:,1] = 2.0 * (mul(X[:m,0], X[:m,1]) - c[0]*c[1]) Df[1:,2] = X[:m,1]**2 - c[1]**2 Df[1:,3] = 2.0 * (-X[:m,0] + c[0]) Df[1:,4] = 2.0 * (-X[:m,1] + c[1]) if z is None: return f, Df # hessf0(Y, y) = (A^-1*Y*A^-1, 0) = (B*YB, 0) H0 = matrix(0.0, (5,5)) H0[0,0] = B[0,0]**2 H0[1,0] = 2.0 * B[0,0] * B[1,0] H0[2,0] = B[1,0]**2 H0[1,1] = 2.0 * ( B[0,0] * B[1,1] + B[1,0]**2 ) H0[2,1] = 2.0 * B[1,0] * B[1,1] H0[2,2] = B[1,1]**2 # hessfi(Y, y) # = ( A^-1*Y*A^-1*b*b'*A^-1 + A^-1*b*b'*A^-1*Y*A^-1 # - A^-1*y*b'*A^-1 - A^-1*b*y'*A^-1, # -2*A^-1*Y*A^-1*b + 2*A^-1*y ) # = ( B*Y*c*c' + c*c'*Y*B - B*y*c' - c*y'*B, -2*B*Y*c + 2*B*y ) # = ( B*(Y*c-y)*c' + c*(Y*c-y)'*B, -2*B*(Y*c - y) ) H1 = matrix(0.0, (5,5)) H1[0,0] = 2.0 * c[0]**2 * B[0,0] H1[1,0] = 2.0 * ( c[0] * c[1] * B[0,0] + c[0]**2 * B[1,0] ) H1[2,0] = 2.0 * c[0] * c[1] * B[1,0] H1[3:,0] = -2.0 * c[0] * B[:,0] H1[1,1] = 2.0 * c[0]**2 * B[1,1] + 4.0 * c[0]*c[1]*B[1,0] + \ 2.0 * c[1]**2 + B[0,0] H1[2,1] = 2.0 * (c[1]**2 * B[1,0] + c[0]*c[1]*B[1,1]) H1[3:,1] = -2.0 * B * c[[1,0]] H1[2,2] = 2.0 * c[1]**2 * B[1,1] H1[3:,2] = -2.0 * c[1] * B[:,1] H1[3:,3:] = 2*B return f, Df, z[0]*H0 + sum(z[1:])*H1
def factor(W, H=None, Df=None): if F['firstcall']: if type(G) is matrix: F['Gs'] = matrix(0.0, G.size) else: F['Gs'] = spmatrix(0.0, G.I, G.J, G.size) if mnl: if type(Df) is matrix: F['Dfs'] = matrix(0.0, Df.size) else: F['Dfs'] = spmatrix(0.0, Df.I, Df.J, Df.size) if (mnl and type(Df) is matrix) or type(G) is matrix or \ type(H) is matrix: F['S'] = matrix(0.0, (n, n)) F['K'] = matrix(0.0, (p, p)) else: F['S'] = spmatrix([], [], [], (n, n), 'd') F['Sf'] = None if type(A) is matrix: F['K'] = matrix(0.0, (p, p)) else: F['K'] = spmatrix([], [], [], (p, p), 'd') # Dfs = Wnl^{-1} * Df if mnl: base.gemm(spmatrix(W['dnli'], list(range(mnl)), list(range(mnl))), Df, F['Dfs'], partial=True) # Gs = Wl^{-1} * G. base.gemm(spmatrix(W['di'], list(range(ml)), list(range(ml))), G, F['Gs'], partial=True) if F['firstcall']: base.syrk(F['Gs'], F['S'], trans='T') if mnl: base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0) if H is not None: F['S'] += H try: if type(F['S']) is matrix: lapack.potrf(F['S']) else: F['Sf'] = cholmod.symbolic(F['S']) cholmod.numeric(F['S'], F['Sf']) except ArithmeticError: F['singular'] = True if type(A) is matrix and type(F['S']) is spmatrix: F['S'] = matrix(0.0, (n, n)) base.syrk(F['Gs'], F['S'], trans='T') if mnl: base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0) base.syrk(A, F['S'], trans='T', beta=1.0) if H is not None: F['S'] += H if type(F['S']) is matrix: lapack.potrf(F['S']) else: F['Sf'] = cholmod.symbolic(F['S']) cholmod.numeric(F['S'], F['Sf']) F['firstcall'] = False else: base.syrk(F['Gs'], F['S'], trans='T', partial=True) if mnl: base.syrk(F['Dfs'], F['S'], trans='T', beta=1.0, partial=True) if H is not None: F['S'] += H if F['singular']: base.syrk(A, F['S'], trans='T', beta=1.0, partial=True) if type(F['S']) is matrix: lapack.potrf(F['S']) else: cholmod.numeric(F['S'], F['Sf']) if type(F['S']) is matrix: # Asct := L^{-1}*A'. Factor K = Asct'*Asct. if type(A) is matrix: Asct = A.T else: Asct = matrix(A.T) blas.trsm(F['S'], Asct) blas.syrk(Asct, F['K'], trans='T') lapack.potrf(F['K']) else: # Asct := L^{-1}*P*A'. Factor K = Asct'*Asct. if type(A) is matrix: Asct = A.T cholmod.solve(F['Sf'], Asct, sys=7) cholmod.solve(F['Sf'], Asct, sys=4) blas.syrk(Asct, F['K'], trans='T') lapack.potrf(F['K']) else: Asct = cholmod.spsolve(F['Sf'], A.T, sys=7) Asct = cholmod.spsolve(F['Sf'], Asct, sys=4) base.syrk(Asct, F['K'], trans='T') Kf = cholmod.symbolic(F['K']) cholmod.numeric(F['K'], Kf) def solve(x, y, z): # Solve # # [ H A' GG'*W^{-1} ] [ ux ] [ bx ] # [ A 0 0 ] * [ uy ] = [ by ] # [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ] # # and return ux, uy, W*uz. # # If not F['singular']: # # K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz ) - by # S*ux = bx + GG'*W^{-1}*W^{-T}*bz - A'*uy # W*uz = W^{-T} * ( GG*ux - bz ). # # If F['singular']: # # K*uy = A * S^{-1} * ( bx + GG'*W^{-1}*W^{-T}*bz + A'*by ) # - by # S*ux = bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y. # W*uz = W^{-T} * ( GG*ux - bz ). # z := W^{-1} * z = W^{-1} * bz scale(z, W, trans='T', inverse='I') # If not F['singular']: # x := L^{-1} * P * (x + GGs'*z) # = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz) # # If F['singular']: # x := L^{-1} * P * (x + GGs'*z + A'*y)) # = L^{-1} * P * (x + GG'*W^{-1}*W^{-T}*bz + A'*y) if mnl: base.gemv(F['Dfs'], z, x, trans='T', beta=1.0) base.gemv(F['Gs'], z, x, offsetx=mnl, trans='T', beta=1.0) if F['singular']: base.gemv(A, y, x, trans='T', beta=1.0) if type(F['S']) is matrix: blas.trsv(F['S'], x) else: cholmod.solve(F['Sf'], x, sys=7) cholmod.solve(F['Sf'], x, sys=4) # y := K^{-1} * (Asc*x - y) # = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz) - by) # (if not F['singular']) # = K^{-1} * (A * S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz + # A'*by) - by) # (if F['singular']). base.gemv(Asct, x, y, trans='T', beta=-1.0) if type(F['K']) is matrix: lapack.potrs(F['K'], y) else: cholmod.solve(Kf, y) # x := P' * L^{-T} * (x - Asc'*y) # = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz - A'*y) # (if not F['singular']) # = S^{-1} * (bx + GG'*W^{-1}*W^{-T}*bz + A'*by - A'*y) # (if F['singular']) base.gemv(Asct, y, x, alpha=-1.0, beta=1.0) if type(F['S']) is matrix: blas.trsv(F['S'], x, trans='T') else: cholmod.solve(F['Sf'], x, sys=5) cholmod.solve(F['Sf'], x, sys=8) # W*z := GGs*x - z = W^{-T} * (GG*x - bz) if mnl: base.gemv(F['Dfs'], x, z, beta=-1.0) base.gemv(F['Gs'], x, z, beta=-1.0, offsety=mnl) return solve