def solve_dist_with_man(man, A, X0, maxiter):
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
@Callable
def cost(S):
# if not(S.P.dtype == np.float):
# raise(ValueError("Non real"))
diff = (A - S.U @ S.P @ S.V.T.conj())
val = rtrace(diff @ diff.T.conj())
# print('val=%f' % val)
return val
@Callable
def egrad(S):
return fr_ambient(-2*A @ S.V @ S.P,
-2*A.T.conj() @ S.U @S.P,
2*(S.P-S.U.T.conj() @ A @ S.V))
@Callable
def ehess(S, xi):
return fr_ambient(-2*A @ (xi.tV @ S.P + S.V @ xi.tP),
-2*A.T.conj() @ (xi.tU @S.P + [email protected]),
2*(xi.tP - xi.tU.T.conj()@[email protected] -
S.U.T.conj()@[email protected]))
prob = Problem(
man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False)
opt = solver.solve(prob, x=X0, Delta_bar=250)
return opt
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
"""This example generates a random 128 x 128 symmetric matrix and finds the
dominant invariant 3 dimensional subspace for this matrix, i.e., it finds
the subspace spanned by the three eigenvectors with the largest
eigenvalues.
"""
num_rows = 128
subspace_dimension = 3
matrix = rnd.randn(num_rows, num_rows)
matrix = 0.5 * (matrix + matrix.T)
cost, egrad, ehess = create_cost_egrad_ehess(
backend, matrix, subspace_dimension)
manifold = Grassmann(num_rows, subspace_dimension)
problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
estimated_spanning_set = solver.solve(
problem, Delta_bar=8*np.sqrt(subspace_dimension))
if quiet:
return
eigenvalues, eigenvectors = la.eig(matrix)
column_indices = np.argsort(eigenvalues)[-subspace_dimension:]
spanning_set = eigenvectors[:, column_indices]
print("Geodesic distance between true and estimated dominant subspace:",
manifold.dist(spanning_set, estimated_spanning_set))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
dimension = 3
num_samples = 200
num_components = 2
samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1])
samples -= samples.mean(axis=0)
cost, egrad, ehess = create_cost_egrad_ehess(backend, samples,
num_components)
manifold = Stiefel(dimension, num_components)
problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
# from pymanopt.solvers import ConjugateGradient
# solver = ConjugateGradient()
estimated_span_matrix = solver.solve(problem)
if quiet:
return
estimated_projector = estimated_span_matrix @ estimated_span_matrix.T
eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples)
indices = np.argsort(eigenvalues)[::-1][:num_components]
span_matrix = eigenvectors[:, indices]
projector = span_matrix @ span_matrix.T
print(
"Frobenius norm error between estimated and closed-form projection "
"matrix:", np.linalg.norm(projector - estimated_projector))
def run(quiet=True):
dimension = 3
num_samples = 200
num_components = 2
samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1])
samples -= samples.mean(axis=0)
samples_ = torch.from_numpy(samples)
@pymanopt.function.PyTorch
def cost(w):
projector = torch.matmul(w, torch.transpose(w, 1, 0))
return torch.norm(samples_ - torch.matmul(samples_, projector)) ** 2
manifold = Stiefel(dimension, num_components)
problem = pymanopt.Problem(manifold, cost, egrad=None, ehess=None)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
# from pymanopt.solvers import ConjugateGradient
# solver = ConjugateGradient()
estimated_span_matrix = solver.solve(problem)
if quiet:
return
estimated_projector = estimated_span_matrix @ estimated_span_matrix.T
eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples)
indices = np.argsort(eigenvalues)[::-1][:num_components]
span_matrix = eigenvectors[:, indices]
projector = span_matrix @ span_matrix.T
print("Frobenius norm error between estimated and closed-form projection "
"matrix:", np.linalg.norm(projector - estimated_projector))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_samples, num_weights = 200, 3
solver = TrustRegions()
manifold = Euclidean(3)
for k in range(5):
samples = rnd.randn(num_samples, num_weights)
targets = rnd.randn(num_samples)
cost, egrad, ehess = create_cost_egrad_ehess(backend, samples, targets)
problem = pymanopt.Problem(manifold,
cost,
egrad=egrad,
ehess=ehess,
verbosity=0)
estimated_weights = solver.solve(problem)
if not quiet:
print("Run {}".format(k + 1))
print("Weights found by pymanopt (top) / "
"closed form solution (bottom)")
print(estimated_weights)
print(la.pinv(samples) @ targets)
print("")
def estimateR_weighted(S, W, D, R0):
'''
estimates the update of the rotation matrix for the second part of the iterations
:param S : shape
:param W : heatmap
:param D : weight of the heatmap
:param R0 : rotation matrix
:return: R the new rotation matrix
'''
A = np.transpose(S)
B = np.transpose(W)
X0 = R0[0:2, :]
store_E = Store()
[m, n] = A.shape
p = B.shape[1]
At = np.zeros([n, m])
At = np.transpose(A)
# we use the optimization on a Stiefel manifold because R is constrained to be othogonal
manifold = Stiefel(n, p, 1)
####################################################################################################################
def cost(X):
'''
cost function of the manifold, the cost is trace(E'*D*E)/2 with E = A*X - B or store_E
:param X : vector
:return f : the cost
'''
if store_E.stored is None:
store_E.stored = np.dot(A, np.transpose(X)) - B
E = store_E.stored
f = np.trace(np.dot(np.transpose(E), np.dot(D, E))) / 2
return f
####################################################################################################################
# setup the problem structure with manifold M and cost
problem = Problem(manifold=manifold, cost=cost, verbosity=0)
# setup the trust region algorithm to solve the problem
TR = TrustRegions(maxiter=10)
# solve the problem
X = TR.solve(problem, X0)
#print('X : ',X)
return np.transpose(X) # return R = X'
def estimate_orth_subspaces(self, DataStruct):
'''
main optimization function
'''
# Grassman point?
if LA.norm(np.dot(self.Q.T, self.Q) - np.eye(self.Q.shape[-1]),
ord='fro') > 1e-4:
self._project_stiefel()
# ----------------------------------------------------------------------- #
# eGrad = grad(cost)
# eHess = hessian(cost)
# Perform optimization
# ----------------------------------------------------------------------- #
# ----------------------------------------------------------------------- #
d, r = np.shape(self.Q) # problem size
print(d)
manif = Stiefel(d, r) # initialize manifold
# instantiate problem
problem = Problem(manifold=manif, cost=self._cost, verbosity=2)
# initialize solver
solver = TrustRegions(mingradnorm=1e-8,
minstepsize=1e-16,
logverbosity=1)
# solve
Xopt, optlog = solver.solve(problem)
opt_subspaces = self._objfn(Xopt)
# Align the axes within a subspace by variance high to low
for j in range(self.numSubspaces):
Aj = DataStruct.A[j]
Qj = opt_subspaces[2].Q[j]
# data projected onto subspace
Aj_proj = np.dot((Aj - np.mean(Aj, 0)), Qj)
if np.size(np.cov(Aj_proj.T)) < 2:
V = 1
else:
V = LA.svd(np.cov(Aj_proj.T))[0]
Qj = np.dot(Qj, V)
opt_subspaces[2].Q[j] = Qj # ranked top to low variance
return opt_subspaces[2]
def ManoptOptimization(A, m):
n = A.shape[0]
T = A.shape[2]
manifold = Stiefel(n, m, k=1)
mycost = lambda x: cost(A, x)
myegrad = lambda x: egrad(A, x)
problem = Problem(manifold=manifold, cost=mycost, egrad=myegrad)
solver = TrustRegions()
print('# Start optimization using solver: trustregion')
Xopt = solver.solve(problem)
return Xopt
def rank_k_correlation_matrix_approximation(A, k):
"""
Returns the matrix with unit-norm columns that is closests to A w.r.t. the
Frobenius norm.
"""
m, n = A.shape
assert m == n, "matrix must be square"
assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric"
manifold = Oblique(k, n)
solver = TrustRegions()
X = T.matrix()
cost = 0.25 * T.sum((T.dot(X.T, X) - A) ** 2)
problem = Problem(manifold=manifold, cost=cost, arg=X)
return solver.solve(problem)
def rank_k_correlation_matrix_approximation(A, k):
"""
Returns the matrix with unit-norm columns that is closests to A w.r.t. the
Frobenius norm.
"""
m, n = A.shape
assert m == n, "matrix must be square"
assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric"
manifold = Oblique(k, n)
solver = TrustRegions()
X = T.matrix()
cost = 0.25 * T.sum((T.dot(X.T, X) - A)**2)
problem = Problem(manifold=manifold, cost=cost, arg=X)
return solver.solve(problem)
def solve_dist_with_man(man, A, X0, maxiter, check_deriv=False):
import pymanopt
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
@pymanopt.function.Callable
def cost(S):
"""
if not(S.P.dtype == np.float):
raise(ValueError("Non real"))
"""
diff = (A - S.Y @ S.P @ S.Y.T.conjugate())
val = trace(diff @ diff.T.conjugate()).real
# print('val=%f' % val)
return val
@pymanopt.function.Callable
def egrad(S):
return psd_ambient(-4 * A @ S.Y @ S.P,
2 * (S.P - S.Y.T.conjugate() @ A @ S.Y))
@pymanopt.function.Callable
def ehess(S, xi):
return psd_ambient(
-4 * A @ (xi.tY @ S.P + S.Y @ xi.tP),
2 * (xi.tP - xi.tY.T.conjugate() @ A @ S.Y -
S.Y.T.conjugate() @ A @ xi.tY))
if check_deriv:
xi = man.randvec(X0)
dlt = 1e-7
S1 = psd_point(X0.Y + dlt * xi.tY, X0.P + dlt * xi.tP)
print((cost(S1) - cost(X0)) / dlt)
print(man.base_inner_ambient(egrad(X0), xi))
h1 = egrad(S1) - egrad(X0)
h1 = psd_ambient(h1.tY / dlt, h1.tP / dlt)
h2 = ehess(X0, xi)
print(check_zero(h1.tY - h2.tY) + check_zero(h1.tP - h2.tP))
return
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False)
opt = solver.solve(prob, x=X0, Delta_bar=250)
return opt
def optimizer_R_v1(W, D, S, R0):
"""
:param W: heatmap constant
:param D: weight of the keypoints
:param S: shape of the object
:param R0: initial R
:return : the optimal R with fixed T and s
the cost is ||(W-(RS))*sqrt(D)|| but in fact is the scale factor is already taken into account in S and T is taken into account in W
"""
# this store object is needed because the manifold optimizer do not works if it is not implemented like that
store = Store()
# -------------------------------------COST FUNCTION-------------------------------------
def cost(R):
"""
:param R: rotation matrix variable
:return : ||(W-(RS))*D^1/2||^2 = tr((W-(RS))*D*(W-(RS))')
"""
if store.stored is None:
store.stored = W - np.dot(R, S)
X = store.stored
f = np.trace(np.dot(X, np.dot(D, np.transpose(X)))) / 2
return f
# ----------------------------------------------------------------------------------------
# we use the optimization on a Stiefel manifold because R is constrained to be othogonal
manifold = Stiefel(3, 2, 1)
# setup the problem structure with manifold M and cost
problem = Problem(manifold=manifold, cost=cost, verbosity=0)
# setup the trust region algorithm to solve the problem
TR = TrustRegions(maxiter=15)
# solve the problem
R_opt = TR.solve(problem, R0)
return np.transpose(R_opt)
def dominant_invariant_subspace(A, p):
"""
Returns an orthonormal basis of the dominant invariant p-subspace of A.
Arguments:
- A
A real, symmetric matrix A of size nxn
- p
integer p < n.
Returns:
A real, orthonormal matrix X of size nxp such that trace(X'*A*X)
is maximized. That is, the columns of X form an orthonormal basis
of a dominant subspace of dimension p of A. These span the same space
as the eigenvectors associated with the largest eigenvalues of A.
Sign is important: 2 is deemed a larger eigenvalue than -5.
"""
# Make sure the input matrix is square and symmetric
n = A.shape[0]
assert type(A) == np.ndarray, 'A must be a numpy array.'
assert np.isreal(A).all(), 'A must be real.'
assert A.shape[1] == n, 'A must be square.'
assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.'
assert p <= n, 'p must be smaller than n.'
# Define the cost on the Grassmann manifold
Gr = Grassmann(n, p)
X = T.matrix()
cost = -T.dot(X.T, T.dot(A, X)).trace()
# Setup the problem
problem = Problem(manifold=Gr, cost=cost, arg=X)
# Create a solver object
solver = TrustRegions()
# Solve
Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p))
return Xopt
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_rows = 1000
rank = 5
low_rank_factor = rnd.randn(num_rows, rank)
matrix = low_rank_factor @ low_rank_factor.T
cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank)
manifold = PSDFixedRank(num_rows, rank)
problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions(maxiter=500, minstepsize=1e-6)
low_rank_factor_estimate = solver.solve(problem)
if quiet:
return
print("Rank of target matrix:", la.matrix_rank(matrix))
matrix_estimate = low_rank_factor_estimate @ low_rank_factor_estimate.T
print("Frobenius norm error of low-rank estimate:",
la.norm(matrix - matrix_estimate))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_rows = 10
rank = 3
matrix = rnd.randn(num_rows, num_rows)
matrix = 0.5 * (matrix + matrix.T)
# Solve the problem with pymanopt.
cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank)
manifold = Oblique(rank, num_rows)
problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
X = solver.solve(problem)
if quiet:
return
C = X.T @ X
print("Diagonal elements:", np.diag(C))
print("Eigenvalues:", np.sort(la.eig(C)[0].real)[::-1])
def dominant_invariant_subspace(A, p):
"""
Returns an orthonormal basis of the dominant invariant p-subspace of A.
Arguments:
- A
A real, symmetric matrix A of size nxn
- p
integer p < n.
Returns:
A real, orthonormal matrix X of size nxp such that trace(X'*A*X)
is maximized. That is, the columns of X form an orthonormal basis
of a dominant subspace of dimension p of A. These span the same space
as the eigenvectors associated with the largest eigenvalues of A.
Sign is important: 2 is deemed a larger eigenvalue than -5.
"""
# Make sure the input matrix is square and symmetric
n = A.shape[0]
assert type(A) == np.ndarray, 'A must be a numpy array.'
assert np.isreal(A).all(), 'A must be real.'
assert A.shape[1] == n, 'A must be square.'
assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.'
assert p <= n, 'p must be smaller than n.'
# Define the cost on the Grassmann manifold
Gr = Grassmann(n, p)
X = T.matrix()
cost = -T.dot(X.T, T.dot(A, X)).trace()
# Setup the problem
problem = Problem(man=Gr, ad_cost=cost, ad_arg=X)
# Create a solver object
solver = TrustRegions()
# Solve
Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p))
return Xopt
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 300
# problem Tr(AXBX^T)
for i in range(1):
dvec = np.array([0, 5, 4, 10])
dvec[0] = n - dvec[1:].sum()
d = dvec[1:].sum()
alpha = randint(1, 10, 2) * .1
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conjugate()
B = make_sym_pos(d)
p = dvec.shape[0] - 1
alpha = randint(1, 10, (p, p + 1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
man = ComplexFlag(dvec, alpha)
@Callable
def cost(X):
return trace(A @ X @ B @ X.T.conjugate()).real
@Callable
def egrad(X):
return 2 * A @ X @ B
@Callable
def ehess(X, H):
return 2 * A @ H @ B
X = man.rand()
if False:
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T.conjugate()).real
print(check_zero(d1 - d2))
prob = Problem(man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
alpha0 = randint(1, 2, (p))
alpha1 = make_simple_alpha(alpha0, 0)
man1 = ComplexFlag(dvec, alpha=alpha1)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha0 = randint(1, 2, (p))
alpha2 = make_simple_alpha(alpha0, -1)
man1 = ComplexFlag(dvec, alpha=alpha2)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
if __name__ == "__main__":
# Generate random data
X = np.random.randn(3, 100)
Y = X[0:1, :] - 2 * X[1:2, :] + np.random.randn(1, 100) + 5
# Cost function is the sqaured test error
w = T.matrix()
b = T.matrix()
cost = T.sum((Y - w.T.dot(X) - b[0, 0])**2)
# A solver that involves the hessian
solver = TrustRegions()
# R^3 x R^1
manifold = Product([Euclidean(3, 1), Euclidean(1, 1)])
# Solve the problem with pymanopt
problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0)
wopt = solver.solve(problem)
print('Weights found by pymanopt (top) / ' 'closed form solution (bottom)')
print(wopt[0].T)
print(wopt[1])
X1 = np.concatenate((X, np.ones((1, 100))), axis=0)
wclosed = np.linalg.inv(X1.dot(X1.T)).dot(X1).dot(Y.T)
print(wclosed[0:3].T)
print(wclosed[3])
# Generate random data
X = np.random.randn(3, 100).astype('float32')
Y = (X[0:1, :] - 2*X[1:2, :] + np.random.randn(1, 100) + 5).astype(
'float32')
# Cost function is the sqaured test error
w = tf.Variable(tf.zeros([3, 1]))
b = tf.Variable(tf.zeros([1]))
cost = tf.reduce_mean(tf.square(Y - tf.matmul(tf.transpose(w), X) - b))
# first-order, second-order
solver = TrustRegions()
# R^3 x R^1
manifold = Product([Euclidean(3, 1), Euclidean(1, 1)])
# Solve the problem with pymanopt
problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0)
wopt = solver.solve(problem)
print('Weights found by pymanopt (top) / '
'closed form solution (bottom)')
print(wopt[0].T)
print(wopt[1])
X1 = np.concatenate((X, np.ones((1, 100))), axis=0)
wclosed = np.linalg.inv(X1.dot(X1.T)).dot(X1).dot(Y.T)
print(wclosed[0:3].T)
print(wclosed[3])
def manifold_fit(self, Y, X):
from pymanopt.manifolds import Rotations, Euclidean, Product
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
# from minimal_varx import make_normalized_G
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p)
self.set_covs(cov_res, cov_xlag)
# m = X.shape[0]
with_c = (self.mm_degree > self.agg_rnk) and (self.m - self.agg_rnk)
C = Euclidean(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk)
RG = Rotations(self.m)
if with_c:
raise (ValueError(
"This option is implemented only for self.agg_rnk == m"))
if with_c:
manifold = Product([RG, C])
else:
manifold = RG
if not with_c:
c_null = np.zeros(
(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk))
else:
c_null = None
if with_c:
def cost(x):
o, c = x
G = make_normalized_G(self, self.m, o, c)
self.calc_states(G)
return self.neg_log_llk
else:
def cost(x):
G = make_normalized_G(self, self.m, x, c_null)
self.calc_states(G)
return self.neg_log_llk
if with_c:
def egrad(x):
o, c = x
grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, o, c)
return [grad_o, grad_c]
else:
def egrad(x):
grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, x,
c_null)
return grad_o
if with_c:
def ehess(x, Heta):
o, c = x
eta = make_normalized_G(self, self.m, Heta[0], Heta[1])
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = self.map_o_c_grad(hess_raw, o, c)
return [hess_o, hess_c]
else:
def ehess(x, Heta):
eta = make_normalized_G(self, self.m, Heta, c_null)
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = self.map_o_c_grad(hess_raw, x, c_null)
return hess_o
if with_c:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
else:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions()
opt = solver.solve(min_mle)
if with_c:
G_opt = make_normalized_G(self, self.m, opt[0], opt[1])
else:
G_opt = make_normalized_G(self, self.m, opt, c_null)
self.calc_H_F_Phi(G_opt, cov_y_xlag)
return opt
def nonlinear_eigenspace(L, k, alpha=1):
"""Nonlinear eigenvalue problem: total energy minimization.
This example is motivated in [1]_ and was adapted from the manopt toolbox
in Matlab.
TODO : check this
Parameters
----------
L : array, shape=(n_channels, n_channels)
Discrete Laplacian operator: the covariance matrix.
alpha : float
Given constant for optimization problem.
k : int
Determines how many eigenvalues are returned.
Returns
-------
Xsol : array, shape=(n_channels, n_channels)
Eigenvectors.
S0 : array
Eigenvalues.
References
----------
.. [1] "A Riemannian Newton Algorithm for Nonlinear Eigenvalue Problems",
Zhi Zhao, Zheng-Jian Bai, and Xiao-Qing Jin, SIAM Journal on Matrix
Analysis and Applications, 36(2), 752-774, 2015.
"""
n = L.shape[0]
assert L.shape[1] == n, 'L must be square.'
# Grassmann manifold description
manifold = Grassmann(n, k)
manifold._dimension = 1 # hack
# A solver that involves the hessian (check if correct TODO)
solver = TrustRegions()
# Cost function evaluation
@pymanopt.function.Callable
def cost(X):
rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X')
val = 0.5 * np.trace(X.T @ (L * X)) + \
(alpha / 4) * (rhoX.T @ mldivide(L, rhoX))
return val
# Euclidean gradient evaluation
@pymanopt.function.Callable
def egrad(X):
rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X')
g = L @ X + alpha * np.diagflat(mldivide(L, rhoX)) @ X
return g
# Euclidean Hessian evaluation
# Note: Manopt automatically converts it to the Riemannian counterpart.
@pymanopt.function.Callable
def ehess(X, U):
rhoX = np.sum(X**2, 1, keepdims=True) # np.diag(X * X')
rhoXdot = 2 * np.sum(X.dot(U), 1)
h = L @ U + alpha * np.diagflat(mldivide(L, rhoXdot)) @ X + \
alpha * np.diagflat(mldivide(L, rhoX)) @ U
return h
# Initialization as suggested in above referenced paper.
# randomly generate starting point for svd
x = np.random.randn(n, k)
[U, S, V] = linalg.svd(x, full_matrices=False)
x = U.dot(V.T)
S0, U0 = linalg.eig(L + alpha * np.diagflat(mldivide(L, np.sum(x**2, 1))))
# Call manoptsolve to automatically call an appropriate solver.
# Note: it calls the trust regions solver as we have all the required
# ingredients, namely, gradient and Hessian, information.
problem = Problem(manifold=manifold,
cost=cost,
egrad=egrad,
ehess=ehess,
verbosity=0)
Xsol = solver.solve(problem, U0)
return S0, Xsol
def egrad(X):
return -ABt
def ehess(X, S):
return manifold.zerovec(X)
problem = Problem(manifold=manifold, cost=cost, egrad=egrad, ehess=ehess)
# problem = Problem(manifold=manifold, cost=cost)
solver = TrustRegions()
# solver = SteepestDescent()
X = solver.solve(problem)
# print(X)
def sol(ABt):
# Compare with the known optimal solution\n",
U, S, Vt = np.linalg.svd(ABt)
UVt = np.dot(U, Vt)
# The determinant of UVt is either 1 or -1, in theory\n",
if abs(1.0 - np.linalg.det(UVt)) < 1e-10:
Xopt = UVt
elif abs(-1.0 - np.linalg.det(UVt)) < 1e-10:
# UVt is in O(n) but not SO(n). This is easily corrected for:\n",
J = np.diag(np.append(np.ones(n - 1), -1))
Xopt = np.dot(np.dot(U, J), Vt)
else:
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 100
d = 20
# problem Tr(AXBX^T)
for i in range(1):
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
B = make_sym_pos(d)
man = RealStiefelWoodbury(n,
d,
A=np.diag(D),
B=np.diag(np.diagonal(B)))
@Callable
def cost(X):
return trace(A @ X @ B @ X.T)
@Callable
def egrad(X):
return 2 * A @ X @ B
@Callable
def ehess(X, H):
return 2 * A @ H @ B
X = man.rand()
if False:
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T)
print(check_zero(d1 - d2))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
man.retr_method = 'geo'
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
# print(opt)
# double check:
print(cost(opt))
min_val = 1e190
min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = RealStiefelWoodbury(n, d, np.eye(n), np.eye(d))
prob1 = Problem(man1, cost, egrad=egrad, ehess=ehess)
man1.retr_method = 'svd'
solver1 = TrustRegions(maxtime=100000, maxiter=100)
opt1 = solver1.solve(prob1, x=XInit, Delta_bar=250)
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 1000
# problem Tr(AXBX^T)
for i in range(1):
dvec = np.array([0, 30, 2, 1])
dvec[0] = 1000 - dvec[1:].sum()
d = dvec[1:].sum()
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
B = make_sym_pos(d)
p = dvec.shape[0]-1
alpha = randint(1, 10, (p, p+1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
man = RealFlag(dvec, alpha=alpha)
@Callable
def cost(X):
return trace(A @ X @ B @ X.T)
@Callable
def egrad(X):
return 2*A @ X @ B
@Callable
def ehess(X, H):
return 2*A @ H @ B
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T)
print(check_zero(d1-d2))
prob = Problem(
man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(
man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
if False:
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
alpha_c = alpha.copy()
alpha_c[:] = 1
man1 = RealFlag(dvec, alpha=alpha_c)
prob = Problem(
man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha_c5 = alpha_c.copy()
alpha_c5[:, 1:] = .5
man1 = RealFlag(dvec, alpha=alpha_c5)
prob = Problem(
man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
def optim_test2():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 100
d = 20
# problem Tr(AXBX^T)
for i in range(1):
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conj()
B = make_sym_pos(d)
alpha = randint(1, 10, 2) * .1
alpha = alpha / alpha[0]
print(alpha)
man = ComplexStiefel(n, d, alpha)
A2 = A @ A
@Callable
def cost(X):
return rtrace(A @ X @ B @ X.T.conj() @ A2 @ X @ B @ X.T.conj() @ A)
@Callable
def egrad(X):
R = 4 * A2 @ X @ B @ X.T.conj() @ A2 @ X @ B
return R
@Callable
def ehess(X, H):
return 4*A2 @ H @ B @ X.T.conj() @ A2 @ X @ B +\
4*A2 @ X @ B @ H.T.conj() @ A2 @ X @ B +\
4*A2 @ X @ B @ X.T.conj() @ A2 @ H @ B
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = rtrace(egrad(X) @ xi.T.conj())
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
prob = Problem(man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
man1 = ComplexStiefel(n, d, alpha=np.array([1, .5]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
def optim_test3():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 200
d = 20
# problem Tr(AXBX^T)
for i in range(1):
B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)]))
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conj()
alpha = randint(1, 10, 2)
alpha = alpha / alpha[0]
print(alpha)
man = ComplexStiefel(n, d, alpha)
cf = 10
B2 = B @ B
@Callable
def cost(X):
return cf * rtrace(
B @ X @ X.T.conj() @ B2 @ X @ X.T.conj() @ B) +\
rtrace(X.T.conj() @ A @ X)
@Callable
def egrad(X):
R = cf * 4 * B2 @ X @ X.T.conj() @ B2 @ X + 2 * A @ X
return R
@Callable
def ehess(X, H):
return 4*cf*B2 @ H @ X.T.conj() @ B2 @ X +\
4*cf*B2 @ X @ H.T.conj() @ B2 @ X +\
4*cf*B2 @ X @ X.T.conj() @ B2 @ H + 2*A @ H
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = rtrace(egrad(X) @ xi.T.conj())
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
man1 = ComplexStiefel(n, d, alpha=np.array([1, .5]))
# man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
return shannon_rate(L, T)
return shannon_capacity_fact_fixed_time
# instantiate manifold for Pymanopt
manifold_fact = Euclidean(n,n)
# instantiate solver for Pymanopt
solver = TrustRegions()
#### find R_max via bisection method ####
# min value T
T1=1e-8
shannon_capacity_fact_fixed_time = make_shannon_capacity_fact_fixed_time(T1)
problem = Problem(manifold=manifold_fact, cost=shannon_capacity_fact_fixed_time, verbosity=0)
L_opt1 = solver.solve(problem)
p_1 = -shannon_capacity_fact_fixed_time(L_opt1)
#print p_1
Sigma_opt1 = np.dot(L_opt1,L_opt1.transpose())
Sigma_opt_norm1 = Sigma_opt1/(np.trace(Sigma_opt1))
eigs, eigvs = np.linalg.eig(Sigma_opt_norm1)
eff_rank1 = np.square(np.sum(np.real(eigs)))/np.sum(np.square(np.real(eigs)))
# max value T
T2=10.
shannon_capacity_fact_fixed_time = make_shannon_capacity_fact_fixed_time(T2)
problem = Problem(manifold=manifold_fact, cost=shannon_capacity_fact_fixed_time, verbosity=0)
L_opt2 = solver.solve(problem)
p_2 = -shannon_capacity_fact_fixed_time(L_opt2)
def optim_test3():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 1000
# problem Tr(AXBX^T)
for i in range(1):
# Stiefel manifold
dvec = np.array([0, 50])
dvec[0] = n - dvec[1:].sum()
d = dvec[1:].sum()
p = dvec.shape[0] - 1
alpha = randint(1, 10, (p, p + 1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)]))
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
man = ComplexFlag(dvec, alpha)
cf = 10
B2 = B @ B
@Callable
def cost(X):
return cf * trace(
B @ X @ st(X) @ B2 @ X @ st(X) @ B) +\
trace(st(X) @ A @ X)
@Callable
def egrad(X):
R = cf * 4 * B2 @ X @ st(X) @ B2 @ X + 2 * A @ X
return R
@Callable
def ehess(X, H):
return 4*cf*B2 @ H @ st(X) @ B2 @ X +\
4*cf*B2 @ X @ st(H) @ B2 @ X +\
4*cf*B2 @ X @ st(X) @ B2 @ H + 2*A @ H
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ st(xi)).real
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
alpha0 = randint(1, 2, (p))
alpha1 = make_simple_alpha(alpha0, 0)
man1 = ComplexFlag(dvec, alpha=alpha1)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha0 = randint(1, 2, (p))
alpha2 = make_simple_alpha(alpha0, -1)
man1 = ComplexFlag(dvec, alpha=alpha2)
# man1 = RealStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
