def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
dimension = 3
num_samples = 200
num_components = 2
samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1])
samples -= samples.mean(axis=0)
cost, egrad, ehess = create_cost_egrad_ehess(backend, samples,
num_components)
manifold = Stiefel(dimension, num_components)
problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
# from pymanopt.solvers import ConjugateGradient
# solver = ConjugateGradient()
estimated_span_matrix = solver.solve(problem)
if quiet:
return
estimated_projector = estimated_span_matrix @ estimated_span_matrix.T
eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples)
indices = np.argsort(eigenvalues)[::-1][:num_components]
span_matrix = eigenvectors[:, indices]
projector = span_matrix @ span_matrix.T
print(
"Frobenius norm error between estimated and closed-form projection "
"matrix:", np.linalg.norm(projector - estimated_projector))
def solve_dist_with_man(man, A, X0, maxiter):
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
@Callable
def cost(S):
# if not(S.P.dtype == np.float):
# raise(ValueError("Non real"))
diff = (A - S.U @ S.P @ S.V.T.conj())
val = rtrace(diff @ diff.T.conj())
# print('val=%f' % val)
return val
@Callable
def egrad(S):
return fr_ambient(-2*A @ S.V @ S.P,
-2*A.T.conj() @ S.U @S.P,
2*(S.P-S.U.T.conj() @ A @ S.V))
@Callable
def ehess(S, xi):
return fr_ambient(-2*A @ (xi.tV @ S.P + S.V @ xi.tP),
-2*A.T.conj() @ (xi.tU @S.P + [email protected]),
2*(xi.tP - xi.tU.T.conj()@[email protected] -
S.U.T.conj()@[email protected]))
prob = Problem(
man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False)
opt = solver.solve(prob, x=X0, Delta_bar=250)
return opt
def run(quiet=True):
dimension = 3
num_samples = 200
num_components = 2
samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1])
samples -= samples.mean(axis=0)
samples_ = torch.from_numpy(samples)
@pymanopt.function.PyTorch
def cost(w):
projector = torch.matmul(w, torch.transpose(w, 1, 0))
return torch.norm(samples_ - torch.matmul(samples_, projector)) ** 2
manifold = Stiefel(dimension, num_components)
problem = pymanopt.Problem(manifold, cost, egrad=None, ehess=None)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
# from pymanopt.solvers import ConjugateGradient
# solver = ConjugateGradient()
estimated_span_matrix = solver.solve(problem)
if quiet:
return
estimated_projector = estimated_span_matrix @ estimated_span_matrix.T
eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples)
indices = np.argsort(eigenvalues)[::-1][:num_components]
span_matrix = eigenvectors[:, indices]
projector = span_matrix @ span_matrix.T
print("Frobenius norm error between estimated and closed-form projection "
"matrix:", np.linalg.norm(projector - estimated_projector))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
"""This example generates a random 128 x 128 symmetric matrix and finds the
dominant invariant 3 dimensional subspace for this matrix, i.e., it finds
the subspace spanned by the three eigenvectors with the largest
eigenvalues.
"""
num_rows = 128
subspace_dimension = 3
matrix = rnd.randn(num_rows, num_rows)
matrix = 0.5 * (matrix + matrix.T)
cost, egrad, ehess = create_cost_egrad_ehess(
backend, matrix, subspace_dimension)
manifold = Grassmann(num_rows, subspace_dimension)
problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
estimated_spanning_set = solver.solve(
problem, Delta_bar=8*np.sqrt(subspace_dimension))
if quiet:
return
eigenvalues, eigenvectors = la.eig(matrix)
column_indices = np.argsort(eigenvalues)[-subspace_dimension:]
spanning_set = eigenvectors[:, column_indices]
print("Geodesic distance between true and estimated dominant subspace:",
manifold.dist(spanning_set, estimated_spanning_set))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_samples, num_weights = 200, 3
solver = TrustRegions()
manifold = Euclidean(3)
for k in range(5):
samples = rnd.randn(num_samples, num_weights)
targets = rnd.randn(num_samples)
cost, egrad, ehess = create_cost_egrad_ehess(backend, samples, targets)
problem = pymanopt.Problem(manifold,
cost,
egrad=egrad,
ehess=ehess,
verbosity=0)
estimated_weights = solver.solve(problem)
if not quiet:
print("Run {}".format(k + 1))
print("Weights found by pymanopt (top) / "
"closed form solution (bottom)")
print(estimated_weights)
print(la.pinv(samples) @ targets)
print("")
def estimateR_weighted(S, W, D, R0):
'''
estimates the update of the rotation matrix for the second part of the iterations
:param S : shape
:param W : heatmap
:param D : weight of the heatmap
:param R0 : rotation matrix
:return: R the new rotation matrix
'''
A = np.transpose(S)
B = np.transpose(W)
X0 = R0[0:2, :]
store_E = Store()
[m, n] = A.shape
p = B.shape[1]
At = np.zeros([n, m])
At = np.transpose(A)
# we use the optimization on a Stiefel manifold because R is constrained to be othogonal
manifold = Stiefel(n, p, 1)
####################################################################################################################
def cost(X):
'''
cost function of the manifold, the cost is trace(E'*D*E)/2 with E = A*X - B or store_E
:param X : vector
:return f : the cost
'''
if store_E.stored is None:
store_E.stored = np.dot(A, np.transpose(X)) - B
E = store_E.stored
f = np.trace(np.dot(np.transpose(E), np.dot(D, E))) / 2
return f
####################################################################################################################
# setup the problem structure with manifold M and cost
problem = Problem(manifold=manifold, cost=cost, verbosity=0)
# setup the trust region algorithm to solve the problem
TR = TrustRegions(maxiter=10)
# solve the problem
X = TR.solve(problem, X0)
#print('X : ',X)
return np.transpose(X) # return R = X'
def estimate_orth_subspaces(self, DataStruct):
'''
main optimization function
'''
# Grassman point?
if LA.norm(np.dot(self.Q.T, self.Q) - np.eye(self.Q.shape[-1]),
ord='fro') > 1e-4:
self._project_stiefel()
# ----------------------------------------------------------------------- #
# eGrad = grad(cost)
# eHess = hessian(cost)
# Perform optimization
# ----------------------------------------------------------------------- #
# ----------------------------------------------------------------------- #
d, r = np.shape(self.Q) # problem size
print(d)
manif = Stiefel(d, r) # initialize manifold
# instantiate problem
problem = Problem(manifold=manif, cost=self._cost, verbosity=2)
# initialize solver
solver = TrustRegions(mingradnorm=1e-8,
minstepsize=1e-16,
logverbosity=1)
# solve
Xopt, optlog = solver.solve(problem)
opt_subspaces = self._objfn(Xopt)
# Align the axes within a subspace by variance high to low
for j in range(self.numSubspaces):
Aj = DataStruct.A[j]
Qj = opt_subspaces[2].Q[j]
# data projected onto subspace
Aj_proj = np.dot((Aj - np.mean(Aj, 0)), Qj)
if np.size(np.cov(Aj_proj.T)) < 2:
V = 1
else:
V = LA.svd(np.cov(Aj_proj.T))[0]
Qj = np.dot(Qj, V)
opt_subspaces[2].Q[j] = Qj # ranked top to low variance
return opt_subspaces[2]
def ManoptOptimization(A, m):
n = A.shape[0]
T = A.shape[2]
manifold = Stiefel(n, m, k=1)
mycost = lambda x: cost(A, x)
myegrad = lambda x: egrad(A, x)
problem = Problem(manifold=manifold, cost=mycost, egrad=myegrad)
solver = TrustRegions()
print('# Start optimization using solver: trustregion')
Xopt = solver.solve(problem)
return Xopt
def rank_k_correlation_matrix_approximation(A, k):
"""
Returns the matrix with unit-norm columns that is closests to A w.r.t. the
Frobenius norm.
"""
m, n = A.shape
assert m == n, "matrix must be square"
assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric"
manifold = Oblique(k, n)
solver = TrustRegions()
X = T.matrix()
cost = 0.25 * T.sum((T.dot(X.T, X) - A) ** 2)
problem = Problem(manifold=manifold, cost=cost, arg=X)
return solver.solve(problem)
def _bootstrap_problem(A, k):
m, n = A.shape
assert m == n, "matrix must be square"
assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric"
manifold = SymFixedRankYY(n, k)
solver = TrustRegions(maxiter=500, minstepsize=1e-6)
return manifold, solver
def rank_k_correlation_matrix_approximation(A, k):
"""
Returns the matrix with unit-norm columns that is closests to A w.r.t. the
Frobenius norm.
"""
m, n = A.shape
assert m == n, "matrix must be square"
assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric"
manifold = Oblique(k, n)
solver = TrustRegions()
X = T.matrix()
cost = 0.25 * T.sum((T.dot(X.T, X) - A)**2)
problem = Problem(manifold=manifold, cost=cost, arg=X)
return solver.solve(problem)
def solve_dist_with_man(man, A, X0, maxiter, check_deriv=False):
import pymanopt
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
@pymanopt.function.Callable
def cost(S):
"""
if not(S.P.dtype == np.float):
raise(ValueError("Non real"))
"""
diff = (A - S.Y @ S.P @ S.Y.T.conjugate())
val = trace(diff @ diff.T.conjugate()).real
# print('val=%f' % val)
return val
@pymanopt.function.Callable
def egrad(S):
return psd_ambient(-4 * A @ S.Y @ S.P,
2 * (S.P - S.Y.T.conjugate() @ A @ S.Y))
@pymanopt.function.Callable
def ehess(S, xi):
return psd_ambient(
-4 * A @ (xi.tY @ S.P + S.Y @ xi.tP),
2 * (xi.tP - xi.tY.T.conjugate() @ A @ S.Y -
S.Y.T.conjugate() @ A @ xi.tY))
if check_deriv:
xi = man.randvec(X0)
dlt = 1e-7
S1 = psd_point(X0.Y + dlt * xi.tY, X0.P + dlt * xi.tP)
print((cost(S1) - cost(X0)) / dlt)
print(man.base_inner_ambient(egrad(X0), xi))
h1 = egrad(S1) - egrad(X0)
h1 = psd_ambient(h1.tY / dlt, h1.tP / dlt)
h2 = ehess(X0, xi)
print(check_zero(h1.tY - h2.tY) + check_zero(h1.tP - h2.tP))
return
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False)
opt = solver.solve(prob, x=X0, Delta_bar=250)
return opt
def optimizer_R_v1(W, D, S, R0):
"""
:param W: heatmap constant
:param D: weight of the keypoints
:param S: shape of the object
:param R0: initial R
:return : the optimal R with fixed T and s
the cost is ||(W-(RS))*sqrt(D)|| but in fact is the scale factor is already taken into account in S and T is taken into account in W
"""
# this store object is needed because the manifold optimizer do not works if it is not implemented like that
store = Store()
# -------------------------------------COST FUNCTION-------------------------------------
def cost(R):
"""
:param R: rotation matrix variable
:return : ||(W-(RS))*D^1/2||^2 = tr((W-(RS))*D*(W-(RS))')
"""
if store.stored is None:
store.stored = W - np.dot(R, S)
X = store.stored
f = np.trace(np.dot(X, np.dot(D, np.transpose(X)))) / 2
return f
# ----------------------------------------------------------------------------------------
# we use the optimization on a Stiefel manifold because R is constrained to be othogonal
manifold = Stiefel(3, 2, 1)
# setup the problem structure with manifold M and cost
problem = Problem(manifold=manifold, cost=cost, verbosity=0)
# setup the trust region algorithm to solve the problem
TR = TrustRegions(maxiter=15)
# solve the problem
R_opt = TR.solve(problem, R0)
return np.transpose(R_opt)
def dominant_invariant_subspace(A, p):
"""
Returns an orthonormal basis of the dominant invariant p-subspace of A.
Arguments:
- A
A real, symmetric matrix A of size nxn
- p
integer p < n.
Returns:
A real, orthonormal matrix X of size nxp such that trace(X'*A*X)
is maximized. That is, the columns of X form an orthonormal basis
of a dominant subspace of dimension p of A. These span the same space
as the eigenvectors associated with the largest eigenvalues of A.
Sign is important: 2 is deemed a larger eigenvalue than -5.
"""
# Make sure the input matrix is square and symmetric
n = A.shape[0]
assert type(A) == np.ndarray, 'A must be a numpy array.'
assert np.isreal(A).all(), 'A must be real.'
assert A.shape[1] == n, 'A must be square.'
assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.'
assert p <= n, 'p must be smaller than n.'
# Define the cost on the Grassmann manifold
Gr = Grassmann(n, p)
X = T.matrix()
cost = -T.dot(X.T, T.dot(A, X)).trace()
# Setup the problem
problem = Problem(manifold=Gr, cost=cost, arg=X)
# Create a solver object
solver = TrustRegions()
# Solve
Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p))
return Xopt
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_rows = 1000
rank = 5
low_rank_factor = rnd.randn(num_rows, rank)
matrix = low_rank_factor @ low_rank_factor.T
cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank)
manifold = PSDFixedRank(num_rows, rank)
problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions(maxiter=500, minstepsize=1e-6)
low_rank_factor_estimate = solver.solve(problem)
if quiet:
return
print("Rank of target matrix:", la.matrix_rank(matrix))
matrix_estimate = low_rank_factor_estimate @ low_rank_factor_estimate.T
print("Frobenius norm error of low-rank estimate:",
la.norm(matrix - matrix_estimate))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True):
num_rows = 10
rank = 3
matrix = rnd.randn(num_rows, num_rows)
matrix = 0.5 * (matrix + matrix.T)
# Solve the problem with pymanopt.
cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank)
manifold = Oblique(rank, num_rows)
problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess)
if quiet:
problem.verbosity = 0
solver = TrustRegions()
X = solver.solve(problem)
if quiet:
return
C = X.T @ X
print("Diagonal elements:", np.diag(C))
print("Eigenvalues:", np.sort(la.eig(C)[0].real)[::-1])
def dominant_invariant_subspace(A, p):
"""
Returns an orthonormal basis of the dominant invariant p-subspace of A.
Arguments:
- A
A real, symmetric matrix A of size nxn
- p
integer p < n.
Returns:
A real, orthonormal matrix X of size nxp such that trace(X'*A*X)
is maximized. That is, the columns of X form an orthonormal basis
of a dominant subspace of dimension p of A. These span the same space
as the eigenvectors associated with the largest eigenvalues of A.
Sign is important: 2 is deemed a larger eigenvalue than -5.
"""
# Make sure the input matrix is square and symmetric
n = A.shape[0]
assert type(A) == np.ndarray, 'A must be a numpy array.'
assert np.isreal(A).all(), 'A must be real.'
assert A.shape[1] == n, 'A must be square.'
assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.'
assert p <= n, 'p must be smaller than n.'
# Define the cost on the Grassmann manifold
Gr = Grassmann(n, p)
X = T.matrix()
cost = -T.dot(X.T, T.dot(A, X)).trace()
# Setup the problem
problem = Problem(man=Gr, ad_cost=cost, ad_arg=X)
# Create a solver object
solver = TrustRegions()
# Solve
Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p))
return Xopt
from pymanopt.solvers import TrustRegions
from pymanopt.manifolds import Euclidean, Product
if __name__ == "__main__":
# Generate random data
X = np.random.randn(3, 100).astype('float32')
Y = (X[0:1, :] - 2*X[1:2, :] + np.random.randn(1, 100) + 5).astype(
'float32')
# Cost function is the sqaured test error
w = tf.Variable(tf.zeros([3, 1]))
b = tf.Variable(tf.zeros([1]))
cost = tf.reduce_mean(tf.square(Y - tf.matmul(tf.transpose(w), X) - b))
# first-order, second-order
solver = TrustRegions()
# R^3 x R^1
manifold = Product([Euclidean(3, 1), Euclidean(1, 1)])
# Solve the problem with pymanopt
problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0)
wopt = solver.solve(problem)
print('Weights found by pymanopt (top) / '
'closed form solution (bottom)')
print(wopt[0].T)
print(wopt[1])
X1 = np.concatenate((X, np.ones((1, 100))), axis=0)
def kindr(Ud, n_clusters, mansolver, init, tol_in, tol_out, max_iter_in,
max_iter_out, disp, do_inner, post_SR, isnrm_row_U, isnrm_col_H,
isbinary_H):
if max_iter_out <= 0:
raise ValueError('Number of iterations should be a positive number,'
' got %d instead' % max_iter_out)
if tol_out <= 0:
raise ValueError('The tolerance should be a positive number,'
' got %d instead' % tol_out)
try:
from pymanopt import Problem
from pymanopt.manifolds import Stiefel, Rotations
from pymanopt.solvers import SteepestDescent, ConjugateGradient, TrustRegions, NelderMead
except ImportError:
raise ValueError(
"KindR needs pymanopt, which is unavailable, try KindAP instead.")
# warnings.warn("KindR solver is unavailable. Transfer to"
# "KindAP instead.")
try:
import autograd.numpy as anp
except ImportError:
warnings.warn(
"Pymanopt needs autograd, which is unavailable,try KindAP instead."
)
idx, center, gerr, numiter = kindap(Ud, n_clusters, init, tol_in,
tol_out, max_iter_in, max_iter_out,
disp, True, post_SR, isnrm_row_U,
isnrm_col_H, isbinary_H)
return idx, center, gerr, numiter
n, d = Ud.shape
k = n_clusters
if d != k:
warnings.warn('Provided more features, expected %d, got %d' % (k, d))
if isnrm_row_U:
Ud = normalize(Ud, axis=1)
# initialization
if isinstance(init, string_types) and init == 'eye':
# Z_0 = -np.identity(k)
U = Ud[:, :k]
elif hasattr(init, '__array__'):
if init.shape[0] != d:
raise ValueError(
'The row size of init should be the same as the total'
'features, got %d instead.' % init.shape[0])
if init.shape[1] != k:
raise ValueError(
'The column size of init should be the same as the total'
'clusters, got %d instead.' % init.shape[1])
U = np.matmul(Ud, np.array(init))
elif isinstance(init, string_types) and init == 'random':
H = sparse.csc_matrix((np.ones(
(n, )), (np.arange(n), np.random.randint(0, k, (n, )))),
shape=(n, k))
smat, sigma, vmat = la.svd(sparse.csc_matrix.dot(Ud.T, H),
full_matrices=False)
z_init = np.matmul(smat, vmat)
U = np.matmul(Ud, z_init)
else:
raise ValueError(
"The init parameter for KindAP should be 'eye','random',"
"or an array. Got a %s with type %s instead." % (init, type(init)))
if isinstance(do_inner, bool) or isinstance(do_inner, int):
do_inner = bool(do_inner)
elif isinstance(do_inner, string_types) and (do_inner
in ["relu", "softmax"]):
pass
else:
raise ValueError("Invalid put do_inner")
H, N = sparse.csc_matrix((n, k)), sparse.csc_matrix((n, k))
dUH = float('inf')
numiter, gerr = [], []
idx = np.ones(n)
crit2 = np.zeros(4)
def cost_n(rotation):
return 0.5 * anp.sum(anp.minimum(anp.matmul(Ud, rotation), 0)**2)
# def cost_softmax(rotation):
# umat = anp.matmul(Ud, rotation)
# exp_umat = anp.exp(umat)
# mat = exp_umat / exp_umat.sum(axis=1).reshape(Ud.shape[0], 1)
# return 0.5 * anp.sum((umat - mat) ** 2)
for n_iter_out in range(max_iter_out):
idxp, Up, Np, Hp = idx, U, N, H
optlog = {}
itr = 0
# inner iterations
if do_inner:
if d == k:
manifold = Rotations(k)
else:
manifold = Stiefel(d, k)
# if do_inner == "softmax":
# cost = cost_softmax
# else:
# cost = cost_n
problem = Problem(manifold=manifold, cost=cost_n, verbosity=0)
if mansolver == "SD":
solver = SteepestDescent(maxiter=max_iter_in,
mingradnorm=tol_in,
logverbosity=2)
elif mansolver == "CG":
solver = ConjugateGradient(maxiter=max_iter_in,
mingradnorm=tol_in,
logverbosity=2)
elif mansolver == "TR":
solver = TrustRegions(maxiter=max_iter_in,
mingradnorm=tol_in,
logverbosity=2)
elif mansolver == "NM":
solver = NelderMead(maxiter=max_iter_in,
mingradnorm=tol_in,
logverbosity=2)
else:
raise ValueError(
"Undefined manifold optimization method, Expect SD, CG, TR or NM, "
"get %s instead" % mansolver)
Z, optlog = solver.solve(problem)
U = np.matmul(Ud, Z)
N = np.maximum(U, 0)
itr = len(optlog['iterations']['iteration'])
numiter.append(itr)
if disp:
print(optlog['iterations']['f(x)'])
print(optlog['stoppingreason'])
else:
N = U
numiter.append(itr)
# project onto H
H, idx = proj_h(N, isnrm_col_H, isbinary_H)
idxchg = sum(idx != idxp)
# project back to Ud
U = proj_ud(H, Ud)
dUHp = dUH
dUH = la.norm(U - H, 'fro')
gerr.append(dUH)
if disp:
print('Outer %3d: %3d dUH: %11.8e idxchg: %6d' %
(n_iter_out + 1, itr, dUH, idxchg))
# stopping criteria of outer iterations
crit2[0] = dUH < 1e-12
crit2[1] = abs(dUH - dUHp) < dUHp * tol_out
crit2[2] = dUH > dUHp
crit2[3] = idxchg == 0
if any(crit2):
if post_SR and do_inner:
do_inner, isbinary_H = False, True
continue
if crit2[2] and not crit2[1]:
idx, H, U, N, dUH = idxp, Hp, Up, Np, dUHp
if disp and optlog and 'stoppingreason' in optlog:
print('\t stop reason:', optlog['stoppingreason'])
break
center = sparse.csc_matrix.dot(
normalize((H != 0), axis=0, norm='l1').T, Ud)
return idx, center, gerr, numiter
def wda(X, y, p=2, reg=1, k=10, solver=None, maxiter=100, verbose=0):
"""
Wasserstein Discriminant Analysis [11]_
The function solves the following optimization problem:
.. math::
P = \\text{arg}\min_P \\frac{\\sum_i W(PX^i,PX^i)}{\\sum_{i,j\\neq i} W(PX^i,PX^j)}
where :
- :math:`P` is a linear projection operator in the Stiefel(p,d) manifold
- :math:`W` is entropic regularized Wasserstein distances
- :math:`X^i` are samples in the dataset corresponding to class i
Parameters
----------
X : numpy.ndarray (n,d)
Training samples
y : np.ndarray (n,)
labels for training samples
p : int, optional
size of dimensionnality reduction
reg : float, optional
Regularization term >0 (entropic regularization)
solver : str, optional
None for steepest decsent or 'TrustRegions' for trust regions algorithm
else shoudl be a pymanopt.sovers
verbose : int, optional
Print information along iterations
Returns
-------
P : (d x p) ndarray
Optimal transportation matrix for the given parameters
proj : fun
projectiuon function including mean centering
References
----------
.. [11] Flamary, R., Cuturi, M., Courty, N., & Rakotomamonjy, A. (2016). Wasserstein Discriminant Analysis. arXiv preprint arXiv:1608.08063.
"""
mx = np.mean(X)
X -= mx.reshape((1, -1))
# data split between classes
d = X.shape[1]
xc = split_classes(X, y)
# compute uniform weighs
wc = [np.ones((x.shape[0]), dtype=np.float32) / x.shape[0] for x in xc]
def cost(P):
# wda loss
loss_b = 0
loss_w = 0
for i, xi in enumerate(xc):
xi = np.dot(xi, P)
for j, xj in enumerate(xc[i:]):
xj = np.dot(xj, P)
M = dist(xi, xj)
G = sinkhorn(wc[i], wc[j + i], M, reg, k)
if j == 0:
loss_w += np.sum(G * M)
else:
loss_b += np.sum(G * M)
# loss inversed because minimization
return loss_w / loss_b
# declare manifold and problem
manifold = Stiefel(d, p)
problem = Problem(manifold=manifold, cost=cost)
# declare solver and solve
if solver is None:
solver = SteepestDescent(maxiter=maxiter, logverbosity=verbose)
elif solver in ['tr', 'TrustRegions']:
solver = TrustRegions(maxiter=maxiter, logverbosity=verbose)
Popt = solver.solve(problem)
def proj(X):
return (X - mx.reshape((1, -1))).dot(Popt)
return Popt, proj
def manifold_fit(self, Y, X):
from pymanopt.manifolds import Rotations, Euclidean, Product
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
# from minimal_varx import make_normalized_G
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p)
self.set_covs(cov_res, cov_xlag)
# m = X.shape[0]
with_c = (self.mm_degree > self.agg_rnk) and (self.m - self.agg_rnk)
C = Euclidean(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk)
RG = Rotations(self.m)
if with_c:
raise (ValueError(
"This option is implemented only for self.agg_rnk == m"))
if with_c:
manifold = Product([RG, C])
else:
manifold = RG
if not with_c:
c_null = np.zeros(
(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk))
else:
c_null = None
if with_c:
def cost(x):
o, c = x
G = make_normalized_G(self, self.m, o, c)
self.calc_states(G)
return self.neg_log_llk
else:
def cost(x):
G = make_normalized_G(self, self.m, x, c_null)
self.calc_states(G)
return self.neg_log_llk
if with_c:
def egrad(x):
o, c = x
grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, o, c)
return [grad_o, grad_c]
else:
def egrad(x):
grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, x,
c_null)
return grad_o
if with_c:
def ehess(x, Heta):
o, c = x
eta = make_normalized_G(self, self.m, Heta[0], Heta[1])
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = self.map_o_c_grad(hess_raw, o, c)
return [hess_o, hess_c]
else:
def ehess(x, Heta):
eta = make_normalized_G(self, self.m, Heta, c_null)
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = self.map_o_c_grad(hess_raw, x, c_null)
return hess_o
if with_c:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
else:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions()
opt = solver.solve(min_mle)
if with_c:
G_opt = make_normalized_G(self, self.m, opt[0], opt[1])
else:
G_opt = make_normalized_G(self, self.m, opt, c_null)
self.calc_H_F_Phi(G_opt, cov_y_xlag)
return opt
def nonlinear_eigenspace(L, k, alpha=1):
"""Nonlinear eigenvalue problem: total energy minimization.
This example is motivated in [1]_ and was adapted from the manopt toolbox
in Matlab.
TODO : check this
Parameters
----------
L : array, shape=(n_channels, n_channels)
Discrete Laplacian operator: the covariance matrix.
alpha : float
Given constant for optimization problem.
k : int
Determines how many eigenvalues are returned.
Returns
-------
Xsol : array, shape=(n_channels, n_channels)
Eigenvectors.
S0 : array
Eigenvalues.
References
----------
.. [1] "A Riemannian Newton Algorithm for Nonlinear Eigenvalue Problems",
Zhi Zhao, Zheng-Jian Bai, and Xiao-Qing Jin, SIAM Journal on Matrix
Analysis and Applications, 36(2), 752-774, 2015.
"""
n = L.shape[0]
assert L.shape[1] == n, 'L must be square.'
# Grassmann manifold description
manifold = Grassmann(n, k)
manifold._dimension = 1 # hack
# A solver that involves the hessian (check if correct TODO)
solver = TrustRegions()
# Cost function evaluation
@pymanopt.function.Callable
def cost(X):
rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X')
val = 0.5 * np.trace(X.T @ (L * X)) + \
(alpha / 4) * (rhoX.T @ mldivide(L, rhoX))
return val
# Euclidean gradient evaluation
@pymanopt.function.Callable
def egrad(X):
rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X')
g = L @ X + alpha * np.diagflat(mldivide(L, rhoX)) @ X
return g
# Euclidean Hessian evaluation
# Note: Manopt automatically converts it to the Riemannian counterpart.
@pymanopt.function.Callable
def ehess(X, U):
rhoX = np.sum(X**2, 1, keepdims=True) # np.diag(X * X')
rhoXdot = 2 * np.sum(X.dot(U), 1)
h = L @ U + alpha * np.diagflat(mldivide(L, rhoXdot)) @ X + \
alpha * np.diagflat(mldivide(L, rhoX)) @ U
return h
# Initialization as suggested in above referenced paper.
# randomly generate starting point for svd
x = np.random.randn(n, k)
[U, S, V] = linalg.svd(x, full_matrices=False)
x = U.dot(V.T)
S0, U0 = linalg.eig(L + alpha * np.diagflat(mldivide(L, np.sum(x**2, 1))))
# Call manoptsolve to automatically call an appropriate solver.
# Note: it calls the trust regions solver as we have all the required
# ingredients, namely, gradient and Hessian, information.
problem = Problem(manifold=manifold,
cost=cost,
egrad=egrad,
ehess=ehess,
verbosity=0)
Xsol = solver.solve(problem, U0)
return S0, Xsol
def optim_test2():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 100
d = 20
# problem Tr(AXBX^T)
for i in range(1):
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conj()
B = make_sym_pos(d)
alpha = randint(1, 10, 2) * .1
alpha = alpha / alpha[0]
print(alpha)
man = ComplexStiefel(n, d, alpha)
A2 = A @ A
@Callable
def cost(X):
return rtrace(A @ X @ B @ X.T.conj() @ A2 @ X @ B @ X.T.conj() @ A)
@Callable
def egrad(X):
R = 4 * A2 @ X @ B @ X.T.conj() @ A2 @ X @ B
return R
@Callable
def ehess(X, H):
return 4*A2 @ H @ B @ X.T.conj() @ A2 @ X @ B +\
4*A2 @ X @ B @ H.T.conj() @ A2 @ X @ B +\
4*A2 @ X @ B @ X.T.conj() @ A2 @ H @ B
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = rtrace(egrad(X) @ xi.T.conj())
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
prob = Problem(man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
man1 = ComplexStiefel(n, d, alpha=np.array([1, .5]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
def wda(X,
y,
p=2,
reg=1,
k=10,
solver=None,
sinkhorn_method='sinkhorn',
maxiter=100,
verbose=0,
P0=None,
normalize=False):
r"""
Wasserstein Discriminant Analysis :ref:`[11] <references-wda>`
The function solves the following optimization problem:
.. math::
\mathbf{P} = \mathop{\arg \min}_\mathbf{P} \quad
\frac{\sum\limits_i W(P \mathbf{X}^i, P \mathbf{X}^i)}{\sum\limits_{i, j \neq i} W(P \mathbf{X}^i, P \mathbf{X}^j)}
where :
- :math:`P` is a linear projection operator in the Stiefel(`p`, `d`) manifold
- :math:`W` is entropic regularized Wasserstein distances
- :math:`\mathbf{X}^i` are samples in the dataset corresponding to class i
**Choosing a Sinkhorn solver**
By default and when using a regularization parameter that is not too small
the default sinkhorn solver should be enough. If you need to use a small
regularization to get sparse cost matrices, you should use the
:py:func:`ot.dr.sinkhorn_log` solver that will avoid numerical
errors, but can be slow in practice.
Parameters
----------
X : ndarray, shape (n, d)
Training samples.
y : ndarray, shape (n,)
Labels for training samples.
p : int, optional
Size of dimensionnality reduction.
reg : float, optional
Regularization term >0 (entropic regularization)
solver : None | str, optional
None for steepest descent or 'TrustRegions' for trust regions algorithm
else should be a pymanopt.solvers
sinkhorn_method : str
method used for the Sinkhorn solver, either 'sinkhorn' or 'sinkhorn_log'
P0 : ndarray, shape (d, p)
Initial starting point for projection.
normalize : bool, optional
Normalise the Wasserstaiun distance by the average distance on P0 (default : False)
verbose : int, optional
Print information along iterations.
Returns
-------
P : ndarray, shape (d, p)
Optimal transportation matrix for the given parameters
proj : callable
Projection function including mean centering.
.. _references-wda:
References
----------
.. [11] Flamary, R., Cuturi, M., Courty, N., & Rakotomamonjy, A. (2016).
Wasserstein Discriminant Analysis. arXiv preprint arXiv:1608.08063.
""" # noqa
if sinkhorn_method.lower() == 'sinkhorn':
sinkhorn_solver = sinkhorn
elif sinkhorn_method.lower() == 'sinkhorn_log':
sinkhorn_solver = sinkhorn_log
else:
raise ValueError("Unknown Sinkhorn method '%s'." % sinkhorn_method)
mx = np.mean(X)
X -= mx.reshape((1, -1))
# data split between classes
d = X.shape[1]
xc = split_classes(X, y)
# compute uniform weighs
wc = [np.ones((x.shape[0]), dtype=np.float32) / x.shape[0] for x in xc]
# pre-compute reg_c,c'
if P0 is not None and normalize:
regmean = np.zeros((len(xc), len(xc)))
for i, xi in enumerate(xc):
xi = np.dot(xi, P0)
for j, xj in enumerate(xc[i:]):
xj = np.dot(xj, P0)
M = dist(xi, xj)
regmean[i, j] = np.sum(M) / (len(xi) * len(xj))
else:
regmean = np.ones((len(xc), len(xc)))
@Autograd
def cost(P):
# wda loss
loss_b = 0
loss_w = 0
for i, xi in enumerate(xc):
xi = np.dot(xi, P)
for j, xj in enumerate(xc[i:]):
xj = np.dot(xj, P)
M = dist(xi, xj)
G = sinkhorn_solver(wc[i], wc[j + i], M, reg * regmean[i, j],
k)
if j == 0:
loss_w += np.sum(G * M)
else:
loss_b += np.sum(G * M)
# loss inversed because minimization
return loss_w / loss_b
# declare manifold and problem
manifold = Stiefel(d, p)
problem = Problem(manifold=manifold, cost=cost)
# declare solver and solve
if solver is None:
solver = SteepestDescent(maxiter=maxiter, logverbosity=verbose)
elif solver in ['tr', 'TrustRegions']:
solver = TrustRegions(maxiter=maxiter, logverbosity=verbose)
Popt = solver.solve(problem, x=P0)
def proj(X):
return (X - mx.reshape((1, -1))).dot(Popt)
return Popt, proj
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 100
d = 20
# problem Tr(AXBX^T)
for i in range(1):
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
B = make_sym_pos(d)
man = RealStiefelWoodbury(n,
d,
A=np.diag(D),
B=np.diag(np.diagonal(B)))
@Callable
def cost(X):
return trace(A @ X @ B @ X.T)
@Callable
def egrad(X):
return 2 * A @ X @ B
@Callable
def ehess(X, H):
return 2 * A @ H @ B
X = man.rand()
if False:
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T)
print(check_zero(d1 - d2))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
man.retr_method = 'geo'
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
# print(opt)
# double check:
print(cost(opt))
min_val = 1e190
min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = RealStiefelWoodbury(n, d, np.eye(n), np.eye(d))
prob1 = Problem(man1, cost, egrad=egrad, ehess=ehess)
man1.retr_method = 'svd'
solver1 = TrustRegions(maxtime=100000, maxiter=100)
opt1 = solver1.solve(prob1, x=XInit, Delta_bar=250)
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 1000
# problem Tr(AXBX^T)
for i in range(1):
dvec = np.array([0, 30, 2, 1])
dvec[0] = 1000 - dvec[1:].sum()
d = dvec[1:].sum()
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
B = make_sym_pos(d)
p = dvec.shape[0]-1
alpha = randint(1, 10, (p, p+1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
man = RealFlag(dvec, alpha=alpha)
@Callable
def cost(X):
return trace(A @ X @ B @ X.T)
@Callable
def egrad(X):
return 2*A @ X @ B
@Callable
def ehess(X, H):
return 2*A @ H @ B
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T)
print(check_zero(d1-d2))
prob = Problem(
man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(
man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
if False:
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
alpha_c = alpha.copy()
alpha_c[:] = 1
man1 = RealFlag(dvec, alpha=alpha_c)
prob = Problem(
man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha_c5 = alpha_c.copy()
alpha_c5[:, 1:] = .5
man1 = RealFlag(dvec, alpha=alpha_c5)
prob = Problem(
man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.manifolds import Euclidean, Product
if __name__ == "__main__":
# Generate random data
X = np.random.randn(3, 100)
Y = X[0:1, :] - 2 * X[1:2, :] + np.random.randn(1, 100) + 5
# Cost function is the sqaured test error
w = T.matrix()
b = T.matrix()
cost = T.sum((Y - w.T.dot(X) - b[0, 0])**2)
# A solver that involves the hessian
solver = TrustRegions()
# R^3 x R^1
manifold = Product([Euclidean(3, 1), Euclidean(1, 1)])
# Solve the problem with pymanopt
problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0)
wopt = solver.solve(problem)
print('Weights found by pymanopt (top) / ' 'closed form solution (bottom)')
print(wopt[0].T)
print(wopt[1])
X1 = np.concatenate((X, np.ones((1, 100))), axis=0)
wclosed = np.linalg.inv(X1.dot(X1.T)).dot(X1).dot(Y.T)
def cost(X):
return -np.tensordot(X, ABt, axes=X.ndim)
def egrad(X):
return -ABt
def ehess(X, S):
return manifold.zerovec(X)
problem = Problem(manifold=manifold, cost=cost, egrad=egrad, ehess=ehess)
# problem = Problem(manifold=manifold, cost=cost)
solver = TrustRegions()
# solver = SteepestDescent()
X = solver.solve(problem)
# print(X)
def sol(ABt):
# Compare with the known optimal solution\n",
U, S, Vt = np.linalg.svd(ABt)
UVt = np.dot(U, Vt)
# The determinant of UVt is either 1 or -1, in theory\n",
if abs(1.0 - np.linalg.det(UVt)) < 1e-10:
Xopt = UVt
elif abs(-1.0 - np.linalg.det(UVt)) < 1e-10:
# UVt is in O(n) but not SO(n). This is easily corrected for:\n",
def optim_test():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 300
# problem Tr(AXBX^T)
for i in range(1):
dvec = np.array([0, 5, 4, 10])
dvec[0] = n - dvec[1:].sum()
d = dvec[1:].sum()
alpha = randint(1, 10, 2) * .1
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conjugate()
B = make_sym_pos(d)
p = dvec.shape[0] - 1
alpha = randint(1, 10, (p, p + 1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
man = ComplexFlag(dvec, alpha)
@Callable
def cost(X):
return trace(A @ X @ B @ X.T.conjugate()).real
@Callable
def egrad(X):
return 2 * A @ X @ B
@Callable
def ehess(X, H):
return 2 * A @ H @ B
X = man.rand()
if False:
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ xi.T.conjugate()).real
print(check_zero(d1 - d2))
prob = Problem(man, cost, egrad=egrad)
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
print(cost(opt))
alpha0 = randint(1, 2, (p))
alpha1 = make_simple_alpha(alpha0, 0)
man1 = ComplexFlag(dvec, alpha=alpha1)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha0 = randint(1, 2, (p))
alpha2 = make_simple_alpha(alpha0, -1)
man1 = ComplexFlag(dvec, alpha=alpha2)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
def optim_test3():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 200
d = 20
# problem Tr(AXBX^T)
for i in range(1):
B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)]))
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T.conj()
alpha = randint(1, 10, 2)
alpha = alpha / alpha[0]
print(alpha)
man = ComplexStiefel(n, d, alpha)
cf = 10
B2 = B @ B
@Callable
def cost(X):
return cf * rtrace(
B @ X @ X.T.conj() @ B2 @ X @ X.T.conj() @ B) +\
rtrace(X.T.conj() @ A @ X)
@Callable
def egrad(X):
R = cf * 4 * B2 @ X @ X.T.conj() @ B2 @ X + 2 * A @ X
return R
@Callable
def ehess(X, H):
return 4*cf*B2 @ H @ X.T.conj() @ B2 @ X +\
4*cf*B2 @ X @ H.T.conj() @ B2 @ X +\
4*cf*B2 @ X @ X.T.conj() @ B2 @ H + 2*A @ H
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = rtrace(egrad(X) @ xi.T.conj())
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
man1 = ComplexStiefel(n, d, alpha=np.array([1, .5]))
# man1 = ComplexStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
Wcd = np.reshape(Wcd_vec,(n,n))
Wnum = np.dot(Wo,Wcd)
Wden = np.dot(Wo,Wcd-Sigma_norm_B)
num = np.linalg.det(Wnum+sigma2*I)
den = np.linalg.det(Wden+sigma2*I)
return -np.log2(num/den)/(2.*T)
def make_shannon_capacity_fact_fixed_time(T):
def shannon_capacity_fact_fixed_time(L):
return shannon_rate(L, T)
return shannon_capacity_fact_fixed_time
# instantiate manifold for Pymanopt
manifold_fact = Euclidean(n,n)
# instantiate solver for Pymanopt
solver = TrustRegions()
#### find R_max via bisection method ####
# min value T
T1=1e-8
shannon_capacity_fact_fixed_time = make_shannon_capacity_fact_fixed_time(T1)
problem = Problem(manifold=manifold_fact, cost=shannon_capacity_fact_fixed_time, verbosity=0)
L_opt1 = solver.solve(problem)
p_1 = -shannon_capacity_fact_fixed_time(L_opt1)
#print p_1
Sigma_opt1 = np.dot(L_opt1,L_opt1.transpose())
Sigma_opt_norm1 = Sigma_opt1/(np.trace(Sigma_opt1))
def optim_test3():
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from pymanopt.function import Callable
n = 1000
# problem Tr(AXBX^T)
for i in range(1):
# Stiefel manifold
dvec = np.array([0, 50])
dvec[0] = n - dvec[1:].sum()
d = dvec[1:].sum()
p = dvec.shape[0] - 1
alpha = randint(1, 10, (p, p + 1)) * .1
alpha0 = randint(1, 10, (p))
# alpha0 = randint(1, 2, (p))
alpha = make_simple_alpha(alpha0, 0)
B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)]))
D = randint(1, 10, n) * 0.02 + 1
OO = random_orthogonal(n)
A = OO @ np.diag(D) @ OO.T
man = ComplexFlag(dvec, alpha)
cf = 10
B2 = B @ B
@Callable
def cost(X):
return cf * trace(
B @ X @ st(X) @ B2 @ X @ st(X) @ B) +\
trace(st(X) @ A @ X)
@Callable
def egrad(X):
R = cf * 4 * B2 @ X @ st(X) @ B2 @ X + 2 * A @ X
return R
@Callable
def ehess(X, H):
return 4*cf*B2 @ H @ st(X) @ B2 @ X +\
4*cf*B2 @ X @ st(H) @ B2 @ X +\
4*cf*B2 @ X @ st(X) @ B2 @ H + 2*A @ H
if False:
X = man.rand()
xi = man.randvec(X)
d1 = num_deriv(man, X, xi, cost)
d2 = trace(egrad(X) @ st(xi)).real
print(check_zero(d1 - d2))
d3 = num_deriv(man, X, xi, egrad)
d4 = ehess(X, xi)
print(check_zero(d3 - d4))
XInit = man.rand()
prob = Problem(man, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=2500)
print(cost(opt))
if False:
# print(opt)
# double check:
# print(cost(opt))
min_val = 1e190
# min_X = None
for i in range(100):
Xi = man.rand()
c = cost(Xi)
if c < min_val:
# min_X = Xi
min_val = c
if i % 1000 == 0:
print('i=%d min=%f' % (i, min_val))
print(min_val)
alpha0 = randint(1, 2, (p))
alpha1 = make_simple_alpha(alpha0, 0)
man1 = ComplexFlag(dvec, alpha=alpha1)
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
alpha0 = randint(1, 2, (p))
alpha2 = make_simple_alpha(alpha0, -1)
man1 = ComplexFlag(dvec, alpha=alpha2)
# man1 = RealStiefel(n, d, alpha=np.array([1, 1]))
prob = Problem(man1, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions(maxtime=100000, maxiter=100)
opt = solver.solve(prob, x=XInit, Delta_bar=250)
