def run(backend=SUPPORTED_BACKENDS[0], quiet=True): dimension = 3 num_samples = 200 num_components = 2 samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1]) samples -= samples.mean(axis=0) cost, egrad, ehess = create_cost_egrad_ehess(backend, samples, num_components) manifold = Stiefel(dimension, num_components) problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess) if quiet: problem.verbosity = 0 solver = TrustRegions() # from pymanopt.solvers import ConjugateGradient # solver = ConjugateGradient() estimated_span_matrix = solver.solve(problem) if quiet: return estimated_projector = estimated_span_matrix @ estimated_span_matrix.T eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples) indices = np.argsort(eigenvalues)[::-1][:num_components] span_matrix = eigenvectors[:, indices] projector = span_matrix @ span_matrix.T print( "Frobenius norm error between estimated and closed-form projection " "matrix:", np.linalg.norm(projector - estimated_projector))
def solve_dist_with_man(man, A, X0, maxiter): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable @Callable def cost(S): # if not(S.P.dtype == np.float): # raise(ValueError("Non real")) diff = (A - S.U @ S.P @ S.V.T.conj()) val = rtrace(diff @ diff.T.conj()) # print('val=%f' % val) return val @Callable def egrad(S): return fr_ambient(-2*A @ S.V @ S.P, -2*A.T.conj() @ S.U @S.P, 2*(S.P-S.U.T.conj() @ A @ S.V)) @Callable def ehess(S, xi): return fr_ambient(-2*A @ (xi.tV @ S.P + S.V @ xi.tP), -2*A.T.conj() @ (xi.tU @S.P + [email protected]), 2*(xi.tP - xi.tU.T.conj()@[email protected] - S.U.T.conj()@[email protected])) prob = Problem( man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False) opt = solver.solve(prob, x=X0, Delta_bar=250) return opt
def run(quiet=True): dimension = 3 num_samples = 200 num_components = 2 samples = np.random.randn(num_samples, dimension) @ np.diag([3, 2, 1]) samples -= samples.mean(axis=0) samples_ = torch.from_numpy(samples) @pymanopt.function.PyTorch def cost(w): projector = torch.matmul(w, torch.transpose(w, 1, 0)) return torch.norm(samples_ - torch.matmul(samples_, projector)) ** 2 manifold = Stiefel(dimension, num_components) problem = pymanopt.Problem(manifold, cost, egrad=None, ehess=None) if quiet: problem.verbosity = 0 solver = TrustRegions() # from pymanopt.solvers import ConjugateGradient # solver = ConjugateGradient() estimated_span_matrix = solver.solve(problem) if quiet: return estimated_projector = estimated_span_matrix @ estimated_span_matrix.T eigenvalues, eigenvectors = np.linalg.eig(samples.T @ samples) indices = np.argsort(eigenvalues)[::-1][:num_components] span_matrix = eigenvectors[:, indices] projector = span_matrix @ span_matrix.T print("Frobenius norm error between estimated and closed-form projection " "matrix:", np.linalg.norm(projector - estimated_projector))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True): """This example generates a random 128 x 128 symmetric matrix and finds the dominant invariant 3 dimensional subspace for this matrix, i.e., it finds the subspace spanned by the three eigenvectors with the largest eigenvalues. """ num_rows = 128 subspace_dimension = 3 matrix = rnd.randn(num_rows, num_rows) matrix = 0.5 * (matrix + matrix.T) cost, egrad, ehess = create_cost_egrad_ehess( backend, matrix, subspace_dimension) manifold = Grassmann(num_rows, subspace_dimension) problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess) if quiet: problem.verbosity = 0 solver = TrustRegions() estimated_spanning_set = solver.solve( problem, Delta_bar=8*np.sqrt(subspace_dimension)) if quiet: return eigenvalues, eigenvectors = la.eig(matrix) column_indices = np.argsort(eigenvalues)[-subspace_dimension:] spanning_set = eigenvectors[:, column_indices] print("Geodesic distance between true and estimated dominant subspace:", manifold.dist(spanning_set, estimated_spanning_set))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True): num_samples, num_weights = 200, 3 solver = TrustRegions() manifold = Euclidean(3) for k in range(5): samples = rnd.randn(num_samples, num_weights) targets = rnd.randn(num_samples) cost, egrad, ehess = create_cost_egrad_ehess(backend, samples, targets) problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess, verbosity=0) estimated_weights = solver.solve(problem) if not quiet: print("Run {}".format(k + 1)) print("Weights found by pymanopt (top) / " "closed form solution (bottom)") print(estimated_weights) print(la.pinv(samples) @ targets) print("")
def estimateR_weighted(S, W, D, R0): ''' estimates the update of the rotation matrix for the second part of the iterations :param S : shape :param W : heatmap :param D : weight of the heatmap :param R0 : rotation matrix :return: R the new rotation matrix ''' A = np.transpose(S) B = np.transpose(W) X0 = R0[0:2, :] store_E = Store() [m, n] = A.shape p = B.shape[1] At = np.zeros([n, m]) At = np.transpose(A) # we use the optimization on a Stiefel manifold because R is constrained to be othogonal manifold = Stiefel(n, p, 1) #################################################################################################################### def cost(X): ''' cost function of the manifold, the cost is trace(E'*D*E)/2 with E = A*X - B or store_E :param X : vector :return f : the cost ''' if store_E.stored is None: store_E.stored = np.dot(A, np.transpose(X)) - B E = store_E.stored f = np.trace(np.dot(np.transpose(E), np.dot(D, E))) / 2 return f #################################################################################################################### # setup the problem structure with manifold M and cost problem = Problem(manifold=manifold, cost=cost, verbosity=0) # setup the trust region algorithm to solve the problem TR = TrustRegions(maxiter=10) # solve the problem X = TR.solve(problem, X0) #print('X : ',X) return np.transpose(X) # return R = X'
def estimate_orth_subspaces(self, DataStruct): ''' main optimization function ''' # Grassman point? if LA.norm(np.dot(self.Q.T, self.Q) - np.eye(self.Q.shape[-1]), ord='fro') > 1e-4: self._project_stiefel() # ----------------------------------------------------------------------- # # eGrad = grad(cost) # eHess = hessian(cost) # Perform optimization # ----------------------------------------------------------------------- # # ----------------------------------------------------------------------- # d, r = np.shape(self.Q) # problem size print(d) manif = Stiefel(d, r) # initialize manifold # instantiate problem problem = Problem(manifold=manif, cost=self._cost, verbosity=2) # initialize solver solver = TrustRegions(mingradnorm=1e-8, minstepsize=1e-16, logverbosity=1) # solve Xopt, optlog = solver.solve(problem) opt_subspaces = self._objfn(Xopt) # Align the axes within a subspace by variance high to low for j in range(self.numSubspaces): Aj = DataStruct.A[j] Qj = opt_subspaces[2].Q[j] # data projected onto subspace Aj_proj = np.dot((Aj - np.mean(Aj, 0)), Qj) if np.size(np.cov(Aj_proj.T)) < 2: V = 1 else: V = LA.svd(np.cov(Aj_proj.T))[0] Qj = np.dot(Qj, V) opt_subspaces[2].Q[j] = Qj # ranked top to low variance return opt_subspaces[2]
def ManoptOptimization(A, m): n = A.shape[0] T = A.shape[2] manifold = Stiefel(n, m, k=1) mycost = lambda x: cost(A, x) myegrad = lambda x: egrad(A, x) problem = Problem(manifold=manifold, cost=mycost, egrad=myegrad) solver = TrustRegions() print('# Start optimization using solver: trustregion') Xopt = solver.solve(problem) return Xopt
def rank_k_correlation_matrix_approximation(A, k): """ Returns the matrix with unit-norm columns that is closests to A w.r.t. the Frobenius norm. """ m, n = A.shape assert m == n, "matrix must be square" assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric" manifold = Oblique(k, n) solver = TrustRegions() X = T.matrix() cost = 0.25 * T.sum((T.dot(X.T, X) - A) ** 2) problem = Problem(manifold=manifold, cost=cost, arg=X) return solver.solve(problem)
def _bootstrap_problem(A, k): m, n = A.shape assert m == n, "matrix must be square" assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric" manifold = SymFixedRankYY(n, k) solver = TrustRegions(maxiter=500, minstepsize=1e-6) return manifold, solver
def rank_k_correlation_matrix_approximation(A, k): """ Returns the matrix with unit-norm columns that is closests to A w.r.t. the Frobenius norm. """ m, n = A.shape assert m == n, "matrix must be square" assert np.allclose(np.sum(A - A.T), 0), "matrix must be symmetric" manifold = Oblique(k, n) solver = TrustRegions() X = T.matrix() cost = 0.25 * T.sum((T.dot(X.T, X) - A)**2) problem = Problem(manifold=manifold, cost=cost, arg=X) return solver.solve(problem)
def solve_dist_with_man(man, A, X0, maxiter, check_deriv=False): import pymanopt from pymanopt import Problem from pymanopt.solvers import TrustRegions @pymanopt.function.Callable def cost(S): """ if not(S.P.dtype == np.float): raise(ValueError("Non real")) """ diff = (A - S.Y @ S.P @ S.Y.T.conjugate()) val = trace(diff @ diff.T.conjugate()).real # print('val=%f' % val) return val @pymanopt.function.Callable def egrad(S): return psd_ambient(-4 * A @ S.Y @ S.P, 2 * (S.P - S.Y.T.conjugate() @ A @ S.Y)) @pymanopt.function.Callable def ehess(S, xi): return psd_ambient( -4 * A @ (xi.tY @ S.P + S.Y @ xi.tP), 2 * (xi.tP - xi.tY.T.conjugate() @ A @ S.Y - S.Y.T.conjugate() @ A @ xi.tY)) if check_deriv: xi = man.randvec(X0) dlt = 1e-7 S1 = psd_point(X0.Y + dlt * xi.tY, X0.P + dlt * xi.tP) print((cost(S1) - cost(X0)) / dlt) print(man.base_inner_ambient(egrad(X0), xi)) h1 = egrad(S1) - egrad(X0) h1 = psd_ambient(h1.tY / dlt, h1.tP / dlt) h2 = ehess(X0, xi) print(check_zero(h1.tY - h2.tY) + check_zero(h1.tP - h2.tP)) return prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=maxiter, use_rand=False) opt = solver.solve(prob, x=X0, Delta_bar=250) return opt
def optimizer_R_v1(W, D, S, R0): """ :param W: heatmap constant :param D: weight of the keypoints :param S: shape of the object :param R0: initial R :return : the optimal R with fixed T and s the cost is ||(W-(RS))*sqrt(D)|| but in fact is the scale factor is already taken into account in S and T is taken into account in W """ # this store object is needed because the manifold optimizer do not works if it is not implemented like that store = Store() # -------------------------------------COST FUNCTION------------------------------------- def cost(R): """ :param R: rotation matrix variable :return : ||(W-(RS))*D^1/2||^2 = tr((W-(RS))*D*(W-(RS))') """ if store.stored is None: store.stored = W - np.dot(R, S) X = store.stored f = np.trace(np.dot(X, np.dot(D, np.transpose(X)))) / 2 return f # ---------------------------------------------------------------------------------------- # we use the optimization on a Stiefel manifold because R is constrained to be othogonal manifold = Stiefel(3, 2, 1) # setup the problem structure with manifold M and cost problem = Problem(manifold=manifold, cost=cost, verbosity=0) # setup the trust region algorithm to solve the problem TR = TrustRegions(maxiter=15) # solve the problem R_opt = TR.solve(problem, R0) return np.transpose(R_opt)
def dominant_invariant_subspace(A, p): """ Returns an orthonormal basis of the dominant invariant p-subspace of A. Arguments: - A A real, symmetric matrix A of size nxn - p integer p < n. Returns: A real, orthonormal matrix X of size nxp such that trace(X'*A*X) is maximized. That is, the columns of X form an orthonormal basis of a dominant subspace of dimension p of A. These span the same space as the eigenvectors associated with the largest eigenvalues of A. Sign is important: 2 is deemed a larger eigenvalue than -5. """ # Make sure the input matrix is square and symmetric n = A.shape[0] assert type(A) == np.ndarray, 'A must be a numpy array.' assert np.isreal(A).all(), 'A must be real.' assert A.shape[1] == n, 'A must be square.' assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.' assert p <= n, 'p must be smaller than n.' # Define the cost on the Grassmann manifold Gr = Grassmann(n, p) X = T.matrix() cost = -T.dot(X.T, T.dot(A, X)).trace() # Setup the problem problem = Problem(manifold=Gr, cost=cost, arg=X) # Create a solver object solver = TrustRegions() # Solve Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p)) return Xopt
def run(backend=SUPPORTED_BACKENDS[0], quiet=True): num_rows = 1000 rank = 5 low_rank_factor = rnd.randn(num_rows, rank) matrix = low_rank_factor @ low_rank_factor.T cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank) manifold = PSDFixedRank(num_rows, rank) problem = pymanopt.Problem(manifold, cost=cost, egrad=egrad, ehess=ehess) if quiet: problem.verbosity = 0 solver = TrustRegions(maxiter=500, minstepsize=1e-6) low_rank_factor_estimate = solver.solve(problem) if quiet: return print("Rank of target matrix:", la.matrix_rank(matrix)) matrix_estimate = low_rank_factor_estimate @ low_rank_factor_estimate.T print("Frobenius norm error of low-rank estimate:", la.norm(matrix - matrix_estimate))
def run(backend=SUPPORTED_BACKENDS[0], quiet=True): num_rows = 10 rank = 3 matrix = rnd.randn(num_rows, num_rows) matrix = 0.5 * (matrix + matrix.T) # Solve the problem with pymanopt. cost, egrad, ehess = create_cost_egrad_ehess(backend, matrix, rank) manifold = Oblique(rank, num_rows) problem = pymanopt.Problem(manifold, cost, egrad=egrad, ehess=ehess) if quiet: problem.verbosity = 0 solver = TrustRegions() X = solver.solve(problem) if quiet: return C = X.T @ X print("Diagonal elements:", np.diag(C)) print("Eigenvalues:", np.sort(la.eig(C)[0].real)[::-1])
def dominant_invariant_subspace(A, p): """ Returns an orthonormal basis of the dominant invariant p-subspace of A. Arguments: - A A real, symmetric matrix A of size nxn - p integer p < n. Returns: A real, orthonormal matrix X of size nxp such that trace(X'*A*X) is maximized. That is, the columns of X form an orthonormal basis of a dominant subspace of dimension p of A. These span the same space as the eigenvectors associated with the largest eigenvalues of A. Sign is important: 2 is deemed a larger eigenvalue than -5. """ # Make sure the input matrix is square and symmetric n = A.shape[0] assert type(A) == np.ndarray, 'A must be a numpy array.' assert np.isreal(A).all(), 'A must be real.' assert A.shape[1] == n, 'A must be square.' assert np.linalg.norm(A-A.T) < n * np.spacing(1), 'A must be symmetric.' assert p <= n, 'p must be smaller than n.' # Define the cost on the Grassmann manifold Gr = Grassmann(n, p) X = T.matrix() cost = -T.dot(X.T, T.dot(A, X)).trace() # Setup the problem problem = Problem(man=Gr, ad_cost=cost, ad_arg=X) # Create a solver object solver = TrustRegions() # Solve Xopt = solver.solve(problem, Delta_bar=8*np.sqrt(p)) return Xopt
from pymanopt.solvers import TrustRegions from pymanopt.manifolds import Euclidean, Product if __name__ == "__main__": # Generate random data X = np.random.randn(3, 100).astype('float32') Y = (X[0:1, :] - 2*X[1:2, :] + np.random.randn(1, 100) + 5).astype( 'float32') # Cost function is the sqaured test error w = tf.Variable(tf.zeros([3, 1])) b = tf.Variable(tf.zeros([1])) cost = tf.reduce_mean(tf.square(Y - tf.matmul(tf.transpose(w), X) - b)) # first-order, second-order solver = TrustRegions() # R^3 x R^1 manifold = Product([Euclidean(3, 1), Euclidean(1, 1)]) # Solve the problem with pymanopt problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0) wopt = solver.solve(problem) print('Weights found by pymanopt (top) / ' 'closed form solution (bottom)') print(wopt[0].T) print(wopt[1]) X1 = np.concatenate((X, np.ones((1, 100))), axis=0)
def kindr(Ud, n_clusters, mansolver, init, tol_in, tol_out, max_iter_in, max_iter_out, disp, do_inner, post_SR, isnrm_row_U, isnrm_col_H, isbinary_H): if max_iter_out <= 0: raise ValueError('Number of iterations should be a positive number,' ' got %d instead' % max_iter_out) if tol_out <= 0: raise ValueError('The tolerance should be a positive number,' ' got %d instead' % tol_out) try: from pymanopt import Problem from pymanopt.manifolds import Stiefel, Rotations from pymanopt.solvers import SteepestDescent, ConjugateGradient, TrustRegions, NelderMead except ImportError: raise ValueError( "KindR needs pymanopt, which is unavailable, try KindAP instead.") # warnings.warn("KindR solver is unavailable. Transfer to" # "KindAP instead.") try: import autograd.numpy as anp except ImportError: warnings.warn( "Pymanopt needs autograd, which is unavailable,try KindAP instead." ) idx, center, gerr, numiter = kindap(Ud, n_clusters, init, tol_in, tol_out, max_iter_in, max_iter_out, disp, True, post_SR, isnrm_row_U, isnrm_col_H, isbinary_H) return idx, center, gerr, numiter n, d = Ud.shape k = n_clusters if d != k: warnings.warn('Provided more features, expected %d, got %d' % (k, d)) if isnrm_row_U: Ud = normalize(Ud, axis=1) # initialization if isinstance(init, string_types) and init == 'eye': # Z_0 = -np.identity(k) U = Ud[:, :k] elif hasattr(init, '__array__'): if init.shape[0] != d: raise ValueError( 'The row size of init should be the same as the total' 'features, got %d instead.' % init.shape[0]) if init.shape[1] != k: raise ValueError( 'The column size of init should be the same as the total' 'clusters, got %d instead.' % init.shape[1]) U = np.matmul(Ud, np.array(init)) elif isinstance(init, string_types) and init == 'random': H = sparse.csc_matrix((np.ones( (n, )), (np.arange(n), np.random.randint(0, k, (n, )))), shape=(n, k)) smat, sigma, vmat = la.svd(sparse.csc_matrix.dot(Ud.T, H), full_matrices=False) z_init = np.matmul(smat, vmat) U = np.matmul(Ud, z_init) else: raise ValueError( "The init parameter for KindAP should be 'eye','random'," "or an array. Got a %s with type %s instead." % (init, type(init))) if isinstance(do_inner, bool) or isinstance(do_inner, int): do_inner = bool(do_inner) elif isinstance(do_inner, string_types) and (do_inner in ["relu", "softmax"]): pass else: raise ValueError("Invalid put do_inner") H, N = sparse.csc_matrix((n, k)), sparse.csc_matrix((n, k)) dUH = float('inf') numiter, gerr = [], [] idx = np.ones(n) crit2 = np.zeros(4) def cost_n(rotation): return 0.5 * anp.sum(anp.minimum(anp.matmul(Ud, rotation), 0)**2) # def cost_softmax(rotation): # umat = anp.matmul(Ud, rotation) # exp_umat = anp.exp(umat) # mat = exp_umat / exp_umat.sum(axis=1).reshape(Ud.shape[0], 1) # return 0.5 * anp.sum((umat - mat) ** 2) for n_iter_out in range(max_iter_out): idxp, Up, Np, Hp = idx, U, N, H optlog = {} itr = 0 # inner iterations if do_inner: if d == k: manifold = Rotations(k) else: manifold = Stiefel(d, k) # if do_inner == "softmax": # cost = cost_softmax # else: # cost = cost_n problem = Problem(manifold=manifold, cost=cost_n, verbosity=0) if mansolver == "SD": solver = SteepestDescent(maxiter=max_iter_in, mingradnorm=tol_in, logverbosity=2) elif mansolver == "CG": solver = ConjugateGradient(maxiter=max_iter_in, mingradnorm=tol_in, logverbosity=2) elif mansolver == "TR": solver = TrustRegions(maxiter=max_iter_in, mingradnorm=tol_in, logverbosity=2) elif mansolver == "NM": solver = NelderMead(maxiter=max_iter_in, mingradnorm=tol_in, logverbosity=2) else: raise ValueError( "Undefined manifold optimization method, Expect SD, CG, TR or NM, " "get %s instead" % mansolver) Z, optlog = solver.solve(problem) U = np.matmul(Ud, Z) N = np.maximum(U, 0) itr = len(optlog['iterations']['iteration']) numiter.append(itr) if disp: print(optlog['iterations']['f(x)']) print(optlog['stoppingreason']) else: N = U numiter.append(itr) # project onto H H, idx = proj_h(N, isnrm_col_H, isbinary_H) idxchg = sum(idx != idxp) # project back to Ud U = proj_ud(H, Ud) dUHp = dUH dUH = la.norm(U - H, 'fro') gerr.append(dUH) if disp: print('Outer %3d: %3d dUH: %11.8e idxchg: %6d' % (n_iter_out + 1, itr, dUH, idxchg)) # stopping criteria of outer iterations crit2[0] = dUH < 1e-12 crit2[1] = abs(dUH - dUHp) < dUHp * tol_out crit2[2] = dUH > dUHp crit2[3] = idxchg == 0 if any(crit2): if post_SR and do_inner: do_inner, isbinary_H = False, True continue if crit2[2] and not crit2[1]: idx, H, U, N, dUH = idxp, Hp, Up, Np, dUHp if disp and optlog and 'stoppingreason' in optlog: print('\t stop reason:', optlog['stoppingreason']) break center = sparse.csc_matrix.dot( normalize((H != 0), axis=0, norm='l1').T, Ud) return idx, center, gerr, numiter
def wda(X, y, p=2, reg=1, k=10, solver=None, maxiter=100, verbose=0): """ Wasserstein Discriminant Analysis [11]_ The function solves the following optimization problem: .. math:: P = \\text{arg}\min_P \\frac{\\sum_i W(PX^i,PX^i)}{\\sum_{i,j\\neq i} W(PX^i,PX^j)} where : - :math:`P` is a linear projection operator in the Stiefel(p,d) manifold - :math:`W` is entropic regularized Wasserstein distances - :math:`X^i` are samples in the dataset corresponding to class i Parameters ---------- X : numpy.ndarray (n,d) Training samples y : np.ndarray (n,) labels for training samples p : int, optional size of dimensionnality reduction reg : float, optional Regularization term >0 (entropic regularization) solver : str, optional None for steepest decsent or 'TrustRegions' for trust regions algorithm else shoudl be a pymanopt.sovers verbose : int, optional Print information along iterations Returns ------- P : (d x p) ndarray Optimal transportation matrix for the given parameters proj : fun projectiuon function including mean centering References ---------- .. [11] Flamary, R., Cuturi, M., Courty, N., & Rakotomamonjy, A. (2016). Wasserstein Discriminant Analysis. arXiv preprint arXiv:1608.08063. """ mx = np.mean(X) X -= mx.reshape((1, -1)) # data split between classes d = X.shape[1] xc = split_classes(X, y) # compute uniform weighs wc = [np.ones((x.shape[0]), dtype=np.float32) / x.shape[0] for x in xc] def cost(P): # wda loss loss_b = 0 loss_w = 0 for i, xi in enumerate(xc): xi = np.dot(xi, P) for j, xj in enumerate(xc[i:]): xj = np.dot(xj, P) M = dist(xi, xj) G = sinkhorn(wc[i], wc[j + i], M, reg, k) if j == 0: loss_w += np.sum(G * M) else: loss_b += np.sum(G * M) # loss inversed because minimization return loss_w / loss_b # declare manifold and problem manifold = Stiefel(d, p) problem = Problem(manifold=manifold, cost=cost) # declare solver and solve if solver is None: solver = SteepestDescent(maxiter=maxiter, logverbosity=verbose) elif solver in ['tr', 'TrustRegions']: solver = TrustRegions(maxiter=maxiter, logverbosity=verbose) Popt = solver.solve(problem) def proj(X): return (X - mx.reshape((1, -1))).dot(Popt) return Popt, proj
def manifold_fit(self, Y, X): from pymanopt.manifolds import Rotations, Euclidean, Product from pymanopt import Problem from pymanopt.solvers import TrustRegions # from minimal_varx import make_normalized_G cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p) self.set_covs(cov_res, cov_xlag) # m = X.shape[0] with_c = (self.mm_degree > self.agg_rnk) and (self.m - self.agg_rnk) C = Euclidean(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk) RG = Rotations(self.m) if with_c: raise (ValueError( "This option is implemented only for self.agg_rnk == m")) if with_c: manifold = Product([RG, C]) else: manifold = RG if not with_c: c_null = np.zeros( (self.mm_degree - self.agg_rnk, self.m - self.agg_rnk)) else: c_null = None if with_c: def cost(x): o, c = x G = make_normalized_G(self, self.m, o, c) self.calc_states(G) return self.neg_log_llk else: def cost(x): G = make_normalized_G(self, self.m, x, c_null) self.calc_states(G) return self.neg_log_llk if with_c: def egrad(x): o, c = x grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, o, c) return [grad_o, grad_c] else: def egrad(x): grad_o, grad_c = self.map_o_c_grad(self._gradient_tensor, x, c_null) return grad_o if with_c: def ehess(x, Heta): o, c = x eta = make_normalized_G(self, self.m, Heta[0], Heta[1]) hess_raw = self.hessian_prod(eta) hess_o, hess_c = self.map_o_c_grad(hess_raw, o, c) return [hess_o, hess_c] else: def ehess(x, Heta): eta = make_normalized_G(self, self.m, Heta, c_null) hess_raw = self.hessian_prod(eta) hess_o, hess_c = self.map_o_c_grad(hess_raw, x, c_null) return hess_o if with_c: min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess) else: min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess) solver = TrustRegions() opt = solver.solve(min_mle) if with_c: G_opt = make_normalized_G(self, self.m, opt[0], opt[1]) else: G_opt = make_normalized_G(self, self.m, opt, c_null) self.calc_H_F_Phi(G_opt, cov_y_xlag) return opt
def nonlinear_eigenspace(L, k, alpha=1): """Nonlinear eigenvalue problem: total energy minimization. This example is motivated in [1]_ and was adapted from the manopt toolbox in Matlab. TODO : check this Parameters ---------- L : array, shape=(n_channels, n_channels) Discrete Laplacian operator: the covariance matrix. alpha : float Given constant for optimization problem. k : int Determines how many eigenvalues are returned. Returns ------- Xsol : array, shape=(n_channels, n_channels) Eigenvectors. S0 : array Eigenvalues. References ---------- .. [1] "A Riemannian Newton Algorithm for Nonlinear Eigenvalue Problems", Zhi Zhao, Zheng-Jian Bai, and Xiao-Qing Jin, SIAM Journal on Matrix Analysis and Applications, 36(2), 752-774, 2015. """ n = L.shape[0] assert L.shape[1] == n, 'L must be square.' # Grassmann manifold description manifold = Grassmann(n, k) manifold._dimension = 1 # hack # A solver that involves the hessian (check if correct TODO) solver = TrustRegions() # Cost function evaluation @pymanopt.function.Callable def cost(X): rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X') val = 0.5 * np.trace(X.T @ (L * X)) + \ (alpha / 4) * (rhoX.T @ mldivide(L, rhoX)) return val # Euclidean gradient evaluation @pymanopt.function.Callable def egrad(X): rhoX = np.sum(X**2, 1, keepdims=True) # diag(X*X') g = L @ X + alpha * np.diagflat(mldivide(L, rhoX)) @ X return g # Euclidean Hessian evaluation # Note: Manopt automatically converts it to the Riemannian counterpart. @pymanopt.function.Callable def ehess(X, U): rhoX = np.sum(X**2, 1, keepdims=True) # np.diag(X * X') rhoXdot = 2 * np.sum(X.dot(U), 1) h = L @ U + alpha * np.diagflat(mldivide(L, rhoXdot)) @ X + \ alpha * np.diagflat(mldivide(L, rhoX)) @ U return h # Initialization as suggested in above referenced paper. # randomly generate starting point for svd x = np.random.randn(n, k) [U, S, V] = linalg.svd(x, full_matrices=False) x = U.dot(V.T) S0, U0 = linalg.eig(L + alpha * np.diagflat(mldivide(L, np.sum(x**2, 1)))) # Call manoptsolve to automatically call an appropriate solver. # Note: it calls the trust regions solver as we have all the required # ingredients, namely, gradient and Hessian, information. problem = Problem(manifold=manifold, cost=cost, egrad=egrad, ehess=ehess, verbosity=0) Xsol = solver.solve(problem, U0) return S0, Xsol
def optim_test2(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 100 d = 20 # problem Tr(AXBX^T) for i in range(1): D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T.conj() B = make_sym_pos(d) alpha = randint(1, 10, 2) * .1 alpha = alpha / alpha[0] print(alpha) man = ComplexStiefel(n, d, alpha) A2 = A @ A @Callable def cost(X): return rtrace(A @ X @ B @ X.T.conj() @ A2 @ X @ B @ X.T.conj() @ A) @Callable def egrad(X): R = 4 * A2 @ X @ B @ X.T.conj() @ A2 @ X @ B return R @Callable def ehess(X, H): return 4*A2 @ H @ B @ X.T.conj() @ A2 @ X @ B +\ 4*A2 @ X @ B @ H.T.conj() @ A2 @ X @ B +\ 4*A2 @ X @ B @ X.T.conj() @ A2 @ H @ B if False: X = man.rand() xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = rtrace(egrad(X) @ xi.T.conj()) print(check_zero(d1 - d2)) d3 = num_deriv(man, X, xi, egrad) d4 = ehess(X, xi) print(check_zero(d3 - d4)) prob = Problem(man, cost, egrad=egrad) XInit = man.rand() prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=2500) print(cost(opt)) if False: # print(opt) # double check: # print(cost(opt)) min_val = 1e190 # min_X = None for i in range(100): Xi = man.rand() c = cost(Xi) if c < min_val: # min_X = Xi min_val = c if i % 1000 == 0: print('i=%d min=%f' % (i, min_val)) print(min_val) man1 = ComplexStiefel(n, d, alpha=np.array([1, 1])) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) man1 = ComplexStiefel(n, d, alpha=np.array([1, .5])) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250)
def wda(X, y, p=2, reg=1, k=10, solver=None, sinkhorn_method='sinkhorn', maxiter=100, verbose=0, P0=None, normalize=False): r""" Wasserstein Discriminant Analysis :ref:`[11] <references-wda>` The function solves the following optimization problem: .. math:: \mathbf{P} = \mathop{\arg \min}_\mathbf{P} \quad \frac{\sum\limits_i W(P \mathbf{X}^i, P \mathbf{X}^i)}{\sum\limits_{i, j \neq i} W(P \mathbf{X}^i, P \mathbf{X}^j)} where : - :math:`P` is a linear projection operator in the Stiefel(`p`, `d`) manifold - :math:`W` is entropic regularized Wasserstein distances - :math:`\mathbf{X}^i` are samples in the dataset corresponding to class i **Choosing a Sinkhorn solver** By default and when using a regularization parameter that is not too small the default sinkhorn solver should be enough. If you need to use a small regularization to get sparse cost matrices, you should use the :py:func:`ot.dr.sinkhorn_log` solver that will avoid numerical errors, but can be slow in practice. Parameters ---------- X : ndarray, shape (n, d) Training samples. y : ndarray, shape (n,) Labels for training samples. p : int, optional Size of dimensionnality reduction. reg : float, optional Regularization term >0 (entropic regularization) solver : None | str, optional None for steepest descent or 'TrustRegions' for trust regions algorithm else should be a pymanopt.solvers sinkhorn_method : str method used for the Sinkhorn solver, either 'sinkhorn' or 'sinkhorn_log' P0 : ndarray, shape (d, p) Initial starting point for projection. normalize : bool, optional Normalise the Wasserstaiun distance by the average distance on P0 (default : False) verbose : int, optional Print information along iterations. Returns ------- P : ndarray, shape (d, p) Optimal transportation matrix for the given parameters proj : callable Projection function including mean centering. .. _references-wda: References ---------- .. [11] Flamary, R., Cuturi, M., Courty, N., & Rakotomamonjy, A. (2016). Wasserstein Discriminant Analysis. arXiv preprint arXiv:1608.08063. """ # noqa if sinkhorn_method.lower() == 'sinkhorn': sinkhorn_solver = sinkhorn elif sinkhorn_method.lower() == 'sinkhorn_log': sinkhorn_solver = sinkhorn_log else: raise ValueError("Unknown Sinkhorn method '%s'." % sinkhorn_method) mx = np.mean(X) X -= mx.reshape((1, -1)) # data split between classes d = X.shape[1] xc = split_classes(X, y) # compute uniform weighs wc = [np.ones((x.shape[0]), dtype=np.float32) / x.shape[0] for x in xc] # pre-compute reg_c,c' if P0 is not None and normalize: regmean = np.zeros((len(xc), len(xc))) for i, xi in enumerate(xc): xi = np.dot(xi, P0) for j, xj in enumerate(xc[i:]): xj = np.dot(xj, P0) M = dist(xi, xj) regmean[i, j] = np.sum(M) / (len(xi) * len(xj)) else: regmean = np.ones((len(xc), len(xc))) @Autograd def cost(P): # wda loss loss_b = 0 loss_w = 0 for i, xi in enumerate(xc): xi = np.dot(xi, P) for j, xj in enumerate(xc[i:]): xj = np.dot(xj, P) M = dist(xi, xj) G = sinkhorn_solver(wc[i], wc[j + i], M, reg * regmean[i, j], k) if j == 0: loss_w += np.sum(G * M) else: loss_b += np.sum(G * M) # loss inversed because minimization return loss_w / loss_b # declare manifold and problem manifold = Stiefel(d, p) problem = Problem(manifold=manifold, cost=cost) # declare solver and solve if solver is None: solver = SteepestDescent(maxiter=maxiter, logverbosity=verbose) elif solver in ['tr', 'TrustRegions']: solver = TrustRegions(maxiter=maxiter, logverbosity=verbose) Popt = solver.solve(problem, x=P0) def proj(X): return (X - mx.reshape((1, -1))).dot(Popt) return Popt, proj
def optim_test(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 100 d = 20 # problem Tr(AXBX^T) for i in range(1): D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T B = make_sym_pos(d) man = RealStiefelWoodbury(n, d, A=np.diag(D), B=np.diag(np.diagonal(B))) @Callable def cost(X): return trace(A @ X @ B @ X.T) @Callable def egrad(X): return 2 * A @ X @ B @Callable def ehess(X, H): return 2 * A @ H @ B X = man.rand() if False: xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = trace(egrad(X) @ xi.T) print(check_zero(d1 - d2)) XInit = man.rand() prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) man.retr_method = 'geo' opt = solver.solve(prob, x=XInit, Delta_bar=250) print(cost(opt)) # print(opt) # double check: print(cost(opt)) min_val = 1e190 min_X = None for i in range(100): Xi = man.rand() c = cost(Xi) if c < min_val: min_X = Xi min_val = c if i % 1000 == 0: print('i=%d min=%f' % (i, min_val)) print(min_val) man1 = RealStiefelWoodbury(n, d, np.eye(n), np.eye(d)) prob1 = Problem(man1, cost, egrad=egrad, ehess=ehess) man1.retr_method = 'svd' solver1 = TrustRegions(maxtime=100000, maxiter=100) opt1 = solver1.solve(prob1, x=XInit, Delta_bar=250)
def optim_test(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 1000 # problem Tr(AXBX^T) for i in range(1): dvec = np.array([0, 30, 2, 1]) dvec[0] = 1000 - dvec[1:].sum() d = dvec[1:].sum() D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T B = make_sym_pos(d) p = dvec.shape[0]-1 alpha = randint(1, 10, (p, p+1)) * .1 alpha0 = randint(1, 10, (p)) # alpha0 = randint(1, 2, (p)) alpha = make_simple_alpha(alpha0, 0) man = RealFlag(dvec, alpha=alpha) @Callable def cost(X): return trace(A @ X @ B @ X.T) @Callable def egrad(X): return 2*A @ X @ B @Callable def ehess(X, H): return 2*A @ H @ B if False: X = man.rand() xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = trace(egrad(X) @ xi.T) print(check_zero(d1-d2)) prob = Problem( man, cost, egrad=egrad) XInit = man.rand() prob = Problem( man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) print(cost(opt)) if False: min_val = 1e190 # min_X = None for i in range(100): Xi = man.rand() c = cost(Xi) if c < min_val: # min_X = Xi min_val = c if i % 1000 == 0: print('i=%d min=%f' % (i, min_val)) print(min_val) alpha_c = alpha.copy() alpha_c[:] = 1 man1 = RealFlag(dvec, alpha=alpha_c) prob = Problem( man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) alpha_c5 = alpha_c.copy() alpha_c5[:, 1:] = .5 man1 = RealFlag(dvec, alpha=alpha_c5) prob = Problem( man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250)
from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.manifolds import Euclidean, Product if __name__ == "__main__": # Generate random data X = np.random.randn(3, 100) Y = X[0:1, :] - 2 * X[1:2, :] + np.random.randn(1, 100) + 5 # Cost function is the sqaured test error w = T.matrix() b = T.matrix() cost = T.sum((Y - w.T.dot(X) - b[0, 0])**2) # A solver that involves the hessian solver = TrustRegions() # R^3 x R^1 manifold = Product([Euclidean(3, 1), Euclidean(1, 1)]) # Solve the problem with pymanopt problem = Problem(manifold=manifold, cost=cost, arg=[w, b], verbosity=0) wopt = solver.solve(problem) print('Weights found by pymanopt (top) / ' 'closed form solution (bottom)') print(wopt[0].T) print(wopt[1]) X1 = np.concatenate((X, np.ones((1, 100))), axis=0) wclosed = np.linalg.inv(X1.dot(X1.T)).dot(X1).dot(Y.T)
def cost(X): return -np.tensordot(X, ABt, axes=X.ndim) def egrad(X): return -ABt def ehess(X, S): return manifold.zerovec(X) problem = Problem(manifold=manifold, cost=cost, egrad=egrad, ehess=ehess) # problem = Problem(manifold=manifold, cost=cost) solver = TrustRegions() # solver = SteepestDescent() X = solver.solve(problem) # print(X) def sol(ABt): # Compare with the known optimal solution\n", U, S, Vt = np.linalg.svd(ABt) UVt = np.dot(U, Vt) # The determinant of UVt is either 1 or -1, in theory\n", if abs(1.0 - np.linalg.det(UVt)) < 1e-10: Xopt = UVt elif abs(-1.0 - np.linalg.det(UVt)) < 1e-10: # UVt is in O(n) but not SO(n). This is easily corrected for:\n",
def optim_test(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 300 # problem Tr(AXBX^T) for i in range(1): dvec = np.array([0, 5, 4, 10]) dvec[0] = n - dvec[1:].sum() d = dvec[1:].sum() alpha = randint(1, 10, 2) * .1 D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T.conjugate() B = make_sym_pos(d) p = dvec.shape[0] - 1 alpha = randint(1, 10, (p, p + 1)) * .1 alpha0 = randint(1, 10, (p)) # alpha0 = randint(1, 2, (p)) alpha = make_simple_alpha(alpha0, 0) man = ComplexFlag(dvec, alpha) @Callable def cost(X): return trace(A @ X @ B @ X.T.conjugate()).real @Callable def egrad(X): return 2 * A @ X @ B @Callable def ehess(X, H): return 2 * A @ H @ B X = man.rand() if False: xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = trace(egrad(X) @ xi.T.conjugate()).real print(check_zero(d1 - d2)) prob = Problem(man, cost, egrad=egrad) XInit = man.rand() prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) print(cost(opt)) alpha0 = randint(1, 2, (p)) alpha1 = make_simple_alpha(alpha0, 0) man1 = ComplexFlag(dvec, alpha=alpha1) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) alpha0 = randint(1, 2, (p)) alpha2 = make_simple_alpha(alpha0, -1) man1 = ComplexFlag(dvec, alpha=alpha2) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250)
def optim_test3(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 200 d = 20 # problem Tr(AXBX^T) for i in range(1): B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)])) D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T.conj() alpha = randint(1, 10, 2) alpha = alpha / alpha[0] print(alpha) man = ComplexStiefel(n, d, alpha) cf = 10 B2 = B @ B @Callable def cost(X): return cf * rtrace( B @ X @ X.T.conj() @ B2 @ X @ X.T.conj() @ B) +\ rtrace(X.T.conj() @ A @ X) @Callable def egrad(X): R = cf * 4 * B2 @ X @ X.T.conj() @ B2 @ X + 2 * A @ X return R @Callable def ehess(X, H): return 4*cf*B2 @ H @ X.T.conj() @ B2 @ X +\ 4*cf*B2 @ X @ H.T.conj() @ B2 @ X +\ 4*cf*B2 @ X @ X.T.conj() @ B2 @ H + 2*A @ H if False: X = man.rand() xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = rtrace(egrad(X) @ xi.T.conj()) print(check_zero(d1 - d2)) d3 = num_deriv(man, X, xi, egrad) d4 = ehess(X, xi) print(check_zero(d3 - d4)) XInit = man.rand() prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=2500) print(cost(opt)) if False: # print(opt) # double check: # print(cost(opt)) min_val = 1e190 # min_X = None for i in range(100): Xi = man.rand() c = cost(Xi) if c < min_val: # min_X = Xi min_val = c if i % 1000 == 0: print('i=%d min=%f' % (i, min_val)) print(min_val) man1 = ComplexStiefel(n, d, alpha=np.array([1, 1])) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) man1 = ComplexStiefel(n, d, alpha=np.array([1, .5])) # man1 = ComplexStiefel(n, d, alpha=np.array([1, 1])) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250)
Wcd = np.reshape(Wcd_vec,(n,n)) Wnum = np.dot(Wo,Wcd) Wden = np.dot(Wo,Wcd-Sigma_norm_B) num = np.linalg.det(Wnum+sigma2*I) den = np.linalg.det(Wden+sigma2*I) return -np.log2(num/den)/(2.*T) def make_shannon_capacity_fact_fixed_time(T): def shannon_capacity_fact_fixed_time(L): return shannon_rate(L, T) return shannon_capacity_fact_fixed_time # instantiate manifold for Pymanopt manifold_fact = Euclidean(n,n) # instantiate solver for Pymanopt solver = TrustRegions() #### find R_max via bisection method #### # min value T T1=1e-8 shannon_capacity_fact_fixed_time = make_shannon_capacity_fact_fixed_time(T1) problem = Problem(manifold=manifold_fact, cost=shannon_capacity_fact_fixed_time, verbosity=0) L_opt1 = solver.solve(problem) p_1 = -shannon_capacity_fact_fixed_time(L_opt1) #print p_1 Sigma_opt1 = np.dot(L_opt1,L_opt1.transpose()) Sigma_opt_norm1 = Sigma_opt1/(np.trace(Sigma_opt1))
def optim_test3(): from pymanopt import Problem from pymanopt.solvers import TrustRegions from pymanopt.function import Callable n = 1000 # problem Tr(AXBX^T) for i in range(1): # Stiefel manifold dvec = np.array([0, 50]) dvec[0] = n - dvec[1:].sum() d = dvec[1:].sum() p = dvec.shape[0] - 1 alpha = randint(1, 10, (p, p + 1)) * .1 alpha0 = randint(1, 10, (p)) # alpha0 = randint(1, 2, (p)) alpha = make_simple_alpha(alpha0, 0) B = np.diag(np.concatenate([randint(1, 10, d), np.zeros(n - d)])) D = randint(1, 10, n) * 0.02 + 1 OO = random_orthogonal(n) A = OO @ np.diag(D) @ OO.T man = ComplexFlag(dvec, alpha) cf = 10 B2 = B @ B @Callable def cost(X): return cf * trace( B @ X @ st(X) @ B2 @ X @ st(X) @ B) +\ trace(st(X) @ A @ X) @Callable def egrad(X): R = cf * 4 * B2 @ X @ st(X) @ B2 @ X + 2 * A @ X return R @Callable def ehess(X, H): return 4*cf*B2 @ H @ st(X) @ B2 @ X +\ 4*cf*B2 @ X @ st(H) @ B2 @ X +\ 4*cf*B2 @ X @ st(X) @ B2 @ H + 2*A @ H if False: X = man.rand() xi = man.randvec(X) d1 = num_deriv(man, X, xi, cost) d2 = trace(egrad(X) @ st(xi)).real print(check_zero(d1 - d2)) d3 = num_deriv(man, X, xi, egrad) d4 = ehess(X, xi) print(check_zero(d3 - d4)) XInit = man.rand() prob = Problem(man, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=2500) print(cost(opt)) if False: # print(opt) # double check: # print(cost(opt)) min_val = 1e190 # min_X = None for i in range(100): Xi = man.rand() c = cost(Xi) if c < min_val: # min_X = Xi min_val = c if i % 1000 == 0: print('i=%d min=%f' % (i, min_val)) print(min_val) alpha0 = randint(1, 2, (p)) alpha1 = make_simple_alpha(alpha0, 0) man1 = ComplexFlag(dvec, alpha=alpha1) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250) alpha0 = randint(1, 2, (p)) alpha2 = make_simple_alpha(alpha0, -1) man1 = ComplexFlag(dvec, alpha=alpha2) # man1 = RealStiefel(n, d, alpha=np.array([1, 1])) prob = Problem(man1, cost, egrad=egrad, ehess=ehess) solver = TrustRegions(maxtime=100000, maxiter=100) opt = solver.solve(prob, x=XInit, Delta_bar=250)