def krprod(factors, indices_list): ''' Implement Khatri Rao Product for given nonzeros' indicies ''' rank = tl.shape(factors[0])[1] nnz = len(indices_list[0]) nonzeros = tl.ones((nnz, rank), **tl.context(factors[0])) for indices, factor in zip(indices_list, factors): nonzeros = nonzeros * factor[indices, :] return torch.sum(nonzeros, dim=1)
def normalize_factors(factors): """Normalizes factors to unit length and returns factor magnitudes Turns ``factors = [|U_1, ... U_n|]`` into ``[weights; |V_1, ... V_n|]``, where the columns of each `V_k` are normalized to unit Euclidean length from the columns of `U_k` with the normalizing constants absorbed into `weights`. In the special case of a symmetric tensor, `weights` holds the eigenvalues of the tensor. Parameters ---------- factors : ndarray list list of matrices, all with the same number of columns i.e.:: for u in U: u[i].shape == (s_i, R) where `R` is fixed while `s_i` can vary with `i` Returns ------- normalized_factors : list of ndarrays list of matrices with the same shape as `factors` weights : ndarray vector of length `R` holding normalizing constants """ # allocate variables for weights, and normalized factors rank = factors[0].shape[1] weights = tl.ones(rank, **tl.context(factors[0])) normalized_factors = [] # normalize columns of factor matrices for factor in factors: scales = tl.norm(factor, axis=0) weights *= scales scales_non_zero = tl.where( scales == 0, tl.ones(tl.shape(scales), **tl.context(factors[0])), scales) normalized_factors.append(factor / scales_non_zero) return normalized_factors, weights
def test_cp_to_tensor_with_weights(): A = tl.reshape(tl.arange(1,5), (2,2)) B = tl.reshape(tl.arange(5,9), (2,2)) weigths = tl.tensor([2,-1], **tl.context(A)) out = cp_to_tensor((weigths, [A,B])) expected = tl.tensor([[-2,-2], [6, 10]]) # computed by hand assert_array_equal(out, expected) (weigths, factors) = random_cp((5, 5, 5), rank=5, normalise_factors=True, full=False) true_res = tl.dot(tl.dot(factors[0], tl.diag(weigths)), tl.transpose(tl.tenalg.khatri_rao(factors[1:]))) true_res = tl.fold(true_res, 0, (5, 5, 5)) res = cp_to_tensor((weigths, factors)) assert_array_almost_equal(true_res, res, err_msg='weights incorrectly incorporated in cp_to_tensor')
def test_vec2factors_1(): """ Test wrapper function from GCP paper""" rank = 2 X = tl.tensor(np.arange(24).reshape((3, 4, 2)), dtype=tl.float32) X_shp = tl.shape(X) X_cntx = tl.context(X) factors = [] for i in range(3): f = tl.transpose(X[:][:][i]) factors.append(f) vec1 = factors2vec(factors) M = vec2factors(vec1,X_shp, rank, X_cntx) vec2 = factors2vec(M[1]) for i in range(X_shp[0]): assert(vec1[i] == vec2[i])
def sparsify_tensor(tensor, card): """Zeros out all elements in the `tensor` except `card` elements with maximum absolute values. Parameters ---------- tensor : ndarray card : int Desired number of non-zero elements in the `tensor` Returns ------- ndarray of shape tensor.shape """ if card >= np.prod(tensor.shape): return tensor bound = tl.sort(tl.abs(tensor), axis = None)[-card] return tl.where(tl.abs(tensor) < bound, tl.zeros(tensor.shape, **tl.context(tensor)), tensor)
def apply(self, module): """Apply an instance of the L1Regularizer to a tensor module Parameters ---------- module : TensorModule module on which to add the regularization Returns ------- TensorModule (with Regularization hook) """ context = tl.context(module.factors[0]) lasso_weights = nn.Parameter(torch.ones(module.rank, **context)) setattr(module, 'lasso_weights', lasso_weights) module.register_forward_hook(self) return module
def __custom_parafac(self, tensor, neg_fac=0, tol=1e-7): """Customized PARAFAC algorithm Parameters ---------- tensor : torch.Tensor The tensor of activity of N neurons, T timepoints and K trials of shape N, T, K neg_fac : int, optional Index of the factor which is allowed to be negative (default is 0) tol : float, optional Threshold for convergence (default is 1e-7) Returns ------- list List of optimized factors """ factors = self.__initialize_factors(tensor, self.rank, self.init, custom=neg_fac) pseudo_inverse = tl.tensor(np.ones((self.rank, self.rank)), **tl.context(tensor)) for iteration in range(self.max_iteration): for mode in range(self.dimension): if mode == neg_fac: factors[mode] = self.__factor(self, tensor, factors, mode, pseudo_inverse) else: factors[mode] = factors[mode] * self.__factor_non_negative( tensor, factors, mode) if (iteration % 25 == 0 or iteration % 25 == 1) and iteration > 1: if self.__get_error(iteration, tensor, factors, tol, self.verbose): break return factors
def hard_thresholding(tensor, number_of_non_zero): """ Proximal operator of the l0 ``norm'' Keeps greater "number_of_non_zero" elements untouched and sets other elements to zero. Parameters ---------- tensor : ndarray number_of_non_zero : int Returns ------- ndarray Thresholded tensor on which the operator has been applied """ tensor_vec = tl.copy(tl.tensor_to_vec(tensor)) sorted_indices = tl.argsort(tl.argsort(tl.abs(tensor_vec), axis=0, descending=True), axis=0) return tl.reshape( tl.where(sorted_indices < number_of_non_zero, tensor_vec, tl.tensor(0, **tl.context(tensor_vec))), tensor.shape)
def partial_tucker(tensor, modes, rank=None, n_iter_max=100, init='svd', tol=10e-5, svd='numpy_svd', random_state=None, verbose=False, mask=None): """Partial tucker decomposition via Higher Order Orthogonal Iteration (HOI) Decomposes `tensor` into a Tucker decomposition exclusively along the provided modes. Parameters ---------- tensor : ndarray modes : int list list of the modes on which to perform the decomposition rank : None, int or int list size of the core tensor, ``(len(ranks) == tensor.ndim)`` if int, the same rank is used for all modes n_iter_max : int maximum number of iteration init : {'svd', 'random'}, or TuckerTensor optional if a TuckerTensor is provided, this is used for initialization svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS tol : float, optional tolerance: the algorithm stops when the variation in the reconstruction error is less than the tolerance random_state : {None, int, np.random.RandomState} verbose : int, optional level of verbosity mask : ndarray array of booleans with the same shape as ``tensor`` should be 0 where the values are missing and 1 everywhere else. Note: if tensor is sparse, then mask should also be sparse with a fill value of 1 (or True). Returns ------- core : ndarray core tensor of the Tucker decomposition factors : ndarray list list of factors of the Tucker decomposition. with ``core.shape[i] == (tensor.shape[i], ranks[i]) for i in modes`` """ if rank is None: message = "No value given for 'rank'. The decomposition will preserve the original size." warnings.warn(message, Warning) rank = [tl.shape(tensor)[mode] for mode in modes] elif isinstance(rank, int): message = "Given only one int for 'rank' instead of a list of {} modes. Using this rank for all modes.".format( len(modes)) warnings.warn(message, Warning) rank = tuple(rank for _ in modes) else: rank = tuple(rank) if mask is not None and init == "svd": message = "Masking occurs after initialization. Therefore, random initialization is recommended." warnings.warn(message, Warning) try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) # SVD init if init == 'svd': factors = [] for index, mode in enumerate(modes): eigenvecs, _, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank[index], random_state=random_state) factors.append(eigenvecs) # The initial core approximation is needed here for the masking step core = multi_mode_dot(tensor, factors, modes=modes, transpose=True) elif init == 'random': rng = tl.check_random_state(random_state) # len(rank) == len(modes) but we still want a core dimension for the modes not optimized core_shape = list(tl.shape(tensor)) for (i, e) in enumerate(modes): core_shape[e] = rank[i] core = tl.tensor(rng.random_sample(core_shape), **tl.context(tensor)) factors = [ tl.tensor(rng.random_sample((tl.shape(tensor)[mode], rank[index])), **tl.context(tensor)) for (index, mode) in enumerate(modes) ] else: (core, factors) = init rec_errors = [] norm_tensor = tl.norm(tensor, 2) for iteration in range(n_iter_max): if mask is not None: tensor = tensor * mask + multi_mode_dot( core, factors, modes=modes, transpose=False) * (1 - mask) for index, mode in enumerate(modes): core_approximation = multi_mode_dot(tensor, factors, modes=modes, skip=index, transpose=True) eigenvecs, _, _ = svd_fun(unfold(core_approximation, mode), n_eigenvecs=rank[index], random_state=random_state) factors[index] = eigenvecs core = multi_mode_dot(tensor, factors, modes=modes, transpose=True) # The factors are orthonormal and therefore do not affect the reconstructed tensor's norm rec_error = sqrt( abs(norm_tensor**2 - tl.norm(core, 2)**2)) / norm_tensor rec_errors.append(rec_error) if iteration > 1: if verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break return (core, factors)
def initialize_tucker(tensor, rank, modes, random_state, init='svd', svd='numpy_svd', non_negative=False): """ Initialize core and factors used in `tucker`. The type of initialization is set using `init`. If `init == 'random'` then initialize factor matrices using `random_state`. If `init == 'svd'` then initialize the `m`th factor matrix using the `rank` left singular vectors of the `m`th unfolding of the input tensor. Parameters ---------- tensor : ndarray rank : int number of components modes : int list random_state : {None, int, np.random.RandomState} init : {'svd', 'random', cptensor}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS non_negative : bool, default is False if True, non-negative factors are returned Returns ------- core : ndarray initialized core tensor factors : list of factors """ try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) # Initialisation if init == 'svd': factors = [] for index, mode in enumerate(modes): U, S, V = svd_fun(unfold(tensor, mode), n_eigenvecs=rank[index], random_state=random_state) if non_negative is True: U = make_svd_non_negative(tensor, U, S, V, nntype="nndsvd") factors.append(U[:, :rank[index]]) # The initial core approximation is needed here for the masking step core = multi_mode_dot(tensor, factors, modes=modes, transpose=True) if non_negative is True: core = tl.abs(core) elif init == 'random': rng = tl.check_random_state(random_state) core = tl.tensor(rng.random_sample(rank) + 0.01, **tl.context(tensor)) # Check this factors = [ tl.tensor(rng.random_sample(s), **tl.context(tensor)) for s in zip(tl.shape(tensor), rank) ] if non_negative is True: factors = [tl.abs(f) for f in factors] core = tl.abs(core) else: (core, factors) = init return core, factors
def non_negative_tucker(tensor, rank, n_iter_max=10, init='svd', tol=10e-5, random_state=None, verbose=False, return_errors=False, normalize_factors=False): """Non-negative Tucker decomposition Iterative multiplicative update, see [2]_ Parameters ---------- tensor : ``ndarray`` rank : None, int or int list size of the core tensor, ``(len(ranks) == tensor.ndim)`` if int, the same rank is used for all modes n_iter_max : int maximum number of iteration init : {'svd', 'random'} random_state : {None, int, np.random.RandomState} verbose : int , optional level of verbosity ranks : None or int list size of the core tensor normalize_factors : if True, aggregates the core which will contain the norms of the factors. Returns ------- core : ndarray positive core of the Tucker decomposition has shape `ranks` factors : ndarray list list of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)`` References ---------- .. [2] Yong-Deok Kim and Seungjin Choi, "Non-negative tucker decomposition", IEEE Conference on Computer Vision and Pattern Recognition s(CVPR), pp 1-8, 2007 """ rank = validate_tucker_rank(tl.shape(tensor), rank=rank) epsilon = 10e-12 # Initialisation if init == 'svd': core, factors = tucker(tensor, rank) nn_factors = [tl.abs(f) for f in factors] nn_core = tl.abs(core) else: rng = tl.check_random_state(random_state) core = tl.tensor(rng.random_sample(rank) + 0.01, **tl.context(tensor)) # Check this factors = [ tl.tensor(rng.random_sample(s), **tl.context(tensor)) for s in zip(tl.shape(tensor), rank) ] nn_factors = [tl.abs(f) for f in factors] nn_core = tl.abs(core) norm_tensor = tl.norm(tensor, 2) rec_errors = [] for iteration in range(n_iter_max): for mode in range(tl.ndim(tensor)): B = tucker_to_tensor((nn_core, nn_factors), skip_factor=mode) B = tl.transpose(unfold(B, mode)) numerator = tl.dot(unfold(tensor, mode), B) numerator = tl.clip(numerator, a_min=epsilon, a_max=None) denominator = tl.dot(nn_factors[mode], tl.dot(tl.transpose(B), B)) denominator = tl.clip(denominator, a_min=epsilon, a_max=None) nn_factors[mode] *= numerator / denominator numerator = tucker_to_tensor((tensor, nn_factors), transpose_factors=True) numerator = tl.clip(numerator, a_min=epsilon, a_max=None) for i, f in enumerate(nn_factors): if i: denominator = mode_dot(denominator, tl.dot(tl.transpose(f), f), i) else: denominator = mode_dot(nn_core, tl.dot(tl.transpose(f), f), i) denominator = tl.clip(denominator, a_min=epsilon, a_max=None) nn_core *= numerator / denominator rec_error = tl.norm(tensor - tucker_to_tensor( (nn_core, nn_factors)), 2) / norm_tensor rec_errors.append(rec_error) if iteration > 1 and verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if iteration > 1 and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break if normalize_factors: nn_core, nn_factors = tucker_normalize((nn_core, nn_factors)) tensor = TuckerTensor((nn_core, nn_factors)) if return_errors: return tensor, rec_errors else: return tensor
def __initialize_factors(self, tensor, svd='numpy_svd', non_negative=False, custom=None): """Initialize random or SVD-guided factors for TCA depending on TCA type Parameters ---------- tensor : torch.Tensor The tensor of activity of N neurons, T timepoints and K trials of shape N, T, K svd : str, optional Type of SVD algorithm to use (default is numpy_svd) non_negative : bool, optional A flag used to specify if factors generated must be strictyl positive (default is False) custom : int, optional A flag used to specify which factor should be strictly positive for 'custom parafac' (default is None) Raises ------ ValueError If svd does not contain a valid SVD algorithm reference If self.init variable does not contain a valid intialization method Returns ------- list List of initialized tensors """ rng = tensorly.random.check_random_state(self.random_state) if self.init == 'random': if custom: factors = [ tl.tensor( rng.random_sample( (tensor.shape[i], self.rank)) * 2 - 1, **tl.context(tensor)) for i in range(self.dimension) ] factors = [ f if int(i) == int(custom) else tl.abs(f) for i, f in enumerate(factors) ] elif non_negative: factors = [ tl.tensor(rng.random_sample((tensor.shape[i], self.rank)), **tl.context(tensor)) for i in range(self.dimension) ] factors = [ tl.abs(f) for f in factors ] # See if this line is useful depending on random function used else: factors = [ tl.tensor( rng.random_sample( (tensor.shape[i], self.rank)) * 2 - 1, **tl.context(tensor)) for i in range(self.dimension) ] return factors elif self.init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) factors = [] for mode in range(tl.ndim(tensor)): U, *_ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank) if tensor.shape[mode] < rank: random_part = tl.tensor( rng.random_sample( (U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) U = tl.concatenate([U, random_part], axis=1) if non_negative or custom == mode: factors.append(tl.abs(U[:, :rank])) else: factors.append(U[:, :rank]) return factors else: raise ValueError( 'Initialization method "{}" not recognized'.format(self.init))
def constrained_parafac(tensor, rank, n_iter_max=100, n_iter_max_inner=10, init='svd', svd='numpy_svd', tol_outer=1e-8, tol_inner=1e-6, random_state=None, verbose=0, return_errors=False, cvg_criterion='abs_rec_error', fixed_modes=None, non_negative=None, l1_reg=None, l2_reg=None, l2_square_reg=None, unimodality=None, normalize=None, simplex=None, normalized_sparsity=None, soft_sparsity=None, smoothness=None, monotonicity=None, hard_sparsity=None): """CANDECOMP/PARAFAC decomposition via alternating optimization of alternating direction method of multipliers (AO-ADMM): Computes a rank-`rank` decomposition of `tensor` [1]_ such that:: tensor = [|weights; factors[0], ..., factors[-1] |], where factors are either penalized or constrained according to the user-defined constraint. In order to compute the factors efficiently, the ADMM algorithm introduces an auxilliary factor which is called factor_aux in the function. Parameters ---------- tensor : ndarray rank : int Number of components. n_iter_max : int Maximum number of iteration for outer loop n_iter_max_inner : int Number of iteration for inner loop init : {'svd', 'random', cptensor}, optional Type of factor matrix initialization. See `initialize_factors`. svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS tol_outer : float, optional (Default: 1e-8) Relative reconstruction error tolerance for outer loop. The algorithm is considered to have found a local minimum when the reconstruction error is less than `tol_outer`. tol_inner : float, optional (Default: 1e-6) Absolute reconstruction error tolerance for factor update during inner loop, i.e. ADMM optimization. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors non_negative : bool or dictionary This constraint is clipping negative values to '0'. If it is True non-negative constraint is applied to all modes. l1_reg : float or list or dictionary, optional l2_reg : float or list or dictionary, optional l2_square_reg : float or list or dictionary, optional unimodality : bool or dictionary, optional If it is True unimodality constraint is applied to all modes. normalize : bool or dictionary, optional This constraint divides all the values by maximum value of the input array. If it is True normalize constraint is applied to all modes. simplex : float or list or dictionary, optional normalized_sparsity : float or list or dictionary, optional soft_sparsity : float or list or dictionary, optional smoothness : float or list or dictionary, optional monotonicity : bool or dictionary, optional hard_sparsity : float or list or dictionary, optional cvg_criterion : {'abs_rec_error', 'rec_error'}, optional Stopping criterion if `tol` is not None. If 'rec_error', algorithm stops at current iteration if ``(previous rec_error - current rec_error) < tol``. If 'abs_rec_error', algorithm terminates when `|previous rec_error - current rec_error| < tol`. fixed_modes : list, default is None A list of modes for which the initial value is not modified. The last mode cannot be fixed due to error computation. Returns ------- CPTensor : (weight, factors) * weights : 1D array of shape (rank, ) * factors : List of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)`` errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009. .. [2] Huang, Kejun, Nicholas D. Sidiropoulos, and Athanasios P. Liavas. "A flexible and efficient algorithmic framework for constrained matrix and tensor factorization." IEEE Transactions on Signal Processing 64.19 (2016): 5052-5065. """ rank = validate_cp_rank(tl.shape(tensor), rank=rank) _, _ = validate_constraints(non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg, unimodality=unimodality, normalize=normalize, simplex=simplex, normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity, smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity, n_const=tl.ndim(tensor)) weights, factors = initialize_constrained_parafac(tensor, rank, init=init, svd=svd, random_state=random_state, non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg, unimodality=unimodality, normalize=normalize, simplex=simplex, normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity, smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity) rec_errors = [] norm_tensor = tl.norm(tensor, 2) if fixed_modes is None: fixed_modes = [] if tl.ndim(tensor) - 1 in fixed_modes: warnings.warn('You asked for fixing the last mode, which is not supported.\n ' 'The last mode will not be fixed. Consider using tl.moveaxis()') fixed_modes.remove(tl.ndim(tensor) - 1) modes_list = [mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes] # ADMM inits dual_variables = [] factors_aux = [] for i in range(len(factors)): dual_variables.append(tl.zeros(tl.shape(factors[i]))) factors_aux.append(tl.transpose(tl.zeros(tl.shape(factors[i])))) for iteration in range(n_iter_max): if verbose > 1: print("Starting iteration", iteration + 1) for mode in modes_list: if verbose > 1: print("Mode", mode, "of", tl.ndim(tensor)) pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor)) for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse * tl.dot(tl.transpose(factor), factor) mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode) factors[mode], factors_aux[mode], dual_variables[mode] = admm(mttkrp, pseudo_inverse, factors[mode], dual_variables[mode], n_iter_max=n_iter_max_inner, n_const=tl.ndim(tensor), order=mode, non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg, unimodality=unimodality, normalize=normalize, simplex=simplex, normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity, smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity, tol=tol_inner) factors_norm = cp_norm((weights, factors)) iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0) * weights) rec_error = tl.sqrt(tl.abs(norm_tensor ** 2 + factors_norm ** 2 - 2 * iprod)) / norm_tensor rec_errors.append(rec_error) constraint_error = 0 for mode in modes_list: constraint_error += tl.norm(factors[mode] - tl.transpose(factors_aux[mode])) / tl.norm(factors[mode]) if tol_outer: if iteration >= 1: rec_error_decrease = rec_errors[-2] - rec_errors[-1] if verbose: print("iteration {}, reconstruction error: {}, decrease = {}".format(iteration, rec_error, rec_error_decrease)) if constraint_error < tol_outer: break if cvg_criterion == 'abs_rec_error': stop_flag = abs(rec_error_decrease) < tol_outer elif cvg_criterion == 'rec_error': stop_flag = rec_error_decrease < tol_outer else: raise TypeError("Unknown convergence criterion") if stop_flag: if verbose: print("PARAFAC converged after {} iterations".format(iteration)) break else: if verbose: print('reconstruction error={}'.format(rec_errors[-1])) cp_tensor = CPTensor((weights, factors)) if return_errors: return cp_tensor, rec_errors else: return cp_tensor
def initialize_constrained_parafac(tensor, rank, init='svd', svd='numpy_svd', random_state=None, non_negative=None, l1_reg=None, l2_reg=None, l2_square_reg=None, unimodality=None, normalize=None, simplex=None, normalized_sparsity=None, soft_sparsity=None, smoothness=None, monotonicity=None, hard_sparsity=None): r"""Initialize factors used in `constrained_parafac`. Parameters ---------- The type of initialization is set using `init`. If `init == 'random'` then initialize factor matrices with uniform distribution using `random_state`. If `init == 'svd'` then initialize the `m`th factor matrix using the `rank` left singular vectors of the `m`th unfolding of the input tensor. If init is a previously initialized `cp tensor`, all the weights are pulled in the last factor and then the weights are set to "1" for the output tensor. Lastly, factors are updated with proximal operator according to the selected constraint(s), so that they satisfy the imposed constraints (does not apply to cptensor initialization). Parameters ---------- tensor : ndarray rank : int random_state : {None, int, np.random.RandomState} init : {'svd', 'random', cptensor}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS non_negative : bool or dictionary This constraint is clipping negative values to '0'. If it is True non-negative constraint is applied to all modes. l1_reg : float or list or dictionary, optional l2_reg : float or list or dictionary, optional l2_square_reg : float or list or dictionary, optional unimodality : bool or dictionary, optional If it is True unimodality constraint is applied to all modes. normalize : bool or dictionary, optional This constraint divides all the values by maximum value of the input array. If it is True normalize constraint is applied to all modes. simplex : float or list or dictionary, optional normalized_sparsity : float or list or dictionary, optional soft_sparsity : float or list or dictionary, optional smoothness : float or list or dictionary, optional monotonicity : bool or dictionary, optional hard_sparsity : float or list or dictionary, optional Returns ------- factors : CPTensor An initial cp tensor. """ n_modes = tl.ndim(tensor) rng = tl.check_random_state(random_state) if init == 'random': weights, factors = random_cp(tl.shape(tensor), rank, normalise_factors=False, **tl.context(tensor)) elif init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) factors = [] for mode in range(tl.ndim(tensor)): U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank) # Put SVD initialization on the same scaling as the tensor in case normalize_factors=False if mode == 0: idx = min(rank, tl.shape(S)[0]) U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx]) if tensor.shape[mode] < rank: random_part = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) U = tl.concatenate([U, random_part], axis=1) factors.append(U[:, :rank]) elif isinstance(init, (tuple, list, CPTensor)): try: weights, factors = CPTensor(init) if tl.all(weights == 1): weights, factors = CPTensor((None, factors)) else: weights_avg = tl.prod(weights) ** (1.0 / tl.shape(weights)[0]) for i in range(len(factors)): factors[i] = factors[i] * weights_avg kt = CPTensor((None, factors)) return kt except ValueError: raise ValueError( 'If initialization method is a mapping, then it must ' 'be possible to convert it to a CPTensor instance' ) else: raise ValueError('Initialization method "{}" not recognized'.format(init)) for i in range(n_modes): factors[i] = proximal_operator(factors[i], non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg, unimodality=unimodality, normalize=normalize, simplex=simplex, normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity, smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity, n_const=n_modes, order=i) kt = CPTensor((None, factors)) return kt
def tensor_train_cross(input_tensor, rank, tol=1e-4, n_iter_max=100): """TT (tensor-train) decomposition via cross-approximation (TTcross) [1] Decomposes `input_tensor` into a sequence of order-3 tensors of given rank. (factors/cores) Rather than directly decompose the whole tensor, we sample fibers based on skeleton decomposition. We initialize a random tensor-train and sweep from left to right and right to left. On each core, we shape the core as a matrix and choose the fibers indices by finding maximum-volume submatrix and update the core. Advantage: faster The main advantage of TTcross is that it doesn't need to evaluate all the entries of the tensor. For a tensor_shape^tensor_order tensor, SVD needs O(tensor_shape^tensor_order) runtime, but TTcross' runtime is linear in tensor_shape and tensor_order, which makes it feasible in high dimension. Disadvantage: less accurate TTcross may underestimate the error, since it only evaluates partial entries of the tensor. Besides, in contrast to its practical fast performance, there is no theoretical guarantee of it convergence. Parameters ---------- input_tensor : tensorly.tensor The tensor to decompose. rank : {int, int list} maximum allowable TT rank of the factors if int, then this is the same for all the factors if int list, then rank[k] is the rank of the kth factor tol : float accuracy threshold for outer while-loop n_iter_max : int maximum iterations of outer while-loop (the 'crosses' or 'sweeps' sampled) Returns ------- factors : TT factors order-3 tensors of the TT decomposition Examples -------- Generate a 5^3 tensor, and decompose it into tensor-train of 3 factors, with rank = [1,3,3,1] >>> tensor = tl.tensor(np.arange(5**3).reshape(5,5,5)) >>> rank = [1, 3, 3, 1] >>> factors = tensor_train_cross(tensor, rank) print the first core: >>> print(factors[0]) .[[[ 24. 0. 4.] [ 49. 25. 29.] [ 74. 50. 54.] [ 99. 75. 79.] [124. 100. 104.]]] Notes ----- Pseudo-code [2]: 1. Initialization tensor_order cores and column indices 2. while (error > tol) 3. update the tensor-train from left to right: for Core 1 to Core tensor_order approximate the skeleton-decomposition by QR and maxvol 4. update the tensor-train from right to left: for Core tensor_order to Core 1 approximate the skeleton-decomposition by QR and maxvol 5. end while Acknowledgement: the main body of the code is modified based on TensorToolbox by Daniele Bigoni References ---------- .. [1] Ivan Oseledets and Eugene Tyrtyshnikov. Tt-cross approximation for multidimensional arrays. LinearAlgebra and its Applications, 432(1):70–88, 2010. .. [2] Sergey Dolgov and Robert Scheichl. A hybrid alternating least squares–tt cross algorithm for parametricpdes. arXiv preprint arXiv:1707.04562, 2017. """ # Check user input for errors tensor_shape = tl.shape(input_tensor) tensor_order = tl.ndim(input_tensor) if isinstance(rank, int): rank = [rank] * (tensor_order + 1) elif tensor_order + 1 != len(rank): message = 'Provided incorrect number of ranks. Should verify len(rank) == tl.ndim(tensor)+1, but len(rank) = {} while tl.ndim(tensor) + 1 = {}'.format( len(rank), tensor_order) raise (ValueError(message)) # Make sure iter's not a tuple but a list rank = list(rank) # Initialize rank if rank[0] != 1: print( 'Provided rank[0] == {} but boundary conditions dictate rank[0] == rank[-1] == 1: setting rank[0] to 1.'.format( rank[0])) rank[0] = 1 if rank[-1] != 1: print( 'Provided rank[-1] == {} but boundary conditions dictate rank[0] == rank[-1] == 1: setting rank[-1] to 1.'.format( rank[0])) # list col_idx: column indices (right indices) for skeleton-decomposition: indicate which columns used in each core. # list row_idx: row indices (left indices) for skeleton-decomposition: indicate which rows used in each core. # Initialize indice: random selection of column indices random_seed = None rng = check_random_state(random_seed) col_idx = [None] * tensor_order for k_col_idx in range(tensor_order - 1): col_idx[k_col_idx] = [] for i in range(rank[k_col_idx + 1]): newidx = tuple([rng.randint(tensor_shape[j]) for j in range(k_col_idx + 1, tensor_order)]) while newidx in col_idx[k_col_idx]: newidx = tuple([rng.randint(tensor_shape[j]) for j in range(k_col_idx + 1, tensor_order)]) col_idx[k_col_idx].append(newidx) # Initialize the cores of tensor-train factor_old = [tl.zeros((rank[k], tensor_shape[k], rank[k + 1]), **tl.context(input_tensor)) for k in range(tensor_order)] factor_new = [tl.tensor(rng.random_sample((rank[k], tensor_shape[k], rank[k + 1])), **tl.context(input_tensor)) for k in range(tensor_order)] iter = 0 error = tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2) threshold = tol * tl.norm(tt_to_tensor(factor_new), 2) for iter in range(n_iter_max): if error < threshold: break factor_old = factor_new factor_new = [None for i in range(tensor_order)] ###################################### # left-to-right step left_to_right_fiberlist = [] # list row_idx: list of (tensor_order-1) of lists of left indices row_idx = [[()]] for k in range(tensor_order - 1): (next_row_idx, fibers_list) = left_right_ttcross_step(input_tensor, k, rank, row_idx, col_idx) # update row indices left_to_right_fiberlist.extend(fibers_list) row_idx.append(next_row_idx) # end left-to-right step ############################################### ############################################### # right-to-left step right_to_left_fiberlist = [] # list col_idx: list (tensor_order-1) of lists of right indices col_idx = [None] * tensor_order col_idx[-1] = [()] for k in range(tensor_order, 1, -1): (next_col_idx, fibers_list, Q_skeleton) = right_left_ttcross_step(input_tensor, k, rank, row_idx, col_idx) # update col indices right_to_left_fiberlist.extend(fibers_list) col_idx[k - 2] = next_col_idx # Compute cores try: factor_new[k - 1] = tl.transpose(Q_skeleton) factor_new[k - 1] = tl.reshape(factor_new[k - 1], (rank[k - 1], tensor_shape[k - 1], rank[k])) except: # The rank should not be larger than the input tensor's size raise (ValueError("The rank is too large compared to the size of the tensor. Try with small rank.")) # Add the last core idx = (slice(None, None, None),) + tuple(zip(*col_idx[0])) core = input_tensor[idx] core = tl.reshape(core, (tensor_shape[0], 1, rank[1])) core = tl.transpose(core, (1, 0, 2)) factor_new[0] = core # end right-to-left step ################################################ # check the error for while-loop error = tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2) threshold = tol * tl.norm(tt_to_tensor(factor_new), 2) print("It: " + str(iter) + "; error: " + str(error) + "; threshold: " + str(threshold)) # check convergence if iter >= n_iter_max: print('Maximum number of iterations reached.') if tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2) > tol * tl.norm(tt_to_tensor(factor_new), 2): print('Low Rank Approximation algorithm did not converge.') return factor_new
def parafac2(tensor_slices, rank, n_iter_max=100, init='random', svd='numpy_svd', normalize_factors=False, tol=1e-8, random_state=None, verbose=False, return_errors=False, n_iter_parafac=5): r"""PARAFAC2 decomposition [1]_ of a third order tensor via alternating least squares (ALS) Computes a rank-`rank` PARAFAC2 decomposition of the third-order tensor defined by `tensor_slices`. The decomposition is on the form :math:`(A [B_i] C)` such that the i-th frontal slice, :math:`X_i`, of :math:`X` is given by .. math:: X_i = B_i diag(a_i) C^T, where :math:`diag(a_i)` is the diagonal matrix whose nonzero entries are equal to the :math:`i`-th row of the :math:`I \times R` factor matrix :math:`A`, :math:`B_i` is a :math:`J_i \times R` factor matrix such that the cross product matrix :math:`B_{i_1}^T B_{i_1}` is constant for all :math:`i`, and :math:`C` is a :math:`K \times R` factor matrix. To compute this decomposition, we reformulate the expression for :math:`B_i` such that .. math:: B_i = P_i B, where :math:`P_i` is a :math:`J_i \times R` orthogonal matrix and :math:`B` is a :math:`R \times R` matrix. An alternative formulation of the PARAFAC2 decomposition is that the tensor element :math:`X_{ijk}` is given by .. math:: X_{ijk} = \sum_{r=1}^R A_{ir} B_{ijr} C_{kr}, with the same constraints hold for :math:`B_i` as above. Parameters ---------- tensor_slices : ndarray or list of ndarrays Either a third order tensor or a list of second order tensors that may have different number of rows. Note that the second mode factor matrices are allowed to change over the first mode, not the third mode as some other implementations use (see note below). rank : int Number of components. n_iter_max : int Maximum number of iteration init : {'svd', 'random', CPTensor, Parafac2Tensor} Type of factor matrix initialization. See `initialize_factors`. svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS normalize_factors : bool (optional) If True, aggregate the weights of each factor in a 1D-tensor of shape (rank, ), which will contain the norms of the factors. Note that there may be some inaccuracies in the component weights. tol : float, optional (Default: 1e-8) Relative reconstruction error tolerance. The algorithm is considered to have found the global minimum when the reconstruction error is less than `tol`. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors n_iter_parafac: int, optional Number of PARAFAC iterations to perform for each PARAFAC2 iteration Returns ------- Parafac2Tensor : (weight, factors, projection_matrices) * weights : 1D array of shape (rank, ) all ones if normalize_factors is False (default), weights of the (normalized) factors otherwise * factors : List of factors of the CP decomposition element `i` is of shape (tensor.shape[i], rank) * projection_matrices : List of projection matrices used to create evolving factors. errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] Kiers, H.A.L., ten Berge, J.M.F. and Bro, R. (1999), PARAFAC2—Part I. A direct fitting algorithm for the PARAFAC2 model. J. Chemometrics, 13: 275-294. Notes ----- This formulation of the PARAFAC2 decomposition is slightly different from the one in [1]_. The difference lies in that here, the second mode changes over the first mode, whereas in [1]_, the second mode changes over the third mode. We made this change since that means that the function accept both lists of matrices and a single nd-array as input without any reordering of the modes. """ weights, factors, projections = initialize_decomposition( tensor_slices, rank, init=init, svd=svd, random_state=random_state) rec_errors = [] norm_tensor = tl.sqrt( sum(tl.norm(tensor_slice, 2)**2 for tensor_slice in tensor_slices)) svd_fun = _get_svd(svd) projected_tensor = tl.zeros([factor.shape[0] for factor in factors], **T.context(factors[0])) for iteration in range(n_iter_max): if verbose: print("Starting iteration", iteration) factors[1] = factors[1] * T.reshape(weights, (1, -1)) weights = T.ones(weights.shape, **tl.context(tensor_slices[0])) projections = _compute_projections(tensor_slices, factors, svd_fun, out=projections) projected_tensor = _project_tensor_slices(tensor_slices, projections, out=projected_tensor) _, factors = parafac(projected_tensor, rank, n_iter_max=n_iter_parafac, init=(weights, factors), svd=svd, orthogonalise=False, verbose=verbose, return_errors=False, normalize_factors=False, mask=None, random_state=random_state, tol=1e-100) if normalize_factors: new_factors = [] for factor in factors: norms = T.norm(factor, axis=0) norms = tl.where( tl.abs(norms) <= tl.eps(factor.dtype), tl.ones(tl.shape(norms), **tl.context(factors[0])), norms) weights = weights * norms new_factors.append(factor / (tl.reshape(norms, (1, -1)))) factors = new_factors if tol: rec_error = _parafac2_reconstruction_error( tensor_slices, (weights, factors, projections)) rec_error /= norm_tensor rec_errors.append(rec_error) if iteration >= 1: if verbose: print('PARAFAC2 reconstruction error={}, variation={}.'. format(rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break else: if verbose: print('PARAFAC2 reconstruction error={}'.format( rec_errors[-1])) parafac2_tensor = Parafac2Tensor((weights, factors, projections)) if return_errors: return parafac2_tensor, rec_errors else: return parafac2_tensor
def parafac(tensor, rank, n_iter_max=100, tol=1e-8, random_state=None, verbose=False, return_errors=False, mode_three_val=[[0.5, 0.5, 0.0], [0.0, 0.5, 0.5]]): """CANDECOMP/PARAFAC decomposition via alternating least squares (ALS) Computes a rank-`rank` decomposition of `tensor` [1]_ such that, ``tensor = [| factors[0], ..., factors[-1] |]``. Parameters ---------- tensor : ndarray rank : int Number of components. n_iter_max : int Maximum number of iteration tol : float, optional (Default: 1e-6) Relative reconstruction error tolerance. The algorithm is considered to have found the global minimum when the reconstruction error is less than `tol`. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors Returns ------- factors : ndarray list List of factors of the CP decomposition element `i` is of shape (tensor.shape[i], rank) errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009. """ factors = initialize_factors(tensor, rank, random_state=random_state) rec_errors = [] norm_tensor = tl.norm(tensor, 2) # Mode-3 values that control the country factors are set using the # mode_three_val argument. fixed_ja = mode_three_val[0] fixed_ko = mode_three_val[1] for iteration in range(n_iter_max): for mode in range(tl.ndim(tensor)): pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor)) factors[2][0] = fixed_ja # set mode-3 values factors[2][1] = fixed_ko # set mode-3 values for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse * tl.dot( tl.transpose(factor), factor) factor = tl.dot(unfold(tensor, mode), khatri_rao(factors, skip_matrix=mode)) factor = tl.transpose( tl.solve(tl.transpose(pseudo_inverse), tl.transpose(factor))) factors[mode] = factor if tol: rec_error = tl.norm(tensor - kruskal_to_tensor(factors), 2) / norm_tensor rec_errors.append(rec_error) if iteration > 1: if verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break if return_errors: return factors, rec_errors else: return factors
def parafac(tensor, rank, n_iter_max=100, init='svd', svd='numpy_svd', normalize_factors=False, tol=1e-8, orthogonalise=False, random_state=None, verbose=0, return_errors=False, non_negative=False, mask=None): """CANDECOMP/PARAFAC decomposition via alternating least squares (ALS) Computes a rank-`rank` decomposition of `tensor` [1]_ such that, ``tensor = [|weights; factors[0], ..., factors[-1] |]``. Parameters ---------- tensor : ndarray rank : int Number of components. n_iter_max : int Maximum number of iteration init : {'svd', 'random'}, optional Type of factor matrix initialization. See `initialize_factors`. svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor of shape (rank, ), which will contain the norms of the factors tol : float, optional (Default: 1e-6) Relative reconstruction error tolerance. The algorithm is considered to have found the global minimum when the reconstruction error is less than `tol`. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors non_negative : bool, optional Perform non_negative PARAFAC. See :func:`non_negative_parafac`. mask : ndarray array of booleans with the same shape as ``tensor`` should be 0 where the values are missing and 1 everywhere else. Note: if tensor is sparse, then mask should also be sparse with a fill value of 1 (or True). Allows for missing values [2]_ Returns ------- KruskalTensor : (weight, factors) * weights : 1D array of shape (rank, ) all ones if normalize_factors is False (default), weights of the (normalized) factors otherwise * factors : List of factors of the CP decomposition element `i` is of shape (tensor.shape[i], rank) errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009. .. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values." Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180. """ epsilon = 10e-12 if orthogonalise and not isinstance(orthogonalise, int): orthogonalise = n_iter_max factors = initialize_factors(tensor, rank, init=init, svd=svd, random_state=random_state, non_negative=non_negative, normalize_factors=normalize_factors) rec_errors = [] norm_tensor = tl.norm(tensor, 2) weights = tl.ones(rank, **tl.context(tensor)) for iteration in range(n_iter_max): if orthogonalise and iteration <= orthogonalise: factors = [ tl.qr(f)[0] if min(tl.shape(f)) >= rank else f for i, f in enumerate(factors) ] if verbose > 1: print("Starting iteration", iteration + 1) for mode in range(tl.ndim(tensor)): if verbose > 1: print("Mode", mode, "of", tl.ndim(tensor)) if non_negative: accum = 1 # khatri_rao(factors).tl.dot(khatri_rao(factors)) # simplifies to multiplications sub_indices = [i for i in range(len(factors)) if i != mode] for i, e in enumerate(sub_indices): if i: accum *= tl.dot(tl.transpose(factors[e]), factors[e]) else: accum = tl.dot(tl.transpose(factors[e]), factors[e]) pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor)) for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse * tl.dot( tl.conj(tl.transpose(factor)), factor) if mask is not None: tensor = tensor * mask + tl.kruskal_to_tensor( (None, factors), mask=1 - mask) mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode) if non_negative: numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None) denominator = tl.dot(factors[mode], accum) denominator = tl.clip(denominator, a_min=epsilon, a_max=None) factor = factors[mode] * numerator / denominator else: factor = tl.transpose( tl.solve(tl.conj(tl.transpose(pseudo_inverse)), tl.transpose(mttkrp))) if normalize_factors: weights = tl.norm(factor, order=2, axis=0) weights = tl.where( tl.abs(weights) <= tl.eps(tensor.dtype), tl.ones(tl.shape(weights), **tl.context(factors[0])), weights) factor = factor / (tl.reshape(weights, (1, -1))) factors[mode] = factor if tol: # ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec> factors_norm = kruskal_norm((weights, factors)) # mttkrp and factor for the last mode. This is equivalent to the # inner product <tensor, factorization> iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights) rec_error = tl.sqrt( tl.abs(norm_tensor**2 + factors_norm**2 - 2 * iprod)) / norm_tensor rec_errors.append(rec_error) if iteration >= 1: if verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break else: if verbose: print('reconstruction error={}'.format(rec_errors[-1])) kruskal_tensor = KruskalTensor((weights, factors)) if return_errors: return kruskal_tensor, rec_errors else: return kruskal_tensor
def initialize_factors(tensor, rank, init='svd', svd='numpy_svd', random_state=None, non_negative=False, normalize_factors=False): r"""Initialize factors used in `parafac`. The type of initialization is set using `init`. If `init == 'random'` then initialize factor matrices using `random_state`. If `init == 'svd'` then initialize the `m`th factor matrix using the `rank` left singular vectors of the `m`th unfolding of the input tensor. Parameters ---------- tensor : ndarray rank : int init : {'svd', 'random'}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS non_negative : bool, default is False if True, non-negative factors are returned Returns ------- factors : ndarray list List of initialized factors of the CP decomposition where element `i` is of shape (tensor.shape[i], rank) """ rng = check_random_state(random_state) if init == 'random': factors = [ tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor)) ] if non_negative: factors = [tl.abs(f) for f in factors] if normalize_factors: factors = [ f / (tl.reshape(tl.norm(f, axis=0), (1, -1)) + 1e-12) for f in factors ] return factors elif init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) factors = [] for mode in range(tl.ndim(tensor)): U, _, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank) if tensor.shape[mode] < rank: # TODO: this is a hack but it seems to do the job for now # factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor)) # factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) # factor[:, :tensor.shape[mode]] = U random_part = tl.tensor( rng.random_sample( (U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) U = tl.concatenate([U, random_part], axis=1) factor = U[:, :rank] if non_negative: factor = tl.abs(factor) if normalize_factors: factor = factor / (tl.reshape(tl.norm(factor, axis=0), (1, -1)) + 1e-12) factors.append(factor) return factors raise ValueError('Initialization method "{}" not recognized'.format(init))
def initialize_nn_cp(tensor, rank, init='svd', svd='numpy_svd', random_state=None, normalize_factors=False, nntype='nndsvda'): r"""Initialize factors used in `parafac`. The type of initialization is set using `init`. If `init == 'random'` then initialize factor matrices using `random_state`. If `init == 'svd'` then initialize the `m`th factor matrix using the `rank` left singular vectors of the `m`th unfolding of the input tensor. Parameters ---------- tensor : ndarray rank : int init : {'svd', 'random'}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS nntype : {'nndsvd', 'nndsvda'} Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default). Returns ------- factors : CPTensor An initial cp tensor. """ rng = tl.check_random_state(random_state) if init == 'random': kt = random_cp(tl.shape(tensor), rank, normalise_factors=False, random_state=rng, **tl.context(tensor)) elif init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) factors = [] for mode in range(tl.ndim(tensor)): U, S, V = svd_fun(unfold(tensor, mode), n_eigenvecs=rank) # Apply nnsvd to make non-negative U = make_svd_non_negative(tensor, U, S, V, nntype) if tensor.shape[mode] < rank: # TODO: this is a hack but it seems to do the job for now random_part = tl.tensor( rng.random_sample( (U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) U = tl.concatenate([U, random_part], axis=1) factors.append(U[:, :rank]) kt = CPTensor((None, factors)) # If the initialisation is a precomputed decomposition, we double check its validity and return it elif isinstance(init, (tuple, list, CPTensor)): # TODO: Test this try: kt = CPTensor(init) except ValueError: raise ValueError( 'If initialization method is a mapping, then it must ' 'be possible to convert it to a CPTensor instance') return kt else: raise ValueError( 'Initialization method "{}" not recognized'.format(init)) # Make decomposition feasible by taking the absolute value of all factor matrices kt.factors = [tl.abs(f) for f in kt[1]] if normalize_factors: kt = cp_normalize(kt) return kt
def non_negative_parafac_hals(tensor, rank, n_iter_max=100, init="svd", svd='numpy_svd', tol=10e-8, random_state=None, sparsity_coefficients=None, fixed_modes=None, nn_modes='all', exact=False, normalize_factors=False, verbose=False, return_errors=False, cvg_criterion='abs_rec_error'): """ Non-negative CP decomposition via HALS Uses Hierarchical ALS (Alternating Least Squares) which updates each factor column-wise (one column at a time while keeping all other columns fixed), see [1]_ Parameters ---------- tensor : ndarray rank : int number of components n_iter_max : int maximum number of iteration init : {'svd', 'random'}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS tol : float, optional tolerance: the algorithm stops when the variation in the reconstruction error is less than the tolerance Default: 1e-8 random_state : {None, int, np.random.RandomState} sparsity_coefficients: array of float (of length the number of modes) The sparsity coefficients on each factor. If set to None, the algorithm is computed without sparsity Default: None, fixed_modes: array of integers (between 0 and the number of modes) Has to be set not to update a factor, 0 and 1 for U and V respectively Default: None nn_modes: None, 'all' or array of integers (between 0 and the number of modes) Used to specify which modes to impose non-negativity constraints on. If 'all', then non-negativity is imposed on all modes. Default: 'all' exact: If it is True, the algorithm gives a results with high precision but it needs high computational cost. If it is False, the algorithm gives an approximate solution Default: False normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor of shape (rank, ), which will contain the norms of the factors verbose: boolean Indicates whether the algorithm prints the successive reconstruction errors or not Default: False return_errors: boolean Indicates whether the algorithm should return all reconstruction errors and computation time of each iteration or not Default: False cvg_criterion : {'abs_rec_error', 'rec_error'}, optional Stopping criterion for ALS, works if `tol` is not None. If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``. If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`. sparsity : float or int random_state : {None, int, np.random.RandomState} Returns ------- factors : ndarray list list of positive factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)`` errors: list A list of reconstruction errors at each iteration of the algorithm. References ---------- .. [1]: N. Gillis and F. Glineur, Accelerated Multiplicative Updates and Hierarchical ALS Algorithms for Nonnegative Matrix Factorization, Neural Computation 24 (4): 1085-1105, 2012. """ weights, factors = initialize_nn_cp(tensor, rank, init=init, svd=svd, random_state=random_state, normalize_factors=normalize_factors) norm_tensor = tl.norm(tensor, 2) n_modes = tl.ndim(tensor) if sparsity_coefficients is None or isinstance(sparsity_coefficients, float): sparsity_coefficients = [sparsity_coefficients] * n_modes if fixed_modes is None: fixed_modes = [] if nn_modes == 'all': nn_modes = set(range(n_modes)) elif nn_modes is None: nn_modes = set() # Avoiding errors for fixed_value in fixed_modes: sparsity_coefficients[fixed_value] = None for mode in range(n_modes): if sparsity_coefficients[mode] is not None: warnings.warn( "Sparsity coefficient is ignored in unconstrained modes.") # Generating the mode update sequence modes = [mode for mode in range(n_modes) if mode not in fixed_modes] # initialisation - declare local varaibles rec_errors = [] # Iteratation for iteration in range(n_iter_max): # One pass of least squares on each updated mode for mode in modes: # Computing Hadamard of cross-products pseudo_inverse = tl.tensor(tl.ones((rank, rank)), **tl.context(tensor)) for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse * tl.dot( tl.transpose(factor), factor) pseudo_inverse = tl.reshape(weights, (-1, 1)) * pseudo_inverse * tl.reshape( weights, (1, -1)) mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode) if mode in nn_modes: # Call the hals resolution with nnls, optimizing the current mode nn_factor, _, _, _ = hals_nnls( tl.transpose(mttkrp), pseudo_inverse, tl.transpose(factors[mode]), n_iter_max=100, sparsity_coefficient=sparsity_coefficients[mode], exact=exact) factors[mode] = tl.transpose(nn_factor) else: factor = tl.solve(tl.transpose(pseudo_inverse), tl.transpose(mttkrp)) factors[mode] = tl.transpose(factor) if normalize_factors and mode != modes[-1]: weights, factors = cp_normalize((weights, factors)) if tol: factors_norm = cp_norm((weights, factors)) iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0)) rec_error = tl.sqrt( tl.abs(norm_tensor**2 + factors_norm**2 - 2 * iprod)) / norm_tensor rec_errors.append(rec_error) if iteration >= 1: rec_error_decrease = rec_errors[-2] - rec_errors[-1] if verbose: print( "iteration {}, reconstruction error: {}, decrease = {}" .format(iteration, rec_error, rec_error_decrease)) if cvg_criterion == 'abs_rec_error': stop_flag = abs(rec_error_decrease) < tol elif cvg_criterion == 'rec_error': stop_flag = rec_error_decrease < tol else: raise TypeError("Unknown convergence criterion") if stop_flag: if verbose: print("PARAFAC converged after {} iterations".format( iteration)) break else: if verbose: print('reconstruction error={}'.format(rec_errors[-1])) if normalize_factors: weights, factors = cp_normalize((weights, factors)) cp_tensor = CPTensor((weights, factors)) if return_errors: return cp_tensor, rec_errors else: return cp_tensor
def make_svd_non_negative(tensor, U, S, V, nntype): """ Use NNDSVD method to transform SVD results into a non-negative form. This method leads to more efficient solving with NNMF [1]. Parameters ---------- tensor : tensor being decomposed U, S, V: SVD factorization results nntype : {'nndsvd', 'nndsvda'} Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default). [1]: Boutsidis & Gallopoulos. Pattern Recognition, 41(4): 1350-1362, 2008. """ # NNDSVD initialization W = tl.zeros_like(U) H = tl.zeros_like(V) # The leading singular triplet is non-negative # so it can be used as is for initialization. W = tl.index_update(W, tl.index[:, 0], tl.sqrt(S[0]) * tl.abs(U[:, 0])) H = tl.index_update(H, tl.index[0, :], tl.sqrt(S[0]) * tl.abs(V[0, :])) for j in range(1, tl.shape(U)[1]): x, y = U[:, j], V[j, :] # extract positive and negative parts of column vectors x_p, y_p = tl.clip(x, a_min=0.0), tl.clip(y, a_min=0.0) x_n, y_n = tl.abs(tl.clip(x, a_max=0.0)), tl.abs(tl.clip(y, a_max=0.0)) # and their norms x_p_nrm, y_p_nrm = tl.norm(x_p), tl.norm(y_p) x_n_nrm, y_n_nrm = tl.norm(x_n), tl.norm(y_n) m_p, m_n = x_p_nrm * y_p_nrm, x_n_nrm * y_n_nrm # choose update if m_p > m_n: u = x_p / x_p_nrm v = y_p / y_p_nrm sigma = m_p else: u = x_n / x_n_nrm v = y_n / y_n_nrm sigma = m_n lbd = tl.sqrt(S[j] * sigma) W = tl.index_update(W, tl.index[:, j], lbd * u) H = tl.index_update(H, tl.index[j, :], lbd * v) # After this point we no longer need H eps = tl.eps(tensor.dtype) if nntype == "nndsvd": W = soft_thresholding(W, eps) elif nntype == "nndsvda": avg = tl.mean(tensor) W = tl.where(W < eps, tl.ones(tl.shape(W), **tl.context(W)) * avg, W) else: raise ValueError( 'Invalid nntype parameter: got %r instead of one of %r' % (nntype, ('nndsvd', 'nndsvda'))) return W
def matrix_product_state(input_tensor, rank, verbose=False): """MPS decomposition via recursive SVD Decomposes `input_tensor` into a sequence of order-3 tensors (factors) -- also known as Tensor-Train decomposition [1]_. Parameters ---------- input_tensor : tensorly.tensor rank : {int, int list} maximum allowable MPS rank of the factors if int, then this is the same for all the factors if int list, then rank[k] is the rank of the kth factor verbose : boolean, optional level of verbosity Returns ------- factors : MPS factors order-3 tensors of the MPS decomposition References ---------- .. [1] Ivan V. Oseledets. "Tensor-train decomposition", SIAM J. Scientific Computing, 33(5):2295–2317, 2011. """ # Check user input for errors tensor_size = input_tensor.shape n_dim = len(tensor_size) if isinstance(rank, int): rank = [rank] * (n_dim + 1) elif n_dim + 1 != len(rank): message = 'Provided incorrect number of ranks. Should verify len(rank) == tl.ndim(tensor)+1, but len(rank) = {} while tl.ndim(tensor) + 1 = {}'.format( len(rank), n_dim) raise (ValueError(message)) # Make sure it's not a tuple but a list rank = list(rank) context = tl.context(input_tensor) # Initialization if rank[0] != 1: print( 'Provided rank[0] == {} but boundaring conditions dictatate rank[0] == rank[-1] == 1: setting rank[0] to 1.' .format(rank[0])) rank[0] = 1 if rank[-1] != 1: print( 'Provided rank[-1] == {} but boundaring conditions dictatate rank[0] == rank[-1] == 1: setting rank[-1] to 1.' .format(rank[0])) unfolding = input_tensor factors = [None] * n_dim # Getting the MPS factors up to n_dim - 1 for k in range(n_dim - 1): # Reshape the unfolding matrix of the remaining factors n_row = int(rank[k] * tensor_size[k]) unfolding = tl.reshape(unfolding, (n_row, -1)) # SVD of unfolding matrix (n_row, n_column) = unfolding.shape current_rank = min(n_row, n_column, rank[k + 1]) U, S, V = tl.partial_svd(unfolding, current_rank) rank[k + 1] = current_rank # Get kth MPS factor factors[k] = tl.reshape(U, (rank[k], tensor_size[k], rank[k + 1])) if (verbose is True): print("MPS factor " + str(k) + " computed with shape " + str(factors[k].shape)) # Get new unfolding matrix for the remaining factors unfolding = tl.reshape(S, (-1, 1)) * V # Getting the last factor (prev_rank, last_dim) = unfolding.shape factors[-1] = tl.reshape(unfolding, (prev_rank, last_dim, 1)) if (verbose is True): print("MPS factor " + str(n_dim - 1) + " computed with shape " + str(factors[n_dim - 1].shape)) return factors
def power_iteration(tensor, n_repeat=10, n_iteration=10, verbose=False): """A single Robust Tensor Power Iteration Parameters ---------- tensor : tl.tensor input tensor to decompose n_repeat : int, default is 10 number of initializations to be tried n_iteration : int, default is 10 number of power iterations verbose : bool level of verbosity Returns ------- (eigenval, best_factor, deflated) eigenval : float the obtained eigenvalue best_factors : tl.tensor list the best estimated eigenvector, for each mode of the input tensor deflated : tl.tensor of same shape as `tensor` the deflated tensor (i.e. without the estimated component) """ order = tl.ndim(tensor) # A list of candidates for each mode best_score = 0 scores = [] for _ in range(n_repeat): factors = [ tl.tensor(np.random.random_sample(s), **tl.context(tensor)) for s in tl.shape(tensor) ] for _ in range(n_iteration): for mode in range(order): factor = tl.tenalg.multi_mode_dot(tensor, factors, skip=mode) factor = factor / tl.norm(factor, 2) factors[mode] = factor score = tl.tenalg.multi_mode_dot(tensor, factors) scores.append(score) #round(score, 2)) if score > best_score: best_score = score best_factors = factors if verbose: print(f'Best score of {n_repeat}: {best_score}') # Refine the init for _ in range(n_iteration): for mode in range(order): factor = tl.tenalg.multi_mode_dot(tensor, best_factors, skip=mode) factor = factor / tl.norm(factor, 2) best_factors[mode] = factor eigenval = tl.tenalg.multi_mode_dot(tensor, best_factors) deflated = tensor - outer(best_factors) * eigenval if verbose: explained = tl.norm(deflated) / tl.norm(tensor) print(f'Eingenvalue: {eigenval}, explained: {explained}') return eigenval, best_factors, deflated
def initialize_decomposition(tensor_slices, rank, init='random', svd='numpy_svd', random_state=None): r"""Initiate a random PARAFAC2 decomposition given rank and tensor slices Parameters ---------- tensor_slices : Iterable of ndarray rank : int init : {'random', 'svd', CPTensor, Parafac2Tensor}, optional random_state : `np.random.RandomState` Returns ------- parafac2_tensor : Parafac2Tensor List of initialized factors of the CP decomposition where element `i` is of shape (tensor.shape[i], rank) """ context = tl.context(tensor_slices[0]) shapes = [m.shape for m in tensor_slices] if init == 'random': return random_parafac2(shapes, rank, full=False, random_state=random_state, **context) elif init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) padded_tensor = _pad_by_zeros(tensor_slices) A = T.ones((padded_tensor.shape[0], rank), **context) unfolded_mode_2 = unfold(padded_tensor, 2) if T.shape(unfolded_mode_2)[0] < rank: raise ValueError( "Cannot perform SVD init if rank ({}) is greater than the number of columns in each tensor slice ({})" .format(rank, T.shape(unfolded_mode_2)[0])) C = svd_fun(unfold(padded_tensor, 2), n_eigenvecs=rank)[0] B = T.eye(rank, **context) projections = _compute_projections(tensor_slices, (A, B, C), svd_fun) return Parafac2Tensor((None, [A, B, C], projections)) elif isinstance(init, (tuple, list, Parafac2Tensor, CPTensor)): try: decomposition = Parafac2Tensor.from_CPTensor( init, parafac2_tensor_ok=True) except ValueError: raise ValueError( 'If initialization method is a mapping, then it must ' 'be possible to convert it to a Parafac2Tensor instance') if decomposition.rank != rank: raise ValueError( 'Cannot init with a decomposition of different rank') return decomposition raise ValueError('Initialization method "{}" not recognized'.format(init))
def parafac(tensor, rank, n_iter_max=100, init='svd', svd='numpy_svd',\ normalize_factors=False, orthogonalise=False,\ tol=1e-8, random_state=None,\ verbose=0, return_errors=False,\ sparsity = None,\ l2_reg = 0, mask=None,\ cvg_criterion = 'abs_rec_error',\ fixed_modes = [], svd_mask_repeats=5, linesearch = False): """CANDECOMP/PARAFAC decomposition via alternating least squares (ALS) Computes a rank-`rank` decomposition of `tensor` [1]_ such that:: tensor = [|weights; factors[0], ..., factors[-1] |]. Parameters ---------- tensor : ndarray rank : int Number of components. n_iter_max : int Maximum number of iteration init : {'svd', 'random'}, optional Type of factor matrix initialization. See `initialize_factors`. svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor of shape (rank, ), which will contain the norms of the factors tol : float, optional (Default: 1e-6) Relative reconstruction error tolerance. The algorithm is considered to have found the global minimum when the reconstruction error is less than `tol`. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors mask : ndarray array of booleans with the same shape as ``tensor`` should be 0 where the values are missing and 1 everywhere else. Note: if tensor is sparse, then mask should also be sparse with a fill value of 1 (or True). Allows for missing values [2]_ cvg_criterion : {'abs_rec_error', 'rec_error'}, optional Stopping criterion for ALS, works if `tol` is not None. If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``. If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`. sparsity : float or int If `sparsity` is not None, we approximate tensor as a sum of low_rank_component and sparse_component, where low_rank_component = cp_to_tensor((weights, factors)). `sparsity` denotes desired fraction or number of non-zero elements in the sparse_component of the `tensor`. fixed_modes : list, default is [] A list of modes for which the initial value is not modified. The last mode cannot be fixed due to error computation. svd_mask_repeats: int If using a tensor with masked values, this initializes using SVD multiple times to remove the effect of these missing values on the initialization. linesearch : bool, default is False Whether to perform line search as proposed by Bro [3]. Returns ------- CPTensor : (weight, factors) * weights : 1D array of shape (rank, ) * all ones if normalize_factors is False (default) * weights of the (normalized) factors otherwise * factors : List of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)`` * sparse_component : nD array of shape tensor.shape. Returns only if `sparsity` is not None. errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009. .. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values." Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180. .. [3] R. Bro, "Multi-Way Analysis in the Food Industry: Models, Algorithms, and Applications", PhD., University of Amsterdam, 1998 """ rank = validate_cp_rank(tl.shape(tensor), rank=rank) if orthogonalise and not isinstance(orthogonalise, int): orthogonalise = n_iter_max if linesearch: acc_pow = 2.0 # Extrapolate to the iteration^(1/acc_pow) ahead acc_fail = 0 # How many times acceleration have failed max_fail = 4 # Increase acc_pow with one after max_fail failure weights, factors = initialize_cp(tensor, rank, init=init, svd=svd, random_state=random_state, normalize_factors=normalize_factors) if mask is not None and init == "svd": for _ in range(svd_mask_repeats): tensor = tensor*mask + tl.cp_to_tensor((weights, factors), mask=1-mask) weights, factors = initialize_cp(tensor, rank, init=init, svd=svd, random_state=random_state, normalize_factors=normalize_factors) rec_errors = [] norm_tensor = tl.norm(tensor, 2) Id = tl.eye(rank, **tl.context(tensor))*l2_reg if tl.ndim(tensor)-1 in fixed_modes: warnings.warn('You asked for fixing the last mode, which is not supported.\n The last mode will not be fixed. Consider using tl.moveaxis()') fixed_modes.remove(tl.ndim(tensor)-1) modes_list = [mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes] if sparsity: sparse_component = tl.zeros_like(tensor) if isinstance(sparsity, float): sparsity = int(sparsity * np.prod(tensor.shape)) else: sparsity = int(sparsity) for iteration in range(n_iter_max): if orthogonalise and iteration <= orthogonalise: factors = [tl.qr(f)[0] if min(tl.shape(f)) >= rank else f for i, f in enumerate(factors)] if linesearch and iteration % 2 == 0: factors_last = [tl.copy(f) for f in factors] weights_last = tl.copy(weights) if verbose > 1: print("Starting iteration", iteration + 1) for mode in modes_list: if verbose > 1: print("Mode", mode, "of", tl.ndim(tensor)) pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor)) for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse*tl.dot(tl.conj(tl.transpose(factor)), factor) pseudo_inverse += Id if not iteration and weights is not None: # Take into account init weights mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode) else: mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode) factor = tl.transpose(tl.solve(tl.conj(tl.transpose(pseudo_inverse)), tl.transpose(mttkrp))) if normalize_factors: scales = tl.norm(factor, 2, axis=0) weights = tl.where(scales==0, tl.ones(tl.shape(scales), **tl.context(factor)), scales) factor = factor / tl.reshape(weights, (1, -1)) factors[mode] = factor # Will we be performing a line search iteration if linesearch and iteration % 2 == 0 and iteration > 5: line_iter = True else: line_iter = False # Calculate the current unnormalized error if we need it if (tol or return_errors) and line_iter is False: unnorml_rec_error, tensor, norm_tensor = error_calc(tensor, norm_tensor, weights, factors, sparsity, mask, mttkrp) else: if mask is not None: tensor = tensor*mask + tl.cp_to_tensor((weights, factors), mask=1-mask) # Start line search if requested. if line_iter is True: jump = iteration ** (1.0 / acc_pow) new_weights = weights_last + (weights - weights_last) * jump new_factors = [factors_last[ii] + (factors[ii] - factors_last[ii])*jump for ii in range(tl.ndim(tensor))] new_rec_error, new_tensor, new_norm_tensor = error_calc(tensor, norm_tensor, new_weights, new_factors, sparsity, mask) if (new_rec_error / new_norm_tensor) < rec_errors[-1]: factors, weights = new_factors, new_weights tensor, norm_tensor = new_tensor, new_norm_tensor unnorml_rec_error = new_rec_error acc_fail = 0 if verbose: print("Accepted line search jump of {}.".format(jump)) else: unnorml_rec_error, tensor, norm_tensor = error_calc(tensor, norm_tensor, weights, factors, sparsity, mask, mttkrp) acc_fail += 1 if verbose: print("Line search failed for jump of {}.".format(jump)) if acc_fail == max_fail: acc_pow += 1.0 acc_fail = 0 if verbose: print("Reducing acceleration.") rec_error = unnorml_rec_error / norm_tensor rec_errors.append(rec_error) if tol: if iteration >= 1: rec_error_decrease = rec_errors[-2] - rec_errors[-1] if verbose: print("iteration {}, reconstruction error: {}, decrease = {}, unnormalized = {}".format(iteration, rec_error, rec_error_decrease, unnorml_rec_error)) if cvg_criterion == 'abs_rec_error': stop_flag = abs(rec_error_decrease) < tol elif cvg_criterion == 'rec_error': stop_flag = rec_error_decrease < tol else: raise TypeError("Unknown convergence criterion") if stop_flag: if verbose: print("PARAFAC converged after {} iterations".format(iteration)) break else: if verbose: print('reconstruction error={}'.format(rec_errors[-1])) cp_tensor = CPTensor((weights, factors)) if sparsity: sparse_component = sparsify_tensor(tensor -\ cp_to_tensor((weights, factors)),\ sparsity) cp_tensor = (cp_tensor, sparse_component) if return_errors: return cp_tensor, rec_errors else: return cp_tensor
def initialize_cp(tensor, rank, init='svd', svd='numpy_svd', random_state=None, non_negative=False, normalize_factors=False): r"""Initialize factors used in `parafac`. The type of initialization is set using `init`. If `init == 'random'` then initialize factor matrices using `random_state`. If `init == 'svd'` then initialize the `m`th factor matrix using the `rank` left singular vectors of the `m`th unfolding of the input tensor. Parameters ---------- tensor : ndarray rank : int init : {'svd', 'random'}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS non_negative : bool, default is False if True, non-negative factors are returned Returns ------- factors : CPTensor An initial cp tensor. """ rng = check_random_state(random_state) if init == 'random': # factors = [tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor))] # kt = CPTensor((None, factors)) return random_cp(tl.shape(tensor), rank, normalise_factors=False, random_state=rng, **tl.context(tensor)) elif init == 'svd': try: svd_fun = tl.SVD_FUNS[svd] except KeyError: message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format( svd, tl.get_backend(), tl.SVD_FUNS) raise ValueError(message) factors = [] for mode in range(tl.ndim(tensor)): U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank) # Put SVD initialization on the same scaling as the tensor in case normalize_factors=False if mode == 0: idx = min(rank, tl.shape(S)[0]) U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx]) if tensor.shape[mode] < rank: # TODO: this is a hack but it seems to do the job for now # factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor)) # factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) # factor[:, :tensor.shape[mode]] = U random_part = tl.tensor( rng.random_sample( (U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor)) U = tl.concatenate([U, random_part], axis=1) factors.append(U[:, :rank]) kt = CPTensor((None, factors)) elif isinstance(init, (tuple, list, CPTensor)): # TODO: Test this try: kt = CPTensor(init) except ValueError: raise ValueError( 'If initialization method is a mapping, then it must ' 'be possible to convert it to a CPTensor instance') else: raise ValueError( 'Initialization method "{}" not recognized'.format(init)) if non_negative: kt.factors = [tl.abs(f) for f in kt[1]] if normalize_factors: kt = cp_normalize(kt) return kt
def randomised_parafac(tensor, rank, n_samples, n_iter_max=100, init='random', svd='numpy_svd', tol=10e-9, max_stagnation=20, random_state=None, verbose=1): """Randomised CP decomposition via sampled ALS Parameters ---------- tensor : ndarray rank : int number of components n_samples : int number of samples per ALS step n_iter_max : int maximum number of iteration init : {'svd', 'random'}, optional svd : str, default is 'numpy_svd' function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS tol : float, optional tolerance: the algorithm stops when the variation in the reconstruction error is less than the tolerance max_stagnation: int, optional, default is 0 if not zero, the maximum allowed number of iterations with no decrease in fit random_state : {None, int, np.random.RandomState}, default is None verbose : int, optional level of verbosity Returns ------- factors : ndarray list list of positive factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)`` References ---------- .. [3] Casey Battaglino, Grey Ballard and Tamara G. Kolda, "A Practical Randomized CP Tensor Decomposition", """ rng = check_random_state(random_state) factors = initialize_factors(tensor, rank, init=init, svd=svd, random_state=random_state) rec_errors = [] n_dims = tl.ndim(tensor) norm_tensor = tl.norm(tensor, 2) min_error = 0 weights = tl.ones(rank, **tl.context(tensor)) for iteration in range(n_iter_max): for mode in range(n_dims): kr_prod, indices_list = sample_khatri_rao(factors, n_samples, skip_matrix=mode, random_state=rng) indices_list = [i.tolist() for i in indices_list] # Keep all the elements of the currently considered mode indices_list.insert(mode, slice(None, None, None)) # MXNet will not be happy if this is a list insteaf of a tuple indices_list = tuple(indices_list) if mode: sampled_unfolding = tensor[indices_list] else: sampled_unfolding = tl.transpose(tensor[indices_list]) pseudo_inverse = tl.dot(tl.transpose(kr_prod), kr_prod) factor = tl.dot(tl.transpose(kr_prod), sampled_unfolding) factor = tl.transpose(tl.solve(pseudo_inverse, factor)) factors[mode] = factor if max_stagnation or tol: rec_error = tl.norm(tensor - kruskal_to_tensor( (weights, factors)), 2) / norm_tensor if not min_error or rec_error < min_error: min_error = rec_error stagnation = -1 stagnation += 1 rec_errors.append(rec_error) if iteration > 1: if verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if (tol and abs(rec_errors[-2] - rec_errors[-1]) < tol) or \ (stagnation and (stagnation > max_stagnation)): if verbose: print('converged in {} iterations.'.format(iteration)) break return KruskalTensor((weights, factors))
def sample_khatri_rao(matrices, n_samples, skip_matrix=None, return_sampled_rows=False, random_state=None): """Random subsample of the Khatri-Rao product of the given list of matrices If one matrix only is given, that matrix is directly returned. Parameters ---------- matrices : ndarray list list of matrices with the same number of columns, i.e.:: for i in len(matrices): matrices[i].shape = (n_i, m) n_samples : int number of samples to be taken from the Khatri-Rao product skip_matrix : None or int, optional, default is None if not None, index of a matrix to skip random_state : None, int or numpy.random.RandomState if int, used to set the seed of the random number generator if numpy.random.RandomState, used to generate random_samples returned_sampled_rows : bool, default is False if True, also returns a list of the rows sampled from the full khatri-rao product Returns ------- sampled_Khatri_Rao : ndarray The sampled matricised tensor Khatri-Rao with `n_samples` rows indices : tuple list a list of indices sampled for each mode indices_kr : int list list of length `n_samples` containing the sampled row indices """ if random_state is None or not isinstance(random_state, np.random.RandomState): rng = check_random_state(random_state) warnings.warn( 'You are creating a new random number generator at each call.\n' 'If you are calling sample_khatri_rao inside a loop this will be slow:' ' best to create a rng outside and pass it as argument (random_state=rng).' ) else: rng = random_state if skip_matrix is not None: matrices = [ matrices[i] for i in range(len(matrices)) if i != skip_matrix ] rank = tl.shape(matrices[0])[1] sizes = [tl.shape(m)[0] for m in matrices] # For each matrix, randomly choose n_samples indices for which to compute the khatri-rao product indices_list = [ rng.randint(0, tl.shape(m)[0], size=n_samples, dtype=int) for m in matrices ] if return_sampled_rows: # Compute corresponding rows of the full khatri-rao product indices_kr = np.zeros((n_samples), dtype=int) for size, indices in zip(sizes, indices_list): indices_kr = indices_kr * size + indices # Compute the Khatri-Rao product for the chosen indices sampled_kr = tl.ones((n_samples, rank), **tl.context(matrices[0])) for indices, matrix in zip(indices_list, matrices): sampled_kr = sampled_kr * matrix[indices, :] if return_sampled_rows: return sampled_kr, indices_list, indices_kr else: return sampled_kr, indices_list
def maxvol(A): """ Find the rxr submatrix of maximal volume in A(nxr), n>=r We want to decompose matrix A as A = A[:,J] * (A[I,J])^-1 * A[I,:] This algorithm helps us find this submatrix A[I,J] from A, which has the largest determinant. We greedily find vector of max norm, and subtract its projection from the rest of rows. Parameters ---------- A: matrix The matrix to find maximal volume Returns ------- row_idx: list of int is the list or rows of A forming the matrix with maximal volume, A_inv: matrix is the inverse of the matrix with maximal volume. References ---------- S. A. Goreinov, I. V. Oseledets, D. V. Savostyanov, E. E. Tyrtyshnikov, N. L. Zamarashkin. How to find a good submatrix.Goreinov, S. A., et al. Matrix Methods: Theory, Algorithms and Applications: Dedicated to the Memory of Gene Golub. 2010. 247-256. Ali Çivril, Malik Magdon-Ismail On selecting a maximum volume sub-matrix of a matrix and related problems Theoretical Computer Science. Volume 410, Issues 47–49, 6 November 2009, Pages 4801-4811 """ (n, r) = tl.shape(A) # The index of row of the submatrix row_idx = tl.zeros(r) # Rest of rows / unselected rows rest_of_rows = tl.tensor(list(range(n)),dtype= tl.int64) # Find r rows iteratively i = 0 A_new = A while i < r: mask = list(range(tl.shape(A_new)[0])) # Compute the square of norm of each row rows_norms = tl.sum(A_new ** 2, axis=1) # If there is only one row of A left, let's just return it. MxNet is not robust about this case. if tl.shape(rows_norms) == (): row_idx[i] = rest_of_rows break # If a row is 0, we delete it. if any(rows_norms == 0): zero_idx = tl.argmin(rows_norms,axis=0) mask.pop(zero_idx) rest_of_rows = rest_of_rows[mask] A_new = A_new[mask,:] continue # Find the row of max norm max_row_idx = tl.argmax(rows_norms, axis=0) max_row = A[rest_of_rows[max_row_idx], :] # Compute the projection of max_row to other rows # projection a to b is computed as: <a,b> / sqrt(|a|*|b|) projection = tl.dot(A_new, tl.transpose(max_row)) normalization = tl.sqrt(rows_norms[max_row_idx] * rows_norms) # make sure normalization vector is of the same shape of projection (causing bugs for MxNet) normalization = tl.reshape(normalization, tl.shape(projection)) projection = projection/normalization # Subtract the projection from A_new: b <- b - a * projection A_new = A_new - A_new * tl.reshape(projection, (tl.shape(A_new)[0], 1)) # Delete the selected row mask.pop(max_row_idx) A_new = A_new[mask,:] # update the row_idx and rest_of_rows row_idx[i] = rest_of_rows[max_row_idx] rest_of_rows = rest_of_rows[mask] i = i + 1 row_idx = tl.tensor(row_idx, dtype=tl.int64) inverse = tl.solve(A[row_idx,:], tl.eye(tl.shape(A[row_idx,:])[0], **tl.context(A))) row_idx = tl.to_numpy(row_idx) return row_idx, inverse