def _parafac2_reconstruction_error(tensor_slices, decomposition):
_validate_parafac2_tensor(decomposition)
squared_error = 0
for idx, tensor_slice in enumerate(tensor_slices):
reconstruction = parafac2_to_slice(decomposition, idx, validate=False)
squared_error += tl.sum((tensor_slice - reconstruction)**2)
return tl.sqrt(squared_error)
def tt_vonneumann_entropy(tensor):
"""Returns the von Neumann entropy of a density matrix (square matrix) in TT tensor form.
Parameters
----------
tensor : (TT tensor)
Data structure
Returns
-------
tt_von_neumann_entropy : order-0 tensor
"""
square_dim = int(tl.sqrt(tl.prod(tl.tensor(tensor.shape))))
tensor = tl.reshape(tt_to_tensor(tensor), (square_dim, square_dim))
return vonneumann_entropy(tensor)
def non_negative_parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
tol=10e-7,
random_state=None,
verbose=0,
normalize_factors=False,
return_errors=False,
mask=None,
orthogonalise=False,
cvg_criterion='abs_rec_error',
fixed_modes=[]):
"""
Non-negative CP decomposition
Uses multiplicative updates, see [2]_
This is the same as parafac(non_negative=True).
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
random_state : {None, int, np.random.RandomState}
verbose : int, optional
level of verbosity
fixed_modes : list, default is []
A list of modes for which the initial value is not modified.
The last mode cannot be fixed due to error computation.
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [2] Amnon Shashua and Tamir Hazan,
"Non-negative tensor factorization with applications to statistics and computer vision",
In Proceedings of the International Conference on Machine Learning (ICML),
pp 792-799, ICML, 2005
"""
epsilon = 10e-12
rank = validate_cp_rank(tl.shape(tensor), rank=rank)
if mask is not None and init == "svd":
message = "Masking occurs after initialization. Therefore, random initialization is recommended."
warnings.warn(message, Warning)
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
weights, factors = initialize_cp(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=True,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
if tl.ndim(tensor) - 1 in fixed_modes:
warnings.warn(
'You asked for fixing the last mode, which is not supported while tol is fixed.\n The last mode will not be fixed. Consider using tl.moveaxis()'
)
fixed_modes.remove(tl.ndim(tensor) - 1)
modes_list = [
mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes
]
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
for i, f in enumerate(factors):
if min(tl.shape(f)) >= rank:
factors[i] = tl.abs(tl.qr(f)[0])
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in modes_list:
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
if mask is not None:
tensor = tensor * mask + tl.cp_to_tensor(
(None, factors), mask=1 - mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
factors[mode] = factor
if normalize_factors:
weights, factors = cp_normalize((weights, factors))
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = cp_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print(
"iteration {}, reconstraction error: {}, decrease = {}"
.format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(
iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def parafac2(tensor_slices,
rank,
n_iter_max=100,
init='random',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
n_iter_parafac=5):
r"""PARAFAC2 decomposition [1]_ of a third order tensor via alternating least squares (ALS)
Computes a rank-`rank` PARAFAC2 decomposition of the third-order tensor defined by
`tensor_slices`. The decomposition is on the form :math:`(A [B_i] C)` such that the
i-th frontal slice, :math:`X_i`, of :math:`X` is given by
.. math::
X_i = B_i diag(a_i) C^T,
where :math:`diag(a_i)` is the diagonal matrix whose nonzero entries are equal to
the :math:`i`-th row of the :math:`I \times R` factor matrix :math:`A`, :math:`B_i`
is a :math:`J_i \times R` factor matrix such that the cross product matrix :math:`B_{i_1}^T B_{i_1}`
is constant for all :math:`i`, and :math:`C` is a :math:`K \times R` factor matrix.
To compute this decomposition, we reformulate the expression for :math:`B_i` such that
.. math::
B_i = P_i B,
where :math:`P_i` is a :math:`J_i \times R` orthogonal matrix and :math:`B` is a
:math:`R \times R` matrix.
An alternative formulation of the PARAFAC2 decomposition is that the tensor element
:math:`X_{ijk}` is given by
.. math::
X_{ijk} = \sum_{r=1}^R A_{ir} B_{ijr} C_{kr},
with the same constraints hold for :math:`B_i` as above.
Parameters
----------
tensor_slices : ndarray or list of ndarrays
Either a third order tensor or a list of second order tensors that may have different number of rows.
Note that the second mode factor matrices are allowed to change over the first mode, not the
third mode as some other implementations use (see note below).
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random', CPTensor, Parafac2Tensor}
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : bool (optional)
If True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors. Note that
there may be some inaccuracies in the component weights.
tol : float, optional
(Default: 1e-8) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
n_iter_parafac: int, optional
Number of PARAFAC iterations to perform for each PARAFAC2 iteration
Returns
-------
Parafac2Tensor : (weight, factors, projection_matrices)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
* projection_matrices : List of projection matrices used to create evolving
factors.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] Kiers, H.A.L., ten Berge, J.M.F. and Bro, R. (1999),
PARAFAC2—Part I. A direct fitting algorithm for the PARAFAC2 model.
J. Chemometrics, 13: 275-294.
Notes
-----
This formulation of the PARAFAC2 decomposition is slightly different from the one in [1]_.
The difference lies in that here, the second mode changes over the first mode, whereas in
[1]_, the second mode changes over the third mode. We made this change since that means
that the function accept both lists of matrices and a single nd-array as input without
any reordering of the modes.
"""
weights, factors, projections = initialize_decomposition(
tensor_slices, rank, init=init, svd=svd, random_state=random_state)
rec_errors = []
norm_tensor = tl.sqrt(
sum(tl.norm(tensor_slice, 2)**2 for tensor_slice in tensor_slices))
svd_fun = _get_svd(svd)
projected_tensor = tl.zeros([factor.shape[0] for factor in factors],
**T.context(factors[0]))
for iteration in range(n_iter_max):
if verbose:
print("Starting iteration", iteration)
factors[1] = factors[1] * T.reshape(weights, (1, -1))
weights = T.ones(weights.shape, **tl.context(tensor_slices[0]))
projections = _compute_projections(tensor_slices,
factors,
svd_fun,
out=projections)
projected_tensor = _project_tensor_slices(tensor_slices,
projections,
out=projected_tensor)
_, factors = parafac(projected_tensor,
rank,
n_iter_max=n_iter_parafac,
init=(weights, factors),
svd=svd,
orthogonalise=False,
verbose=verbose,
return_errors=False,
normalize_factors=False,
mask=None,
random_state=random_state,
tol=1e-100)
if normalize_factors:
new_factors = []
for factor in factors:
norms = T.norm(factor, axis=0)
norms = tl.where(
tl.abs(norms) <= tl.eps(factor.dtype),
tl.ones(tl.shape(norms), **tl.context(factors[0])), norms)
weights = weights * norms
new_factors.append(factor / (tl.reshape(norms, (1, -1))))
factors = new_factors
if tol:
rec_error = _parafac2_reconstruction_error(
tensor_slices, (weights, factors, projections))
rec_error /= norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('PARAFAC2 reconstruction error={}, variation={}.'.
format(rec_errors[-1],
rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('PARAFAC2 reconstruction error={}'.format(
rec_errors[-1]))
parafac2_tensor = Parafac2Tensor((weights, factors, projections))
if return_errors:
return parafac2_tensor, rec_errors
else:
return parafac2_tensor
def maxvol(A):
""" Find the rxr submatrix of maximal volume in A(nxr), n>=r
We want to decompose matrix A as
A = A[:,J] * (A[I,J])^-1 * A[I,:]
This algorithm helps us find this submatrix A[I,J] from A, which has the largest determinant.
We greedily find vector of max norm, and subtract its projection from the rest of rows.
Parameters
----------
A: matrix
The matrix to find maximal volume
Returns
-------
row_idx: list of int
is the list or rows of A forming the matrix with maximal volume,
A_inv: matrix
is the inverse of the matrix with maximal volume.
References
----------
S. A. Goreinov, I. V. Oseledets, D. V. Savostyanov, E. E. Tyrtyshnikov, N. L. Zamarashkin.
How to find a good submatrix.Goreinov, S. A., et al.
Matrix Methods: Theory, Algorithms and Applications: Dedicated to the Memory of Gene Golub. 2010. 247-256.
Ali Çivril, Malik Magdon-Ismail
On selecting a maximum volume sub-matrix of a matrix and related problems
Theoretical Computer Science. Volume 410, Issues 47–49, 6 November 2009, Pages 4801-4811
"""
(n, r) = tl.shape(A)
# The index of row of the submatrix
row_idx = tl.zeros(r)
# Rest of rows / unselected rows
rest_of_rows = tl.tensor(list(range(n)),dtype= tl.int64)
# Find r rows iteratively
i = 0
A_new = A
while i < r:
mask = list(range(tl.shape(A_new)[0]))
# Compute the square of norm of each row
rows_norms = tl.sum(A_new ** 2, axis=1)
# If there is only one row of A left, let's just return it. MxNet is not robust about this case.
if tl.shape(rows_norms) == ():
row_idx[i] = rest_of_rows
break
# If a row is 0, we delete it.
if any(rows_norms == 0):
zero_idx = tl.argmin(rows_norms,axis=0)
mask.pop(zero_idx)
rest_of_rows = rest_of_rows[mask]
A_new = A_new[mask,:]
continue
# Find the row of max norm
max_row_idx = tl.argmax(rows_norms, axis=0)
max_row = A[rest_of_rows[max_row_idx], :]
# Compute the projection of max_row to other rows
# projection a to b is computed as: <a,b> / sqrt(|a|*|b|)
projection = tl.dot(A_new, tl.transpose(max_row))
normalization = tl.sqrt(rows_norms[max_row_idx] * rows_norms)
# make sure normalization vector is of the same shape of projection (causing bugs for MxNet)
normalization = tl.reshape(normalization, tl.shape(projection))
projection = projection/normalization
# Subtract the projection from A_new: b <- b - a * projection
A_new = A_new - A_new * tl.reshape(projection, (tl.shape(A_new)[0], 1))
# Delete the selected row
mask.pop(max_row_idx)
A_new = A_new[mask,:]
# update the row_idx and rest_of_rows
row_idx[i] = rest_of_rows[max_row_idx]
rest_of_rows = rest_of_rows[mask]
i = i + 1
row_idx = tl.tensor(row_idx, dtype=tl.int64)
inverse = tl.solve(A[row_idx,:],
tl.eye(tl.shape(A[row_idx,:])[0], **tl.context(A)))
row_idx = tl.to_numpy(row_idx)
return row_idx, inverse
def non_negative_parafac_hals(tensor,
rank,
n_iter_max=100,
init="svd",
svd='numpy_svd',
tol=10e-8,
random_state=None,
sparsity_coefficients=None,
fixed_modes=None,
nn_modes='all',
exact=False,
normalize_factors=False,
verbose=False,
return_errors=False,
cvg_criterion='abs_rec_error'):
"""
Non-negative CP decomposition via HALS
Uses Hierarchical ALS (Alternating Least Squares) which updates each factor column-wise (one column at a time while keeping all other columns fixed), see [1]_
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
Default: 1e-8
random_state : {None, int, np.random.RandomState}
sparsity_coefficients: array of float (of length the number of modes)
The sparsity coefficients on each factor.
If set to None, the algorithm is computed without sparsity
Default: None,
fixed_modes: array of integers (between 0 and the number of modes)
Has to be set not to update a factor, 0 and 1 for U and V respectively
Default: None
nn_modes: None, 'all' or array of integers (between 0 and the number of modes)
Used to specify which modes to impose non-negativity constraints on.
If 'all', then non-negativity is imposed on all modes.
Default: 'all'
exact: If it is True, the algorithm gives a results with high precision but it needs high computational cost.
If it is False, the algorithm gives an approximate solution
Default: False
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
verbose: boolean
Indicates whether the algorithm prints the successive
reconstruction errors or not
Default: False
return_errors: boolean
Indicates whether the algorithm should return all reconstruction errors
and computation time of each iteration or not
Default: False
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion for ALS, works if `tol` is not None.
If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`.
sparsity : float or int
random_state : {None, int, np.random.RandomState}
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
errors: list
A list of reconstruction errors at each iteration of the algorithm.
References
----------
.. [1]: N. Gillis and F. Glineur, Accelerated Multiplicative Updates and
Hierarchical ALS Algorithms for Nonnegative Matrix Factorization,
Neural Computation 24 (4): 1085-1105, 2012.
"""
weights, factors = initialize_nn_cp(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
normalize_factors=normalize_factors)
norm_tensor = tl.norm(tensor, 2)
n_modes = tl.ndim(tensor)
if sparsity_coefficients is None or isinstance(sparsity_coefficients,
float):
sparsity_coefficients = [sparsity_coefficients] * n_modes
if fixed_modes is None:
fixed_modes = []
if nn_modes == 'all':
nn_modes = set(range(n_modes))
elif nn_modes is None:
nn_modes = set()
# Avoiding errors
for fixed_value in fixed_modes:
sparsity_coefficients[fixed_value] = None
for mode in range(n_modes):
if sparsity_coefficients[mode] is not None:
warnings.warn(
"Sparsity coefficient is ignored in unconstrained modes.")
# Generating the mode update sequence
modes = [mode for mode in range(n_modes) if mode not in fixed_modes]
# initialisation - declare local varaibles
rec_errors = []
# Iteratation
for iteration in range(n_iter_max):
# One pass of least squares on each updated mode
for mode in modes:
# Computing Hadamard of cross-products
pseudo_inverse = tl.tensor(tl.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
pseudo_inverse = tl.reshape(weights,
(-1, 1)) * pseudo_inverse * tl.reshape(
weights, (1, -1))
mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode)
if mode in nn_modes:
# Call the hals resolution with nnls, optimizing the current mode
nn_factor, _, _, _ = hals_nnls(
tl.transpose(mttkrp),
pseudo_inverse,
tl.transpose(factors[mode]),
n_iter_max=100,
sparsity_coefficient=sparsity_coefficients[mode],
exact=exact)
factors[mode] = tl.transpose(nn_factor)
else:
factor = tl.solve(tl.transpose(pseudo_inverse),
tl.transpose(mttkrp))
factors[mode] = tl.transpose(factor)
if normalize_factors and mode != modes[-1]:
weights, factors = cp_normalize((weights, factors))
if tol:
factors_norm = cp_norm((weights, factors))
iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0))
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print(
"iteration {}, reconstruction error: {}, decrease = {}"
.format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(
iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
if normalize_factors:
weights, factors = cp_normalize((weights, factors))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
orthogonalise=False,
random_state=None,
verbose=0,
return_errors=False,
non_negative=False,
mask=None):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [|weights; factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool, optional
Perform non_negative PARAFAC. See :func:`non_negative_parafac`.
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True). Allows for missing values [2]_
Returns
-------
KruskalTensor : (weight, factors)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values."
Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180.
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=non_negative,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
weights = tl.ones(rank, **tl.context(tensor))
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factors = [
tl.qr(f)[0] if min(tl.shape(f)) >= rank else f
for i, f in enumerate(factors)
]
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in range(tl.ndim(tensor)):
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
if non_negative:
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.conj(tl.transpose(factor)), factor)
if mask is not None:
tensor = tensor * mask + tl.kruskal_to_tensor(
(None, factors), mask=1 - mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
if non_negative:
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
else:
factor = tl.transpose(
tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
if normalize_factors:
weights = tl.norm(factor, order=2, axis=0)
weights = tl.where(
tl.abs(weights) <= tl.eps(tensor.dtype),
tl.ones(tl.shape(weights), **tl.context(factors[0])),
weights)
factor = factor / (tl.reshape(weights, (1, -1)))
factors[mode] = factor
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = kruskal_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
kruskal_tensor = KruskalTensor((weights, factors))
if return_errors:
return kruskal_tensor, rec_errors
else:
return kruskal_tensor
def error_calc(tensor, norm_tensor, weights, factors, sparsity, mask, mttkrp=None):
r""" Perform the error calculation. Different forms are used here depending upon
the available information. If `mttkrp=None` or masking is being performed, then the
full tensor must be constructed. Otherwise, the mttkrp is used to reduce the calculation cost.
Parameters
----------
tensor : tensor
norm_tensor : float
The l2 norm of tensor.
weights : tensor
The current CP weights
factors : tensor
The current CP factors
sparsity : float or int
Whether we allow for a sparse component
mask : bool
Whether masking is being performed.
mttkrp : tensor or None
The mttkrp product, if available.
Returns
-------
unnorml_rec_error : float
The unnormalized reconstruction error.
tensor : tensor
The tensor, in case it has been updated by masking.
norm_tensor: float
The tensor norm, in case it has been updated by masking.
"""
# If we have to update the mask we already have to build the full tensor
if (mask is not None) or (mttkrp is None):
low_rank_component = cp_to_tensor((weights, factors))
# Update the tensor based on the mask
if mask is not None:
tensor = tensor*mask + low_rank_component*(1-mask)
norm_tensor = tl.norm(tensor, 2)
if sparsity:
sparse_component = sparsify_tensor(tensor - low_rank_component, sparsity)
else:
sparse_component = 0.0
unnorml_rec_error = tl.norm(tensor - low_rank_component - sparse_component, 2)
else:
if sparsity:
low_rank_component = cp_to_tensor((weights, factors))
sparse_component = sparsify_tensor(tensor - low_rank_component, sparsity)
unnorml_rec_error = tl.norm(tensor - low_rank_component - sparse_component, 2)
else:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = cp_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp*factors[-1], axis=0)*weights)
unnorml_rec_error = tl.sqrt(tl.abs(norm_tensor**2 + factors_norm**2 - 2*iprod))
return unnorml_rec_error, tensor, norm_tensor
def make_svd_non_negative(tensor, U, S, V, nntype):
""" Use NNDSVD method to transform SVD results into a non-negative form. This
method leads to more efficient solving with NNMF [1].
Parameters
----------
tensor : tensor being decomposed
U, S, V: SVD factorization results
nntype : {'nndsvd', 'nndsvda'}
Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default).
[1]: Boutsidis & Gallopoulos. Pattern Recognition, 41(4): 1350-1362, 2008.
"""
# NNDSVD initialization
W = tl.zeros_like(U)
H = tl.zeros_like(V)
# The leading singular triplet is non-negative
# so it can be used as is for initialization.
W = tl.index_update(W, tl.index[:, 0], tl.sqrt(S[0]) * tl.abs(U[:, 0]))
H = tl.index_update(H, tl.index[0, :], tl.sqrt(S[0]) * tl.abs(V[0, :]))
for j in range(1, tl.shape(U)[1]):
x, y = U[:, j], V[j, :]
# extract positive and negative parts of column vectors
x_p, y_p = tl.clip(x, a_min=0.0), tl.clip(y, a_min=0.0)
x_n, y_n = tl.abs(tl.clip(x, a_max=0.0)), tl.abs(tl.clip(y, a_max=0.0))
# and their norms
x_p_nrm, y_p_nrm = tl.norm(x_p), tl.norm(y_p)
x_n_nrm, y_n_nrm = tl.norm(x_n), tl.norm(y_n)
m_p, m_n = x_p_nrm * y_p_nrm, x_n_nrm * y_n_nrm
# choose update
if m_p > m_n:
u = x_p / x_p_nrm
v = y_p / y_p_nrm
sigma = m_p
else:
u = x_n / x_n_nrm
v = y_n / y_n_nrm
sigma = m_n
lbd = tl.sqrt(S[j] * sigma)
W = tl.index_update(W, tl.index[:, j], lbd * u)
H = tl.index_update(H, tl.index[j, :], lbd * v)
# After this point we no longer need H
eps = tl.eps(tensor.dtype)
if nntype == "nndsvd":
W = soft_thresholding(W, eps)
elif nntype == "nndsvda":
avg = tl.mean(tensor)
W = tl.where(W < eps, tl.ones(tl.shape(W), **tl.context(W)) * avg, W)
else:
raise ValueError(
'Invalid nntype parameter: got %r instead of one of %r' %
(nntype, ('nndsvd', 'nndsvda')))
return W
def _common_snippet_2(core, norm_tensor):
# The factors are orthonormal and therefore do not affect the
# reconstructed tensor's norm
return tl.sqrt(tl.abs(norm_tensor**2 - tl.norm(core, 2)**2)) / norm_tensor
def constrained_parafac(tensor, rank, n_iter_max=100, n_iter_max_inner=10,
init='svd', svd='numpy_svd',
tol_outer=1e-8, tol_inner=1e-6, random_state=None,
verbose=0, return_errors=False,
cvg_criterion='abs_rec_error',
fixed_modes=None, non_negative=None, l1_reg=None,
l2_reg=None, l2_square_reg=None, unimodality=None, normalize=None,
simplex=None, normalized_sparsity=None, soft_sparsity=None,
smoothness=None, monotonicity=None, hard_sparsity=None):
"""CANDECOMP/PARAFAC decomposition via alternating optimization of
alternating direction method of multipliers (AO-ADMM):
Computes a rank-`rank` decomposition of `tensor` [1]_ such that::
tensor = [|weights; factors[0], ..., factors[-1] |],
where factors are either penalized or constrained according to the user-defined constraint.
In order to compute the factors efficiently, the ADMM algorithm
introduces an auxilliary factor which is called factor_aux in the function.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration for outer loop
n_iter_max_inner : int
Number of iteration for inner loop
init : {'svd', 'random', cptensor}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol_outer : float, optional
(Default: 1e-8) Relative reconstruction error tolerance for outer loop. The
algorithm is considered to have found a local minimum when the
reconstruction error is less than `tol_outer`.
tol_inner : float, optional
(Default: 1e-6) Absolute reconstruction error tolerance for factor update during inner loop, i.e. ADMM optimization.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool or dictionary
This constraint is clipping negative values to '0'. If it is True non-negative constraint is applied to all modes.
l1_reg : float or list or dictionary, optional
l2_reg : float or list or dictionary, optional
l2_square_reg : float or list or dictionary, optional
unimodality : bool or dictionary, optional
If it is True unimodality constraint is applied to all modes.
normalize : bool or dictionary, optional
This constraint divides all the values by maximum value of the input array. If it is True normalize constraint
is applied to all modes.
simplex : float or list or dictionary, optional
normalized_sparsity : float or list or dictionary, optional
soft_sparsity : float or list or dictionary, optional
smoothness : float or list or dictionary, optional
monotonicity : bool or dictionary, optional
hard_sparsity : float or list or dictionary, optional
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion if `tol` is not None.
If 'rec_error', algorithm stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', algorithm terminates when `|previous rec_error - current rec_error| < tol`.
fixed_modes : list, default is None
A list of modes for which the initial value is not modified.
The last mode cannot be fixed due to error computation.
Returns
-------
CPTensor : (weight, factors)
* weights : 1D array of shape (rank, )
* factors : List of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)``
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM
REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Huang, Kejun, Nicholas D. Sidiropoulos, and Athanasios P. Liavas.
"A flexible and efficient algorithmic framework for constrained matrix and tensor factorization." IEEE
Transactions on Signal Processing 64.19 (2016): 5052-5065.
"""
rank = validate_cp_rank(tl.shape(tensor), rank=rank)
_, _ = validate_constraints(non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize, simplex=simplex,
normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity,
n_const=tl.ndim(tensor))
weights, factors = initialize_constrained_parafac(tensor, rank, init=init, svd=svd,
random_state=random_state, non_negative=non_negative,
l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize,
simplex=simplex,
normalized_sparsity=normalized_sparsity,
soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity,
hard_sparsity=hard_sparsity)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
if fixed_modes is None:
fixed_modes = []
if tl.ndim(tensor) - 1 in fixed_modes:
warnings.warn('You asked for fixing the last mode, which is not supported.\n '
'The last mode will not be fixed. Consider using tl.moveaxis()')
fixed_modes.remove(tl.ndim(tensor) - 1)
modes_list = [mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes]
# ADMM inits
dual_variables = []
factors_aux = []
for i in range(len(factors)):
dual_variables.append(tl.zeros(tl.shape(factors[i])))
factors_aux.append(tl.transpose(tl.zeros(tl.shape(factors[i]))))
for iteration in range(n_iter_max):
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in modes_list:
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(tl.transpose(factor), factor)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
factors[mode], factors_aux[mode], dual_variables[mode] = admm(mttkrp, pseudo_inverse, factors[mode], dual_variables[mode],
n_iter_max=n_iter_max_inner, n_const=tl.ndim(tensor),
order=mode, non_negative=non_negative, l1_reg=l1_reg,
l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize,
simplex=simplex, normalized_sparsity=normalized_sparsity,
soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity,
hard_sparsity=hard_sparsity, tol=tol_inner)
factors_norm = cp_norm((weights, factors))
iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0) * weights)
rec_error = tl.sqrt(tl.abs(norm_tensor ** 2 + factors_norm ** 2 - 2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
constraint_error = 0
for mode in modes_list:
constraint_error += tl.norm(factors[mode] - tl.transpose(factors_aux[mode])) / tl.norm(factors[mode])
if tol_outer:
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print("iteration {}, reconstruction error: {}, decrease = {}".format(iteration,
rec_error,
rec_error_decrease))
if constraint_error < tol_outer:
break
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol_outer
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol_outer
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
tol=1e-8,
orthogonalise=False,
random_state=None,
verbose=False,
return_errors=False,
non_negative=False):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [| factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool, optional
Perform non_negative PARAFAC. See :func:`non_negative_parafac`.
Returns
-------
factors : ndarray list
List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=non_negative)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factor = [tl.qr(factor)[0] for factor in factors]
if verbose:
print("Starting iteration", iteration)
for mode in range(tl.ndim(tensor)):
if verbose:
print("Mode", mode, "of", tl.ndim(tensor))
if non_negative:
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.conj(tl.transpose(factor)), factor)
#factor = tl.dot(unfold(tensor, mode), khatri_rao(factors, skip_matrix=mode).conj())
mttkrp = tl.tenalg.unfolding_dot_khatri_rao(tensor, factors, mode)
if non_negative:
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
else:
factor = tl.transpose(
tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
factors[mode] = factor
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
# This is ||kruskal_to_tensor(factors)||^2
factors_norm = tl.sum(
tl.prod(
tl.stack([tl.dot(tl.transpose(f), f) for f in factors], 0),
0))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(mttkrp * factor)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
if return_errors:
return factors, rec_errors
else:
return factors
def non_negative_parafac(tensor, rank, n_iter_max=100, init='svd', svd='numpy_svd',
tol=10e-7, random_state=None, verbose=0, normalize_factors=False,
return_errors=False, mask=None, orthogonalise=False, cvg_criterion='abs_rec_error'):
"""
Non-negative CP decomposition
Uses multiplicative updates, see [2]_
This is the same as parafac(non_negative=True).
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
random_state : {None, int, np.random.RandomState}
verbose : int, optional
level of verbosity
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [2] Amnon Shashua and Tamir Hazan,
"Non-negative tensor factorization with applications to statistics and computer vision",
In Proceedings of the International Conference on Machine Learning (ICML),
pp 792-799, ICML, 2005
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor, rank, init=init, svd=svd,
random_state=random_state,
non_negative=True,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
weights = tl.ones(rank, **tl.context(tensor))
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
for i, f in enumerate(factors):
if min(tl.shape(f)) >= rank:
factors[i] = tl.abs(tl.qr(f)[0])
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in range(tl.ndim(tensor)):
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
if mask is not None:
tensor = tensor*mask + tl.kruskal_to_tensor((None, factors), mask=1-mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
if normalize_factors:
weights = tl.norm(factor, order=2, axis=0)
weights = tl.where(tl.abs(weights) <= tl.eps(tensor.dtype),
tl.ones(tl.shape(weights), **tl.context(factors[0])),
weights)
factor = factor/(tl.reshape(weights, (1, -1)))
factors[mode] = factor
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = kruskal_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp*factor, axis=0)*weights)
rec_error = tl.sqrt(tl.abs(norm_tensor**2 + factors_norm**2 - 2*iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print("iteration {}, reconstraction error: {}, decrease = {}".format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
kruskal_tensor = KruskalTensor((weights, factors))
if return_errors:
return kruskal_tensor, rec_errors
else:
return kruskal_tensor
def parafac(tensor, rank, n_iter_max=100, init='svd', svd='numpy_svd',\
normalize_factors=False, orthogonalise=False,\
tol=1e-8, random_state=None,\
verbose=0, return_errors=False,\
non_negative=False,\
sparsity = None,\
l2_reg = 0, mask=None,\
cvg_criterion = 'abs_rec_error'):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [|weights; factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True). Allows for missing values [2]_
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion for ALS, works if `tol` is not None.
If 'rec_error', ALS stops at current iteration if (previous rec_error - current rec_error) < tol.
If 'abs_rec_error', ALS terminates when |previous rec_error - current rec_error| < tol.
sparsity : float or int
If `sparsity` is not None, we approximate tensor as a sum of low_rank_component and sparse_component, where low_rank_component = kruskal_to_tensor((weights, factors)). `sparsity` denotes desired fraction or number of non-zero elements in the sparse_component of the `tensor`.
Returns
-------
KruskalTensor : (weight, factors)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
* sparse_component : nD array of shape tensor.shape. Returns only if `sparsity` is not None.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values."
Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180.
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor, rank, init=init, svd=svd,
random_state=random_state,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
weights = tl.ones(rank, **tl.context(tensor))
Id = tl.eye(rank, **tl.context(tensor))*l2_reg
if sparsity:
sparse_component = tl.zeros_like(tensor)
if isinstance(sparsity, float):
sparsity = int(sparsity * np.prod(tensor.shape))
else:
sparsity = int(sparsity)
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factors = [tl.qr(f)[0] if min(tl.shape(f)) >= rank else f for i, f in enumerate(factors)]
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in range(tl.ndim(tensor)):
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse*tl.dot(tl.conj(tl.transpose(factor)), factor)
pseudo_inverse += Id
if mask is not None:
tensor = tensor*mask + tl.kruskal_to_tensor((None, factors), mask=1-mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
factor = tl.transpose(tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
if normalize_factors:
weights = tl.norm(factor, order=2, axis=0)
weights = tl.where(tl.abs(weights) <= tl.eps(tensor.dtype),
tl.ones(tl.shape(weights), **tl.context(factors[0])),
weights)
factor = factor/(tl.reshape(weights, (1, -1)))
factors[mode] = factor
if tol:
if sparsity:
low_rank_component = kruskal_to_tensor((weights, factors))
sparse_component = sparsify_tensor(tensor - low_rank_component, sparsity)
unnorml_rec_error = tl.norm(tensor - low_rank_component - sparse_component, 2)
else:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = kruskal_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp*factor, axis=0)*weights)
unnorml_rec_error = tl.sqrt(tl.abs(norm_tensor**2 + factors_norm**2 - 2*iprod))
rec_error = unnorml_rec_error / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print("iteration {}, reconstraction error: {}, decrease = {}, unnormalized = {}".format(iteration, rec_error, rec_error_decrease, unnorml_rec_error))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
kruskal_tensor = KruskalTensor((weights, factors))
if sparsity:
sparse_component = sparsify_tensor(tensor -\
kruskal_to_tensor((weights, factors)),\
sparsity)
kruskal_tensor = (kruskal_tensor, sparse_component)
if return_errors:
return kruskal_tensor, rec_errors
else:
return kruskal_tensor
def fista(UtM,
UtU,
x=None,
n_iter_max=100,
non_negative=True,
sparsity_coef=0,
lr=None,
tol=10e-8):
"""
Fast Iterative Shrinkage Thresholding Algorithm (FISTA)
Computes an approximate (nonnegative) solution for Ux=M linear system.
Parameters
----------
UtM : ndarray
Pre-computed product of the transposed of U and M
UtU : ndarray
Pre-computed product of the transposed of U and U
x : init
Default: None
n_iter_max : int
Maximum number of iteration
Default: 100
non_negative : bool, default is False
if True, result will be non-negative
lr : float
learning rate
Default : None
sparsity_coef : float or None
tol : float
stopping criterion
Returns
-------
x : approximate solution such that Ux = M
Notes
-----
We solve the following problem :math: `1/2 ||m - Ux ||_2^2 + \\lambda |x|_1`
Reference
----------
[1] : Beck, A., & Teboulle, M. (2009). A fast iterative
shrinkage-thresholding algorithm for linear inverse problems.
SIAM journal on imaging sciences, 2(1), 183-202.
"""
if sparsity_coef is None:
sparsity_coef = 0
if x is None:
x = tl.zeros(tl.shape(UtM), **tl.context(UtM))
if lr is None:
lr = 1 / (tl.partial_svd(UtU)[1][0])
# Parameters
momentum_old = tl.tensor(1.0)
norm_0 = 0.0
x_update = tl.copy(x)
for iteration in range(n_iter_max):
if isinstance(UtU, list):
x_gradient = -UtM + tl.tenalg.multi_mode_dot(
x_update, UtU, transpose=False) + sparsity_coef
else:
x_gradient = -UtM + tl.dot(UtU, x_update) + sparsity_coef
if non_negative is True:
x_gradient = tl.where(lr * x_gradient < x_update, x_gradient,
x_update / lr)
x_new = x_update - lr * x_gradient
momentum = (1 + tl.sqrt(1 + 4 * momentum_old**2)) / 2
x_update = x_new + ((momentum_old - 1) / momentum) * (x_new - x)
momentum_old = momentum
x = tl.copy(x_new)
norm = tl.norm(lr * x_gradient)
if iteration == 1:
norm_0 = norm
if norm < tol * norm_0:
break
return x
def parafac2(
tensor_slices,
rank,
n_iter_max=2000,
init='random',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
absolute_tol=1e-13,
nn_modes=None,
random_state=None,
verbose=False,
return_errors=False,
n_iter_parafac=5,
):
r"""PARAFAC2 decomposition [1]_ of a third order tensor via alternating least squares (ALS)
Computes a rank-`rank` PARAFAC2 decomposition of the third-order tensor defined by
`tensor_slices`. The decomposition is on the form :math:`(A [B_i] C)` such that the
i-th frontal slice, :math:`X_i`, of :math:`X` is given by
.. math::
X_i = B_i diag(a_i) C^T,
where :math:`diag(a_i)` is the diagonal matrix whose nonzero entries are equal to
the :math:`i`-th row of the :math:`I \times R` factor matrix :math:`A`, :math:`B_i`
is a :math:`J_i \times R` factor matrix such that the cross product matrix :math:`B_{i_1}^T B_{i_1}`
is constant for all :math:`i`, and :math:`C` is a :math:`K \times R` factor matrix.
To compute this decomposition, we reformulate the expression for :math:`B_i` such that
.. math::
B_i = P_i B,
where :math:`P_i` is a :math:`J_i \times R` orthogonal matrix and :math:`B` is a
:math:`R \times R` matrix.
An alternative formulation of the PARAFAC2 decomposition is that the tensor element
:math:`X_{ijk}` is given by
.. math::
X_{ijk} = \sum_{r=1}^R A_{ir} B_{ijr} C_{kr},
with the same constraints hold for :math:`B_i` as above.
Parameters
----------
tensor_slices : ndarray or list of ndarrays
Either a third order tensor or a list of second order tensors that may have different number of rows.
Note that the second mode factor matrices are allowed to change over the first mode, not the
third mode as some other implementations use (see note below).
rank : int
Number of components.
n_iter_max : int, optional
(Default: 2000) Maximum number of iteration
.. versionchanged:: 0.6.1
Previously, the default maximum number of iterations was 100.
init : {'svd', 'random', CPTensor, Parafac2Tensor}
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : bool (optional)
If True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors. Note that
there may be some inaccuracies in the component weights.
tol : float, optional
(Default: 1e-8) Relative reconstruction error decrease tolerance. The
algorithm is considered to have converged when
:math:`\left|\| X - \hat{X}_{n-1} \|^2 - \| X - \hat{X}_{n} \|^2\right| < \epsilon \| X - \hat{X}_{n-1} \|^2`.
That is, when the relative change in sum of squared error is less
than the tolerance.
.. versionchanged:: 0.6.1
Previously, the stopping condition was
:math:`\left|\| X - \hat{X}_{n-1} \| - \| X - \hat{X}_{n} \|\right| < \epsilon`.
absolute_tol : float, optional
(Default: 1e-13) Absolute reconstruction error tolearnce. The algorithm
is considered to have converged when
:math:`\left|\| X - \hat{X}_{n-1} \|^2 - \| X - \hat{X}_{n} \|^2\right| < \epsilon_\text{abs}`.
That is, when the relative sum of squared error is less than the specified tolerance.
The absolute tolerance is necessary for stopping the algorithm when used on noise-free
data that follows the PARAFAC2 constraint.
If None, then the machine precision + 1000 will be used.
nn_modes: None, 'all' or array of integers
(Default: None) Used to specify which modes to impose non-negativity constraints on.
We cannot impose non-negativity constraints on the the B-mode (mode 1) with the ALS
algorithm, so if this mode is among the constrained modes, then a warning will be shown
(see notes for more info).
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
n_iter_parafac : int, optional
Number of PARAFAC iterations to perform for each PARAFAC2 iteration
Returns
-------
Parafac2Tensor : (weight, factors, projection_matrices)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
* projection_matrices : List of projection matrices used to create evolving
factors.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] Kiers, H.A.L., ten Berge, J.M.F. and Bro, R. (1999),
PARAFAC2—Part I. A direct fitting algorithm for the PARAFAC2 model.
J. Chemometrics, 13: 275-294.
Notes
-----
This formulation of the PARAFAC2 decomposition is slightly different from the one in [1]_.
The difference lies in that here, the second mode changes over the first mode, whereas in
[1]_, the second mode changes over the third mode. We made this change since that means
that the function accept both lists of matrices and a single nd-array as input without
any reordering of the modes.
Because of the reformulation above, :math:`B_i = P_i B`, the :math:`B_i` matrices
cannot be constrained to be non-negative with ALS. If this mode is constrained to be
non-negative, then :math:`B` will be non-negative, but not the orthogonal `P_i` matrices.
Consequently, the `B_i` matrices are unlikely to be non-negative.
"""
weights, factors, projections = initialize_decomposition(
tensor_slices, rank, init=init, svd=svd, random_state=random_state)
rec_errors = []
norm_tensor = tl.sqrt(
sum(tl.norm(tensor_slice, 2)**2 for tensor_slice in tensor_slices))
svd_fun = _get_svd(svd)
if absolute_tol is None:
absolute_tol = tl.eps(factors[0].dtype) * 1000
# If nn_modes is set, we use HALS, otherwise, we use the standard parafac implementation.
if nn_modes is None:
def parafac_updates(X, w, f):
return parafac(X,
rank,
n_iter_max=n_iter_parafac,
init=(w, f),
svd=svd,
orthogonalise=False,
verbose=verbose,
return_errors=False,
normalize_factors=False,
mask=None,
random_state=random_state,
tol=1e-100)[1]
else:
if nn_modes == 'all' or 1 in nn_modes:
warn(
"Mode `1` of PARAFAC2 fitted with ALS cannot be constrained to be truly non-negative. See the documentation for more info."
)
def parafac_updates(X, w, f):
return non_negative_parafac_hals(X,
rank,
n_iter_max=n_iter_parafac,
init=(w, f),
svd=svd,
nn_modes=nn_modes,
verbose=verbose,
return_errors=False,
tol=1e-100)[1]
projected_tensor = tl.zeros([factor.shape[0] for factor in factors],
**T.context(factors[0]))
for iteration in range(n_iter_max):
if verbose:
print("Starting iteration", iteration)
factors[1] = factors[1] * T.reshape(weights, (1, -1))
weights = T.ones(weights.shape, **tl.context(tensor_slices[0]))
projections = _compute_projections(tensor_slices,
factors,
svd_fun,
out=projections)
projected_tensor = _project_tensor_slices(tensor_slices,
projections,
out=projected_tensor)
factors = parafac_updates(projected_tensor, weights, factors)
if normalize_factors:
new_factors = []
for factor in factors:
norms = T.norm(factor, axis=0)
norms = tl.where(
tl.abs(norms) <= tl.eps(factor.dtype),
tl.ones(tl.shape(norms), **tl.context(factors[0])), norms)
weights = weights * norms
new_factors.append(factor / (tl.reshape(norms, (1, -1))))
factors = new_factors
if tol:
rec_error = _parafac2_reconstruction_error(
tensor_slices, (weights, factors, projections))
rec_error /= norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('PARAFAC2 reconstruction error={}, variation={}.'.
format(rec_errors[-1],
rec_errors[-2] - rec_errors[-1]))
if abs(rec_errors[-2]**2 - rec_errors[-1]**2) < (
tol *
rec_errors[-2]**2) or rec_errors[-1]**2 < absolute_tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('PARAFAC2 reconstruction error={}'.format(
rec_errors[-1]))
parafac2_tensor = Parafac2Tensor((weights, factors, projections))
if return_errors:
return parafac2_tensor, rec_errors
else:
return parafac2_tensor
