def krprod(factors, indices_list):
''' Implement Khatri Rao Product for given nonzeros' indicies '''
rank = tl.shape(factors[0])[1]
nnz = len(indices_list[0])
nonzeros = tl.ones((nnz, rank), **tl.context(factors[0]))
for indices, factor in zip(indices_list, factors):
nonzeros = nonzeros * factor[indices, :]
return torch.sum(nonzeros, dim=1)
def normalize_factors(factors):
"""Normalizes factors to unit length and returns factor magnitudes
Turns ``factors = [|U_1, ... U_n|]`` into ``[weights; |V_1, ... V_n|]``,
where the columns of each `V_k` are normalized to unit Euclidean length
from the columns of `U_k` with the normalizing constants absorbed into
`weights`. In the special case of a symmetric tensor, `weights` holds the
eigenvalues of the tensor.
Parameters
----------
factors : ndarray list
list of matrices, all with the same number of columns
i.e.::
for u in U:
u[i].shape == (s_i, R)
where `R` is fixed while `s_i` can vary with `i`
Returns
-------
normalized_factors : list of ndarrays
list of matrices with the same shape as `factors`
weights : ndarray
vector of length `R` holding normalizing constants
"""
# allocate variables for weights, and normalized factors
rank = factors[0].shape[1]
weights = tl.ones(rank, **tl.context(factors[0]))
normalized_factors = []
# normalize columns of factor matrices
for factor in factors:
scales = tl.norm(factor, axis=0)
weights *= scales
scales_non_zero = tl.where(
scales == 0, tl.ones(tl.shape(scales), **tl.context(factors[0])),
scales)
normalized_factors.append(factor / scales_non_zero)
return normalized_factors, weights
def test_cp_to_tensor_with_weights():
A = tl.reshape(tl.arange(1,5), (2,2))
B = tl.reshape(tl.arange(5,9), (2,2))
weigths = tl.tensor([2,-1], **tl.context(A))
out = cp_to_tensor((weigths, [A,B]))
expected = tl.tensor([[-2,-2], [6, 10]]) # computed by hand
assert_array_equal(out, expected)
(weigths, factors) = random_cp((5, 5, 5), rank=5, normalise_factors=True, full=False)
true_res = tl.dot(tl.dot(factors[0], tl.diag(weigths)),
tl.transpose(tl.tenalg.khatri_rao(factors[1:])))
true_res = tl.fold(true_res, 0, (5, 5, 5))
res = cp_to_tensor((weigths, factors))
assert_array_almost_equal(true_res, res,
err_msg='weights incorrectly incorporated in cp_to_tensor')
def test_vec2factors_1():
""" Test wrapper function from GCP paper"""
rank = 2
X = tl.tensor(np.arange(24).reshape((3, 4, 2)), dtype=tl.float32)
X_shp = tl.shape(X)
X_cntx = tl.context(X)
factors = []
for i in range(3):
f = tl.transpose(X[:][:][i])
factors.append(f)
vec1 = factors2vec(factors)
M = vec2factors(vec1,X_shp, rank, X_cntx)
vec2 = factors2vec(M[1])
for i in range(X_shp[0]):
assert(vec1[i] == vec2[i])
def sparsify_tensor(tensor, card):
"""Zeros out all elements in the `tensor` except `card` elements with maximum absolute values.
Parameters
----------
tensor : ndarray
card : int
Desired number of non-zero elements in the `tensor`
Returns
-------
ndarray of shape tensor.shape
"""
if card >= np.prod(tensor.shape):
return tensor
bound = tl.sort(tl.abs(tensor), axis = None)[-card]
return tl.where(tl.abs(tensor) < bound, tl.zeros(tensor.shape, **tl.context(tensor)), tensor)
def apply(self, module):
"""Apply an instance of the L1Regularizer to a tensor module
Parameters
----------
module : TensorModule
module on which to add the regularization
Returns
-------
TensorModule (with Regularization hook)
"""
context = tl.context(module.factors[0])
lasso_weights = nn.Parameter(torch.ones(module.rank, **context))
setattr(module, 'lasso_weights', lasso_weights)
module.register_forward_hook(self)
return module
def __custom_parafac(self, tensor, neg_fac=0, tol=1e-7):
"""Customized PARAFAC algorithm
Parameters
----------
tensor : torch.Tensor
The tensor of activity of N neurons, T timepoints and K trials of shape N, T, K
neg_fac : int, optional
Index of the factor which is allowed to be negative (default is 0)
tol : float, optional
Threshold for convergence (default is 1e-7)
Returns
-------
list
List of optimized factors
"""
factors = self.__initialize_factors(tensor,
self.rank,
self.init,
custom=neg_fac)
pseudo_inverse = tl.tensor(np.ones((self.rank, self.rank)),
**tl.context(tensor))
for iteration in range(self.max_iteration):
for mode in range(self.dimension):
if mode == neg_fac:
factors[mode] = self.__factor(self, tensor, factors, mode,
pseudo_inverse)
else:
factors[mode] = factors[mode] * self.__factor_non_negative(
tensor, factors, mode)
if (iteration % 25 == 0 or iteration % 25 == 1) and iteration > 1:
if self.__get_error(iteration, tensor, factors, tol,
self.verbose):
break
return factors
def hard_thresholding(tensor, number_of_non_zero):
"""
Proximal operator of the l0 ``norm''
Keeps greater "number_of_non_zero" elements untouched and sets other elements to zero.
Parameters
----------
tensor : ndarray
number_of_non_zero : int
Returns
-------
ndarray
Thresholded tensor on which the operator has been applied
"""
tensor_vec = tl.copy(tl.tensor_to_vec(tensor))
sorted_indices = tl.argsort(tl.argsort(tl.abs(tensor_vec),
axis=0,
descending=True),
axis=0)
return tl.reshape(
tl.where(sorted_indices < number_of_non_zero, tensor_vec,
tl.tensor(0, **tl.context(tensor_vec))), tensor.shape)
def partial_tucker(tensor,
modes,
rank=None,
n_iter_max=100,
init='svd',
tol=10e-5,
svd='numpy_svd',
random_state=None,
verbose=False,
mask=None):
"""Partial tucker decomposition via Higher Order Orthogonal Iteration (HOI)
Decomposes `tensor` into a Tucker decomposition exclusively along the provided modes.
Parameters
----------
tensor : ndarray
modes : int list
list of the modes on which to perform the decomposition
rank : None, int or int list
size of the core tensor, ``(len(ranks) == tensor.ndim)``
if int, the same rank is used for all modes
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, or TuckerTensor optional
if a TuckerTensor is provided, this is used for initialization
svd : str, default is 'numpy_svd'
function to use to compute the SVD,
acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
random_state : {None, int, np.random.RandomState}
verbose : int, optional
level of verbosity
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True).
Returns
-------
core : ndarray
core tensor of the Tucker decomposition
factors : ndarray list
list of factors of the Tucker decomposition.
with ``core.shape[i] == (tensor.shape[i], ranks[i]) for i in modes``
"""
if rank is None:
message = "No value given for 'rank'. The decomposition will preserve the original size."
warnings.warn(message, Warning)
rank = [tl.shape(tensor)[mode] for mode in modes]
elif isinstance(rank, int):
message = "Given only one int for 'rank' instead of a list of {} modes. Using this rank for all modes.".format(
len(modes))
warnings.warn(message, Warning)
rank = tuple(rank for _ in modes)
else:
rank = tuple(rank)
if mask is not None and init == "svd":
message = "Masking occurs after initialization. Therefore, random initialization is recommended."
warnings.warn(message, Warning)
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
# SVD init
if init == 'svd':
factors = []
for index, mode in enumerate(modes):
eigenvecs, _, _ = svd_fun(unfold(tensor, mode),
n_eigenvecs=rank[index],
random_state=random_state)
factors.append(eigenvecs)
# The initial core approximation is needed here for the masking step
core = multi_mode_dot(tensor, factors, modes=modes, transpose=True)
elif init == 'random':
rng = tl.check_random_state(random_state)
# len(rank) == len(modes) but we still want a core dimension for the modes not optimized
core_shape = list(tl.shape(tensor))
for (i, e) in enumerate(modes):
core_shape[e] = rank[i]
core = tl.tensor(rng.random_sample(core_shape), **tl.context(tensor))
factors = [
tl.tensor(rng.random_sample((tl.shape(tensor)[mode], rank[index])),
**tl.context(tensor))
for (index, mode) in enumerate(modes)
]
else:
(core, factors) = init
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
for iteration in range(n_iter_max):
if mask is not None:
tensor = tensor * mask + multi_mode_dot(
core, factors, modes=modes, transpose=False) * (1 - mask)
for index, mode in enumerate(modes):
core_approximation = multi_mode_dot(tensor,
factors,
modes=modes,
skip=index,
transpose=True)
eigenvecs, _, _ = svd_fun(unfold(core_approximation, mode),
n_eigenvecs=rank[index],
random_state=random_state)
factors[index] = eigenvecs
core = multi_mode_dot(tensor, factors, modes=modes, transpose=True)
# The factors are orthonormal and therefore do not affect the reconstructed tensor's norm
rec_error = sqrt(
abs(norm_tensor**2 - tl.norm(core, 2)**2)) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
return (core, factors)
def initialize_tucker(tensor,
rank,
modes,
random_state,
init='svd',
svd='numpy_svd',
non_negative=False):
"""
Initialize core and factors used in `tucker`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
number of components
modes : int list
random_state : {None, int, np.random.RandomState}
init : {'svd', 'random', cptensor}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
core : ndarray
initialized core tensor
factors : list of factors
"""
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
# Initialisation
if init == 'svd':
factors = []
for index, mode in enumerate(modes):
U, S, V = svd_fun(unfold(tensor, mode),
n_eigenvecs=rank[index],
random_state=random_state)
if non_negative is True:
U = make_svd_non_negative(tensor, U, S, V, nntype="nndsvd")
factors.append(U[:, :rank[index]])
# The initial core approximation is needed here for the masking step
core = multi_mode_dot(tensor, factors, modes=modes, transpose=True)
if non_negative is True:
core = tl.abs(core)
elif init == 'random':
rng = tl.check_random_state(random_state)
core = tl.tensor(rng.random_sample(rank) + 0.01,
**tl.context(tensor)) # Check this
factors = [
tl.tensor(rng.random_sample(s), **tl.context(tensor))
for s in zip(tl.shape(tensor), rank)
]
if non_negative is True:
factors = [tl.abs(f) for f in factors]
core = tl.abs(core)
else:
(core, factors) = init
return core, factors
def non_negative_tucker(tensor,
rank,
n_iter_max=10,
init='svd',
tol=10e-5,
random_state=None,
verbose=False,
return_errors=False,
normalize_factors=False):
"""Non-negative Tucker decomposition
Iterative multiplicative update, see [2]_
Parameters
----------
tensor : ``ndarray``
rank : None, int or int list
size of the core tensor, ``(len(ranks) == tensor.ndim)``
if int, the same rank is used for all modes
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}
random_state : {None, int, np.random.RandomState}
verbose : int , optional
level of verbosity
ranks : None or int list
size of the core tensor
normalize_factors : if True, aggregates the core which will contain the norms of the factors.
Returns
-------
core : ndarray
positive core of the Tucker decomposition
has shape `ranks`
factors : ndarray list
list of factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [2] Yong-Deok Kim and Seungjin Choi,
"Non-negative tucker decomposition",
IEEE Conference on Computer Vision and Pattern Recognition s(CVPR),
pp 1-8, 2007
"""
rank = validate_tucker_rank(tl.shape(tensor), rank=rank)
epsilon = 10e-12
# Initialisation
if init == 'svd':
core, factors = tucker(tensor, rank)
nn_factors = [tl.abs(f) for f in factors]
nn_core = tl.abs(core)
else:
rng = tl.check_random_state(random_state)
core = tl.tensor(rng.random_sample(rank) + 0.01,
**tl.context(tensor)) # Check this
factors = [
tl.tensor(rng.random_sample(s), **tl.context(tensor))
for s in zip(tl.shape(tensor), rank)
]
nn_factors = [tl.abs(f) for f in factors]
nn_core = tl.abs(core)
norm_tensor = tl.norm(tensor, 2)
rec_errors = []
for iteration in range(n_iter_max):
for mode in range(tl.ndim(tensor)):
B = tucker_to_tensor((nn_core, nn_factors), skip_factor=mode)
B = tl.transpose(unfold(B, mode))
numerator = tl.dot(unfold(tensor, mode), B)
numerator = tl.clip(numerator, a_min=epsilon, a_max=None)
denominator = tl.dot(nn_factors[mode], tl.dot(tl.transpose(B), B))
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
nn_factors[mode] *= numerator / denominator
numerator = tucker_to_tensor((tensor, nn_factors),
transpose_factors=True)
numerator = tl.clip(numerator, a_min=epsilon, a_max=None)
for i, f in enumerate(nn_factors):
if i:
denominator = mode_dot(denominator, tl.dot(tl.transpose(f), f),
i)
else:
denominator = mode_dot(nn_core, tl.dot(tl.transpose(f), f), i)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
nn_core *= numerator / denominator
rec_error = tl.norm(tensor - tucker_to_tensor(
(nn_core, nn_factors)), 2) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1 and verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if iteration > 1 and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
if normalize_factors:
nn_core, nn_factors = tucker_normalize((nn_core, nn_factors))
tensor = TuckerTensor((nn_core, nn_factors))
if return_errors:
return tensor, rec_errors
else:
return tensor
def __initialize_factors(self,
tensor,
svd='numpy_svd',
non_negative=False,
custom=None):
"""Initialize random or SVD-guided factors for TCA depending on TCA type
Parameters
----------
tensor : torch.Tensor
The tensor of activity of N neurons, T timepoints and K trials of shape N, T, K
svd : str, optional
Type of SVD algorithm to use (default is numpy_svd)
non_negative : bool, optional
A flag used to specify if factors generated must be strictyl positive (default is False)
custom : int, optional
A flag used to specify which factor should be strictly positive for 'custom parafac' (default is None)
Raises
------
ValueError
If svd does not contain a valid SVD algorithm reference
If self.init variable does not contain a valid intialization method
Returns
-------
list
List of initialized tensors
"""
rng = tensorly.random.check_random_state(self.random_state)
if self.init == 'random':
if custom:
factors = [
tl.tensor(
rng.random_sample(
(tensor.shape[i], self.rank)) * 2 - 1,
**tl.context(tensor)) for i in range(self.dimension)
]
factors = [
f if int(i) == int(custom) else tl.abs(f)
for i, f in enumerate(factors)
]
elif non_negative:
factors = [
tl.tensor(rng.random_sample((tensor.shape[i], self.rank)),
**tl.context(tensor))
for i in range(self.dimension)
]
factors = [
tl.abs(f) for f in factors
] # See if this line is useful depending on random function used
else:
factors = [
tl.tensor(
rng.random_sample(
(tensor.shape[i], self.rank)) * 2 - 1,
**tl.context(tensor)) for i in range(self.dimension)
]
return factors
elif self.init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, *_ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
if tensor.shape[mode] < rank:
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
if non_negative or custom == mode:
factors.append(tl.abs(U[:, :rank]))
else:
factors.append(U[:, :rank])
return factors
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(self.init))
def constrained_parafac(tensor, rank, n_iter_max=100, n_iter_max_inner=10,
init='svd', svd='numpy_svd',
tol_outer=1e-8, tol_inner=1e-6, random_state=None,
verbose=0, return_errors=False,
cvg_criterion='abs_rec_error',
fixed_modes=None, non_negative=None, l1_reg=None,
l2_reg=None, l2_square_reg=None, unimodality=None, normalize=None,
simplex=None, normalized_sparsity=None, soft_sparsity=None,
smoothness=None, monotonicity=None, hard_sparsity=None):
"""CANDECOMP/PARAFAC decomposition via alternating optimization of
alternating direction method of multipliers (AO-ADMM):
Computes a rank-`rank` decomposition of `tensor` [1]_ such that::
tensor = [|weights; factors[0], ..., factors[-1] |],
where factors are either penalized or constrained according to the user-defined constraint.
In order to compute the factors efficiently, the ADMM algorithm
introduces an auxilliary factor which is called factor_aux in the function.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration for outer loop
n_iter_max_inner : int
Number of iteration for inner loop
init : {'svd', 'random', cptensor}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol_outer : float, optional
(Default: 1e-8) Relative reconstruction error tolerance for outer loop. The
algorithm is considered to have found a local minimum when the
reconstruction error is less than `tol_outer`.
tol_inner : float, optional
(Default: 1e-6) Absolute reconstruction error tolerance for factor update during inner loop, i.e. ADMM optimization.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool or dictionary
This constraint is clipping negative values to '0'. If it is True non-negative constraint is applied to all modes.
l1_reg : float or list or dictionary, optional
l2_reg : float or list or dictionary, optional
l2_square_reg : float or list or dictionary, optional
unimodality : bool or dictionary, optional
If it is True unimodality constraint is applied to all modes.
normalize : bool or dictionary, optional
This constraint divides all the values by maximum value of the input array. If it is True normalize constraint
is applied to all modes.
simplex : float or list or dictionary, optional
normalized_sparsity : float or list or dictionary, optional
soft_sparsity : float or list or dictionary, optional
smoothness : float or list or dictionary, optional
monotonicity : bool or dictionary, optional
hard_sparsity : float or list or dictionary, optional
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion if `tol` is not None.
If 'rec_error', algorithm stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', algorithm terminates when `|previous rec_error - current rec_error| < tol`.
fixed_modes : list, default is None
A list of modes for which the initial value is not modified.
The last mode cannot be fixed due to error computation.
Returns
-------
CPTensor : (weight, factors)
* weights : 1D array of shape (rank, )
* factors : List of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)``
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM
REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Huang, Kejun, Nicholas D. Sidiropoulos, and Athanasios P. Liavas.
"A flexible and efficient algorithmic framework for constrained matrix and tensor factorization." IEEE
Transactions on Signal Processing 64.19 (2016): 5052-5065.
"""
rank = validate_cp_rank(tl.shape(tensor), rank=rank)
_, _ = validate_constraints(non_negative=non_negative, l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize, simplex=simplex,
normalized_sparsity=normalized_sparsity, soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity, hard_sparsity=hard_sparsity,
n_const=tl.ndim(tensor))
weights, factors = initialize_constrained_parafac(tensor, rank, init=init, svd=svd,
random_state=random_state, non_negative=non_negative,
l1_reg=l1_reg, l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize,
simplex=simplex,
normalized_sparsity=normalized_sparsity,
soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity,
hard_sparsity=hard_sparsity)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
if fixed_modes is None:
fixed_modes = []
if tl.ndim(tensor) - 1 in fixed_modes:
warnings.warn('You asked for fixing the last mode, which is not supported.\n '
'The last mode will not be fixed. Consider using tl.moveaxis()')
fixed_modes.remove(tl.ndim(tensor) - 1)
modes_list = [mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes]
# ADMM inits
dual_variables = []
factors_aux = []
for i in range(len(factors)):
dual_variables.append(tl.zeros(tl.shape(factors[i])))
factors_aux.append(tl.transpose(tl.zeros(tl.shape(factors[i]))))
for iteration in range(n_iter_max):
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in modes_list:
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(tl.transpose(factor), factor)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
factors[mode], factors_aux[mode], dual_variables[mode] = admm(mttkrp, pseudo_inverse, factors[mode], dual_variables[mode],
n_iter_max=n_iter_max_inner, n_const=tl.ndim(tensor),
order=mode, non_negative=non_negative, l1_reg=l1_reg,
l2_reg=l2_reg, l2_square_reg=l2_square_reg,
unimodality=unimodality, normalize=normalize,
simplex=simplex, normalized_sparsity=normalized_sparsity,
soft_sparsity=soft_sparsity,
smoothness=smoothness, monotonicity=monotonicity,
hard_sparsity=hard_sparsity, tol=tol_inner)
factors_norm = cp_norm((weights, factors))
iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0) * weights)
rec_error = tl.sqrt(tl.abs(norm_tensor ** 2 + factors_norm ** 2 - 2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
constraint_error = 0
for mode in modes_list:
constraint_error += tl.norm(factors[mode] - tl.transpose(factors_aux[mode])) / tl.norm(factors[mode])
if tol_outer:
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print("iteration {}, reconstruction error: {}, decrease = {}".format(iteration,
rec_error,
rec_error_decrease))
if constraint_error < tol_outer:
break
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol_outer
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol_outer
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def initialize_constrained_parafac(tensor, rank, init='svd', svd='numpy_svd',
random_state=None, non_negative=None, l1_reg=None,
l2_reg=None, l2_square_reg=None, unimodality=None, normalize=None,
simplex=None, normalized_sparsity=None,
soft_sparsity=None, smoothness=None, monotonicity=None,
hard_sparsity=None):
r"""Initialize factors used in `constrained_parafac`.
Parameters
----------
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices with uniform distribution using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor. If init is a previously initialized `cp tensor`, all
the weights are pulled in the last factor and then the weights are set to "1" for the output tensor.
Lastly, factors are updated with proximal operator according to the selected constraint(s), so that they satisfy the
imposed constraints (does not apply to cptensor initialization).
Parameters
----------
tensor : ndarray
rank : int
random_state : {None, int, np.random.RandomState}
init : {'svd', 'random', cptensor}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool or dictionary
This constraint is clipping negative values to '0'. If it is True non-negative constraint is applied to all modes.
l1_reg : float or list or dictionary, optional
l2_reg : float or list or dictionary, optional
l2_square_reg : float or list or dictionary, optional
unimodality : bool or dictionary, optional
If it is True unimodality constraint is applied to all modes.
normalize : bool or dictionary, optional
This constraint divides all the values by maximum value of the input array. If it is True normalize constraint
is applied to all modes.
simplex : float or list or dictionary, optional
normalized_sparsity : float or list or dictionary, optional
soft_sparsity : float or list or dictionary, optional
smoothness : float or list or dictionary, optional
monotonicity : bool or dictionary, optional
hard_sparsity : float or list or dictionary, optional
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
n_modes = tl.ndim(tensor)
rng = tl.check_random_state(random_state)
if init == 'random':
weights, factors = random_cp(tl.shape(tensor), rank, normalise_factors=False, **tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Put SVD initialization on the same scaling as the tensor in case normalize_factors=False
if mode == 0:
idx = min(rank, tl.shape(S)[0])
U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx])
if tensor.shape[mode] < rank:
random_part = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
elif isinstance(init, (tuple, list, CPTensor)):
try:
weights, factors = CPTensor(init)
if tl.all(weights == 1):
weights, factors = CPTensor((None, factors))
else:
weights_avg = tl.prod(weights) ** (1.0 / tl.shape(weights)[0])
for i in range(len(factors)):
factors[i] = factors[i] * weights_avg
kt = CPTensor((None, factors))
return kt
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance'
)
else:
raise ValueError('Initialization method "{}" not recognized'.format(init))
for i in range(n_modes):
factors[i] = proximal_operator(factors[i], non_negative=non_negative, l1_reg=l1_reg,
l2_reg=l2_reg, l2_square_reg=l2_square_reg, unimodality=unimodality,
normalize=normalize, simplex=simplex, normalized_sparsity=normalized_sparsity,
soft_sparsity=soft_sparsity, smoothness=smoothness,
monotonicity=monotonicity, hard_sparsity=hard_sparsity, n_const=n_modes, order=i)
kt = CPTensor((None, factors))
return kt
def tensor_train_cross(input_tensor, rank, tol=1e-4, n_iter_max=100):
"""TT (tensor-train) decomposition via cross-approximation (TTcross) [1]
Decomposes `input_tensor` into a sequence of order-3 tensors of given rank. (factors/cores)
Rather than directly decompose the whole tensor, we sample fibers based on skeleton decomposition.
We initialize a random tensor-train and sweep from left to right and right to left.
On each core, we shape the core as a matrix and choose the fibers indices by finding maximum-volume submatrix and update the core.
Advantage: faster
The main advantage of TTcross is that it doesn't need to evaluate all the entries of the tensor.
For a tensor_shape^tensor_order tensor, SVD needs O(tensor_shape^tensor_order) runtime, but TTcross' runtime is linear in tensor_shape and tensor_order, which makes it feasible in high dimension.
Disadvantage: less accurate
TTcross may underestimate the error, since it only evaluates partial entries of the tensor.
Besides, in contrast to its practical fast performance, there is no theoretical guarantee of it convergence.
Parameters
----------
input_tensor : tensorly.tensor
The tensor to decompose.
rank : {int, int list}
maximum allowable TT rank of the factors
if int, then this is the same for all the factors
if int list, then rank[k] is the rank of the kth factor
tol : float
accuracy threshold for outer while-loop
n_iter_max : int
maximum iterations of outer while-loop (the 'crosses' or 'sweeps' sampled)
Returns
-------
factors : TT factors
order-3 tensors of the TT decomposition
Examples
--------
Generate a 5^3 tensor, and decompose it into tensor-train of 3 factors, with rank = [1,3,3,1]
>>> tensor = tl.tensor(np.arange(5**3).reshape(5,5,5))
>>> rank = [1, 3, 3, 1]
>>> factors = tensor_train_cross(tensor, rank)
print the first core:
>>> print(factors[0])
.[[[ 24. 0. 4.]
[ 49. 25. 29.]
[ 74. 50. 54.]
[ 99. 75. 79.]
[124. 100. 104.]]]
Notes
-----
Pseudo-code [2]:
1. Initialization tensor_order cores and column indices
2. while (error > tol)
3. update the tensor-train from left to right:
for Core 1 to Core tensor_order
approximate the skeleton-decomposition by QR and maxvol
4. update the tensor-train from right to left:
for Core tensor_order to Core 1
approximate the skeleton-decomposition by QR and maxvol
5. end while
Acknowledgement: the main body of the code is modified based on TensorToolbox by Daniele Bigoni
References
----------
.. [1] Ivan Oseledets and Eugene Tyrtyshnikov. Tt-cross approximation for multidimensional arrays.
LinearAlgebra and its Applications, 432(1):70–88, 2010.
.. [2] Sergey Dolgov and Robert Scheichl. A hybrid alternating least squares–tt cross algorithm for parametricpdes.
arXiv preprint arXiv:1707.04562, 2017.
"""
# Check user input for errors
tensor_shape = tl.shape(input_tensor)
tensor_order = tl.ndim(input_tensor)
if isinstance(rank, int):
rank = [rank] * (tensor_order + 1)
elif tensor_order + 1 != len(rank):
message = 'Provided incorrect number of ranks. Should verify len(rank) == tl.ndim(tensor)+1, but len(rank) = {} while tl.ndim(tensor) + 1 = {}'.format(
len(rank), tensor_order)
raise (ValueError(message))
# Make sure iter's not a tuple but a list
rank = list(rank)
# Initialize rank
if rank[0] != 1:
print(
'Provided rank[0] == {} but boundary conditions dictate rank[0] == rank[-1] == 1: setting rank[0] to 1.'.format(
rank[0]))
rank[0] = 1
if rank[-1] != 1:
print(
'Provided rank[-1] == {} but boundary conditions dictate rank[0] == rank[-1] == 1: setting rank[-1] to 1.'.format(
rank[0]))
# list col_idx: column indices (right indices) for skeleton-decomposition: indicate which columns used in each core.
# list row_idx: row indices (left indices) for skeleton-decomposition: indicate which rows used in each core.
# Initialize indice: random selection of column indices
random_seed = None
rng = check_random_state(random_seed)
col_idx = [None] * tensor_order
for k_col_idx in range(tensor_order - 1):
col_idx[k_col_idx] = []
for i in range(rank[k_col_idx + 1]):
newidx = tuple([rng.randint(tensor_shape[j]) for j in range(k_col_idx + 1, tensor_order)])
while newidx in col_idx[k_col_idx]:
newidx = tuple([rng.randint(tensor_shape[j]) for j in range(k_col_idx + 1, tensor_order)])
col_idx[k_col_idx].append(newidx)
# Initialize the cores of tensor-train
factor_old = [tl.zeros((rank[k], tensor_shape[k], rank[k + 1]),
**tl.context(input_tensor)) for k in range(tensor_order)]
factor_new = [tl.tensor(rng.random_sample((rank[k], tensor_shape[k], rank[k + 1])),
**tl.context(input_tensor)) for k in range(tensor_order)]
iter = 0
error = tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2)
threshold = tol * tl.norm(tt_to_tensor(factor_new), 2)
for iter in range(n_iter_max):
if error < threshold:
break
factor_old = factor_new
factor_new = [None for i in range(tensor_order)]
######################################
# left-to-right step
left_to_right_fiberlist = []
# list row_idx: list of (tensor_order-1) of lists of left indices
row_idx = [[()]]
for k in range(tensor_order - 1):
(next_row_idx, fibers_list) = left_right_ttcross_step(input_tensor, k, rank, row_idx, col_idx)
# update row indices
left_to_right_fiberlist.extend(fibers_list)
row_idx.append(next_row_idx)
# end left-to-right step
###############################################
###############################################
# right-to-left step
right_to_left_fiberlist = []
# list col_idx: list (tensor_order-1) of lists of right indices
col_idx = [None] * tensor_order
col_idx[-1] = [()]
for k in range(tensor_order, 1, -1):
(next_col_idx, fibers_list, Q_skeleton) = right_left_ttcross_step(input_tensor, k, rank, row_idx, col_idx)
# update col indices
right_to_left_fiberlist.extend(fibers_list)
col_idx[k - 2] = next_col_idx
# Compute cores
try:
factor_new[k - 1] = tl.transpose(Q_skeleton)
factor_new[k - 1] = tl.reshape(factor_new[k - 1], (rank[k - 1], tensor_shape[k - 1], rank[k]))
except:
# The rank should not be larger than the input tensor's size
raise (ValueError("The rank is too large compared to the size of the tensor. Try with small rank."))
# Add the last core
idx = (slice(None, None, None),) + tuple(zip(*col_idx[0]))
core = input_tensor[idx]
core = tl.reshape(core, (tensor_shape[0], 1, rank[1]))
core = tl.transpose(core, (1, 0, 2))
factor_new[0] = core
# end right-to-left step
################################################
# check the error for while-loop
error = tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2)
threshold = tol * tl.norm(tt_to_tensor(factor_new), 2)
print("It: " + str(iter) + "; error: " + str(error) + "; threshold: " + str(threshold))
# check convergence
if iter >= n_iter_max:
print('Maximum number of iterations reached.')
if tl.norm(tt_to_tensor(factor_old) - tt_to_tensor(factor_new), 2) > tol * tl.norm(tt_to_tensor(factor_new), 2):
print('Low Rank Approximation algorithm did not converge.')
return factor_new
def parafac2(tensor_slices,
rank,
n_iter_max=100,
init='random',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
n_iter_parafac=5):
r"""PARAFAC2 decomposition [1]_ of a third order tensor via alternating least squares (ALS)
Computes a rank-`rank` PARAFAC2 decomposition of the third-order tensor defined by
`tensor_slices`. The decomposition is on the form :math:`(A [B_i] C)` such that the
i-th frontal slice, :math:`X_i`, of :math:`X` is given by
.. math::
X_i = B_i diag(a_i) C^T,
where :math:`diag(a_i)` is the diagonal matrix whose nonzero entries are equal to
the :math:`i`-th row of the :math:`I \times R` factor matrix :math:`A`, :math:`B_i`
is a :math:`J_i \times R` factor matrix such that the cross product matrix :math:`B_{i_1}^T B_{i_1}`
is constant for all :math:`i`, and :math:`C` is a :math:`K \times R` factor matrix.
To compute this decomposition, we reformulate the expression for :math:`B_i` such that
.. math::
B_i = P_i B,
where :math:`P_i` is a :math:`J_i \times R` orthogonal matrix and :math:`B` is a
:math:`R \times R` matrix.
An alternative formulation of the PARAFAC2 decomposition is that the tensor element
:math:`X_{ijk}` is given by
.. math::
X_{ijk} = \sum_{r=1}^R A_{ir} B_{ijr} C_{kr},
with the same constraints hold for :math:`B_i` as above.
Parameters
----------
tensor_slices : ndarray or list of ndarrays
Either a third order tensor or a list of second order tensors that may have different number of rows.
Note that the second mode factor matrices are allowed to change over the first mode, not the
third mode as some other implementations use (see note below).
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random', CPTensor, Parafac2Tensor}
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : bool (optional)
If True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors. Note that
there may be some inaccuracies in the component weights.
tol : float, optional
(Default: 1e-8) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
n_iter_parafac: int, optional
Number of PARAFAC iterations to perform for each PARAFAC2 iteration
Returns
-------
Parafac2Tensor : (weight, factors, projection_matrices)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
* projection_matrices : List of projection matrices used to create evolving
factors.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] Kiers, H.A.L., ten Berge, J.M.F. and Bro, R. (1999),
PARAFAC2—Part I. A direct fitting algorithm for the PARAFAC2 model.
J. Chemometrics, 13: 275-294.
Notes
-----
This formulation of the PARAFAC2 decomposition is slightly different from the one in [1]_.
The difference lies in that here, the second mode changes over the first mode, whereas in
[1]_, the second mode changes over the third mode. We made this change since that means
that the function accept both lists of matrices and a single nd-array as input without
any reordering of the modes.
"""
weights, factors, projections = initialize_decomposition(
tensor_slices, rank, init=init, svd=svd, random_state=random_state)
rec_errors = []
norm_tensor = tl.sqrt(
sum(tl.norm(tensor_slice, 2)**2 for tensor_slice in tensor_slices))
svd_fun = _get_svd(svd)
projected_tensor = tl.zeros([factor.shape[0] for factor in factors],
**T.context(factors[0]))
for iteration in range(n_iter_max):
if verbose:
print("Starting iteration", iteration)
factors[1] = factors[1] * T.reshape(weights, (1, -1))
weights = T.ones(weights.shape, **tl.context(tensor_slices[0]))
projections = _compute_projections(tensor_slices,
factors,
svd_fun,
out=projections)
projected_tensor = _project_tensor_slices(tensor_slices,
projections,
out=projected_tensor)
_, factors = parafac(projected_tensor,
rank,
n_iter_max=n_iter_parafac,
init=(weights, factors),
svd=svd,
orthogonalise=False,
verbose=verbose,
return_errors=False,
normalize_factors=False,
mask=None,
random_state=random_state,
tol=1e-100)
if normalize_factors:
new_factors = []
for factor in factors:
norms = T.norm(factor, axis=0)
norms = tl.where(
tl.abs(norms) <= tl.eps(factor.dtype),
tl.ones(tl.shape(norms), **tl.context(factors[0])), norms)
weights = weights * norms
new_factors.append(factor / (tl.reshape(norms, (1, -1))))
factors = new_factors
if tol:
rec_error = _parafac2_reconstruction_error(
tensor_slices, (weights, factors, projections))
rec_error /= norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('PARAFAC2 reconstruction error={}, variation={}.'.
format(rec_errors[-1],
rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('PARAFAC2 reconstruction error={}'.format(
rec_errors[-1]))
parafac2_tensor = Parafac2Tensor((weights, factors, projections))
if return_errors:
return parafac2_tensor, rec_errors
else:
return parafac2_tensor
def parafac(tensor,
rank,
n_iter_max=100,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
mode_three_val=[[0.5, 0.5, 0.0], [0.0, 0.5, 0.5]]):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [| factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
Returns
-------
factors : ndarray list
List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
"""
factors = initialize_factors(tensor, rank, random_state=random_state)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
# Mode-3 values that control the country factors are set using the
# mode_three_val argument.
fixed_ja = mode_three_val[0]
fixed_ko = mode_three_val[1]
for iteration in range(n_iter_max):
for mode in range(tl.ndim(tensor)):
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
factors[2][0] = fixed_ja # set mode-3 values
factors[2][1] = fixed_ko # set mode-3 values
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
factor = tl.dot(unfold(tensor, mode),
khatri_rao(factors, skip_matrix=mode))
factor = tl.transpose(
tl.solve(tl.transpose(pseudo_inverse), tl.transpose(factor)))
factors[mode] = factor
if tol:
rec_error = tl.norm(tensor - kruskal_to_tensor(factors),
2) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
if return_errors:
return factors, rec_errors
else:
return factors
def parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
orthogonalise=False,
random_state=None,
verbose=0,
return_errors=False,
non_negative=False,
mask=None):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [|weights; factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool, optional
Perform non_negative PARAFAC. See :func:`non_negative_parafac`.
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True). Allows for missing values [2]_
Returns
-------
KruskalTensor : (weight, factors)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values."
Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180.
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=non_negative,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
weights = tl.ones(rank, **tl.context(tensor))
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factors = [
tl.qr(f)[0] if min(tl.shape(f)) >= rank else f
for i, f in enumerate(factors)
]
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in range(tl.ndim(tensor)):
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
if non_negative:
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.conj(tl.transpose(factor)), factor)
if mask is not None:
tensor = tensor * mask + tl.kruskal_to_tensor(
(None, factors), mask=1 - mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
if non_negative:
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
else:
factor = tl.transpose(
tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
if normalize_factors:
weights = tl.norm(factor, order=2, axis=0)
weights = tl.where(
tl.abs(weights) <= tl.eps(tensor.dtype),
tl.ones(tl.shape(weights), **tl.context(factors[0])),
weights)
factor = factor / (tl.reshape(weights, (1, -1)))
factors[mode] = factor
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = kruskal_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
kruskal_tensor = KruskalTensor((weights, factors))
if return_errors:
return kruskal_tensor, rec_errors
else:
return kruskal_tensor
def initialize_factors(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
non_negative=False,
normalize_factors=False):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : ndarray list
List of initialized factors of the CP decomposition where element `i`
is of shape (tensor.shape[i], rank)
"""
rng = check_random_state(random_state)
if init == 'random':
factors = [
tl.tensor(rng.random_sample((tensor.shape[i], rank)),
**tl.context(tensor)) for i in range(tl.ndim(tensor))
]
if non_negative:
factors = [tl.abs(f) for f in factors]
if normalize_factors:
factors = [
f / (tl.reshape(tl.norm(f, axis=0), (1, -1)) + 1e-12)
for f in factors
]
return factors
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, _, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
# factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor))
# factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
# factor[:, :tensor.shape[mode]] = U
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factor = U[:, :rank]
if non_negative:
factor = tl.abs(factor)
if normalize_factors:
factor = factor / (tl.reshape(tl.norm(factor, axis=0),
(1, -1)) + 1e-12)
factors.append(factor)
return factors
raise ValueError('Initialization method "{}" not recognized'.format(init))
def initialize_nn_cp(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
normalize_factors=False,
nntype='nndsvda'):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
nntype : {'nndsvd', 'nndsvda'}
Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default).
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
rng = tl.check_random_state(random_state)
if init == 'random':
kt = random_cp(tl.shape(tensor),
rank,
normalise_factors=False,
random_state=rng,
**tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, V = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Apply nnsvd to make non-negative
U = make_svd_non_negative(tensor, U, S, V, nntype)
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
kt = CPTensor((None, factors))
# If the initialisation is a precomputed decomposition, we double check its validity and return it
elif isinstance(init, (tuple, list, CPTensor)):
# TODO: Test this
try:
kt = CPTensor(init)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance')
return kt
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(init))
# Make decomposition feasible by taking the absolute value of all factor matrices
kt.factors = [tl.abs(f) for f in kt[1]]
if normalize_factors:
kt = cp_normalize(kt)
return kt
def non_negative_parafac_hals(tensor,
rank,
n_iter_max=100,
init="svd",
svd='numpy_svd',
tol=10e-8,
random_state=None,
sparsity_coefficients=None,
fixed_modes=None,
nn_modes='all',
exact=False,
normalize_factors=False,
verbose=False,
return_errors=False,
cvg_criterion='abs_rec_error'):
"""
Non-negative CP decomposition via HALS
Uses Hierarchical ALS (Alternating Least Squares) which updates each factor column-wise (one column at a time while keeping all other columns fixed), see [1]_
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
Default: 1e-8
random_state : {None, int, np.random.RandomState}
sparsity_coefficients: array of float (of length the number of modes)
The sparsity coefficients on each factor.
If set to None, the algorithm is computed without sparsity
Default: None,
fixed_modes: array of integers (between 0 and the number of modes)
Has to be set not to update a factor, 0 and 1 for U and V respectively
Default: None
nn_modes: None, 'all' or array of integers (between 0 and the number of modes)
Used to specify which modes to impose non-negativity constraints on.
If 'all', then non-negativity is imposed on all modes.
Default: 'all'
exact: If it is True, the algorithm gives a results with high precision but it needs high computational cost.
If it is False, the algorithm gives an approximate solution
Default: False
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
verbose: boolean
Indicates whether the algorithm prints the successive
reconstruction errors or not
Default: False
return_errors: boolean
Indicates whether the algorithm should return all reconstruction errors
and computation time of each iteration or not
Default: False
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion for ALS, works if `tol` is not None.
If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`.
sparsity : float or int
random_state : {None, int, np.random.RandomState}
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
errors: list
A list of reconstruction errors at each iteration of the algorithm.
References
----------
.. [1]: N. Gillis and F. Glineur, Accelerated Multiplicative Updates and
Hierarchical ALS Algorithms for Nonnegative Matrix Factorization,
Neural Computation 24 (4): 1085-1105, 2012.
"""
weights, factors = initialize_nn_cp(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
normalize_factors=normalize_factors)
norm_tensor = tl.norm(tensor, 2)
n_modes = tl.ndim(tensor)
if sparsity_coefficients is None or isinstance(sparsity_coefficients,
float):
sparsity_coefficients = [sparsity_coefficients] * n_modes
if fixed_modes is None:
fixed_modes = []
if nn_modes == 'all':
nn_modes = set(range(n_modes))
elif nn_modes is None:
nn_modes = set()
# Avoiding errors
for fixed_value in fixed_modes:
sparsity_coefficients[fixed_value] = None
for mode in range(n_modes):
if sparsity_coefficients[mode] is not None:
warnings.warn(
"Sparsity coefficient is ignored in unconstrained modes.")
# Generating the mode update sequence
modes = [mode for mode in range(n_modes) if mode not in fixed_modes]
# initialisation - declare local varaibles
rec_errors = []
# Iteratation
for iteration in range(n_iter_max):
# One pass of least squares on each updated mode
for mode in modes:
# Computing Hadamard of cross-products
pseudo_inverse = tl.tensor(tl.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
pseudo_inverse = tl.reshape(weights,
(-1, 1)) * pseudo_inverse * tl.reshape(
weights, (1, -1))
mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode)
if mode in nn_modes:
# Call the hals resolution with nnls, optimizing the current mode
nn_factor, _, _, _ = hals_nnls(
tl.transpose(mttkrp),
pseudo_inverse,
tl.transpose(factors[mode]),
n_iter_max=100,
sparsity_coefficient=sparsity_coefficients[mode],
exact=exact)
factors[mode] = tl.transpose(nn_factor)
else:
factor = tl.solve(tl.transpose(pseudo_inverse),
tl.transpose(mttkrp))
factors[mode] = tl.transpose(factor)
if normalize_factors and mode != modes[-1]:
weights, factors = cp_normalize((weights, factors))
if tol:
factors_norm = cp_norm((weights, factors))
iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0))
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print(
"iteration {}, reconstruction error: {}, decrease = {}"
.format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(
iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
if normalize_factors:
weights, factors = cp_normalize((weights, factors))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def make_svd_non_negative(tensor, U, S, V, nntype):
""" Use NNDSVD method to transform SVD results into a non-negative form. This
method leads to more efficient solving with NNMF [1].
Parameters
----------
tensor : tensor being decomposed
U, S, V: SVD factorization results
nntype : {'nndsvd', 'nndsvda'}
Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default).
[1]: Boutsidis & Gallopoulos. Pattern Recognition, 41(4): 1350-1362, 2008.
"""
# NNDSVD initialization
W = tl.zeros_like(U)
H = tl.zeros_like(V)
# The leading singular triplet is non-negative
# so it can be used as is for initialization.
W = tl.index_update(W, tl.index[:, 0], tl.sqrt(S[0]) * tl.abs(U[:, 0]))
H = tl.index_update(H, tl.index[0, :], tl.sqrt(S[0]) * tl.abs(V[0, :]))
for j in range(1, tl.shape(U)[1]):
x, y = U[:, j], V[j, :]
# extract positive and negative parts of column vectors
x_p, y_p = tl.clip(x, a_min=0.0), tl.clip(y, a_min=0.0)
x_n, y_n = tl.abs(tl.clip(x, a_max=0.0)), tl.abs(tl.clip(y, a_max=0.0))
# and their norms
x_p_nrm, y_p_nrm = tl.norm(x_p), tl.norm(y_p)
x_n_nrm, y_n_nrm = tl.norm(x_n), tl.norm(y_n)
m_p, m_n = x_p_nrm * y_p_nrm, x_n_nrm * y_n_nrm
# choose update
if m_p > m_n:
u = x_p / x_p_nrm
v = y_p / y_p_nrm
sigma = m_p
else:
u = x_n / x_n_nrm
v = y_n / y_n_nrm
sigma = m_n
lbd = tl.sqrt(S[j] * sigma)
W = tl.index_update(W, tl.index[:, j], lbd * u)
H = tl.index_update(H, tl.index[j, :], lbd * v)
# After this point we no longer need H
eps = tl.eps(tensor.dtype)
if nntype == "nndsvd":
W = soft_thresholding(W, eps)
elif nntype == "nndsvda":
avg = tl.mean(tensor)
W = tl.where(W < eps, tl.ones(tl.shape(W), **tl.context(W)) * avg, W)
else:
raise ValueError(
'Invalid nntype parameter: got %r instead of one of %r' %
(nntype, ('nndsvd', 'nndsvda')))
return W
def matrix_product_state(input_tensor, rank, verbose=False):
"""MPS decomposition via recursive SVD
Decomposes `input_tensor` into a sequence of order-3 tensors (factors)
-- also known as Tensor-Train decomposition [1]_.
Parameters
----------
input_tensor : tensorly.tensor
rank : {int, int list}
maximum allowable MPS rank of the factors
if int, then this is the same for all the factors
if int list, then rank[k] is the rank of the kth factor
verbose : boolean, optional
level of verbosity
Returns
-------
factors : MPS factors
order-3 tensors of the MPS decomposition
References
----------
.. [1] Ivan V. Oseledets. "Tensor-train decomposition", SIAM J. Scientific Computing, 33(5):2295–2317, 2011.
"""
# Check user input for errors
tensor_size = input_tensor.shape
n_dim = len(tensor_size)
if isinstance(rank, int):
rank = [rank] * (n_dim + 1)
elif n_dim + 1 != len(rank):
message = 'Provided incorrect number of ranks. Should verify len(rank) == tl.ndim(tensor)+1, but len(rank) = {} while tl.ndim(tensor) + 1 = {}'.format(
len(rank), n_dim)
raise (ValueError(message))
# Make sure it's not a tuple but a list
rank = list(rank)
context = tl.context(input_tensor)
# Initialization
if rank[0] != 1:
print(
'Provided rank[0] == {} but boundaring conditions dictatate rank[0] == rank[-1] == 1: setting rank[0] to 1.'
.format(rank[0]))
rank[0] = 1
if rank[-1] != 1:
print(
'Provided rank[-1] == {} but boundaring conditions dictatate rank[0] == rank[-1] == 1: setting rank[-1] to 1.'
.format(rank[0]))
unfolding = input_tensor
factors = [None] * n_dim
# Getting the MPS factors up to n_dim - 1
for k in range(n_dim - 1):
# Reshape the unfolding matrix of the remaining factors
n_row = int(rank[k] * tensor_size[k])
unfolding = tl.reshape(unfolding, (n_row, -1))
# SVD of unfolding matrix
(n_row, n_column) = unfolding.shape
current_rank = min(n_row, n_column, rank[k + 1])
U, S, V = tl.partial_svd(unfolding, current_rank)
rank[k + 1] = current_rank
# Get kth MPS factor
factors[k] = tl.reshape(U, (rank[k], tensor_size[k], rank[k + 1]))
if (verbose is True):
print("MPS factor " + str(k) + " computed with shape " +
str(factors[k].shape))
# Get new unfolding matrix for the remaining factors
unfolding = tl.reshape(S, (-1, 1)) * V
# Getting the last factor
(prev_rank, last_dim) = unfolding.shape
factors[-1] = tl.reshape(unfolding, (prev_rank, last_dim, 1))
if (verbose is True):
print("MPS factor " + str(n_dim - 1) + " computed with shape " +
str(factors[n_dim - 1].shape))
return factors
def power_iteration(tensor, n_repeat=10, n_iteration=10, verbose=False):
"""A single Robust Tensor Power Iteration
Parameters
----------
tensor : tl.tensor
input tensor to decompose
n_repeat : int, default is 10
number of initializations to be tried
n_iteration : int, default is 10
number of power iterations
verbose : bool
level of verbosity
Returns
-------
(eigenval, best_factor, deflated)
eigenval : float
the obtained eigenvalue
best_factors : tl.tensor list
the best estimated eigenvector, for each mode of the input tensor
deflated : tl.tensor of same shape as `tensor`
the deflated tensor (i.e. without the estimated component)
"""
order = tl.ndim(tensor)
# A list of candidates for each mode
best_score = 0
scores = []
for _ in range(n_repeat):
factors = [
tl.tensor(np.random.random_sample(s), **tl.context(tensor))
for s in tl.shape(tensor)
]
for _ in range(n_iteration):
for mode in range(order):
factor = tl.tenalg.multi_mode_dot(tensor, factors, skip=mode)
factor = factor / tl.norm(factor, 2)
factors[mode] = factor
score = tl.tenalg.multi_mode_dot(tensor, factors)
scores.append(score) #round(score, 2))
if score > best_score:
best_score = score
best_factors = factors
if verbose:
print(f'Best score of {n_repeat}: {best_score}')
# Refine the init
for _ in range(n_iteration):
for mode in range(order):
factor = tl.tenalg.multi_mode_dot(tensor, best_factors, skip=mode)
factor = factor / tl.norm(factor, 2)
best_factors[mode] = factor
eigenval = tl.tenalg.multi_mode_dot(tensor, best_factors)
deflated = tensor - outer(best_factors) * eigenval
if verbose:
explained = tl.norm(deflated) / tl.norm(tensor)
print(f'Eingenvalue: {eigenval}, explained: {explained}')
return eigenval, best_factors, deflated
def initialize_decomposition(tensor_slices,
rank,
init='random',
svd='numpy_svd',
random_state=None):
r"""Initiate a random PARAFAC2 decomposition given rank and tensor slices
Parameters
----------
tensor_slices : Iterable of ndarray
rank : int
init : {'random', 'svd', CPTensor, Parafac2Tensor}, optional
random_state : `np.random.RandomState`
Returns
-------
parafac2_tensor : Parafac2Tensor
List of initialized factors of the CP decomposition where element `i`
is of shape (tensor.shape[i], rank)
"""
context = tl.context(tensor_slices[0])
shapes = [m.shape for m in tensor_slices]
if init == 'random':
return random_parafac2(shapes,
rank,
full=False,
random_state=random_state,
**context)
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
padded_tensor = _pad_by_zeros(tensor_slices)
A = T.ones((padded_tensor.shape[0], rank), **context)
unfolded_mode_2 = unfold(padded_tensor, 2)
if T.shape(unfolded_mode_2)[0] < rank:
raise ValueError(
"Cannot perform SVD init if rank ({}) is greater than the number of columns in each tensor slice ({})"
.format(rank,
T.shape(unfolded_mode_2)[0]))
C = svd_fun(unfold(padded_tensor, 2), n_eigenvecs=rank)[0]
B = T.eye(rank, **context)
projections = _compute_projections(tensor_slices, (A, B, C), svd_fun)
return Parafac2Tensor((None, [A, B, C], projections))
elif isinstance(init, (tuple, list, Parafac2Tensor, CPTensor)):
try:
decomposition = Parafac2Tensor.from_CPTensor(
init, parafac2_tensor_ok=True)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a Parafac2Tensor instance')
if decomposition.rank != rank:
raise ValueError(
'Cannot init with a decomposition of different rank')
return decomposition
raise ValueError('Initialization method "{}" not recognized'.format(init))
def parafac(tensor, rank, n_iter_max=100, init='svd', svd='numpy_svd',\
normalize_factors=False, orthogonalise=False,\
tol=1e-8, random_state=None,\
verbose=0, return_errors=False,\
sparsity = None,\
l2_reg = 0, mask=None,\
cvg_criterion = 'abs_rec_error',\
fixed_modes = [],
svd_mask_repeats=5,
linesearch = False):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that::
tensor = [|weights; factors[0], ..., factors[-1] |].
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True). Allows for missing values [2]_
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion for ALS, works if `tol` is not None.
If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`.
sparsity : float or int
If `sparsity` is not None, we approximate tensor as a sum of low_rank_component and sparse_component, where low_rank_component = cp_to_tensor((weights, factors)). `sparsity` denotes desired fraction or number of non-zero elements in the sparse_component of the `tensor`.
fixed_modes : list, default is []
A list of modes for which the initial value is not modified.
The last mode cannot be fixed due to error computation.
svd_mask_repeats: int
If using a tensor with masked values, this initializes using SVD multiple times to
remove the effect of these missing values on the initialization.
linesearch : bool, default is False
Whether to perform line search as proposed by Bro [3].
Returns
-------
CPTensor : (weight, factors)
* weights : 1D array of shape (rank, )
* all ones if normalize_factors is False (default)
* weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape ``(tensor.shape[i], rank)``
* sparse_component : nD array of shape tensor.shape. Returns only if `sparsity` is not None.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM
REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values."
Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180.
.. [3] R. Bro, "Multi-Way Analysis in the Food Industry: Models, Algorithms, and
Applications", PhD., University of Amsterdam, 1998
"""
rank = validate_cp_rank(tl.shape(tensor), rank=rank)
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
if linesearch:
acc_pow = 2.0 # Extrapolate to the iteration^(1/acc_pow) ahead
acc_fail = 0 # How many times acceleration have failed
max_fail = 4 # Increase acc_pow with one after max_fail failure
weights, factors = initialize_cp(tensor, rank, init=init, svd=svd,
random_state=random_state,
normalize_factors=normalize_factors)
if mask is not None and init == "svd":
for _ in range(svd_mask_repeats):
tensor = tensor*mask + tl.cp_to_tensor((weights, factors), mask=1-mask)
weights, factors = initialize_cp(tensor, rank, init=init, svd=svd, random_state=random_state, normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
Id = tl.eye(rank, **tl.context(tensor))*l2_reg
if tl.ndim(tensor)-1 in fixed_modes:
warnings.warn('You asked for fixing the last mode, which is not supported.\n The last mode will not be fixed. Consider using tl.moveaxis()')
fixed_modes.remove(tl.ndim(tensor)-1)
modes_list = [mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes]
if sparsity:
sparse_component = tl.zeros_like(tensor)
if isinstance(sparsity, float):
sparsity = int(sparsity * np.prod(tensor.shape))
else:
sparsity = int(sparsity)
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factors = [tl.qr(f)[0] if min(tl.shape(f)) >= rank else f for i, f in enumerate(factors)]
if linesearch and iteration % 2 == 0:
factors_last = [tl.copy(f) for f in factors]
weights_last = tl.copy(weights)
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in modes_list:
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse*tl.dot(tl.conj(tl.transpose(factor)), factor)
pseudo_inverse += Id
if not iteration and weights is not None:
# Take into account init weights
mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode)
else:
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
factor = tl.transpose(tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
if normalize_factors:
scales = tl.norm(factor, 2, axis=0)
weights = tl.where(scales==0, tl.ones(tl.shape(scales), **tl.context(factor)), scales)
factor = factor / tl.reshape(weights, (1, -1))
factors[mode] = factor
# Will we be performing a line search iteration
if linesearch and iteration % 2 == 0 and iteration > 5:
line_iter = True
else:
line_iter = False
# Calculate the current unnormalized error if we need it
if (tol or return_errors) and line_iter is False:
unnorml_rec_error, tensor, norm_tensor = error_calc(tensor, norm_tensor, weights, factors, sparsity, mask, mttkrp)
else:
if mask is not None:
tensor = tensor*mask + tl.cp_to_tensor((weights, factors), mask=1-mask)
# Start line search if requested.
if line_iter is True:
jump = iteration ** (1.0 / acc_pow)
new_weights = weights_last + (weights - weights_last) * jump
new_factors = [factors_last[ii] + (factors[ii] - factors_last[ii])*jump for ii in range(tl.ndim(tensor))]
new_rec_error, new_tensor, new_norm_tensor = error_calc(tensor, norm_tensor, new_weights, new_factors, sparsity, mask)
if (new_rec_error / new_norm_tensor) < rec_errors[-1]:
factors, weights = new_factors, new_weights
tensor, norm_tensor = new_tensor, new_norm_tensor
unnorml_rec_error = new_rec_error
acc_fail = 0
if verbose:
print("Accepted line search jump of {}.".format(jump))
else:
unnorml_rec_error, tensor, norm_tensor = error_calc(tensor, norm_tensor, weights, factors, sparsity, mask, mttkrp)
acc_fail += 1
if verbose:
print("Line search failed for jump of {}.".format(jump))
if acc_fail == max_fail:
acc_pow += 1.0
acc_fail = 0
if verbose:
print("Reducing acceleration.")
rec_error = unnorml_rec_error / norm_tensor
rec_errors.append(rec_error)
if tol:
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print("iteration {}, reconstruction error: {}, decrease = {}, unnormalized = {}".format(iteration, rec_error, rec_error_decrease, unnorml_rec_error))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
cp_tensor = CPTensor((weights, factors))
if sparsity:
sparse_component = sparsify_tensor(tensor -\
cp_to_tensor((weights, factors)),\
sparsity)
cp_tensor = (cp_tensor, sparse_component)
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def initialize_cp(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
non_negative=False,
normalize_factors=False):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
rng = check_random_state(random_state)
if init == 'random':
# factors = [tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor))]
# kt = CPTensor((None, factors))
return random_cp(tl.shape(tensor),
rank,
normalise_factors=False,
random_state=rng,
**tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Put SVD initialization on the same scaling as the tensor in case normalize_factors=False
if mode == 0:
idx = min(rank, tl.shape(S)[0])
U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx])
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
# factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor))
# factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
# factor[:, :tensor.shape[mode]] = U
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
kt = CPTensor((None, factors))
elif isinstance(init, (tuple, list, CPTensor)):
# TODO: Test this
try:
kt = CPTensor(init)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance')
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(init))
if non_negative:
kt.factors = [tl.abs(f) for f in kt[1]]
if normalize_factors:
kt = cp_normalize(kt)
return kt
def randomised_parafac(tensor,
rank,
n_samples,
n_iter_max=100,
init='random',
svd='numpy_svd',
tol=10e-9,
max_stagnation=20,
random_state=None,
verbose=1):
"""Randomised CP decomposition via sampled ALS
Parameters
----------
tensor : ndarray
rank : int
number of components
n_samples : int
number of samples per ALS step
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
max_stagnation: int, optional, default is 0
if not zero, the maximum allowed number
of iterations with no decrease in fit
random_state : {None, int, np.random.RandomState}, default is None
verbose : int, optional
level of verbosity
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [3] Casey Battaglino, Grey Ballard and Tamara G. Kolda,
"A Practical Randomized CP Tensor Decomposition",
"""
rng = check_random_state(random_state)
factors = initialize_factors(tensor,
rank,
init=init,
svd=svd,
random_state=random_state)
rec_errors = []
n_dims = tl.ndim(tensor)
norm_tensor = tl.norm(tensor, 2)
min_error = 0
weights = tl.ones(rank, **tl.context(tensor))
for iteration in range(n_iter_max):
for mode in range(n_dims):
kr_prod, indices_list = sample_khatri_rao(factors,
n_samples,
skip_matrix=mode,
random_state=rng)
indices_list = [i.tolist() for i in indices_list]
# Keep all the elements of the currently considered mode
indices_list.insert(mode, slice(None, None, None))
# MXNet will not be happy if this is a list insteaf of a tuple
indices_list = tuple(indices_list)
if mode:
sampled_unfolding = tensor[indices_list]
else:
sampled_unfolding = tl.transpose(tensor[indices_list])
pseudo_inverse = tl.dot(tl.transpose(kr_prod), kr_prod)
factor = tl.dot(tl.transpose(kr_prod), sampled_unfolding)
factor = tl.transpose(tl.solve(pseudo_inverse, factor))
factors[mode] = factor
if max_stagnation or tol:
rec_error = tl.norm(tensor - kruskal_to_tensor(
(weights, factors)), 2) / norm_tensor
if not min_error or rec_error < min_error:
min_error = rec_error
stagnation = -1
stagnation += 1
rec_errors.append(rec_error)
if iteration > 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if (tol and abs(rec_errors[-2] - rec_errors[-1]) < tol) or \
(stagnation and (stagnation > max_stagnation)):
if verbose:
print('converged in {} iterations.'.format(iteration))
break
return KruskalTensor((weights, factors))
def sample_khatri_rao(matrices,
n_samples,
skip_matrix=None,
return_sampled_rows=False,
random_state=None):
"""Random subsample of the Khatri-Rao product of the given list of matrices
If one matrix only is given, that matrix is directly returned.
Parameters
----------
matrices : ndarray list
list of matrices with the same number of columns, i.e.::
for i in len(matrices):
matrices[i].shape = (n_i, m)
n_samples : int
number of samples to be taken from the Khatri-Rao product
skip_matrix : None or int, optional, default is None
if not None, index of a matrix to skip
random_state : None, int or numpy.random.RandomState
if int, used to set the seed of the random number generator
if numpy.random.RandomState, used to generate random_samples
returned_sampled_rows : bool, default is False
if True, also returns a list of the rows sampled from the full
khatri-rao product
Returns
-------
sampled_Khatri_Rao : ndarray
The sampled matricised tensor Khatri-Rao with `n_samples` rows
indices : tuple list
a list of indices sampled for each mode
indices_kr : int list
list of length `n_samples` containing the sampled row indices
"""
if random_state is None or not isinstance(random_state,
np.random.RandomState):
rng = check_random_state(random_state)
warnings.warn(
'You are creating a new random number generator at each call.\n'
'If you are calling sample_khatri_rao inside a loop this will be slow:'
' best to create a rng outside and pass it as argument (random_state=rng).'
)
else:
rng = random_state
if skip_matrix is not None:
matrices = [
matrices[i] for i in range(len(matrices)) if i != skip_matrix
]
rank = tl.shape(matrices[0])[1]
sizes = [tl.shape(m)[0] for m in matrices]
# For each matrix, randomly choose n_samples indices for which to compute the khatri-rao product
indices_list = [
rng.randint(0, tl.shape(m)[0], size=n_samples, dtype=int)
for m in matrices
]
if return_sampled_rows:
# Compute corresponding rows of the full khatri-rao product
indices_kr = np.zeros((n_samples), dtype=int)
for size, indices in zip(sizes, indices_list):
indices_kr = indices_kr * size + indices
# Compute the Khatri-Rao product for the chosen indices
sampled_kr = tl.ones((n_samples, rank), **tl.context(matrices[0]))
for indices, matrix in zip(indices_list, matrices):
sampled_kr = sampled_kr * matrix[indices, :]
if return_sampled_rows:
return sampled_kr, indices_list, indices_kr
else:
return sampled_kr, indices_list
def maxvol(A):
""" Find the rxr submatrix of maximal volume in A(nxr), n>=r
We want to decompose matrix A as
A = A[:,J] * (A[I,J])^-1 * A[I,:]
This algorithm helps us find this submatrix A[I,J] from A, which has the largest determinant.
We greedily find vector of max norm, and subtract its projection from the rest of rows.
Parameters
----------
A: matrix
The matrix to find maximal volume
Returns
-------
row_idx: list of int
is the list or rows of A forming the matrix with maximal volume,
A_inv: matrix
is the inverse of the matrix with maximal volume.
References
----------
S. A. Goreinov, I. V. Oseledets, D. V. Savostyanov, E. E. Tyrtyshnikov, N. L. Zamarashkin.
How to find a good submatrix.Goreinov, S. A., et al.
Matrix Methods: Theory, Algorithms and Applications: Dedicated to the Memory of Gene Golub. 2010. 247-256.
Ali Çivril, Malik Magdon-Ismail
On selecting a maximum volume sub-matrix of a matrix and related problems
Theoretical Computer Science. Volume 410, Issues 47–49, 6 November 2009, Pages 4801-4811
"""
(n, r) = tl.shape(A)
# The index of row of the submatrix
row_idx = tl.zeros(r)
# Rest of rows / unselected rows
rest_of_rows = tl.tensor(list(range(n)),dtype= tl.int64)
# Find r rows iteratively
i = 0
A_new = A
while i < r:
mask = list(range(tl.shape(A_new)[0]))
# Compute the square of norm of each row
rows_norms = tl.sum(A_new ** 2, axis=1)
# If there is only one row of A left, let's just return it. MxNet is not robust about this case.
if tl.shape(rows_norms) == ():
row_idx[i] = rest_of_rows
break
# If a row is 0, we delete it.
if any(rows_norms == 0):
zero_idx = tl.argmin(rows_norms,axis=0)
mask.pop(zero_idx)
rest_of_rows = rest_of_rows[mask]
A_new = A_new[mask,:]
continue
# Find the row of max norm
max_row_idx = tl.argmax(rows_norms, axis=0)
max_row = A[rest_of_rows[max_row_idx], :]
# Compute the projection of max_row to other rows
# projection a to b is computed as: <a,b> / sqrt(|a|*|b|)
projection = tl.dot(A_new, tl.transpose(max_row))
normalization = tl.sqrt(rows_norms[max_row_idx] * rows_norms)
# make sure normalization vector is of the same shape of projection (causing bugs for MxNet)
normalization = tl.reshape(normalization, tl.shape(projection))
projection = projection/normalization
# Subtract the projection from A_new: b <- b - a * projection
A_new = A_new - A_new * tl.reshape(projection, (tl.shape(A_new)[0], 1))
# Delete the selected row
mask.pop(max_row_idx)
A_new = A_new[mask,:]
# update the row_idx and rest_of_rows
row_idx[i] = rest_of_rows[max_row_idx]
rest_of_rows = rest_of_rows[mask]
i = i + 1
row_idx = tl.tensor(row_idx, dtype=tl.int64)
inverse = tl.solve(A[row_idx,:],
tl.eye(tl.shape(A[row_idx,:])[0], **tl.context(A)))
row_idx = tl.to_numpy(row_idx)
return row_idx, inverse
