def initialize_factors(tensor, rank, init='svd', svd='numpy_svd', random_state=None, non_negative=False):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : ndarray list
List of initialized factors of the CP decomposition where element `i`
is of shape (tensor.shape[i], rank)
"""
rng = check_random_state(random_state)
if init == 'random':
factors = [tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor))]
if non_negative:
return [tl.abs(f) for f in factors]
else:
return factors
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, _, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
# factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor))
# factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
# factor[:, :tensor.shape[mode]] = U
random_part = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
if non_negative:
factors.append(tl.abs(U[:, :rank]))
else:
factors.append(U[:, :rank])
return factors
raise ValueError('Initialization method "{}" not recognized'.format(init))
def initialize_tucker(tensor, rank, modes, random_state, init='svd', svd='numpy_svd', non_negative= False):
"""
Initialize core and factors used in `tucker`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
number of components
modes : int list
random_state : {None, int, np.random.RandomState}
init : {'svd', 'random', cptensor}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
core : ndarray
initialized core tensor
factors : list of factors
"""
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
# Initialisation
if init == 'svd':
factors = []
for index, mode in enumerate(modes):
U, S, V = svd_fun(unfold(tensor, mode), n_eigenvecs=rank[index], random_state=random_state)
if non_negative is True:
U = make_svd_non_negative(tensor, U, S, V, nntype="nndsvd")
factors.append(U[:, :rank[index]])
# The initial core approximation is needed here for the masking step
core = multi_mode_dot(tensor, factors, modes=modes, transpose=True)
if non_negative is True:
core = tl.abs(core)
elif init == 'random':
rng = tl.check_random_state(random_state)
core = tl.tensor(rng.random_sample(rank) + 0.01, **tl.context(tensor)) # Check this
factors = [tl.tensor(rng.random_sample(s), **tl.context(tensor)) for s in zip(tl.shape(tensor), rank)]
if non_negative is True:
factors = [tl.abs(f) for f in factors]
core = tl.abs(core)
else:
(core, factors) = init
return core, factors
def test_tucker():
"""Test for the Tucker decomposition"""
rng = check_random_state(1234)
tol_norm_2 = 10e-3
tol_max_abs = 10e-1
tensor = tl.tensor(rng.random_sample((3, 4, 3)))
core, factors = tucker(tensor, rank=None, n_iter_max=200, verbose=True)
reconstructed_tensor = tucker_to_tensor((core, factors))
norm_rec = tl.norm(reconstructed_tensor, 2)
norm_tensor = tl.norm(tensor, 2)
assert ((norm_rec - norm_tensor) / norm_rec < tol_norm_2)
# Test the max abs difference between the reconstruction and the tensor
assert (tl.max(tl.abs(reconstructed_tensor - tensor)) < tol_max_abs)
# Test the shape of the core and factors
ranks = [2, 3, 1]
core, factors = tucker(tensor, rank=ranks, n_iter_max=100, verbose=1)
for i, rank in enumerate(ranks):
assert_equal(factors[i].shape, (tensor.shape[i], ranks[i]),
err_msg="factors[{}].shape={}, expected {}".format(
i, factors[i].shape, (tensor.shape[i], ranks[i])))
assert_equal(tl.shape(core)[i],
rank,
err_msg="Core.shape[{}]={}, "
"expected {}".format(i, core.shape[i], rank))
# Random and SVD init should converge to a similar solution
tol_norm_2 = 10e-1
tol_max_abs = 10e-1
core_svd, factors_svd = tucker(tensor,
rank=[3, 4, 3],
n_iter_max=200,
init='svd',
verbose=1)
core_random, factors_random = tucker(tensor,
rank=[3, 4, 3],
n_iter_max=200,
init='random',
random_state=1234)
rec_svd = tucker_to_tensor((core_svd, factors_svd))
rec_random = tucker_to_tensor((core_random, factors_random))
error = tl.norm(rec_svd - rec_random, 2)
error /= tl.norm(rec_svd, 2)
assert_(error < tol_norm_2,
'norm 2 of difference between svd and random init too high')
assert_(
tl.max(tl.abs(rec_svd - rec_random)) < tol_max_abs,
'abs norm of difference between svd and random init too high')
def soft_sparsity_prox(tensor, threshold):
"""
Projects the input tensor on the set of tensors with l1 norm smaller than threshold, using Soft Thresholding.
Parameters
----------
tensor : ndarray
threshold :
Returns
-------
ndarray
References
----------
.. [1]: Schenker, C., Cohen, J. E., & Acar, E. (2020). A Flexible Optimization Framework for
Regularized Matrix-Tensor Factorizations with Linear Couplings.
IEEE Journal of Selected Topics in Signal Processing.
Notes
-----
.. math::
\\begin{equation}
\\lambda: prox_\\lambda (||tensor||_1) \\leq parameter
\\end{equation}
"""
return simplex_prox(tl.abs(tensor), threshold) * tl.sign(tensor)
def test_parafac_power_iteration():
"""Test for symmetric Parafac optimized with robust tensor power iterations"""
rng = check_random_state(1234)
tol_norm_2 = 10e-1
tol_max_abs = 10e-1
shape = (5, 3, 4)
rank = 4
tensor = random_cp(shape, rank=rank, full=True, random_state=rng)
ktensor = parafac_power_iteration(tensor,
rank=10,
n_repeat=10,
n_iteration=10)
rec = tl.cp_to_tensor(ktensor)
error = tl.norm(rec - tensor, 2) / tl.norm(tensor, 2)
assert_(
error < tol_norm_2,
f'Norm 2 of reconstruction error={error} higher than tol={tol_norm_2}.'
)
error = tl.max(tl.abs(rec - tensor))
assert_(
error < tol_max_abs,
f'Absolute norm of reconstruction error={error} higher than tol={tol_max_abs}.'
)
def initialize_factors(tensor, rank, random_state=None, non_negative=False):
"""Initialize factors used in `parafac`.
Factor matrices are initialized using `random_state`.
Parameters
----------
tensor : ndarray
rank : int
random_state: int
set to ensure reproducibility
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : ndarray list
List of initialized factors of the CP decomposition where element `i`
is of shape (tensor.shape[i], rank)
"""
rng = check_random_state(random_state)
factors = [
tl.tensor(rng.random_sample((tensor.shape[i], rank)),
**tl.context(tensor)) for i in range(tl.ndim(tensor))
]
if non_negative:
return [tl.abs(f) for f in factors]
else:
return factors
raise ValueError('Initialization method "{}" not recognized'.format(init))
def test_parafac2_slice_and_tensor_input():
rank = 3
random_parafac2_tensor = random_parafac2(shapes=[(15, 30)
for _ in range(25)],
rank=rank,
random_state=1234)
tensor = parafac2_to_tensor(random_parafac2_tensor)
slices = parafac2_to_slices(random_parafac2_tensor)
slice_rec = parafac2(slices,
rank,
random_state=1234,
normalize_factors=False,
n_iter_max=100)
slice_rec_tensor = parafac2_to_tensor(slice_rec)
tensor_rec = parafac2(tensor,
rank,
random_state=1234,
normalize_factors=False,
n_iter_max=100)
tensor_rec_tensor = parafac2_to_tensor(tensor_rec)
assert tl.max(tl.abs(slice_rec_tensor - tensor_rec_tensor)) < 1e-8
def test_partial_tucker():
"""Test for the Partial Tucker decomposition"""
rng = tl.check_random_state(1234)
tol_norm_2 = 10e-3
tol_max_abs = 10e-1
tensor = tl.tensor(rng.random_sample((3, 4, 3)))
modes = [1, 2]
core, factors = partial_tucker(tensor, modes, rank=None, n_iter_max=200, verbose=True)
reconstructed_tensor = multi_mode_dot(core, factors, modes=modes)
norm_rec = tl.norm(reconstructed_tensor, 2)
norm_tensor = tl.norm(tensor, 2)
assert_((norm_rec - norm_tensor)/norm_rec < tol_norm_2)
# Test the max abs difference between the reconstruction and the tensor
assert_(tl.max(tl.abs(norm_rec - norm_tensor)) < tol_max_abs)
# Test the shape of the core and factors
ranks = [3, 1]
core, factors = partial_tucker(tensor, modes=modes, rank=ranks, n_iter_max=100, verbose=1)
for i, rank in enumerate(ranks):
assert_equal(factors[i].shape, (tensor.shape[i+1], ranks[i]),
err_msg="factors[{}].shape={}, expected {}".format(
i, factors[i].shape, (tensor.shape[i+1], ranks[i])))
assert_equal(core.shape, [tensor.shape[0]]+ranks, err_msg="Core.shape={}, "
"expected {}".format(core.shape, [tensor.shape[0]]+ranks))
# Test random_state fixes the core and the factor matrices
core1, factors1 = partial_tucker(tensor, modes=modes, rank=ranks, random_state=0)
core2, factors2 = partial_tucker(tensor, modes=modes, rank=ranks, random_state=0)
assert_array_equal(core1, core2)
for factor1, factor2 in zip(factors1, factors2):
assert_array_equal(factor1, factor2)
def test_parafac2_normalize_factors():
rng = check_random_state(1234)
rank = 2 # Rank 2 so we only need to test rank of minimum and maximum
random_parafac2_tensor = random_parafac2(
shapes=[(15 + rng.randint(5), 30) for _ in range(25)],
rank=rank,
random_state=rng,
)
random_parafac2_tensor.factors[0] = random_parafac2_tensor.factors[0] + 0.1
norms = tl.ones(rank)
for factor in random_parafac2_tensor.factors:
norms = norms * tl.norm(factor, axis=0)
slices = parafac2_to_tensor(random_parafac2_tensor)
unnormalized_rec = parafac2(slices,
rank,
random_state=rng,
normalize_factors=False)
assert unnormalized_rec.weights[0] == 1
normalized_rec = parafac2(slices,
rank,
random_state=rng,
normalize_factors=True,
n_iter_max=1000)
assert tl.max(tl.abs(T.norm(normalized_rec.factors[0], axis=0) - 1)) < 1e-5
assert abs(tl.max(norms) -
tl.max(normalized_rec.weights)) / tl.max(norms) < 1e-2
assert abs(tl.min(norms) -
tl.min(normalized_rec.weights)) / tl.min(norms) < 1e-2
def test_symmetric_parafac_power_iteration(monkeypatch):
"""Test for symmetric Parafac optimized with robust tensor power iterations"""
rng = tl.check_random_state(1234)
tol_norm_2 = 10e-1
tol_max_abs = 10e-1
size = 5
rank = 4
true_factor = tl.tensor(rng.random_sample((size, rank)))
true_weights = tl.ones(rank)
tensor = tl.cp_to_tensor((true_weights, [true_factor] * 3))
weights, factor = symmetric_parafac_power_iteration(tensor,
rank=10,
n_repeat=10,
n_iteration=10)
rec = tl.cp_to_tensor((weights, [factor] * 3))
error = tl.norm(rec - tensor, 2)
error /= tl.norm(tensor, 2)
assert_(error < tol_norm_2, 'norm 2 of reconstruction higher than tol')
# Test the max abs difference between the reconstruction and the tensor
assert_(
tl.max(tl.abs(rec - tensor)) < tol_max_abs,
'abs norm of reconstruction error higher than tol')
assert_class_wrapper_correctly_passes_arguments(
monkeypatch,
symmetric_parafac_power_iteration,
SymmetricCP,
ignore_args={},
rank=3)
def test_parafac2_to_tensor():
rng = check_random_state(1234)
rank = 3
I = 25
J = 15
K = 30
weights, factors, projections = random_parafac2(shapes=[(J, K)] * I,
rank=rank,
random_state=rng)
constructed_tensor = parafac2_to_tensor((weights, factors, projections))
tensor_manual = T.zeros((I, J, K), **T.context(weights))
for i in range(I):
Bi = T.dot(projections[i], factors[1])
for j in range(J):
for k in range(K):
for r in range(rank):
tensor_manual = tl.index_update(
tensor_manual, tl.index[i, j,
k], tensor_manual[i, j, k] +
factors[0][i][r] * Bi[j][r] * factors[2][k][r])
assert_(tl.max(tl.abs(constructed_tensor - tensor_manual)) < 1e-6)
def sparsify_tensor(tensor, card):
"""Zeros out all elements in the `tensor` except `card` elements with maximum absolute values.
Parameters
----------
tensor : ndarray
card : int
Desired number of non-zero elements in the `tensor`
Returns
-------
ndarray of shape tensor.shape
"""
if card >= np.prod(tensor.shape):
return tensor
bound = tl.sort(tl.abs(tensor), axis = None)[-card]
return tl.where(tl.abs(tensor) < bound, tl.zeros(tensor.shape, **tl.context(tensor)), tensor)
def test_cp_norm():
"""Test for cp_norm
"""
shape = (8, 5, 6, 4)
rank = 25
cp_tensor = random_cp(shape=shape, rank=rank,
full=False, normalise_factors=True)
tol = 10e-5
rec = tl.cp_to_tensor(cp_tensor)
true_res = tl.norm(rec, 2)
res = cp_norm(cp_tensor)
assert_(tl.abs(true_res - res) <= tol)
def test_kruskal_norm():
"""Test for kruskal_norm
"""
shape = (8, 5, 6, 4)
rank = 25
kruskal_tensor = random_kruskal(shape=shape,
rank=rank,
full=False,
normalise_factors=True)
tol = 10e-5
rec = tl.kruskal_to_tensor(kruskal_tensor)
true_res = tl.norm(rec, 2)
res = kruskal_norm(kruskal_tensor)
assert_(tl.to_numpy(tl.abs(true_res - res)) <= tol)
def tol_check():
above_tol = tf.greater(tl.abs(dparams.variation), tol)
# Print convergence iterations
if verbose:
print_op = tf.cond(above_tol,
true_fn=lambda: tf.constant(True),
false_fn=lambda: tf.print(
'converged in',
dparams.index,
'iterations.',
output_stream=sys.stdout))
with tf.control_dependencies([print_op]):
above_tol = tf.identity(above_tol)
return tf.logical_and(condition, above_tol)
def test_pad_by_zeros():
"""Test that if we pad a tensor by zeros, then it doesn't change.
This failed for TensorFlow at some point.
"""
rng = check_random_state(1234)
rank = 3
I = 25
J = 15
K = 30
weights, factors, projections = random_parafac2(shapes=[(J, K)] * I,
rank=rank,
random_state=rng)
constructed_tensor = parafac2_to_tensor((weights, factors, projections))
padded_tensor = _pad_by_zeros(constructed_tensor)
assert_(tl.max(tl.abs(constructed_tensor - padded_tensor)) < 1e-10)
def test_partial_tucker():
"""Test for the Partial Tucker decomposition"""
rng = check_random_state(1234)
tol_norm_2 = 10e-3
tol_max_abs = 10e-1
tensor = tl.tensor(rng.random_sample((3, 4, 3)))
modes = [1, 2]
for svd_func in tl.SVD_FUNS:
if tl.get_backend() == 'tensorflow_graph' and svd_func == 'numpy_svd':
continue # TODO(craymichael)
core, factors = partial_tucker(tensor,
modes,
rank=None,
n_iter_max=200,
svd=svd_func,
verbose=True)
reconstructed_tensor = multi_mode_dot(core, factors, modes=modes)
norm_rec = tl.to_numpy(tl.norm(reconstructed_tensor, 2))
norm_tensor = tl.to_numpy(tl.norm(tensor, 2))
assert_((norm_rec - norm_tensor) / norm_rec < tol_norm_2)
# Test the max abs difference between the reconstruction and the tensor
assert_(
tl.to_numpy(tl.max(tl.abs(norm_rec - norm_tensor))) < tol_max_abs)
# Test the shape of the core and factors
ranks = [3, 1]
core, factors = partial_tucker(tensor,
modes=modes,
rank=ranks,
n_iter_max=100,
svd=svd_func,
verbose=1)
for i, rank in enumerate(ranks):
assert_equal(
factors[i].shape, (tensor.shape[i + 1], ranks[i]),
err_msg="factors[{}].shape={}, expected {} (svd=\"{}\")".
format(i, factors[i].shape, (tensor.shape[i + 1], ranks[i]),
svd_func))
assert_equal(core.shape, [tensor.shape[0]] + ranks,
err_msg="Core.shape={}, "
"expected {} (svd=\"{}\")".format(
core.shape, [tensor.shape[0]] + ranks, svd_func))
def _congruence_coefficient_slow(A, B, absolute_value):
_, r = tl.shape(A)
corr_matrix = _tucker_congruence(A, B)
if absolute_value:
corr_matrix = tl.abs(corr_matrix)
corr_matrix = tl.to_numpy(corr_matrix)
best_corr = None
best_permutation = None
for permutation in itertools.permutations(range(r)):
corr = 0
for i, j in zip(range(r), permutation):
corr += corr_matrix[i, j] / r
if best_corr is None or corr > best_corr:
best_corr = corr
best_permutation = permutation
return best_corr, best_permutation
def soft_thresholding(tensor, threshold):
"""Soft-thresholding operator
sign(tensor) * max[abs(tensor) - threshold, 0]
Parameters
----------
tensor : ndarray
threshold : float or ndarray with shape tensor.shape
* If float the threshold is applied to the whole tensor
* If ndarray, one threshold is applied per elements, 0 values are ignored
Returns
-------
ndarray
thresholded tensor on which the operator has been applied
Examples
--------
Basic shrinkage
>>> import tensorly.backend as T
>>> from tensorly.tenalg.proximal import soft_thresholding
>>> tensor = tl.tensor([[1, -2, 1.5], [-4, 3, -0.5]])
>>> soft_thresholding(tensor, 1.1)
array([[ 0. , -0.9, 0.4],
[-2.9, 1.9, 0. ]])
Example with missing values
>>> mask = tl.tensor([[0, 0, 1], [1, 0, 1]])
>>> soft_thresholding(tensor, mask*1.1)
array([[ 1. , -2. , 0.4],
[-2.9, 3. , 0. ]])
See also
--------
inplace_soft_thresholding : Inplace version of the soft-thresholding operator
svd_thresholding : SVD-thresholding operator
"""
return tl.sign(tensor) * tl.clip(tl.abs(tensor) - threshold, a_min=0)
def hard_thresholding(tensor, number_of_non_zero):
"""
Proximal operator of the l0 ``norm''
Keeps greater "number_of_non_zero" elements untouched and sets other elements to zero.
Parameters
----------
tensor : ndarray
number_of_non_zero : int
Returns
-------
ndarray
Thresholded tensor on which the operator has been applied
"""
tensor_vec = tl.copy(tl.tensor_to_vec(tensor))
sorted_indices = tl.argsort(tl.argsort(tl.abs(tensor_vec),
axis=0,
descending=True),
axis=0)
return tl.reshape(
tl.where(sorted_indices < number_of_non_zero, tensor_vec,
tl.tensor(0, **tl.context(tensor_vec))), tensor.shape)
def coupled_matrix_tensor_3d_factorization(tensor_3d,
matrix,
rank,
init='svd',
n_iter_max=100,
normalize_factors=False):
"""
Calculates a coupled matrix and tensor factorization of 3rd order tensor and matrix which are
coupled in first mode.
Assume you have tensor_3d = [[lambda; A, B, C]] and matrix = [[gamma; A, V]], which are
coupled in 1st mode. With coupled matrix and tensor factorization (CTMF), the normalized
factor matrices A, B, C for the CP decomposition of X, the normalized matrix V and the
weights lambda_ and gamma are found. This implementation only works for a coupling in the
first mode.
Solution is found via alternating least squares (ALS) as described in Figure 5 of
@article{acar2011all,
title={All-at-once optimization for coupled matrix and tensor factorizations},
author={Acar, Evrim and Kolda, Tamara G and Dunlavy, Daniel M},
journal={arXiv preprint arXiv:1105.3422},
year={2011}
}
Notes
-----
In the paper, the columns of the factor matrices are not normalized and therefore weights are
not included in the algorithm.
Parameters
----------
tensor_3d : tl.tensor or CP tensor
3rd order tensor X = [[A, B, C]]
matrix : tl.tensor or CP tensor
matrix that is coupled with tensor in first mode: Y = [[A, V]]
rank : int
rank for CP decomposition of X
Returns
-------
tensor_3d_pred : CPTensor
tensor_3d_pred = [[lambda; A,B,C]]
matrix_pred : CPTensor
matrix_pred = [[gamma; A,V]]
rec_errors : list
contains the reconstruction error of each iteration:
error = 1 / 2 * | X - [[ lambda_; A, B, C ]] | ^ 2 + 1 / 2 * | Y - [[ gamma; A, V ]] | ^ 2
Examples
--------
A = tl.tensor([[1, 2], [3, 4]])
B = tl.tensor([[1, 0], [0, 2]])
C = tl.tensor([[2, 0], [0, 1]])
V = tl.tensor([[2, 0], [0, 1]])
R = 2
X = (None, [A, B, C])
Y = (None, [A, V])
tensor_3d_pred, matrix_pred = cmtf_als_for_third_order_tensor(X, Y, R)
"""
rank = validate_cp_rank(tl.shape(tensor_3d), rank=rank)
# initialize values
tensor_cp = initialize_cp(tensor_3d, rank, init=init)
rec_errors = []
# alternating least squares
# note that the order of the khatri rao product is reversed since tl.unfold has another order
# than assumed in paper
for iteration in range(n_iter_max):
V = tl.transpose(tl.lstsq(tensor_cp.factors[0], matrix)[0])
# Loop over modes of the tensor
for ii in range(tl.ndim(tensor_3d)):
kr = khatri_rao(tensor_cp.factors, skip_matrix=ii)
unfolded = tl.unfold(tensor_3d, ii)
# If we are at the coupled mode, concat the matrix
if ii == 0:
kr = tl.concatenate((kr, V), axis=0)
unfolded = tl.concatenate((unfolded, matrix), axis=1)
tensor_cp.factors[ii] = tl.transpose(
tl.lstsq(kr, tl.transpose(unfolded))[0])
error_new = tl.norm(tensor_3d - cp_to_tensor(tensor_cp))**2 + tl.norm(
matrix - cp_to_tensor((None, [tensor_cp.factors[0], V])))**2
if iteration > 0 and (tl.abs(error_new - error_old) / error_old <= 1e-8
or error_new < 1e-5):
break
error_old = error_new
rec_errors.append(error_new)
matrix_pred = CPTensor((None, [tensor_cp.factors[0], V]))
if normalize_factors:
tensor_cp = cp_normalize(tensor_cp)
matrix_pred = cp_normalize(matrix_pred)
return tensor_cp, matrix_pred, rec_errors
def parafac2(tensor_slices,
rank,
n_iter_max=100,
init='random',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
n_iter_parafac=5):
r"""PARAFAC2 decomposition [1]_ of a third order tensor via alternating least squares (ALS)
Computes a rank-`rank` PARAFAC2 decomposition of the third-order tensor defined by
`tensor_slices`. The decomposition is on the form :math:`(A [B_i] C)` such that the
i-th frontal slice, :math:`X_i`, of :math:`X` is given by
.. math::
X_i = B_i diag(a_i) C^T,
where :math:`diag(a_i)` is the diagonal matrix whose nonzero entries are equal to
the :math:`i`-th row of the :math:`I \times R` factor matrix :math:`A`, :math:`B_i`
is a :math:`J_i \times R` factor matrix such that the cross product matrix :math:`B_{i_1}^T B_{i_1}`
is constant for all :math:`i`, and :math:`C` is a :math:`K \times R` factor matrix.
To compute this decomposition, we reformulate the expression for :math:`B_i` such that
.. math::
B_i = P_i B,
where :math:`P_i` is a :math:`J_i \times R` orthogonal matrix and :math:`B` is a
:math:`R \times R` matrix.
An alternative formulation of the PARAFAC2 decomposition is that the tensor element
:math:`X_{ijk}` is given by
.. math::
X_{ijk} = \sum_{r=1}^R A_{ir} B_{ijr} C_{kr},
with the same constraints hold for :math:`B_i` as above.
Parameters
----------
tensor_slices : ndarray or list of ndarrays
Either a third order tensor or a list of second order tensors that may have different number of rows.
Note that the second mode factor matrices are allowed to change over the first mode, not the
third mode as some other implementations use (see note below).
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random', CPTensor, Parafac2Tensor}
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : bool (optional)
If True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors. Note that
there may be some inaccuracies in the component weights.
tol : float, optional
(Default: 1e-8) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
n_iter_parafac: int, optional
Number of PARAFAC iterations to perform for each PARAFAC2 iteration
Returns
-------
Parafac2Tensor : (weight, factors, projection_matrices)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
* projection_matrices : List of projection matrices used to create evolving
factors.
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] Kiers, H.A.L., ten Berge, J.M.F. and Bro, R. (1999),
PARAFAC2—Part I. A direct fitting algorithm for the PARAFAC2 model.
J. Chemometrics, 13: 275-294.
Notes
-----
This formulation of the PARAFAC2 decomposition is slightly different from the one in [1]_.
The difference lies in that here, the second mode changes over the first mode, whereas in
[1]_, the second mode changes over the third mode. We made this change since that means
that the function accept both lists of matrices and a single nd-array as input without
any reordering of the modes.
"""
weights, factors, projections = initialize_decomposition(
tensor_slices, rank, init=init, svd=svd, random_state=random_state)
rec_errors = []
norm_tensor = tl.sqrt(
sum(tl.norm(tensor_slice, 2)**2 for tensor_slice in tensor_slices))
svd_fun = _get_svd(svd)
projected_tensor = tl.zeros([factor.shape[0] for factor in factors],
**T.context(factors[0]))
for iteration in range(n_iter_max):
if verbose:
print("Starting iteration", iteration)
factors[1] = factors[1] * T.reshape(weights, (1, -1))
weights = T.ones(weights.shape, **tl.context(tensor_slices[0]))
projections = _compute_projections(tensor_slices,
factors,
svd_fun,
out=projections)
projected_tensor = _project_tensor_slices(tensor_slices,
projections,
out=projected_tensor)
_, factors = parafac(projected_tensor,
rank,
n_iter_max=n_iter_parafac,
init=(weights, factors),
svd=svd,
orthogonalise=False,
verbose=verbose,
return_errors=False,
normalize_factors=False,
mask=None,
random_state=random_state,
tol=1e-100)
if normalize_factors:
new_factors = []
for factor in factors:
norms = T.norm(factor, axis=0)
norms = tl.where(
tl.abs(norms) <= tl.eps(factor.dtype),
tl.ones(tl.shape(norms), **tl.context(factors[0])), norms)
weights = weights * norms
new_factors.append(factor / (tl.reshape(norms, (1, -1))))
factors = new_factors
if tol:
rec_error = _parafac2_reconstruction_error(
tensor_slices, (weights, factors, projections))
rec_error /= norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('PARAFAC2 reconstruction error={}, variation={}.'.
format(rec_errors[-1],
rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('PARAFAC2 reconstruction error={}'.format(
rec_errors[-1]))
parafac2_tensor = Parafac2Tensor((weights, factors, projections))
if return_errors:
return parafac2_tensor, rec_errors
else:
return parafac2_tensor
def non_negative_parafac_hals(tensor,
rank,
n_iter_max=100,
init="svd",
svd='numpy_svd',
tol=10e-8,
random_state=None,
sparsity_coefficients=None,
fixed_modes=None,
nn_modes='all',
exact=False,
normalize_factors=False,
verbose=False,
return_errors=False,
cvg_criterion='abs_rec_error'):
"""
Non-negative CP decomposition via HALS
Uses Hierarchical ALS (Alternating Least Squares) which updates each factor column-wise (one column at a time while keeping all other columns fixed), see [1]_
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
Default: 1e-8
random_state : {None, int, np.random.RandomState}
sparsity_coefficients: array of float (of length the number of modes)
The sparsity coefficients on each factor.
If set to None, the algorithm is computed without sparsity
Default: None,
fixed_modes: array of integers (between 0 and the number of modes)
Has to be set not to update a factor, 0 and 1 for U and V respectively
Default: None
nn_modes: None, 'all' or array of integers (between 0 and the number of modes)
Used to specify which modes to impose non-negativity constraints on.
If 'all', then non-negativity is imposed on all modes.
Default: 'all'
exact: If it is True, the algorithm gives a results with high precision but it needs high computational cost.
If it is False, the algorithm gives an approximate solution
Default: False
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
verbose: boolean
Indicates whether the algorithm prints the successive
reconstruction errors or not
Default: False
return_errors: boolean
Indicates whether the algorithm should return all reconstruction errors
and computation time of each iteration or not
Default: False
cvg_criterion : {'abs_rec_error', 'rec_error'}, optional
Stopping criterion for ALS, works if `tol` is not None.
If 'rec_error', ALS stops at current iteration if ``(previous rec_error - current rec_error) < tol``.
If 'abs_rec_error', ALS terminates when `|previous rec_error - current rec_error| < tol`.
sparsity : float or int
random_state : {None, int, np.random.RandomState}
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
errors: list
A list of reconstruction errors at each iteration of the algorithm.
References
----------
.. [1]: N. Gillis and F. Glineur, Accelerated Multiplicative Updates and
Hierarchical ALS Algorithms for Nonnegative Matrix Factorization,
Neural Computation 24 (4): 1085-1105, 2012.
"""
weights, factors = initialize_nn_cp(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
normalize_factors=normalize_factors)
norm_tensor = tl.norm(tensor, 2)
n_modes = tl.ndim(tensor)
if sparsity_coefficients is None or isinstance(sparsity_coefficients,
float):
sparsity_coefficients = [sparsity_coefficients] * n_modes
if fixed_modes is None:
fixed_modes = []
if nn_modes == 'all':
nn_modes = set(range(n_modes))
elif nn_modes is None:
nn_modes = set()
# Avoiding errors
for fixed_value in fixed_modes:
sparsity_coefficients[fixed_value] = None
for mode in range(n_modes):
if sparsity_coefficients[mode] is not None:
warnings.warn(
"Sparsity coefficient is ignored in unconstrained modes.")
# Generating the mode update sequence
modes = [mode for mode in range(n_modes) if mode not in fixed_modes]
# initialisation - declare local varaibles
rec_errors = []
# Iteratation
for iteration in range(n_iter_max):
# One pass of least squares on each updated mode
for mode in modes:
# Computing Hadamard of cross-products
pseudo_inverse = tl.tensor(tl.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
pseudo_inverse = tl.reshape(weights,
(-1, 1)) * pseudo_inverse * tl.reshape(
weights, (1, -1))
mttkrp = unfolding_dot_khatri_rao(tensor, (weights, factors), mode)
if mode in nn_modes:
# Call the hals resolution with nnls, optimizing the current mode
nn_factor, _, _, _ = hals_nnls(
tl.transpose(mttkrp),
pseudo_inverse,
tl.transpose(factors[mode]),
n_iter_max=100,
sparsity_coefficient=sparsity_coefficients[mode],
exact=exact)
factors[mode] = tl.transpose(nn_factor)
else:
factor = tl.solve(tl.transpose(pseudo_inverse),
tl.transpose(mttkrp))
factors[mode] = tl.transpose(factor)
if normalize_factors and mode != modes[-1]:
weights, factors = cp_normalize((weights, factors))
if tol:
factors_norm = cp_norm((weights, factors))
iprod = tl.sum(tl.sum(mttkrp * factors[-1], axis=0))
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print(
"iteration {}, reconstruction error: {}, decrease = {}"
.format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(
iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
if normalize_factors:
weights, factors = cp_normalize((weights, factors))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def initialize_nn_cp(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
normalize_factors=False,
nntype='nndsvda'):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
nntype : {'nndsvd', 'nndsvda'}
Whether to fill small values with 0.0 (nndsvd), or the tensor mean (nndsvda, default).
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
rng = tl.check_random_state(random_state)
if init == 'random':
kt = random_cp(tl.shape(tensor),
rank,
normalise_factors=False,
random_state=rng,
**tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, V = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Apply nnsvd to make non-negative
U = make_svd_non_negative(tensor, U, S, V, nntype)
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
kt = CPTensor((None, factors))
# If the initialisation is a precomputed decomposition, we double check its validity and return it
elif isinstance(init, (tuple, list, CPTensor)):
# TODO: Test this
try:
kt = CPTensor(init)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance')
return kt
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(init))
# Make decomposition feasible by taking the absolute value of all factor matrices
kt.factors = [tl.abs(f) for f in kt[1]]
if normalize_factors:
kt = cp_normalize(kt)
return kt
def non_negative_tucker(tensor,
rank,
n_iter_max=10,
init='svd',
tol=10e-5,
random_state=None,
verbose=False,
return_errors=False,
normalize_factors=False):
"""Non-negative Tucker decomposition
Iterative multiplicative update, see [2]_
Parameters
----------
tensor : ``ndarray``
rank : None, int or int list
size of the core tensor, ``(len(ranks) == tensor.ndim)``
if int, the same rank is used for all modes
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}
random_state : {None, int, np.random.RandomState}
verbose : int , optional
level of verbosity
ranks : None or int list
size of the core tensor
normalize_factors : if True, aggregates the core which will contain the norms of the factors.
Returns
-------
core : ndarray
positive core of the Tucker decomposition
has shape `ranks`
factors : ndarray list
list of factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [2] Yong-Deok Kim and Seungjin Choi,
"Non-negative tucker decomposition",
IEEE Conference on Computer Vision and Pattern Recognition s(CVPR),
pp 1-8, 2007
"""
rank = validate_tucker_rank(tl.shape(tensor), rank=rank)
epsilon = 10e-12
# Initialisation
if init == 'svd':
core, factors = tucker(tensor, rank)
nn_factors = [tl.abs(f) for f in factors]
nn_core = tl.abs(core)
else:
rng = tl.check_random_state(random_state)
core = tl.tensor(rng.random_sample(rank) + 0.01,
**tl.context(tensor)) # Check this
factors = [
tl.tensor(rng.random_sample(s), **tl.context(tensor))
for s in zip(tl.shape(tensor), rank)
]
nn_factors = [tl.abs(f) for f in factors]
nn_core = tl.abs(core)
norm_tensor = tl.norm(tensor, 2)
rec_errors = []
for iteration in range(n_iter_max):
for mode in range(tl.ndim(tensor)):
B = tucker_to_tensor((nn_core, nn_factors), skip_factor=mode)
B = tl.transpose(unfold(B, mode))
numerator = tl.dot(unfold(tensor, mode), B)
numerator = tl.clip(numerator, a_min=epsilon, a_max=None)
denominator = tl.dot(nn_factors[mode], tl.dot(tl.transpose(B), B))
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
nn_factors[mode] *= numerator / denominator
numerator = tucker_to_tensor((tensor, nn_factors),
transpose_factors=True)
numerator = tl.clip(numerator, a_min=epsilon, a_max=None)
for i, f in enumerate(nn_factors):
if i:
denominator = mode_dot(denominator, tl.dot(tl.transpose(f), f),
i)
else:
denominator = mode_dot(nn_core, tl.dot(tl.transpose(f), f), i)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
nn_core *= numerator / denominator
rec_error = tl.norm(tensor - tucker_to_tensor(
(nn_core, nn_factors)), 2) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1 and verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if iteration > 1 and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
if normalize_factors:
nn_core, nn_factors = tucker_normalize((nn_core, nn_factors))
tensor = TuckerTensor((nn_core, nn_factors))
if return_errors:
return tensor, rec_errors
else:
return tensor
def non_negative_parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
tol=10e-7,
random_state=None,
verbose=0,
normalize_factors=False,
return_errors=False,
mask=None,
orthogonalise=False,
cvg_criterion='abs_rec_error',
fixed_modes=[]):
"""
Non-negative CP decomposition
Uses multiplicative updates, see [2]_
This is the same as parafac(non_negative=True).
Parameters
----------
tensor : ndarray
rank : int
number of components
n_iter_max : int
maximum number of iteration
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
tol : float, optional
tolerance: the algorithm stops when the variation in
the reconstruction error is less than the tolerance
random_state : {None, int, np.random.RandomState}
verbose : int, optional
level of verbosity
fixed_modes : list, default is []
A list of modes for which the initial value is not modified.
The last mode cannot be fixed due to error computation.
Returns
-------
factors : ndarray list
list of positive factors of the CP decomposition
element `i` is of shape ``(tensor.shape[i], rank)``
References
----------
.. [2] Amnon Shashua and Tamir Hazan,
"Non-negative tensor factorization with applications to statistics and computer vision",
In Proceedings of the International Conference on Machine Learning (ICML),
pp 792-799, ICML, 2005
"""
epsilon = 10e-12
rank = validate_cp_rank(tl.shape(tensor), rank=rank)
if mask is not None and init == "svd":
message = "Masking occurs after initialization. Therefore, random initialization is recommended."
warnings.warn(message, Warning)
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
weights, factors = initialize_cp(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=True,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
if tl.ndim(tensor) - 1 in fixed_modes:
warnings.warn(
'You asked for fixing the last mode, which is not supported while tol is fixed.\n The last mode will not be fixed. Consider using tl.moveaxis()'
)
fixed_modes.remove(tl.ndim(tensor) - 1)
modes_list = [
mode for mode in range(tl.ndim(tensor)) if mode not in fixed_modes
]
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
for i, f in enumerate(factors):
if min(tl.shape(f)) >= rank:
factors[i] = tl.abs(tl.qr(f)[0])
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in modes_list:
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
if mask is not None:
tensor = tensor * mask + tl.cp_to_tensor(
(None, factors), mask=1 - mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
factors[mode] = factor
if normalize_factors:
weights, factors = cp_normalize((weights, factors))
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = cp_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
rec_error_decrease = rec_errors[-2] - rec_errors[-1]
if verbose:
print(
"iteration {}, reconstraction error: {}, decrease = {}"
.format(iteration, rec_error, rec_error_decrease))
if cvg_criterion == 'abs_rec_error':
stop_flag = abs(rec_error_decrease) < tol
elif cvg_criterion == 'rec_error':
stop_flag = rec_error_decrease < tol
else:
raise TypeError("Unknown convergence criterion")
if stop_flag:
if verbose:
print("PARAFAC converged after {} iterations".format(
iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
cp_tensor = CPTensor((weights, factors))
if return_errors:
return cp_tensor, rec_errors
else:
return cp_tensor
def initialize_cp(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
non_negative=False,
normalize_factors=False):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
rng = check_random_state(random_state)
if init == 'random':
# factors = [tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor))]
# kt = CPTensor((None, factors))
return random_cp(tl.shape(tensor),
rank,
normalise_factors=False,
random_state=rng,
**tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Put SVD initialization on the same scaling as the tensor in case normalize_factors=False
if mode == 0:
idx = min(rank, tl.shape(S)[0])
U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx])
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
# factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor))
# factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
# factor[:, :tensor.shape[mode]] = U
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
kt = CPTensor((None, factors))
elif isinstance(init, (tuple, list, CPTensor)):
# TODO: Test this
try:
kt = CPTensor(init)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance')
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(init))
if non_negative:
kt.factors = [tl.abs(f) for f in kt[1]]
if normalize_factors:
kt = cp_normalize(kt)
return kt
def __initialize_factors(self,
tensor,
svd='numpy_svd',
non_negative=False,
custom=None):
"""Initialize random or SVD-guided factors for TCA depending on TCA type
Parameters
----------
tensor : torch.Tensor
The tensor of activity of N neurons, T timepoints and K trials of shape N, T, K
svd : str, optional
Type of SVD algorithm to use (default is numpy_svd)
non_negative : bool, optional
A flag used to specify if factors generated must be strictyl positive (default is False)
custom : int, optional
A flag used to specify which factor should be strictly positive for 'custom parafac' (default is None)
Raises
------
ValueError
If svd does not contain a valid SVD algorithm reference
If self.init variable does not contain a valid intialization method
Returns
-------
list
List of initialized tensors
"""
rng = tensorly.random.check_random_state(self.random_state)
if self.init == 'random':
if custom:
factors = [
tl.tensor(
rng.random_sample(
(tensor.shape[i], self.rank)) * 2 - 1,
**tl.context(tensor)) for i in range(self.dimension)
]
factors = [
f if int(i) == int(custom) else tl.abs(f)
for i, f in enumerate(factors)
]
elif non_negative:
factors = [
tl.tensor(rng.random_sample((tensor.shape[i], self.rank)),
**tl.context(tensor))
for i in range(self.dimension)
]
factors = [
tl.abs(f) for f in factors
] # See if this line is useful depending on random function used
else:
factors = [
tl.tensor(
rng.random_sample(
(tensor.shape[i], self.rank)) * 2 - 1,
**tl.context(tensor)) for i in range(self.dimension)
]
return factors
elif self.init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, *_ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
if tensor.shape[mode] < rank:
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
if non_negative or custom == mode:
factors.append(tl.abs(U[:, :rank]))
else:
factors.append(U[:, :rank])
return factors
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(self.init))
def parafac(tensor,
rank,
n_iter_max=100,
init='svd',
svd='numpy_svd',
normalize_factors=False,
tol=1e-8,
orthogonalise=False,
random_state=None,
verbose=0,
return_errors=False,
non_negative=False,
mask=None):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [|weights; factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
init : {'svd', 'random'}, optional
Type of factor matrix initialization. See `initialize_factors`.
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
normalize_factors : if True, aggregate the weights of each factor in a 1D-tensor
of shape (rank, ), which will contain the norms of the factors
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
non_negative : bool, optional
Perform non_negative PARAFAC. See :func:`non_negative_parafac`.
mask : ndarray
array of booleans with the same shape as ``tensor`` should be 0 where
the values are missing and 1 everywhere else. Note: if tensor is
sparse, then mask should also be sparse with a fill value of 1 (or
True). Allows for missing values [2]_
Returns
-------
KruskalTensor : (weight, factors)
* weights : 1D array of shape (rank, )
all ones if normalize_factors is False (default),
weights of the (normalized) factors otherwise
* factors : List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] T.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
.. [2] Tomasi, Giorgio, and Rasmus Bro. "PARAFAC and missing values."
Chemometrics and Intelligent Laboratory Systems 75.2 (2005): 163-180.
"""
epsilon = 10e-12
if orthogonalise and not isinstance(orthogonalise, int):
orthogonalise = n_iter_max
factors = initialize_factors(tensor,
rank,
init=init,
svd=svd,
random_state=random_state,
non_negative=non_negative,
normalize_factors=normalize_factors)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
weights = tl.ones(rank, **tl.context(tensor))
for iteration in range(n_iter_max):
if orthogonalise and iteration <= orthogonalise:
factors = [
tl.qr(f)[0] if min(tl.shape(f)) >= rank else f
for i, f in enumerate(factors)
]
if verbose > 1:
print("Starting iteration", iteration + 1)
for mode in range(tl.ndim(tensor)):
if verbose > 1:
print("Mode", mode, "of", tl.ndim(tensor))
if non_negative:
accum = 1
# khatri_rao(factors).tl.dot(khatri_rao(factors))
# simplifies to multiplications
sub_indices = [i for i in range(len(factors)) if i != mode]
for i, e in enumerate(sub_indices):
if i:
accum *= tl.dot(tl.transpose(factors[e]), factors[e])
else:
accum = tl.dot(tl.transpose(factors[e]), factors[e])
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.conj(tl.transpose(factor)), factor)
if mask is not None:
tensor = tensor * mask + tl.kruskal_to_tensor(
(None, factors), mask=1 - mask)
mttkrp = unfolding_dot_khatri_rao(tensor, (None, factors), mode)
if non_negative:
numerator = tl.clip(mttkrp, a_min=epsilon, a_max=None)
denominator = tl.dot(factors[mode], accum)
denominator = tl.clip(denominator, a_min=epsilon, a_max=None)
factor = factors[mode] * numerator / denominator
else:
factor = tl.transpose(
tl.solve(tl.conj(tl.transpose(pseudo_inverse)),
tl.transpose(mttkrp)))
if normalize_factors:
weights = tl.norm(factor, order=2, axis=0)
weights = tl.where(
tl.abs(weights) <= tl.eps(tensor.dtype),
tl.ones(tl.shape(weights), **tl.context(factors[0])),
weights)
factor = factor / (tl.reshape(weights, (1, -1)))
factors[mode] = factor
if tol:
# ||tensor - rec||^2 = ||tensor||^2 + ||rec||^2 - 2*<tensor, rec>
factors_norm = kruskal_norm((weights, factors))
# mttkrp and factor for the last mode. This is equivalent to the
# inner product <tensor, factorization>
iprod = tl.sum(tl.sum(mttkrp * factor, axis=0) * weights)
rec_error = tl.sqrt(
tl.abs(norm_tensor**2 + factors_norm**2 -
2 * iprod)) / norm_tensor
rec_errors.append(rec_error)
if iteration >= 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
else:
if verbose:
print('reconstruction error={}'.format(rec_errors[-1]))
kruskal_tensor = KruskalTensor((weights, factors))
if return_errors:
return kruskal_tensor, rec_errors
else:
return kruskal_tensor
def test_non_negative_parafac_hals():
"""Test for non-negative PARAFAC HALS
TODO: more rigorous test
"""
tol_norm_2 = 10e-1
tol_max_abs = 1
rng = tl.check_random_state(1234)
tensor = tl.tensor(rng.random_sample((3, 3, 3)) + 1)
res = parafac(tensor, rank=3, n_iter_max=120)
nn_res = non_negative_parafac_hals(tensor,
rank=3,
n_iter_max=100,
tol=10e-4,
init='svd',
verbose=0)
# Make sure all components are positive
_, nn_factors = nn_res
for factor in nn_factors:
assert_(tl.all(factor >= 0))
reconstructed_tensor = tl.cp_to_tensor(res)
nn_reconstructed_tensor = tl.cp_to_tensor(nn_res)
error = tl.norm(reconstructed_tensor - nn_reconstructed_tensor, 2)
error /= tl.norm(reconstructed_tensor, 2)
assert_(error < tol_norm_2, 'norm 2 of reconstruction higher than tol')
# Test the max abs difference between the reconstruction and the tensor
assert_(
tl.max(tl.abs(reconstructed_tensor - nn_reconstructed_tensor)) <
tol_max_abs, 'abs norm of reconstruction error higher than tol')
# Test fixing mode 0 or 1 with given init
fixed_tensor = random_cp((3, 3, 3), rank=2)
for factor in fixed_tensor[1]:
factor = tl.abs(factor)
rec_svd_fixed_mode_0 = non_negative_parafac_hals(tensor,
rank=2,
n_iter_max=2,
init=fixed_tensor,
fixed_modes=[0])
rec_svd_fixed_mode_1 = non_negative_parafac_hals(tensor,
rank=2,
n_iter_max=2,
init=fixed_tensor,
fixed_modes=[1])
# Check if modified after 2 iterations
assert_array_equal(
rec_svd_fixed_mode_0.factors[0],
fixed_tensor.factors[0],
err_msg='Fixed mode 0 was modified in candecomp_parafac')
assert_array_equal(
rec_svd_fixed_mode_1.factors[1],
fixed_tensor.factors[1],
err_msg='Fixed mode 1 was modified in candecomp_parafac')
res_svd = non_negative_parafac_hals(tensor,
rank=3,
n_iter_max=100,
tol=10e-4,
init='svd')
res_random = non_negative_parafac_hals(tensor,
rank=3,
n_iter_max=100,
tol=10e-4,
init='random',
verbose=0)
rec_svd = tl.cp_to_tensor(res_svd)
rec_random = tl.cp_to_tensor(res_random)
error = tl.norm(rec_svd - rec_random, 2)
error /= tl.norm(rec_svd, 2)
assert_(error < tol_norm_2,
'norm 2 of difference between svd and random init too high')
assert_(
tl.max(tl.abs(rec_svd - rec_random)) < tol_max_abs,
'abs norm of difference between svd and random init too high')
# Regression test: used wrong variable for convergence checking
# Used mttkrp*factor instead of mttkrp*factors[-1], which resulted in
# error when mode 2 was not constrained and erroneous convergence checking
# when mode 2 was constrained.
tensor = tl.tensor(rng.random_sample((3, 3, 3)) + 1)
nn_estimate, errs = non_negative_parafac_hals(tensor,
rank=2,
n_iter_max=2,
tol=1e-10,
init='svd',
verbose=0,
nn_modes={
0,
},
return_errors=True)