def als(X, rank, **kwargs):
"""
Alternating least-sqaures algorithm to compute the CP decomposition.
Parameters
----------
X : tensor_mixin
The tensor to be decomposed.
rank : int
Tensor rank of the decomposition.
init : {'random', 'nvecs'}, optional
The initialization method to use.
- random : Factor matrices are initialized randomly.
- nvecs : Factor matrices are initialzed via HOSVD.
(default 'nvecs')
max_iter : int, optional
Maximium number of iterations of the ALS algorithm.
(default 500)
fit_method : {'full', None}
The method to compute the fit of the factorization
- 'full' : Compute least-squares fit of the dense approximation of.
X and X.
- None : Do not compute the fit of the factorization, but iterate
until ``max_iter`` (Useful for large-scale tensors).
(default 'full')
conv : float
Convergence tolerance on difference of fit between iterations
(default 1e-5)
Returns
-------
P : ktensor
Rank ``rank`` factorization of X. ``P.U[i]`` corresponds to the factor
matrix for the i-th mode. ``P.lambda[i]`` corresponds to the weight
of the i-th mode.
fit : float
Fit of the factorization compared to ``X``
itr : int
Number of iterations that were needed until convergence
exectimes : ndarray of floats
Time needed for each single iteration
Examples
--------
Create random dense tensor
>>> from sktensor import dtensor, ktensor
>>> U = [np.random.rand(i,3) for i in (20, 10, 14)]
>>> T = dtensor(ktensor(U).toarray())
Compute rank-3 CP decomposition of ``T`` with ALS
>>> P, fit, itr, _ = als(T, 3)
Result is a decomposed tensor stored as a Kruskal operator
>>> type(P)
<class 'sktensor.ktensor.ktensor'>
Factorization should be close to original data
>>> np.allclose(T, P.totensor())
True
References
----------
.. [1] Kolda, T. G. & Bader, B. W.
Tensor Decompositions and Applications.
SIAM Rev. 51, 455–500 (2009).
.. [2] Harshman, R. A.
Foundations of the PARAFAC procedure: models and conditions for an 'explanatory' multimodal factor analysis.
UCLA Working Papers in Phonetics 16, (1970).
.. [3] Carroll, J. D., Chang, J. J.
Analysis of individual differences in multidimensional scaling via an N-way generalization of 'Eckart-Young' decomposition.
Psychometrika 35, 283–319 (1970).
"""
# init options
ainit = kwargs.pop('init', _DEF_INIT)
maxiter = kwargs.pop('max_iter', _DEF_MAXITER)
fit_method = kwargs.pop('fit_method', _DEF_FIT_METHOD)
conv = kwargs.pop('conv', _DEF_CONV)
dtype = kwargs.pop('dtype', _DEF_TYPE)
if not len(kwargs) == 0:
raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))
N = X.ndim
normX = norm(X)
U = _init(ainit, X, N, rank, dtype)
fit = 0
exectimes = []
for itr in range(maxiter):
tic = time.clock()
fitold = fit
for n in range(N):
Unew = X.uttkrp(U, n)
Y = ones((rank, rank), dtype=dtype)
for i in (list(range(n)) + list(range(n + 1, N))):
Y = Y * dot(U[i].T, U[i])
Unew = Unew.dot(pinv(Y))
# Normalize
if itr == 0:
lmbda = sqrt((Unew**2).sum(axis=0))
else:
lmbda = Unew.max(axis=0)
lmbda[lmbda < 1] = 1
U[n] = Unew / lmbda
P = ktensor(U, lmbda)
if fit_method == 'full':
normresidual = normX**2 + P.norm()**2 - 2 * P.innerprod(X)
fit = 1 - (normresidual / normX**2)
else:
fit = itr
fitchange = abs(fitold - fit)
exectimes.append(time.clock() - tic)
#print('fitchange:', fitchange)
_log.debug('[%3d] fit: %.5f | delta: %7.1e | secs: %.5f' %
(itr, fit, fitchange, exectimes[-1]))
if itr > 0 and fitchange < conv:
break
return P, fit, itr, array(exectimes)
def fold_in(X, Uold, rank, **kwargs):
# init options
ainit = kwargs.pop('init', _DEF_INIT)
maxiter = kwargs.pop('max_iter', _DEF_MAXITER)
fit_method = kwargs.pop('fit_method', _DEF_FIT_METHOD)
conv = kwargs.pop('conv', _DEF_CONV)
dtype = kwargs.pop('dtype', _DEF_TYPE)
if not len(kwargs) == 0:
raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))
N = X.ndim
normX = norm(X)
U = _init(ainit, X, N, Uold, rank, dtype)
fit = 0
O = U[1]
exectimes = []
for itr in range(maxiter):
#print(itr)
tic = time.clock()
fitold = fit
n = 1 # Mode 1 is the array that changes
Unew = X.uttkrp(U, n)
'''
# Can't implement because of memory error
n, p = U[0].shape
m, pC = U[2].shape
print('n -> U[0], m-> U[2] ROWS', n, m)
C = np.einsum('ij, kj -> ikj', U[0], U[2]).reshape(m * n, p)
nk, ni, nj = X.shape
jk = nk * nj
sess = tf.Session()
with sess.as_default():
Xnew = tf.reshape(X, [1,jk])
Xnew = Xnew.eval()
print('C shape:', C.shape)
print('Xnew shape:', Xnew.shape)
Z = Xnew.dot(pinv(C))
Unew = (Unew.dot(Z)).dot(inv(Unew))
'''
Y = ones((rank, rank), dtype=dtype)
for i in (list(range(n)) + list(range(n + 1, N))):
Y = Y * dot(U[i].T, U[i])
Unew = Unew.dot(pinv(Y))
#O = O*Unew/(O.dot(Y))
# Normalize
if itr == 0:
lmbda = sqrt((Unew**2).sum(axis=0))
else:
lmbda = Unew.max(axis=0)
lmbda[lmbda < 1] = 1
'''
if itr == 0:
lmbda = sqrt((O ** 2).sum(axis=0))
else:
lmbda = O.max(axis=0)
lmbda[lmbda < 1] = 1
'''
U[1] = Unew / lmbda
#U[1] = O / lmbda
P = ktensor(U, lmbda)
if fit_method == 'full':
normresidual = normX**2 + P.norm()**2 - 2 * P.innerprod(X)
#normresidual = normX ** 2 + np.linalg.norm(U[1]) ** 2 - 2 * (np.linalg.norm(U[1]))*(X)
fit = 1 - (normresidual / normX**2)
else:
fit = itr
fitchange = abs(fitold - fit)
exectimes.append(time.clock() - tic)
if itr > 0 and fitchange < conv:
break
return U[1], P
def als(X, Yl, rank, **kwargs):
"""
Alternating least-sqaures algorithm to compute the CP decomposition taking into
consideration the labels of the set
Yl -> lx1
X -> pxuxu
"""
# init options
ainit = kwargs.pop('init', _DEF_INIT)
maxiter = kwargs.pop('max_iter', _DEF_MAXITER)
fit_method = kwargs.pop('fit_method', _DEF_FIT_METHOD)
conv = kwargs.pop('conv', _DEF_CONV)
dtype = kwargs.pop('dtype', _DEF_TYPE)
if not len(kwargs) == 0:
raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))
N = X.ndim
normX = norm(X)
Yl = np.asarray(Yl)
Yl = np.reshape(Yl, (-1, 1))
normYl = np.linalg.norm(Yl)
U = _init(ainit, X, N, rank, dtype)
fit = 0
vecX = np.reshape(X, (np.product(X.shape), ))
W = ones((rank, 1), dtype=dtype)
l = Yl.shape[0]
p = 182
D = np.zeros((l, p))
for i in range(l):
for j in range(l):
if i == j:
D[i, j] = 1
for itr in range(maxiter):
fitold = fit
for n in range(N):
Unew = X.uttkrp(U, n)
# Y is ZtZ
Y = ones((rank, rank), dtype=dtype)
for i in (list(range(n)) + list(range(n + 1, N))):
Y = Y * dot(U[i].T, U[i])
if n != 1:
# Updates remain the same for U0,U2
Unew = Unew.dot(pinv(Y))
else:
Ip = np.identity(p)
IptIp = dot(Ip.T, Ip)
GtG = np.kron(Y, IptIp)
vecA = np.reshape(U[1], (np.product(U[1].shape), 1))
GtvecX1 = dot(GtG, vecA)
L = np.kron(W.T, D)
LtL = dot(L.T, L)
Sum1 = inv(GtG + LtL)
dot0 = dot(L.T, Yl)
Sum2 = GtvecX1 + dot0
vecA = dot(Sum1, Sum2)
Unew = np.reshape(vecA, (p, rank))
# Normalize
if itr == 0:
lmbda = sqrt((Unew**2).sum(axis=0))
else:
lmbda = Unew.max(axis=0)
lmbda[lmbda < 1] = 1
U[n] = Unew / lmbda
# update W
AtDt = dot(U[1].T, D.T)
DA = dot(D, U[1])
inv1 = inv(dot(AtDt, DA))
#print('ok inv')
dot2 = dot(AtDt, Yl)
W = dot(inv1, dot2)
P = ktensor(U, lmbda)
A = U[1]
Ai = A[146:]
ypred = dot(Ai, W)
ypred[abs(ypred) > 0.5] = 1
ypred[abs(ypred) < 0.5] = 0
DAW = dot(DA, W)
normDAW = np.linalg.norm(DAW)
if fit_method == 'full':
normresidual1 = normX**2 + P.norm()**2 - 2 * P.innerprod(X)
normresidual2 = normYl**2 + normDAW**2 - 2 * dot(Yl.T, DAW)
normresidual = normresidual1 + normresidual2
#fit = 1 - (normresidual / normX ** 2)
fit = normresidual
else:
fit = itr
fitchange = abs(fitold - fit) / fitold
#print('fitchange:',fitchange)
if itr > 0 and fitchange < conv:
#print(ypred)
break
#print(ypred)
ypred[abs(ypred) > 0.5] = 1
ypred[abs(ypred) < 0.5] = 0
#print(ypred)
return P, ypred, fit, itr
def orth_als(X, rank, **kwargs):
"""
Orthogonalized Alternating least-sqaures algorithm to compute the CP decomposition.
Orth-ALS is a variant of standard ALS where the factor estimates are
orthogonalized before the ALS step. The orthogonalization may be continued till the end, or
up to a fixed number of iterations.
For more details about Orth-ALS, see reference [4].
Parameters
----------
X : tensor_mixin
The tensor to be decomposed.
rank : int
Tensor rank of the decomposition.
init : {'random', 'nvecs'}, optional
The initialization method to use.
- random : Factor matrices are initialized randomly.
- nvecs : Factor matrices are initialzed via HOSVD.
(default 'nvecs')
max_iter : int, optional
Maximium number of iterations of the ALS algorithm.
(default 500)
stop_orth: int, optional
Number of iterations till which orthogonalization is to be continued
(default 5)
fit_method : {'full', None}
The method to compute the fit of the factorization
- 'full' : Compute least-squares fit of the dense approximation of.
X and X.
- None : Do not compute the fit of the factorization, but iterate
until ``max_iter`` (Useful for large-scale tensors).
(default 'full')
conv : float
Convergence tolerance on difference of fit between iterations
(default 1e-5)
Returns
-------
P : ktensor
Rank ``rank`` factorization of X. ``P.U[i]`` corresponds to the factor
matrix for the i-th mode. ``P.lambda[i]`` corresponds to the weight
of the i-th mode.
fit : float
Fit of the factorization compared to ``X``
itr : int
Number of iterations that were needed until convergence
exectimes : ndarray of floats
Time needed for each single iteration
Examples
--------
Create random dense tensor
>>> from sktensor import dtensor, ktensor
>>> U = [np.random.rand(i,3) for i in (20, 10, 14)]
>>> T = dtensor(ktensor(U).toarray())
Compute rank-3 CP decomposition of ``T`` with Orth-ALS/ Hybrid-ALS
>>> P, fit, itr, _ = als(T, 3)
Result is a decomposed tensor stored as a Kruskal operator
>>> type(P)
<class 'sktensor.ktensor.ktensor'>
Factorization should be close to original data
>>> np.allclose(T, P.totensor())
True
References
----------
.. [1] Kolda, T. G. & Bader, B. W.
Tensor Decompositions and Applications.
SIAM Rev. 51, 455–500 (2009).
.. [2] Harshman, R. A.
Foundations of the PARAFAC procedure: models and conditions for an 'explanatory' multimodal factor analysis.
UCLA Working Papers in Phonetics 16, (1970).
.. [3] Carroll, J. D., Chang, J. J.
Analysis of individual differences in multidimensional scaling
via an N-way generalization of 'Eckart-Young' decomposition.
Psychometrika 35, 283–319 (1970).
.. [4] V. Sharan, G. Valiant
Orthogonalized ALS: A Theoretically Principled Tensor Decomposition Algorithm for Practical Use
arXiv:1703.01804, 2017
"""
# init options
ainit = kwargs.pop('init', _DEF_INIT)
maxiter = kwargs.pop('max_iter', _DEF_MAXITER)
fit_method = kwargs.pop('fit_method', _DEF_FIT_METHOD)
conv = kwargs.pop('conv', _DEF_CONV)
stop_orth = kwargs.pop('stop_orth', _DEF_STOP_ORTH)
dtype = kwargs.pop('dtype', _DEF_TYPE)
if not len(kwargs) == 0:
raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))
N = X.ndim
normX = norm(X)
logging.info('')
U = _init(ainit, X, N, rank, dtype)
logging.info('')
fit = 0
exectimes = []
for itr in range(maxiter):
tic = time.clock()
fitold = fit
if itr != 0 and itr < stop_orth:
t = U[0].shape
dim_max = t[0] - 1
for n in range(N):
t = U[n].shape
if (t[0] - 1) < dim_max:
dim_max = t[0] - 1
for n in range(N):
Q = U[n]
t = Q.shape
J = t[1]
count = 0
for i in range(J):
if LA.norm(Q[:, i] == 0):
count += 1
# _log.debug(
# 'Zero norm, mode %d' % (n)
# )
Q[:, i] = rand(t[0])
Q[:, i] = Q[:, i] / LA.norm(Q[:, i])
else:
Q[:, i] = Q[:, i] / LA.norm(Q[:, i])
if i <= dim_max:
for j in range(i + 1, J):
Q[:, j] = Q[:, j] - 1 * np.inner(Q[:, j],
Q[:, i]) * Q[:, i]
U[n] = Q
_log.debug('Zero norm, mode %d, count %d' % (n, count))
for n in range(N):
Unew = X.uttkrp(U, n)
Y = ones((rank, rank), dtype=dtype)
for i in (list(range(n)) + list(range(n + 1, N))):
Y = Y * dot(U[i].T, U[i])
Unew = Unew.dot(pinv(Y))
# Normalize
if itr == 0:
lmbda = sqrt((Unew**2).sum(axis=0))
else:
lmbda = Unew.max(axis=0)
lmbda[lmbda < 1] = 1
U[n] = Unew / lmbda
P = ktensor(U, lmbda)
if fit_method == 'full':
normresidual = normX**2 + P.norm()**2 - 2 * P.innerprod(X)
fit = 1 - (normresidual / normX**2)
else:
fit = itr
fitchange = abs(fitold - fit)
exectimes.append(time.clock() - tic)
_log.debug('[%3d] fit: %.5f | delta: %7.1e | secs: %.5f' %
(itr, fit, fitchange, exectimes[-1]))
if itr > 0 and fitchange < conv:
break
return P, fit, itr, array(exectimes)