def testSquareFrobeniusNorm():
zeroCount = 2
rowIndices = np.array([1, 2])
colIndices = np.array([0, 0])
rowSize = 6
colSize = 6
M = coo_matrix((ones(zeroCount),(rowIndices, colIndices)), shape=(rowSize, colSize), dtype=np.uint8).tolil()
assert squareFrobeniusNormOfSparse(M) == 2
def testFitNorm():
X = coo_matrix((ones(4),([0, 1, 2, 2], [1, 1, 0, 1])), shape=(3, 3), dtype=np.uint8).tolil()
A = np.array([[0.9, 0.1],
[0.8, 0.2],
[0.1, 0.9]])
R = np.array([[0.9, 0.1],
[0.1, 0.9]])
expectedNorm = norm(X - dot(A,dot(R, A.T)))**2
assert_almost_equal(fitNorm(X, A, R), expectedNorm)
assert_almost_equal(fitNormWithoutNormX(X, A, R) + squareFrobeniusNormOfSparse(X), expectedNorm)
def testSquareFrobeniusNorm():
zeroCount = 2
rowIndices = np.array([1, 2])
colIndices = np.array([0, 0])
rowSize = 6
colSize = 6
M = coo_matrix((ones(zeroCount), (rowIndices, colIndices)),
shape=(rowSize, colSize),
dtype=np.uint8).tolil()
assert squareFrobeniusNormOfSparse(M) == 2
def testFitNorm():
X = coo_matrix((ones(4), ([0, 1, 2, 2], [1, 1, 0, 1])),
shape=(3, 3),
dtype=np.uint8).tolil()
A = np.array([[0.9, 0.1], [0.8, 0.2], [0.1, 0.9]])
R = np.array([[0.9, 0.1], [0.1, 0.9]])
expectedNorm = norm(X - dot(A, dot(R, A.T)))**2
assert_almost_equal(fitNorm(X, A, R), expectedNorm)
assert_almost_equal(
fitNormWithoutNormX(X, A, R) + squareFrobeniusNormOfSparse(X),
expectedNorm)
def testMatrixFitNorm():
A = np.array([[0.1, 0.1, 0.1], [0.1, 0.1, 0.001], [0.2, 0.1, 0.1],
[0.1, 0.3, 0.1], [0.4, 0.1, 0.1], [0.001, 0.01, 0.1]])
V = np.array([[0.1, 0.4, 0.1, 0.1], [0.01, 0.3, 0.1, 0.3],
[0.1, 0.01, 0.4, 0.001]])
D = coo_matrix((ones(6), ([0, 1, 2, 3, 4, 5], [0, 1, 1, 2, 3, 3])),
shape=(6, 4),
dtype=np.uint8).tocsr()
expectedNorm = norm(D - dot(A, V))**2
assert_almost_equal(matrixFitNorm(D, A, V), expectedNorm)
assert_almost_equal(
squareFrobeniusNormOfSparse(D) + matrixFitNormWithoutNormD(D, A, V),
expectedNorm)
def testMatrixFitNorm():
A = np.array([[0.1, 0.1, 0.1],
[0.1, 0.1, 0.001],
[0.2, 0.1, 0.1],
[0.1, 0.3, 0.1],
[0.4, 0.1, 0.1],
[0.001, 0.01, 0.1]])
V = np.array([[0.1, 0.4, 0.1, 0.1],
[0.01, 0.3, 0.1, 0.3],
[0.1, 0.01, 0.4, 0.001]])
D = coo_matrix((ones(6),([0, 1, 2, 3, 4, 5], [0, 1, 1, 2, 3, 3])), shape=(6, 4), dtype=np.uint8).tocsr()
expectedNorm = norm(D - dot(A,V))**2
assert_almost_equal(matrixFitNorm(D, A, V), expectedNorm)
assert_almost_equal(squareFrobeniusNormOfSparse(D) + matrixFitNormWithoutNormD(D, A, V), expectedNorm)
def rescal(X, D, rank, **kwargs):
"""
RESCAL
Factors a three-way tensor X such that each frontal slice
X_k = A * R_k * A.T. The frontal slices of a tensor are
N x N matrices that correspond to the adjacency matrices
of the relational graph for a particular relation.
For a full description of the algorithm see:
Maximilian Nickel, Volker Tresp, Hans-Peter-Kriegel,
"A Three-Way Model for Collective Learning on Multi-Relational Data",
ICML 2011, Bellevue, WA, USA
Parameters
----------
X : list
List of frontal slices X_k of the tensor X. The shape of each X_k is ('N', 'N')
D : matrix
A sparse matrix involved in the tensor factorization (aims to incorporate
the entity-term matrix aka document-term matrix)
rank : int
Rank of the factorization
lmbda : float, optional
Regularization parameter for A and R_k factor matrices. 0 by default
init : string, optional
Initialization method of the factor matrices. 'nvecs' (default)
initializes A based on the eigenvectors of X. 'random' initializes
the factor matrices randomly.
proj : boolean, optional
Whether or not to use the QR decomposition when computing R_k.
True by default
maxIter : int, optional
Maximium number of iterations of the ALS algorithm. 50 by default.
conv : float, optional
Stop when residual of factorization is less than conv. 1e-5 by default
Returns
-------
A : ndarray
matrix of latent embeddings for entities A
R : list
list of 'M' arrays of shape ('rank', 'rank') corresponding to the factor matrices R_k
f : float
function value of the factorization
iter : int
number of iterations until convergence
exectimes : ndarray
execution times to compute the updates in each iteration
V : ndarray
matrix of latent embeddings for words V
"""
# init options
ainit = kwargs.pop('init', __DEF_INIT)
proj = kwargs.pop('proj', __DEF_PROJ)
maxIter = kwargs.pop('maxIter', __DEF_MAXITER)
conv = kwargs.pop('conv', __DEF_CONV)
lmbda = kwargs.pop('lmbda', __DEF_LMBDA)
preheatnum = kwargs.pop('preheatnum', __DEF_PREHEATNUM)
if not len(kwargs) == 0:
raise ValueError( 'Unknown keywords (%s)' % (kwargs.keys()) )
sz = X[0].shape
dtype = X[0].dtype
n = sz[0]
_log.debug('[Config] rank: %d | maxIter: %d | conv: %7.1e | lmbda: %7.1e' % (rank,
maxIter, conv, lmbda))
# precompute norms of X
normX = [squareFrobeniusNormOfSparseBoolean(M) for M in X]
sumNormX = sum(normX)
normD = squareFrobeniusNormOfSparse(D)
_log.debug('[Algorithm] The tensor norm: %.5f' % sumNormX)
_log.debug('[Algorithm] The extended matrix norm: %.5f' % normD)
# initialize A
if ainit == 'random':
_log.debug('[Algorithm] The random initialization will be performed.')
A = array(rand(n, rank), dtype=np.float64)
elif ainit == 'nvecs':
_log.debug('[Algorithm] The eigenvector based initialization will be performed.')
tic = time.clock()
avgX = X[0] + X[0].T
for i in range(1, len(X)):
avgX = avgX + (X[i] + X[i].T)
toc = time.clock()
elapsed = toc - tic
_log.debug('Initializing tensor slices by summation required secs: %.5f' % elapsed)
tic = time.clock()
eigvalsX, A = eigsh(avgX.tocsc(), rank)
toc = time.clock()
elapsed = toc - tic
_log.debug('eigenvector decomposition required secs: %.5f' % elapsed)
else :
raise 'Unknown init option ("%s")' % ainit
# initialize R
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else :
raise 'Projection via QR decomposition is required; pass proj=true'
_log.debug('[Algorithm] Finished initialization.')
# compute factorization
fit = fitchange = fitold = 0
exectimes = []
for iterNum in xrange(maxIter):
tic = time.clock()
V = updateV(A, D, lmbda)
A = updateA(X, A, R, V, D, lmbda)
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else :
raise 'Projection via QR decomposition is required; pass proj=true'
# compute fit values
fit = 0
tensorFit = 0
regularizedFit = 0
extRegularizedFit = 0
regRFit = 0
fitDAV = 0
if iterNum >= preheatnum:
if lmbda != 0:
for i in xrange(len(R)):
regRFit += norm(R[i])**2
regularizedFit = lmbda*(norm(A)**2) + lmbda*regRFit
if lmbda != 0:
extRegularizedFit = lmbda*(norm(V)**2)
fitDAV = normD + matrixFitNormWithoutNormD(D, A, V)
for i in xrange(len(R)):
tensorFit += (normX[i] + fitNormWithoutNormX(X[i], A, R[i]))
fit = 0.5*tensorFit
fit += regularizedFit
fit /= sumNormX
fit += (0.5*fitDAV + extRegularizedFit)/normD
else :
_log.debug('[Algorithm] Preheating is going on.')
toc = time.clock()
exectimes.append( toc - tic )
fitchange = abs(fitold - fit)
_log.debug('[%3d] total fit: %.10f | tensor fit: %.10f | matrix fit: %.10f | delta: %.10f | secs: %.5f' % (iterNum,
fit, tensorFit, fitDAV, fitchange, exectimes[-1]))
fitold = fit
if iterNum > preheatnum and fitchange < conv:
break
return A, R, fit, iterNum+1, array(exectimes), V
def rescal(X, rank, **kwargs):
"""
RESCAL
Factors a three-way tensor X such that each frontal slice
X_k = A * R_k * A.T. The frontal slices of a tensor are
N x N matrices that correspond to the adjacency matrices
of the relational graph for a particular relation.
For a full description of the algorithm see:
Maximilian Nickel, Volker Tresp, Hans-Peter-Kriegel,
"A Three-Way Model for Collective Learning on Multi-Relational Data",
ICML 2011, Bellevue, WA, USA
Parameters
----------
X : list
List of frontal slices X_k of the tensor X. The shape of each X_k is ('N', 'N')
rank : int
Rank of the factorization
lmbda : float, optional
Regularization parameter for A and R_k factor matrices. 0 by default
init : string, optional
Initialization method of the factor matrices. 'nvecs' (default)
initializes A based on the eigenvectors of X. 'random' initializes
the factor matrices randomly.
proj : boolean, optional
Whether or not to use the QR decomposition when computing R_k.
True by default
maxIter : int, optional
Maximium number of iterations of the ALS algorithm. 50 by default.
conv : float, optional
Stop when residual of factorization is less than conv. 1e-5 by default
Returns
-------
A : ndarray
matrix of latent embeddings A
R : list
list of 'M' arrays of shape ('rank', 'rank') corresponding to the factor matrices R_k
f : float
function value of the factorization
iter : int
number of iterations until convergence
exectimes : ndarray
execution times to compute the updates in each iteration
"""
# init options
ainit = kwargs.pop('init', __DEF_INIT)
proj = kwargs.pop('proj', __DEF_PROJ)
maxIter = kwargs.pop('maxIter', __DEF_MAXITER)
conv = kwargs.pop('conv', __DEF_CONV)
lmbda = kwargs.pop('lmbda', __DEF_LMBDA)
preheatnum = kwargs.pop('preheatnum', __DEF_PREHEATNUM)
if not len(kwargs) == 0:
raise ValueError('Unknown keywords (%s)' % (kwargs.keys()))
sz = X[0].shape
dtype = X[0].dtype
n = sz[0]
_log.debug('[Config] rank: %d | maxIter: %d | conv: %7.1e | lmbda: %7.1e' %
(rank, maxIter, conv, lmbda))
# precompute norms of X
normX = [squareFrobeniusNormOfSparse(M) for M in X]
sumNormX = sum(normX)
_log.debug('[Algorithm] The tensor norm: %.5f' % sumNormX)
# initialize A
if ainit == 'random':
_log.debug('[Algorithm] The random initialization will be performed.')
A = array(rand(n, rank), dtype=np.float64)
elif ainit == 'nvecs':
_log.debug(
'[Algorithm] The eigenvector based initialization will be performed.'
)
avgX = lil_matrix((n, n))
for i in range(len(X)):
avgX += (X[i] + X[i].T)
eigvals, A = eigsh(avgX, rank)
else:
raise 'Unknown init option ("%s")' % ainit
# initialize R
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else:
raise 'Projection via QR decomposition is required; pass proj=true'
_log.debug('[Algorithm] Finished initialization.')
# compute factorization
fit = fitchange = fitold = 0
exectimes = []
for iterNum in xrange(maxIter):
tic = time.clock()
A = updateA(X, A, R, lmbda)
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else:
raise 'Projection via QR decomposition is required; pass proj=true'
# compute fit values
fit = 0
regularizedFit = 0
regRFit = 0
if iterNum >= preheatnum:
if lmbda != 0:
for i in xrange(len(R)):
regRFit += norm(R[i])**2
regularizedFit = lmbda * (norm(A)**2) + lmbda * regRFit
for i in xrange(len(R)):
fit += (normX[i] + fitNormWithoutNormX(X[i], A, R[i]))
fit *= 0.5
fit += regularizedFit
fit /= sumNormX
else:
_log.debug('[Algorithm] Preheating is going on.')
toc = time.clock()
exectimes.append(toc - tic)
fitchange = abs(fitold - fit)
_log.debug('[%3d] total fit: %.10f | delta: %.10f | secs: %.5f' %
(iterNum, fit, fitchange, exectimes[-1]))
fitold = fit
if iterNum > preheatnum and fitchange < conv:
break
return A, R, fit, iterNum + 1, array(exectimes)
def matrixFitNorm(D, A, V):
"""
Computes the Frobenius norm of the fitting matrix ||D - A*V||,
where D is a sparse matrix
"""
return squareFrobeniusNormOfSparse(D) + matrixFitNormWithoutNormD(D, A, V)
def matrixFitNorm(D, A, V):
"""
Computes the Frobenius norm of the fitting matrix ||D - A*V||,
where D is a sparse matrix
"""
return squareFrobeniusNormOfSparse(D) + matrixFitNormWithoutNormD(D, A, V)
def rescal(X, rank, **kwargs):
"""
RESCAL
Factors a three-way tensor X such that each frontal slice
X_k = A * R_k * A.T. The frontal slices of a tensor are
N x N matrices that correspond to the adjacency matrices
of the relational graph for a particular relation.
For a full description of the algorithm see:
Maximilian Nickel, Volker Tresp, Hans-Peter-Kriegel,
"A Three-Way Model for Collective Learning on Multi-Relational Data",
ICML 2011, Bellevue, WA, USA
Parameters
----------
X : list
List of frontal slices X_k of the tensor X. The shape of each X_k is ('N', 'N')
rank : int
Rank of the factorization
lmbda : float, optional
Regularization parameter for A and R_k factor matrices. 0 by default
init : string, optional
Initialization method of the factor matrices. 'nvecs' (default)
initializes A based on the eigenvectors of X. 'random' initializes
the factor matrices randomly.
proj : boolean, optional
Whether or not to use the QR decomposition when computing R_k.
True by default
maxIter : int, optional
Maximium number of iterations of the ALS algorithm. 50 by default.
conv : float, optional
Stop when residual of factorization is less than conv. 1e-5 by default
Returns
-------
A : ndarray
matrix of latent embeddings A
R : list
list of 'M' arrays of shape ('rank', 'rank') corresponding to the factor matrices R_k
f : float
function value of the factorization
iter : int
number of iterations until convergence
exectimes : ndarray
execution times to compute the updates in each iteration
"""
# init options
ainit = kwargs.pop("init", __DEF_INIT)
proj = kwargs.pop("proj", __DEF_PROJ)
maxIter = kwargs.pop("maxIter", __DEF_MAXITER)
conv = kwargs.pop("conv", __DEF_CONV)
lmbda = kwargs.pop("lmbda", __DEF_LMBDA)
preheatnum = kwargs.pop("preheatnum", __DEF_PREHEATNUM)
if not len(kwargs) == 0:
raise ValueError("Unknown keywords (%s)" % (kwargs.keys()))
sz = X[0].shape
dtype = X[0].dtype
n = sz[0]
_log.debug("[Config] rank: %d | maxIter: %d | conv: %7.1e | lmbda: %7.1e" % (rank, maxIter, conv, lmbda))
# precompute norms of X
normX = [squareFrobeniusNormOfSparse(M) for M in X]
sumNormX = sum(normX)
_log.debug("[Algorithm] The tensor norm: %.5f" % sumNormX)
# initialize A
if ainit == "random":
_log.debug("[Algorithm] The random initialization will be performed.")
A = array(rand(n, rank), dtype=np.float64)
elif ainit == "nvecs":
_log.debug("[Algorithm] The eigenvector based initialization will be performed.")
avgX = lil_matrix((n, n))
for i in range(len(X)):
avgX += X[i] + X[i].T
eigvals, A = eigsh(avgX, rank)
else:
raise 'Unknown init option ("%s")' % ainit
# initialize R
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else:
raise "Projection via QR decomposition is required; pass proj=true"
_log.debug("[Algorithm] Finished initialization.")
# compute factorization
fit = fitchange = fitold = 0
exectimes = []
for iterNum in xrange(maxIter):
tic = time.clock()
A = updateA(X, A, R, lmbda)
if proj:
Q, A2 = qr(A)
X2 = __projectSlices(X, Q)
R = __updateR(X2, A2, lmbda)
else:
raise "Projection via QR decomposition is required; pass proj=true"
# compute fit values
fit = 0
regularizedFit = 0
regRFit = 0
if iterNum >= preheatnum:
if lmbda != 0:
for i in xrange(len(R)):
regRFit += norm(R[i]) ** 2
regularizedFit = lmbda * (norm(A) ** 2) + lmbda * regRFit
for i in xrange(len(R)):
fit += normX[i] + fitNormWithoutNormX(X[i], A, R[i])
fit *= 0.5
fit += regularizedFit
fit /= sumNormX
else:
_log.debug("[Algorithm] Preheating is going on.")
toc = time.clock()
exectimes.append(toc - tic)
fitchange = abs(fitold - fit)
_log.debug("[%3d] total fit: %.10f | delta: %.10f | secs: %.5f" % (iterNum, fit, fitchange, exectimes[-1]))
fitold = fit
if iterNum > preheatnum and fitchange < conv:
break
return A, R, fit, iterNum + 1, array(exectimes)
