def fit(self, X):
"""
Parameter:
X: array-like, ndarray of shape (I1, I2, ..., n_samples)
----------
Return:
self
"""
dim_in = X.shape
n_spl = dim_in[-1]
self.n_dim = X.ndim
self.Xmean = np.mean(X, axis=-1)
X_ = np.zeros(X.shape)
# init
Phi = dict()
Us = dict() # eigenvectors
Vs = dict() # eigenvalues
cums = dict() # cumulative distribution of eigenvalues
tPs = []
for i in range(n_spl):
X_[..., i] = X[..., i] - self.Xmean
for j in range(self.n_dim - 1):
X_i = unfold(X_[..., i], mode=j)
if j not in Phi:
Phi[j] = 0
Phi[j] = Phi[j] + np.dot(X_i, X_i.T)
for i in range(self.n_dim - 1):
eig_vals, eig_vecs = np.linalg.eig(Phi[i])
idx_sorted = eig_vals.argsort()[::-1]
Us[i] = eig_vecs[:, idx_sorted]
Vs[i] = eig_vals[idx_sorted]
sum_ = np.sum(Vs[i])
for j in range(Vs[i].shape[0] - 1, 0, -1):
if np.sum(Vs[i][j:]) / sum_ > (1 - self.var_exp):
cums[i] = j + 1
break
tPs.append(Us[i][:, :cums[i]].T)
for i_iter in range(self.max_iter):
Phi = dict()
for i in range(self.n_dim - 1):
if i not in Phi:
Phi[i] = 0
for j in range(n_spl):
X_i = X_[..., j]
Xi_ = multi_mode_dot(
X_i, [tPs[m] for m in range(self.n_dim - 1) if m != i],
modes=[m for m in range(self.n_dim - 1) if m != i])
tXi = unfold(Xi_, i)
Phi[i] = np.dot(tXi, tXi.T) + Phi[i]
eig_vals, eig_vecs = np.linalg.eig(Phi[i])
idx_sorted = eig_vals.argsort()[::-1]
tPs[i] = eig_vecs[:, idx_sorted]
tPs[i] = tPs[i][:, :cums[i]].T
self.tPs = tPs
return self
def transform(self, x):
"""Perform dimension reduction on X
Args:
x (array-like tensor): Data to perform dimension reduction, shape (n_samples, I_1, I_2, ..., I_N).
Returns:
array-like tensor:
Projected data in lower dimension, shape (n_samples, P_1, P_2, ..., P_N) if self.return_vector==False.
If self.return_vector==True, features will be sorted based on their explained variance ratio, shape
(n_samples, P_1 * P_2 * ... * P_N) if self.n_components is None, and shape (n_samples, n_components)
if self.n_component is a valid integer.
"""
_check_tensor_dim_shape(x, self.n_dims, self.shape_in)
x = x - self.mean_
# projected tensor in lower dimensions
x_projected = multi_mode_dot(x,
self.proj_mats,
modes=[m for m in range(1, self.n_dims)])
if self.return_vector:
x_projected = unfold(x_projected, mode=0)
x_projected = x_projected[:, self.idx_order]
if isinstance(self.n_components, int):
if self.n_components > np.prod(self.shape_out):
self.n_components = np.prod(self.shape_out)
warn_msg = "n_components exceeds the maximum number, all features will be returned."
logging.warning(warn_msg)
warnings.warn(warn_msg)
x_projected = x_projected[:, :self.n_components]
return x_projected
def derivativeDict(X, G, A, listofmatrices, Pold, Qold, setting, alpha, theta,
n, t):
#The function is used to compute the derivative of the objective function with respect the nth dictionary matrix
#The parameters are tensors
listoffactors = list(listofmatrices)
#Pnew=T.tensor(np.copy(mxnet_backend.to_numpy(Pold)))
#Qnew=T.tensor(np.copy(mxnet_backend.to_numpy(Qold)))
Pnew = tl.tensor(Pold)
Qnew = tl.tensor(Qold)
if (setting == "Single"):
listoffactors[n] = np.identity(X.shape[n])
WidehatX = Tensor_matrixproduct(
X, Operations_listmatrices(listoffactors, "Transpose"))
listoffactors[n] = tl.tensor(np.identity(G.shape[n]))
B = Tensor_matrixproduct(G, listoffactors)
#B=unfold(Offset(G),mode)
#pdb.set_trace()
Pnew = Pnew + tl.tensor(
np.dot(mxnet_backend.to_numpy(unfold(WidehatX, n)),
mxnet_backend.to_numpy(unfold(G, n)).T))
Qnew = Qnew + tl.tensor(
np.dot(mxnet_backend.to_numpy(unfold(B, n)),
mxnet_backend.to_numpy(unfold(B, n)).T))
Res = -Pnew / t + tl.tensor(
np.dot(mxnet_backend.to_numpy(A), mxnet_backend.to_numpy(Qnew)) /
t) + alpha * (1 - theta) * A
return Res
if (setting == "MiniBatch"):
rho = len(X)
for r in range(rho):
listoffactors[n] = tl.tensor(np.identity(X[r].shape[n]))
WidehatX = Tensor_matrixproduct(
X[r], Operations_listmatrices(listoffactors, "Transpose"))
Pnew = Pnew + tl.tensor(
np.dot(mxnet_backend.to_numpy(unfold(WidehatX, n)),
mxnet_backend.to_numpy(unfold(G[r], n)).T))
listoffactors[n] = tl.tensor(np.identity(G[r].shape[n]))
B = Tensor_matrixproduct(G[r], listoffactors)
Qnew = Qnew + tl.tensor(
np.dot(mxnet_backend.to_numpy(unfold(B, n)),
mxnet_backend.to_numpy(unfold(B, n).T)))
Res = -Pnew / t + tl.tensor(
np.dot(mxnet_backend.to_numpy(A), mxnet_backend.to_numpy(Qnew)) /
t) + alpha * (1 - theta) * A
return Res
def svd_init(tensor, modes, ranks):
factors = []
for index, mode in enumerate(modes):
eigenvecs, _, _ = svd_fun(unfold(tensor, mode),
n_eigenvecs=ranks[index])
factors.append(eigenvecs)
#print("factor mode: ", index)
return factors
def derivativeDict(X,G,A,listofmatrices,alpha,theta,n):#the parameters are tensors
listoffactors=list(listofmatrices)
listoffactors[n]=tl.tensor(np.identity(X.shape[n]))
WidehatX=Tensor_matrixproduct(X,Operations_listmatrices(listoffactors,"Transpose"))
listoffactors[n]=tl.tensor(np.identity(G.shape[n]))
B=unfold(Tensor_matrixproduct(G,listoffactors),n)
Result=tl.tensor(np.dot(mxnet_backend.to_numpy(unfold(WidehatX,n)),mxnet_backend.to_numpy(unfold(G,n)).T))-tl.tensor(np.dot(mxnet_backend.to_numpy(A),np.dot(mxnet_backend.to_numpy(B),mxnet_backend.to_numpy(B).T)))+alpha*(1-theta)*A
#Res1=np.dot(A,np.dot(B,B.T))
#Res2=alpha*(1-theta)*A
return Result
def HOSVD(Tensor,Coretensorsize):#The parameter is a tensor
N=len(Tensor.shape)
listofmatrices=[]
for n in range(N):
U,s,V=np.linalg.svd(mxnet_backend.to_numpy(unfold(Tensor,n)),full_matrices=True)
A=U[:,0:Coretensorsize[n]]
listofmatrices.append(A)
Coretensor=Tensor_matrixproduct(tl.tensor(Tensor),Operations_listmatrices(listofmatrices,"Transpose"))
#Coretensor=mxnet_backend.to_numpy(Coretensor)
#Coretensor=Tensor_matrixproduct(Tensor,listofmatrices)
Coretensor=tl.tensor(Coretensor)
return Coretensor,listofmatrices
def main():
# X = tnsr in the tutorial
X = np.moveaxis(
np.array([[[1, 3], [2, 4]], [[5, 7], [6, 8]], [[9, 11], [10, 12]]]), 0,
2)
A = np.linalg.svd(t_base.unfold(X, 0))[0] # output A matches tutorial
print("A = \n", A)
B = np.linalg.svd(t_base.unfold(X, 1))[0] # output B matches tutorial
print("B = \n", B)
C = np.linalg.svd(t_base.unfold(X, 2))[0] # output C matches tutorial
print("C = \n", C)
g = t_alg.mode_dot(t_alg.mode_dot(t_alg.mode_dot(X, A.T, 0), B.T, 1), C.T,
2)
print("g = \n", g)
g2 = t_decomp.tucker(X, ranks=(2, 2, 3)).core
print("g2 = \n", g)
return 0
def saveFactorMatrices(partition):
"""
Spark job to solve for and save each Ci factor matrix.
"""
ret = []
rows = list(partition)
error = 0.0
for row in rows:
label = row[0]
Xi = row[1]
Ki = Xi.shape[0]
dashIdx = label.rindex('-')
dotIdx = label.rindex('.')
labelId = int(label[dashIdx + 1:dotIdx])
# solve for Ci
Ci = np.zeros((Ki, R))
ZiTZic = tensorOps.ZTZ(A, B)
XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B], skip_matrix=0))
if regularization > 0:
ZiTZic = ZiTZic + regulParam * eye
Ci = solve(ZiTZic.T, XiZic.T).T
#print Ci
if outputDir != '':
# save Ci
filename = './Ci-' + str(labelId)
np.save(filename, Ci)
# save A & B
if labelId == 0:
filename = './A'
np.save(filename, A)
filename = './B'
np.save(filename, B)
error = error + np.square(norm(Xi - kruskal_to_tensor([Ci, A, B]), 2))
if outputDir != '':
subprocess.call(['hadoop fs -moveFromLocal ' + './*.npy ' + outputDir],
shell=True)
ret.append(['error', error])
return ret
def _cp_initialize(tensor, rank, init):
""" Parameter initialization methods for CP decomposition
"""
if rank <=0:
raise ValueError('Trying to fit a rank-{} model. Rank must be a positive integer.'.format(rank))
if isinstance(init, list):
_validate_factors(init)
factors = [fctr.copy() for fctr in init]
elif init is 'randn':
factors = [np.random.randn(tensor.shape[i], rank) for i in range(tensor.ndim)]
elif init is 'rand':
factors = [np.random.rand(tensor.shape[i], rank) for i in range(tensor.ndim)]
elif init is 'svd':
factors = []
for mode in range(tensor.ndim):
u, s, _ = np.linalg.svd(unfold(tensor, mode), full_matrices=False)
factors.append(u[:, :rank]*np.sqrt(s[:rank]))
else:
raise ValueError('initialization method not recognized')
return factors
def rowNormCMatrix(partition):
"""
Calculate squared row norm of C factor matrices
"""
ret = []
rows = list(partition)
# dalia
for row in rows:
label = row[0]
Xi = row[1]
Ki = Xi.shape[0]
Ci = np.zeros((Ki,R))
ZiTZic = tensorOps.ZTZ(A, B)
XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B], skip_matrix=0))
if regularization > 0:
ZiTZic = ZiTZic + regulParam * eye
Ci = solve(ZiTZic.T, XiZic.T).T
dashIdx=label.rindex('-')
dotIdx=label.rindex('.')
labelId=int(label[dashIdx+1:dotIdx])
rowNormCi = np.square(np.linalg.norm(Ci, axis=1))
ret.append([labelId, rowNormCi])
return ret
def select_top_weight(weights, select_ratio: float = 0.05):
"""Select top weights in magnitude, and the rest of weights will be zeros
Args:
weights (array-like): model weights, can be a vector or a higher order tensor
select_ratio (float, optional): ratio of top weights to be selected. Defaults to 0.05.
Returns:
array-like: top weights in the same shape with the input model weights
"""
if type(weights) != np.ndarray:
weights = np.array(weights)
orig_shape = weights.shape
if len(orig_shape) > 1:
weights = unfold(weights, mode=0)[0]
n_top_weights = int(weights.size * select_ratio)
top_weight_idx = (-1 * abs(weights)).argsort()[:n_top_weights]
top_weights = np.zeros(weights.size)
top_weights[top_weight_idx] = weights[top_weight_idx]
if len(orig_shape) > 1:
top_weights = fold(top_weights, mode=0, shape=orig_shape)
return top_weights
def parafac(tensor,
rank,
n_iter_max=100,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
mode_three_val=[[0.5, 0.5, 0.0], [0.0, 0.5, 0.5]]):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [| factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
Returns
-------
factors : ndarray list
List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
"""
factors = initialize_factors(tensor, rank, random_state=random_state)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
# Mode-3 values that control the country factors are set using the
# mode_three_val argument.
fixed_ja = mode_three_val[0]
fixed_ko = mode_three_val[1]
for iteration in range(n_iter_max):
for mode in range(tl.ndim(tensor)):
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
factors[2][0] = fixed_ja # set mode-3 values
factors[2][1] = fixed_ko # set mode-3 values
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
factor = tl.dot(unfold(tensor, mode),
khatri_rao(factors, skip_matrix=mode))
factor = tl.transpose(
tl.solve(tl.transpose(pseudo_inverse), tl.transpose(factor)))
factors[mode] = factor
if tol:
rec_error = tl.norm(tensor - kruskal_to_tensor(factors),
2) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
if return_errors:
return factors, rec_errors
else:
return factors
def _fit(self, x):
"""Solve MPCA"""
shape_ = x.shape # shape of input data
n_dims = x.ndim
self.shape_in = shape_[1:]
self.mean_ = np.mean(x, axis=0)
x = x - self.mean_
# init
shape_out = ()
proj_mats = []
# get the output tensor shape based on the cumulative distribution of eigen values for each mode
for i in range(1, n_dims):
mode_data_mat = unfold(x, mode=i)
singular_vec_left, singular_val, singular_vec_right = linalg.svd(
mode_data_mat, full_matrices=False)
eig_values = np.square(singular_val)
idx_sorted = (-1 * eig_values).argsort()
cum = eig_values[idx_sorted]
tot_var = np.sum(cum)
for j in range(1, cum.shape[0] + 1):
if np.sum(cum[:j]) / tot_var > self.var_ratio:
shape_out += (j, )
break
proj_mats.append(
singular_vec_left[:, idx_sorted][:, :shape_out[i - 1]].T)
# set n_components to the maximum n_features if it is None
if self.n_components is None:
self.n_components = int(np.prod(shape_out))
for i_iter in range(self.max_iter):
for i in range(1, n_dims): # ith mode
x_projected = multi_mode_dot(
x, [proj_mats[m] for m in range(n_dims - 1) if m != i - 1],
modes=[m for m in range(1, n_dims) if m != i])
mode_data_mat = unfold(x_projected, i)
singular_vec_left, singular_val, singular_vec_right = linalg.svd(
mode_data_mat, full_matrices=False)
eig_values = np.square(singular_val)
idx_sorted = (-1 * eig_values).argsort()
proj_mats[i - 1] = (
singular_vec_left[:, idx_sorted][:, :shape_out[i - 1]]).T
x_projected = multi_mode_dot(x,
proj_mats,
modes=[m for m in range(1, n_dims)])
x_proj_unfold = unfold(
x_projected, mode=0
) # unfold the tensor projection to shape (n_samples, n_features)
# x_proj_cov = np.diag(np.dot(x_proj_unfold.T, x_proj_unfold)) # covariance of unfolded features
x_proj_cov = np.sum(np.multiply(x_proj_unfold.T, x_proj_unfold.T),
axis=1) # memory saving computing covariance
idx_order = (-1 * x_proj_cov).argsort()
self.proj_mats = proj_mats
self.idx_order = idx_order
self.shape_out = shape_out
self.n_dims = n_dims
return self
def initialize_cp(tensor,
rank,
init='svd',
svd='numpy_svd',
random_state=None,
non_negative=False,
normalize_factors=False):
r"""Initialize factors used in `parafac`.
The type of initialization is set using `init`. If `init == 'random'` then
initialize factor matrices using `random_state`. If `init == 'svd'` then
initialize the `m`th factor matrix using the `rank` left singular vectors
of the `m`th unfolding of the input tensor.
Parameters
----------
tensor : ndarray
rank : int
init : {'svd', 'random'}, optional
svd : str, default is 'numpy_svd'
function to use to compute the SVD, acceptable values in tensorly.SVD_FUNS
non_negative : bool, default is False
if True, non-negative factors are returned
Returns
-------
factors : CPTensor
An initial cp tensor.
"""
rng = check_random_state(random_state)
if init == 'random':
# factors = [tl.tensor(rng.random_sample((tensor.shape[i], rank)), **tl.context(tensor)) for i in range(tl.ndim(tensor))]
# kt = CPTensor((None, factors))
return random_cp(tl.shape(tensor),
rank,
normalise_factors=False,
random_state=rng,
**tl.context(tensor))
elif init == 'svd':
try:
svd_fun = tl.SVD_FUNS[svd]
except KeyError:
message = 'Got svd={}. However, for the current backend ({}), the possible choices are {}'.format(
svd, tl.get_backend(), tl.SVD_FUNS)
raise ValueError(message)
factors = []
for mode in range(tl.ndim(tensor)):
U, S, _ = svd_fun(unfold(tensor, mode), n_eigenvecs=rank)
# Put SVD initialization on the same scaling as the tensor in case normalize_factors=False
if mode == 0:
idx = min(rank, tl.shape(S)[0])
U = tl.index_update(U, tl.index[:, :idx], U[:, :idx] * S[:idx])
if tensor.shape[mode] < rank:
# TODO: this is a hack but it seems to do the job for now
# factor = tl.tensor(np.zeros((U.shape[0], rank)), **tl.context(tensor))
# factor[:, tensor.shape[mode]:] = tl.tensor(rng.random_sample((U.shape[0], rank - tl.shape(tensor)[mode])), **tl.context(tensor))
# factor[:, :tensor.shape[mode]] = U
random_part = tl.tensor(
rng.random_sample(
(U.shape[0], rank - tl.shape(tensor)[mode])),
**tl.context(tensor))
U = tl.concatenate([U, random_part], axis=1)
factors.append(U[:, :rank])
kt = CPTensor((None, factors))
elif isinstance(init, (tuple, list, CPTensor)):
# TODO: Test this
try:
kt = CPTensor(init)
except ValueError:
raise ValueError(
'If initialization method is a mapping, then it must '
'be possible to convert it to a CPTensor instance')
else:
raise ValueError(
'Initialization method "{}" not recognized'.format(init))
if non_negative:
kt.factors = [tl.abs(f) for f in kt[1]]
if normalize_factors:
kt = cp_normalize(kt)
return kt
def CyclicBlocCoordinateTucker_set(X_set, Coretensorsize, Pre_existingfactors,
Pre_existingG_set, Pre_existingP,
Pre_existingQ, Nonnegative, Reprojectornot,
Setting, Minibatchnumber, step, alpha,
theta, max_iter, epsilon, period,
pool): #All the parameters are arrays,
#This function is used to perform the online setting either for single or minibatch processing
if (Nonnegative == True):
Positivity = TestPositivity(X_set)
if (Positivity == False):
raise Exception(
"You decide to perform a nonnegative decomposition while some of your tensors present negative entries"
)
L = len(X_set)
if (Setting == "Single"):
for t in range(L):
X = X_set[t]
Pre_existingG = Pre_existingG_set[t]
Gresult, Inferredfactorsresult = CyclicBlocCoordinateTucker_single(
X, Coretensorsize, Pre_existingfactors, Pre_existingG,
Pre_existingP, Pre_existingQ, Nonnegative, Setting, t + 1,
step, alpha, theta, max_iter, epsilon, period, pool)
#print(len(Inferredfactorsresult))
N = len(list(X.shape))
Pre_existingG = np.copy(Gresult)
G = tl.tensor(Gresult)
for n in range(N):
listoffactors = list(Inferredfactorsresult)
#print("Point I")
#print(listoffactors[n].shape)
listoffactors[n] = np.identity(X.shape[n])
#print("Point II")
#print(listoffactors[n].shape)
WidehatX = Tensor_matrixproduct(
X, Operations_listmatrices(listoffactors, "Transpose"))
listoffactors[n] = tl.tensor(np.identity(G.shape[n]))
B = Tensor_matrixproduct(tl.tensor(G), listoffactors)
Pre_existingP[n] = Pre_existingP[n] + tl.backend.dot(
unfold(WidehatX, n),
unfold(G, n).T)
Pre_existingQ[n] = Pre_existingQ[n] + tl.backend.dot(
unfold(B, n),
unfold(B, n).T)
if (Reprojectornot == True):
Gresult = []
for t in range(L):
X = X_set[t]
G_init = Pre_existingG_set[t]
G = Sparse_coding(
tl.tensor(X), tl.tensor(G_init),
Operations_listmatrices(Inferredfactorsresult,
"Tensorize"), Nonnegative, Setting,
step, max_iter, alpha, theta, epsilon, pool)
Gresult.append(G)
return Gresult, Inferredfactorsresult
if (Reprojectornot == False):
return Inferredfactorsresult
if (Setting == "MiniBatch"):
X_setdivided = SplitToMinibatch(X_set, Minibatchnumber)
Pre_existingGsetdivided = SplitToMinibatch(Pre_existingG_set,
Minibatchnumber)
Pre_existingPold = []
Pre_existingPnew = list(Pre_existingP)
Pre_existingQold = []
Pre_existingQnew = list(Pre_existingQ)
#for mininb in range(Minibatchnumber):
for mininb in range(len(Minibatchnumber)):
print("We are minibatch")
print("The minibatch processed is")
print(mininb)
X_minibatch = X_setdivided[mininb]
Pre_existingG_minibatch = Pre_existingGsetdivided[mininb]
Pre_existingPold = Pre_existingPnew
Pre_existingQold = Pre_existingQnew
Gresult, Inferredfactorsresult = CyclicBlocCoordinateTucker_single(
X_minibatch, Coretensorsize, Pre_existingfactors,
Pre_existingG_minibatch, Pre_existingPold, Pre_existingQold,
Nonnegative, Setting, mininb + 1, step, alpha, theta, max_iter,
epsilon, period, pool)
if (mininb != len(Minibatchnumber) - 1):
X_minibatchold = Operations_listmatrices(
X_setdivided[mininb], "Tensorize")
N = len(list(X_minibatchold[0].shape))
Inferredfactorsresult = Operations_listmatrices(
Inferredfactorsresult, "Tensorize")
minibatchsize = len(X_minibatchold)
N = len(X_minibatchold[0].shape)
Gactivationcoeff = list(Gresult)
for n in range(N):
for r in range(minibatchsize):
X = X_minibatchold[r]
G = Gactivationcoeff[r]
listoffactors = list(Inferredfactorsresult)
listoffactors[n] = np.identity(X.shape[n])
WidehatX = Tensor_matrixproduct(
tl.tensor(X),
Operations_listmatrices(listoffactors,
"Transpose"))
listoffactors[n] = np.identity(G.shape[n])
B = Tensor_matrixproduct(
tl.tensor(G),
Operations_listmatrices(listoffactors,
"Tensorize"))
Pre_existingPnew[
n] = Pre_existingPold[n] + tl.backend.dot(
unfold(WidehatX, n),
unfold(G, n).T)
Pre_existingQnew[
n] = Pre_existingQold[n] + tl.backend.dot(
unfold(B, n),
unfold(B, n).T)
if (Reprojectornot == True):
for mininb in range(len(Minibatchnumber)):
X_minibatch = Operations_listmatrices(X_setdivided[mininb],
"Tensorize")
G_init = Operations_listmatrices(
Pre_existingGsetdivided[mininb], "Tensorize")
G = Sparse_coding(X_minibatch, G_init, Inferredfactorsresult,
Nonnegative, Setting, step, max_iter, alpha,
theta, epsilon, pool)
for activation_coeff in G:
Gresult.append(activation_coeff)
return Gresult, Inferredfactorsresult
if (Reprojectornot == False):
return Inferredfactorsresult
def cp_als(tensor, rank, nonneg=False, init=None, tol=1e-6,
min_time=0, max_time=np.inf, n_iter_max=1000, print_every=0.3,
prepend_print='\r', append_print=''):
""" Fit CP decomposition by alternating least-squares.
Args
----
tensor : ndarray
data to be approximated by CP decomposition
rank : int
number of components in the CP decomposition model
Keyword Args
------------
nonneg : bool
(default = False)
if True, use alternating non-negative least squares to fit tensor
init : str or ktensor
specified initialization procedure for factor matrices
{'randn','rand','svd'}
tol : float
convergence criterion
n_iter_max : int
maximum number of optimizations iterations before aborting
(default = 1000)
print_every : float
how often (in seconds) to print progress. If <= 0 then don't print anything.
(default = -1)
Returns
-------
factors : list of ndarray
estimated low-rank decomposition (in kruskal tensor format)
info : dict
information about the fit / optimization convergence
"""
# default initialization method
if init is None:
init = 'randn' if nonneg is False else 'rand'
# intialize factor matrices
factors = _cp_initialize(tensor, rank, init)
# setup optimization
converged = False
norm_tensor = tensorly.tenalg.norm(tensor, 2)
t_elapsed = [0]
rec_errors = [_compute_squared_recon_error(tensor, factors, norm_tensor)]
# setup alternating solver
solver = _nnls_solver if nonneg else _ls_solver
# initial print statement
verbose = print_every > 0
print_counter = 0 # time to print next progress
if verbose:
print(prepend_print+'iter=0, error={0:.4f}'.format(rec_errors[-1]), end=append_print)
# main loop
t0 = time()
for iteration in range(n_iter_max):
# alternating optimization over modes
for mode in range(tensor.ndim):
# reduce grammians
G = np.ones((rank, rank))
for i, f in enumerate(factors):
if i != mode:
G *= np.dot(f.T, f)
# form unfolding and khatri-rao product
unf = unfold(tensor, mode)
kr = khatri_rao(factors, skip_matrix=mode)
# update factor
factors[mode] = solver(G, np.dot(unf, kr), warm_start=factors[mode].T)
# renormalize factors
factors = standardize_factors(factors, sort_factors=False)
# check convergence
rec_errors.append(_compute_squared_recon_error(tensor, factors, norm_tensor))
t_elapsed.append(time() - t0)
# break loop if converged
converged = abs(rec_errors[-2] - rec_errors[-1]) < tol
if converged and (time()-t0)>min_time:
if verbose: print(prepend_print+'converged in {} iterations.'.format(iteration+1), end=append_print)
break
# display progress
if verbose and (time()-t0)/print_every > print_counter:
print_str = 'iter={0:d}, error={1:.4f}, variation={2:.4f}'.format(
iteration+1, rec_errors[-1], rec_errors[-2] - rec_errors[-1])
print(prepend_print+print_str, end=append_print)
print_counter += print_every
# stop early if over time
if (time()-t0)>max_time:
break
if not converged and verbose:
print('gave up after {} iterations and {} seconds'.format(iteration, time()-t0), end=append_print)
# return optimized factors and info
return factors, { 'err_hist' : rec_errors,
't_hist' : t_elapsed,
'err_final' : rec_errors[-1],
'converged' : converged,
'iterations' : len(rec_errors) }
def unfold(tensor, mode=0):
return tl_base.unfold(tensor, mode)
def setData(self,
Y=None,
X=None,
X_r=None,
X_o=None,
basis=None,
nbasis=None,
**kwargs):
self.Y = Y
self.n = Y.shape[0]
self.t = SP.zeros([
Y.ndim - 1,
], dtype=int)
for i in range(1, Y.ndim):
self.t[i - 1] = Y.shape[i]
self.nt = SP.prod(Y.shape)
if X_r != None:
X = X_r
if X != None:
self.covar_r.X = X
if X_o != None:
self.covar_o.X = X_o
if (basis == None and nbasis == None):
self.Y_hat, self.basis = partial_tucker(Y,
modes=range(1, Y.ndim),
ranks=self.bn,
init='svd',
tol=10e-5)
reconstruction = SP.zeros(self.Y.shape)
for i in range(self.Y_hat.shape[0]):
reconstruction[i, :] = tl.tucker_to_tensor(
self.Y_hat[i, :], self.basis)
res = Y - reconstruction
temp, self.nbasis = partial_tucker(res,
modes=range(1, Y.ndim),
ranks=self.nbn,
init='svd',
tol=10e-5)
elif (basis != None and nbasis == None):
self.basis = basis
b = []
for i in range(len(basis)):
b.append(basis[i].T)
self.Y_hat = tl.tenalg.multi_mode_dot(Y, b, modes=range(1, Y.ndim))
reconstruction = SP.zeros(self.Y.shape)
for i in range(self.Y_hat.shape[0]):
reconstruction[i, :] = tl.tucker_to_tensor(
self.Y_hat[i, :], self.basis)
res = Y - reconstruction
temp, self.nbasis = partial_tucker(res,
modes=range(1, Y.ndim),
ranks=self.nbn,
init='svd',
tol=10e-5)
elif (basis == None and nbasis != None):
self.Y_hat, self.basis = partial_tucker(Y,
modes=range(1, Y.ndim),
ranks=self.bn,
init='svd',
tol=10e-5)
reconstruction = SP.zeros(self.Y.shape)
for i in range(self.Y_hat.shape[0]):
reconstruction[i, :] = tl.tucker_to_tensor(
self.Y_hat[i, :], self.basis)
res = Y - reconstruction
self.nbasis = nbasis
nb = []
for i in range(len(nbasis)):
nb.append(nbasis[i].T)
temp = tl.tenalg.multi_mode_dot(res, nb, modes=range(1, Y.ndim))
elif (basis != None and nbasis != None):
self.basis = basis
b = []
for i in range(len(basis)):
b.append(basis[i].T)
self.Y_hat = tl.tenalg.multi_mode_dot(Y, b, modes=range(1, Y.ndim))
reconstruction = SP.zeros(self.Y.shape)
for i in range(self.Y_hat.shape[0]):
reconstruction[i, :] = tl.tucker_to_tensor(
self.Y_hat[i, :], self.basis)
res = Y - reconstruction
self.nbasis = nbasis
nb = []
for i in range(len(nbasis)):
nb.append(nbasis[i].T)
temp = tl.tenalg.multi_mode_dot(res, nb, modes=range(1, Y.ndim))
for i in range(Y.ndim - 1):
self.covar_c[i].X = unfold(self.Y_hat, mode=i + 1)
for i in range(Y.ndim - 1):
self.covar_s[i].X = unfold(temp, mode=i + 1)
self._invalidate_cache()
def MPCA_(X, variance_explained=0.97, max_iter=1):
"""
Parameter:
X: array-like, ndarray of shape (I1, I2, ..., n_samples)
variance_explained: ration of variance to keep (between 0 and 1)
max_iter: max number of iteration
----------
Return:
tPs: list of projection matrices
"""
dim_in = X.shape
n_spl = dim_in[-1]
n_dim = X.ndim
# Is = dim_in[:-1]
Xmean = np.mean(X, axis=-1)
X_ = np.zeros(X.shape)
# init
Phi = dict()
Us = dict() # eigenvectors
Vs = dict() # eigenvalues
cums = dict() # cumulative distribution of eigenvalues
tPs = []
for i in range(n_spl):
X_[..., i] = X[..., i] - Xmean
for j in range(n_dim - 1):
X_i = unfold(X_[..., i], mode=j)
if j not in Phi:
Phi[j] = 0
Phi[j] = Phi[j] + np.dot(X_i, X_i.T)
for i in range(n_dim - 1):
eig_vals, eig_vecs = np.linalg.eig(Phi[i])
idx_sorted = eig_vals.argsort()[::-1]
Us[i] = eig_vecs[:, idx_sorted]
Vs[i] = eig_vals[idx_sorted]
sum_ = np.sum(Vs[i])
for j in range(Vs[i].shape[0] - 1, 0, -1):
if np.sum(Vs[i][j:]) / sum_ > (1 - variance_explained):
cums[i] = j + 1
break
tPs.append(Us[i][:, :cums[i]].T)
for i_iter in range(max_iter):
Phi = dict()
for i in range(n_dim - 1):
# dim_in_ = dim_in[i]
if i not in Phi:
Phi[i] = 0
for j in range(n_spl):
X_i = X_[..., j]
Xi_ = multi_mode_dot(
X_i, [tPs[m] for m in range(n_dim - 1) if m != i],
modes=[m for m in range(n_dim) if m != i])
tXi = unfold(Xi_, i)
Phi[i] = np.dot(tXi, tXi.T) + Phi[i]
eig_vals, eig_vecs = np.linalg.eig(Phi[i])
idx_sorted = eig_vals.argsort()[::-1]
tPs[i] = eig_vecs[:, idx_sorted]
tPs[i] = tPs[i][:, :cums[i]].T
return tPs
def singleModeALSstep(partition):
"""
Runs a single step of Alternating Least Squares to solve for one of A (mode = 1),
B (mode = 2), or C (mode = 3) matrix.
"""
'''
if decompMode == 1:
print 'Solving for A....'
elif decompMode == 2:
print 'Solving for B....'
elif decompMode == 3:
print 'Solving for Ci...'
'''
ret = []
rows = list(partition)
ZiTZi = 0
XiZi = 0
error = 0.0
for row in rows:
label = row[0]
Xi = row[1]
Ki = Xi.shape[0]
# make sure not to skip over slice if we're calculating error on full tensor
# if (sketching > 0 or (decompMode == 3 and errorCalcSketchingRate < 1)) and not (decompMode == 3 and errorCalcSketchingRate == 1) and not (decompMode == 3 and onUpdateWeightLoop):
if ((sketching > 0 and sketchingRate < 1.0) or (decompMode == 3 and errorCalcSketchingRate < 1)) and not (decompMode == 3 and errorCalcSketchingRate == 1) and not (decompMode == 3 and onUpdateWeightLoop):
dashIdx=label.rindex('-')
dotIdx=label.rindex('.')
labelId=int(label[dashIdx+1:dotIdx])
minIndex = labelId
maxIndex = labelId + Ki - 1
# dalia - IS THIS A PROBLEM? THIS WILL SELECT ROWS OF C WHEN CALCULATING FULL ERROR, BUT NOT SURE THESE ROWS ARE USED
selectRowsC = sketchingRowsC[(sketchingRowsC >= minIndex) & (sketchingRowsC <= maxIndex)]
selectRowsC = selectRowsC - minIndex
if len(selectRowsC) == 0:
continue;
# always solve for Ci first!
Ci = np.zeros((Ki,R))
# if sketching == 1 or sketching == 3:
# if (decompMode < 3 and (sketching == 1 or sketching >= 3)) or (decompMode == 3 and 0 < errorCalcSketchingRate < 1) and not onUpdateWeightLoop:
if (decompMode < 3 and (sketching == 1 or sketching >= 3) and sketchingRate < 1.0) or (decompMode == 3 and 0 < errorCalcSketchingRate < 1) and not onUpdateWeightLoop:
ZiTZic = tensorOps.ZTZ(A[sketchingRowsA,:], B[sketchingRowsB,:])
XiZic = np.dot(unfold(Xi[:,sketchingRowsA,:][:,:,sketchingRowsB], 0), khatri_rao([Ci, A[sketchingRowsA,:], B[sketchingRowsB,:]], skip_matrix=0))
'''
if (decompMode == 3):
print 'Solving for partial C'
'''
# don't need a sketching == 2, since else is the same
else:
'''
if (decompMode == 3):
print 'Solving for full C'
'''
ZiTZic = tensorOps.ZTZ(A, B)
XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B], skip_matrix=0))
#ZiTZic = tensorOps.ZTZ(A, B)
#XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B], skip_matrix=0))
if regularization > 0:
ZiTZic = ZiTZic + regulParam * eye
# I don't have Ci yet...
#if regularization == 2:
# XiZi = XiZi + regulParam * Ci
Ci = solve(ZiTZic.T, XiZic.T).T
# print 'Xi=\n',Xi
# print 'new Ci=\n',Ci
if decompMode == 1:
# if sketching == 1 or sketching >= 3:
if (sketching == 1 or sketching >= 3) and sketchingRate < 1.0:
ZiTZi = ZiTZi + tensorOps.ZTZ(B[sketchingRowsB,:], Ci[selectRowsC,:])
XiZi = XiZi + np.dot(unfold(Xi[selectRowsC,:,:][:,:,sketchingRowsB], 1), khatri_rao([Ci[selectRowsC,:], A, B[sketchingRowsB,:]], skip_matrix=1))
elif sketching == 2:
ZiTZi = ZiTZi + tensorOps.ZTZ(B, Ci[selectRowsC,:])
XiZi = XiZi + np.dot(unfold(Xi[selectRowsC,:,:], 1), khatri_rao([Ci[selectRowsC,:], A, B], skip_matrix=1))
else:
ZiTZi = ZiTZi + tensorOps.ZTZ(B, Ci)
# XiZi = XiZi + tensorOps.unfolded_3D_matrix_multiply(decompMode, Xi, Ci, B, I, J, Ki, R)
XiZi = XiZi + np.dot(unfold(Xi, 1), khatri_rao([Ci, A, B], skip_matrix=1))
elif decompMode == 2:
# if sketching == 1 or sketching >= 3:
if (sketching == 1 or sketching >= 3) and sketchingRate < 1.0:
ZiTZi = ZiTZi + tensorOps.ZTZ(A[sketchingRowsA,:], Ci[selectRowsC,:])
XiZi = XiZi + np.dot(unfold(Xi[selectRowsC,:,:][:,sketchingRowsA,:], 2), khatri_rao([Ci[selectRowsC,:], A[sketchingRowsA,:], B], skip_matrix=2))
elif sketching == 2:
ZiTZi = ZiTZi + tensorOps.ZTZ(A, Ci[selectRowsC,:])
XiZi = XiZi + np.dot(unfold(Xi[selectRowsC,:,:], 2), khatri_rao([Ci[selectRowsC,:], A, B], skip_matrix=2))
else:
ZiTZi = ZiTZi + tensorOps.ZTZ(A, Ci)
# XiZi = XiZi + tensorOps.unfolded_3D_matrix_multiply(decompMode, Xi, Ci, A, I, J, Ki, R)
XiZi = XiZi + np.dot(unfold(Xi, 2), khatri_rao([Ci, A, B], skip_matrix=2))
elif decompMode == 3:
# if sketching == 1 or sketching == 3:
if 0 < errorCalcSketchingRate < 1 and not onUpdateWeightLoop:
error = error + np.square(norm(Xi[selectRowsC,:,:][:,sketchingRowsA,:][:,:,sketchingRowsB] - kruskal_to_tensor([Ci[selectRowsC,:], A[sketchingRowsA,:], B[sketchingRowsB,:]]), 2))
#print 'Error calc with partial C'
elif sketching == 2:
error = error + np.square(norm(Xi[selectRowsC,:,:] - kruskal_to_tensor([Ci[selectRowsC,:], A, B]), 2))
else:
#print 'Error calc with full C'
error = error + np.square(norm(Xi - kruskal_to_tensor([Ci, A, B]), 2))
#print 'local error =',np.square(norm(Xi - kruskal_to_tensor([Ci, A, B]), 2))
else:
print 'Unknown decomposition mode. Catastrophic error. Failing now...'
if (len(rows) > 0) and (decompMode < 3):
ret.append(['ZTZ',ZiTZi])
ret.append(['XZ',XiZi])
elif (decompMode == 3):
ret.append(['error',error])
# print 'cumulative error =',error
del ZiTZi, XiZi
return ret
X = np.array([[[2, 1], [4, 3]], [[6, 5], [8, 7]]])
(K, I, J) = X.shape
R = 2
print 'X:\n--\n', X
C = np.random.rand(2, 2)
print ''
Xm = unfolded_3D_matrix_multiply(1, X, B, A, I, J, K, R)
#ans =
# 172 304
# 268 472
print 'UMM 1:\n------\n', Xm
# print 'X=',X
# print 'unfolded(1)=',unfold(X, 1)
# print 'KR=',khatri_rao([C, A, B], skip_matrix=0)
print 'Tly 1:\n-----\n', np.dot(unfold(X, 1),
khatri_rao([C, B, A], skip_matrix=0))
print ''
Xm = unfolded_3D_matrix_multiply(2, X, B, A, I, J, K, R)
#ans =
# 280 472
# 232 388
print 'UMM 2:\n------\n', Xm
# print 'X=',X
# print 'unfolded(2)=',unfold(X, 2)
# print 'KR=',khatri_rao([A, C, B], skip_matrix=1)
print 'Tly 2:\n-----\n', np.dot(unfold(X, 2),
khatri_rao([B, C, A], skip_matrix=1))
print ''
def _fit(self, x):
"""Solve MPCA"""
shape_ = x.shape # shape of input data
n_samples = shape_[0] # number of samples
n_dims = x.ndim
self.shape_in = shape_[1:]
self.mean_ = np.mean(x, axis=0)
x = x - self.mean_
# init
phi = dict()
shape_out = ()
proj_mats = []
for i in range(1, n_dims):
for j in range(n_samples):
# unfold the j_th tensor along the i_th mode
x_ji = unfold(x[j, :], mode=i - 1)
if i not in phi:
phi[i] = 0
phi[i] = phi[i] + np.dot(x_ji, x_ji.T)
# get the output tensor shape based on the cumulative distribution of eigenvalues for each mode
for i in range(1, n_dims):
eig_values, eig_vectors = np.linalg.eig(phi[i])
idx_sorted = (-1 * eig_values).argsort()
cum = eig_values[idx_sorted]
var_tot = np.sum(cum)
for j in range(1, cum.shape[0] + 1):
if np.sum(cum[:j]) / var_tot > self.var_ratio:
shape_out += (j, )
break
proj_mats.append(eig_vectors[:,
idx_sorted][:, :shape_out[i - 1]].T)
# set n_components to the maximum n_features if it is None
if self.n_components is None:
self.n_components = int(np.prod(shape_out))
for i_iter in range(self.max_iter):
phi = dict()
for i in range(1, n_dims): # ith mode
if i not in phi:
phi[i] = 0
for j in range(n_samples):
xj = multi_mode_dot(
x[j, :], # jth tensor/sample
[
proj_mats[m]
for m in range(n_dims - 1) if m != i - 1
],
modes=[m for m in range(n_dims - 1) if m != i - 1],
)
xj_unfold = unfold(xj, i - 1)
phi[i] = np.dot(xj_unfold, xj_unfold.T) + phi[i]
eig_values, eig_vectors = np.linalg.eig(phi[i])
idx_sorted = (-1 * eig_values).argsort()
proj_mats[i -
1] = (eig_vectors[:, idx_sorted][:, :shape_out[i -
1]]).T
x_projected = multi_mode_dot(x,
proj_mats,
modes=[m for m in range(1, n_dims)])
x_proj_unfold = unfold(
x_projected, mode=0
) # unfold the tensor projection to shape (n_samples, n_features)
# x_proj_cov = np.diag(np.dot(x_proj_unfold.T, x_proj_unfold)) # covariance of unfolded features
x_proj_cov = np.sum(np.multiply(x_proj_unfold, x_proj_unfold),
axis=1) # memory saving computing covariance
idx_order = (-1 * x_proj_cov).argsort()
self.proj_mats = proj_mats
self.idx_order = idx_order
self.shape_out = shape_out
self.n_dims = n_dims
return self
def HOOI(tensor, r1, r2, num_iter=500, error_print=True, tol=10e-5):
"""
U: [r1, S, 1, 1]
Core:[r2,r1, kernel_w, kernel_h]
V: [T, r2, 1, 1]
"""
w_out_channel, w_in_channel, kernel_w, kernel_h = [i for i in tensor.shape]
# compute sparse ratio of W
sparse_ratio = (tensor < 0.005).astype(np.float32).mean()
print 'sparse ratio is ', sparse_ratio
print tensor.shape, tensor.min(), tensor.max()
for i in np.arange(-0.1, 0.282, 0.03):
boolvalue = (tensor > i) & (tensor < (i + 0.03))
ratio = boolvalue.astype(np.float32).mean()
print ratio
# tucker-2 decomposition
ranks = [r2, r1]
### tucker step1: HOSVD init
factors = []
for mode in range(2):
eigenvecs, _, _ = partial_svd(unfold(tensor, mode),
n_eigenvecs=ranks[mode])
factors.append(eigenvecs)
factors.append(np.eye(kernel_w))
factors.append(np.eye(kernel_h))
### HOOI decomposition
rec_errors = []
norm_tensor = norm(tensor, 2)
for iteration in range(num_iter):
for mode in range(2):
core_approximation = tucker_to_tensor(tensor,
factors,
skip_factor=mode,
transpose_factors=True)
eigenvecs, _, _ = partial_svd(unfold(core_approximation, mode),
n_eigenvecs=ranks[mode])
factors[mode] = eigenvecs
core = tucker_to_tensor(tensor, factors, transpose_factors=True)
reconstruct_tensor = tucker_to_tensor(
core, factors, transpose_factors=False) # reconstruct tensor
rec_error1 = norm(tensor - reconstruct_tensor, 2) / norm_tensor
rec_errors.append(rec_error1)
if iteration > 1:
if error_print:
print('reconsturction error={}, norm_tensor={}, variation={}.'.
format(rec_errors[-1], rec_error1,
rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if error_print:
print('converged in {} iterations.'.format(iteration))
break
#print tensor.shape,core.shape,factors[0].shape,factors[1].shape
Core = core
U = factors[1].transpose((1, 0)).reshape((r1, w_in_channel, 1, 1))
V = factors[0].reshape((w_out_channel, r2, 1, 1))
#print Core.shape,U.shape,V.shape
return Core, V, U
def cp_sparse(tensor, rank, penalties, nonneg=False, init=None, warmstart=True,
tol=1e-6, min_time=0, max_time=np.inf, n_iter_max=1000, print_every=0.3,
prepend_print='\r', append_print=''):
""" Fit CP decomposition by alternating least-squares.
Args
----
tensor : ndarray
data to be approximated by CP decomposition
rank : int
number of components in the CP decomposition model
Keyword Args
------------
nonneg : bool
(default = False)
if True, use alternating non-negative least squares to fit tensor
init : str or ktensor
specified initialization procedure for factor matrices
{'randn','rand','svd'}
tol : float
convergence criterion
n_iter_max : int
maximum number of optimizations iterations before aborting
(default = 1000)
print_every : float
how often (in seconds) to print progress. If <= 0 then don't print anything.
(default = -1)
Returns
-------
factors : list of ndarray
estimated low-rank decomposition (in kruskal tensor format)
info : dict
information about the fit / optimization convergence
"""
# default initialization method
if init is None:
init = 'randn' if nonneg is False else 'rand'
# initialize factors
if warmstart:
factors, _ = cp_als(tensor, rank, nonneg=nonneg, tol=tol)
else:
factors = _cp_initialize(tensor, rank, init)
def _compute_penalty(_factors):
return np.sum([lam*np.sum(np.abs(f)) for lam, f in zip(penalties, _factors)])
# setup optimization
converged = False
norm_tensor = tensorly.tenalg.norm(tensor, 2)
t_elapsed = [0]
obj_history = [_compute_squared_recon_error(tensor, factors, norm_tensor) + _compute_penalty(factors)]
# initial print statement
verbose = print_every > 0
print_counter = 0 # time to print next progress
if verbose:
print(prepend_print+'iter=0, error={0:.4f}'.format(obj_history[-1]), end=append_print)
# gradient descent params
linesearch_iters = 100
# main loop
t0 = time()
for iteration in range(n_iter_max):
# alternating optimization over modes
for mode in range(tensor.ndim):
# current optimization state
stepsize = 1.0
old_obj = obj_history[-1]
fctr = factors[mode].copy()
# keep track of positive and negative elements
if not nonneg:
pos = fctr > 0
neg = fctr < 0
# form unfolding and khatri-rao product
unf = unfold(tensor, mode)
kr = khatri_rao(factors, skip_matrix=mode)
# calculate gradient
kr_t_kr = np.dot(kr.T, kr)
gradient = np.dot(fctr, kr_t_kr) - np.dot(unf, kr)
# proximal gradient update
new_obj = np.inf
for liter in range(linesearch_iters):
# take gradient step
new_fctr = fctr - stepsize*gradient
# iterative soft-thresholding
if nonneg:
new_fctr -= stepsize*penalties[mode]
new_fctr[new_fctr<0] = 0.0
else:
new_fctr[pos] -= stepsize*penalties[mode]
new_fctr[neg] += stepsize*penalties[mode]
sign_changes = (new_factor > 0 & neg) | (new_factor < 0 & pos)
new_fctr[sign_changes] = 0.0
# calculate new error
factors[mode] = new_fctr
new_obj = _compute_squared_recon_error(tensor, factors, norm_tensor) + _compute_penalty(factors)
# break if error went down
if new_obj < old_obj:
factors[mode] = new_fctr
break
# decrease step size if error went up
else:
stepsize /= 2.0
# give up if too many iterations
if liter == (linesearch_iters - 1):
factors[mode] = fctr
new_obj = old_obj
# renormalize factors
factors = standardize_factors(factors, sort_factors=False)
# check convergence
t_elapsed.append(time() - t0)
obj_history.append(new_obj)
# break loop if converged
converged = abs(obj_history[-2] - obj_history[-1]) < tol
if converged and (time()-t0)>min_time:
if verbose: print(prepend_print+'converged in {} iterations.'.format(iteration+1), end=append_print)
break
# display progress
if verbose and (time()-t0)/print_every > print_counter:
print_str = 'iter={0:d}, error={1:.4f}, variation={2:.4f}'.format(
iteration+1, obj_history[-1], obj_history[-2] - obj_history[-1])
print(prepend_print+print_str, end=append_print)
print_counter += print_every
# stop early if over time
if (time()-t0)>max_time:
break
if not converged and verbose:
print('gave up after {} iterations and {} seconds'.format(iteration, time()-t0), end=append_print)
# return optimized factors and info
return factors, { 'err_hist' : obj_history,
't_hist' : t_elapsed,
'err_final' : obj_history[-1],
'converged' : converged,
'iterations' : len(obj_history) }
def singleModeALSstep(partition):
"""
Runs a single step of Alternating Least Squares to solve for one of A (mode = 1),
B (mode = 2), or C (mode = 3) matrix.
"""
'''
if decompMode == 1:
print 'Solving for A....'
elif decompMode == 2:
print 'Solving for B....'
elif decompMode == 3:
print 'Solving for Ci...'
'''
ret = []
rows = list(partition)
ZiTZi = 0
XiZi = 0
error = 0.0
for row in rows:
label = row[0]
Xi = row[1]
Ki = Xi.shape[0]
if sketching > 0:
dashIdx = label.rindex('-')
dotIdx = label.rindex('.')
labelId = int(label[dashIdx + 1:dotIdx])
minIndex = labelId
maxIndex = labelId + Ki - 1
selectRowsC = sketchingRowsC[(sketchingRowsC >= minIndex)
& (sketchingRowsC <= maxIndex)]
selectRowsC = selectRowsC - minIndex
if len(selectRowsC) == 0:
continue
# always solve for Ci first!
Ci = np.zeros((Ki, R))
if sketching == 1 or sketching == 3:
ZiTZic = tensorOps.ZTZ(A[sketchingRowsA, :], B[sketchingRowsB, :])
XiZic = np.dot(
unfold(Xi[:, sketchingRowsA, :][:, :, sketchingRowsB], 0),
khatri_rao([Ci, A[sketchingRowsA, :], B[sketchingRowsB, :]],
skip_matrix=0))
# don't need a sketching == 2, since else is the same
else:
ZiTZic = tensorOps.ZTZ(A, B)
XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B],
skip_matrix=0))
#ZiTZic = tensorOps.ZTZ(A, B)
#XiZic = np.dot(unfold(Xi, 0), khatri_rao([Ci, A, B], skip_matrix=0))
if regularization > 0:
ZiTZic = ZiTZic + regulParam * eye
# I don't have Ci yet...
#if regularization == 2:
# XiZi = XiZi + regulParam * Ci
Ci = solve(ZiTZic.T, XiZic.T).T
if decompMode == 1:
if sketching == 1 or sketching == 3:
ZiTZi = ZiTZi + tensorOps.ZTZ(B[sketchingRowsB, :],
Ci[selectRowsC, :])
XiZi = XiZi + np.dot(
unfold(Xi[selectRowsC, :, :][:, :, sketchingRowsB], 1),
khatri_rao([Ci[selectRowsC, :], A, B[sketchingRowsB, :]],
skip_matrix=1))
elif sketching == 2:
ZiTZi = ZiTZi + tensorOps.ZTZ(B, Ci[selectRowsC, :])
XiZi = XiZi + np.dot(
unfold(Xi[selectRowsC, :, :], 1),
khatri_rao([Ci[selectRowsC, :], A, B], skip_matrix=1))
else:
ZiTZi = ZiTZi + tensorOps.ZTZ(B, Ci)
# XiZi = XiZi + tensorOps.unfolded_3D_matrix_multiply(decompMode, Xi, Ci, B, I, J, Ki, R)
XiZi = XiZi + np.dot(unfold(Xi, 1),
khatri_rao([Ci, A, B], skip_matrix=1))
elif decompMode == 2:
if sketching == 1 or sketching == 3:
ZiTZi = ZiTZi + tensorOps.ZTZ(A[sketchingRowsA, :],
Ci[selectRowsC, :])
XiZi = XiZi + np.dot(
unfold(Xi[selectRowsC, :, :][:, sketchingRowsA, :], 2),
khatri_rao([Ci[selectRowsC, :], A[sketchingRowsA, :], B],
skip_matrix=2))
elif sketching == 2:
ZiTZi = ZiTZi + tensorOps.ZTZ(A, Ci[selectRowsC, :])
XiZi = XiZi + np.dot(
unfold(Xi[selectRowsC, :, :], 2),
khatri_rao([Ci[selectRowsC, :], A, B], skip_matrix=2))
else:
ZiTZi = ZiTZi + tensorOps.ZTZ(A, Ci)
# XiZi = XiZi + tensorOps.unfolded_3D_matrix_multiply(decompMode, Xi, Ci, A, I, J, Ki, R)
XiZi = XiZi + np.dot(unfold(Xi, 2),
khatri_rao([Ci, A, B], skip_matrix=2))
elif decompMode == 3:
if sketching == 1 or sketching == 3:
error = error + np.square(
norm(
Xi[selectRowsC, :, :][:, sketchingRowsA, :]
[:, :, sketchingRowsB] - kruskal_to_tensor([
Ci[selectRowsC, :], A[sketchingRowsA, :],
B[sketchingRowsB, :]
]), 2))
elif sketching == 2:
error = error + np.square(
norm(
Xi[selectRowsC, :, :] -
kruskal_to_tensor([Ci[selectRowsC, :], A, B]), 2))
else:
error = error + np.square(
norm(Xi - kruskal_to_tensor([Ci, A, B]), 2))
else:
print 'Unknown decomposition mode. Catastrophic error. Failing now...'
if (len(rows) > 0) and (decompMode < 3):
ret.append(['ZTZ', ZiTZi])
ret.append(['XZ', XiZi])
elif (decompMode == 3):
ret.append(['error', error])
del ZiTZi, XiZi
return ret
def RobustsubspaceLearning_Single(X, Pre_existingprojectionmatrices,
Pre_existingenergymatrices, Pre_existingmean,
beta, alpha, p): #All parameters are arrays
Tensor = tl.tensor(X)
listoffactors = list(Pre_existingprojectionmatrices)
listoffactors = Operations_listmatrices(listoffactors, "Tensorize")
Energymatrices = list(Pre_existingenergymatrices)
Mean = np.copy(Pre_existingmean)
N = len(list(Tensor.shape))
R = Tensor - Tensor_matrixproduct(
X, Operations_listmatrices(listoffactors, "Transposetimes"))
Weightmatriceslist = []
for n in range(N):
Eigenvalue = np.linalg.eig(Energymatrices[n])[0]
U = listoffactors[n]
[I, J] = np.array(U.shape, dtype=int)
Xn = unfold(Tensor, mode=n)
[In, Jn] = np.array(Xn.shape, dtype=int)
Weightmatrix = np.zeros((In, Jn))
Sigma = np.zeros((In, Jn))
for i in range(In):
for j in range(Jn):
Sigma[i, j] = np.max(
np.multiply(np.sqrt(np.abs(Eigenvalue[1:p])),
mxnet_backend.to_numpy(U[1:p, i])))
k = beta * Sigma
if (n == 1):
R = R.T
for i in range(In):
for j in range(Jn):
#Weightmatrix[i,j]=1/(1+np.power(mxnet_backend.to_numpy(R[i,j])/np.maximum(k[i,j],0.001),2)):#This was the initial line
Weightmatrix[i, j] = 1 / (
1 + np.power(R[i, j] / np.maximum(k[i, j], 0.001), 2))
Weightmatriceslist.append(Weightmatrix)
W = np.minimum(Weightmatriceslist[0], Weightmatriceslist[1].T)
WeightTensor = tl.tensor(
np.multiply(np.sqrt(mxnet_backend.to_numpy(W)),
mxnet_backend.to_numpy(Tensor)))
Mean = alpha * Mean + (1 - alpha) * mxnet_backend.to_numpy(WeightTensor)
Projectionmatricesresult = []
Energymatreicesresult = []
for n in range(N):
Xn = unfold(WeightTensor, mode=n)
Covariancematrix = np.dot(
np.dot(
mxnet_backend.to_numpy(listoffactors[n]).T, Energymatrices[n]),
mxnet_backend.to_numpy(listoffactors[n]))
Covariancematrix = alpha * Covariancematrix + (1 - alpha) * np.dot(
mxnet_backend.to_numpy(Xn),
mxnet_backend.to_numpy(Xn).T)
[Un, diagn, V] = np.linalg.svd(Covariancematrix)
diagn = diagn / np.power(tl.norm(Xn, 2), 2)
indices = np.argsort(diagn)
indices = np.flip(indices, axis=0)
[J, I] = np.array(listoffactors[n].shape, dtype=int)
Unew = np.zeros((J, I))
for j in range(J):
Unew[j, :] = Un[indices[j], :]
Sn = np.diag(diagn)
Projectionmatricesresult.append(Unew)
Energymatreicesresult.append(Sn)
return Projectionmatricesresult, Energymatreicesresult, Mean, WeightTensor
def robust_pca(X,
mask=None,
tol=10e-7,
reg_E=1,
reg_J=1,
mu_init=10e-5,
mu_max=10e9,
learning_rate=1.1,
n_iter_max=100,
verbose=1):
"""Robust Tensor PCA via ALM with support for missing values
Decomposes a tensor `X` into the sum of a low-rank component `D`
and a sparse component `E`.
Parameters
----------
X : ndarray
tensor data of shape (n_samples, N1, ..., NS)
mask : ndarray
array of booleans with the same shape as `X`
should be zero where the values are missing and 1 everywhere else
tol : float
convergence value
reg_E : float, optional, default is 1
regularisation on the sparse part `E`
reg_J : float, optional, default is 1
regularisation on the low rank part `D`
mu_init : float, optional, default is 10e-5
initial value for mu
mu_max : float, optional, default is 10e9
maximal value for mu
learning_rate : float, optional, default is 1.1
percentage increase of mu at each iteration
n_iter_max : int, optional, default is 100
maximum number of iteration
verbose : int, default is 1
level of verbosity
Returns
-------
(D, E)
Robust decomposition of `X`
D : `X`-like array
low-rank part
E : `X`-like array
sparse error part
Notes
-----
The problem we solve is, for an input tensor :math:`\\tilde X`:
.. math::
:nowrap:
\\begin{equation*}
\\begin{aligned}
& \\min_{\\{J_i\\}, \\tilde D, \\tilde E}
& & \\sum_{i=1}^N \\text{reg}_J \\|J_i\\|_* + \\text{reg}_E \\|E\\|_1 \\\\
& \\text{subject to}
& & \\tilde X = \\tilde A + \\tilde E \\\\
& & & A_{[i]} = J_i, \\text{ for each } i \\in \\{1, 2, \\cdots, N\\}\\\\
\\end{aligned}
\\end{equation*}
"""
if mask is None:
mask = 1
# Initialise the decompositions
D = T.zeros_like(X, **T.context(X)) # low rank part
E = T.zeros_like(X, **T.context(X)) # sparse part
L_x = T.zeros_like(X, **T.context(
X)) # Lagrangian variables for the (X - D - E - L_x/mu) term
J = [T.zeros_like(X, **T.context(X))
for _ in range(T.ndim(X))] # Low-rank modes of X
L = [T.zeros_like(X, **T.context(X))
for _ in range(T.ndim(X))] # Lagrangian or J
# Norm of the reconstructions at each iteration
rec_X = []
rec_D = []
mu = mu_init
for iteration in range(n_iter_max):
for i in range(T.ndim(X)):
J[i] = fold(
svd_thresholding(
unfold(D, i) + unfold(L[i], i) / mu, reg_J / mu), i,
X.shape)
D = L_x / mu + X - E
for i in range(T.ndim(X)):
D += J[i] - L[i] / mu
D /= (T.ndim(X) + 1)
E = soft_thresholding(X - D + L_x / mu, mask * reg_E / mu)
# Update the lagrangian multipliers
for i in range(T.ndim(X)):
L[i] += mu * (D - J[i])
L_x += mu * (X - D - E)
mu = min(mu * learning_rate, mu_max)
# Evolution of the reconstruction errors
rec_X.append(T.norm(X - D - E, 2))
rec_D.append(max([T.norm(low_rank - D, 2) for low_rank in J]))
# Convergence check
if iteration > 1:
if max(rec_X[-1], rec_D[-1]) <= tol:
if verbose:
print('\nConverged in {} iterations'.format(iteration))
break
else:
print("[INFO] iter:", iteration, " error:",
(max(rec_X[-1], rec_D[-1]).item()))
return D, E
X = np.load(outdir + "X.npy")
T = X.shape[-1]
M = X.ndim - 1
z = np.loadtxt(outdir + "vec_z.txt")
U = [np.loadtxt(outdir + f"U_{i}.txt") for i in range(M)]
ranks = [U[i].shape[1] for i in range(M)]
Z = np.zeros((T, *ranks))
for t in range(T):
Z[t] = z[t].reshape(ranks)
plt.plot(z)
plt.show()
for m in range(M):
pred = np.zeros((T, X.shape[m]))
for t in range(T):
print(unfold(Z[t], m).shape, U[m].shape)
print(kronecker(U, skip_matrix=m, reverse=True).T.shape)
X_n = U[m] @ unfold(Z[t], m) @ kronecker(
U, skip_matrix=m, reverse=True).T
pred[t] = X_n[:, m]
# plt.plot((X, -1)[:, i])
plt.plot(pred)
plt.show()
matU = kronecker(U, reverse=True)
predict = z @ matU.T
print(predict.shape)
print(unfold(X, -1).shape)
for i in range(unfold(X, -1).shape[1]):
plt.plot(unfold(X, -1)[:, i])
plt.plot(predict[:, i])
# -*- coding: utf-8 -*-
"""
Basic tensor operations
=======================
Example on how to use :mod:`tensorly.base` to perform basic tensor operations.
"""
import matplotlib.pyplot as plt
from tensorly.base import unfold, fold
import numpy as np
###########################################################################
# A tensor is simply a numpy array
tensor = np.arange(24).reshape((3, 4, 2))
print('* original tensor:\n{}'.format(tensor))
###########################################################################
# Unfolding a tensor is easy
for mode in range(tensor.ndim):
print('* mode-{} unfolding:\n{}'.format(mode, unfold(tensor, mode)))
###########################################################################
# Re-folding the tensor is as easy:
for mode in range(tensor.ndim):
unfolding = unfold(tensor, mode)
folded = fold(unfolding, mode, tensor.shape)
print(np.all(folded == tensor))
