def get_cp_factors(self):
if self.pretrained is not None:
mat_dict = scipy.io.loadmat(self.pretrained)
if mat_dict['R'][0][0] != self.rank:
print('WRONG FACTORS, do not correspond to desired rank')
PU_z, PU_cout, PU_cin = [Ui[0] for Ui in mat_dict['P_bals_epc_U']]
Plmbda = mat_dict['P_bals_epc_lambda'].ravel()
f_cin = np.array(PU_cin)
f_cout = np.array(PU_cout)
f_z = (np.array(PU_z)*(Plmbda))
else:
bias = self.bias
if isinstance(self.layer, nn.Sequential):
# Tensorly case
_, (f_cout, f_cin, f_z) = parafac(kruskal_to_tensor((None, self.weight)),
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd = None,
cvg_criterion = 'rec_error')
else:
# Tensorly case
_, (f_cout, f_cin, f_z) = parafac(self.weight,
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd = None,
cvg_criterion = 'rec_error')
# # Reshape factor matrices to 4D weight tensors
# f_cin: (cin, rank) -> (rank, cin, 1, 1)
# f_z: (z, rank) -> (rank, 1, h, w)
# f_cout: (count, rank) -> (count, rank, 1, 1)
# Pytorh case
f_cin = f_cin.t().unsqueeze_(2).unsqueeze_(3).contiguous()
f_z = torch.einsum('hwr->rhw', f_z.resize_((*self.kernel_size, self.rank))\
).unsqueeze_(1).contiguous()
f_cout = f_cout.unsqueeze_(2).unsqueeze_(3).contiguous()
return [f_cin, f_z, f_cout], [None, None, bias]
def get_cp_factors(self):
tl.set_backend('pytorch')
bias = self.bias
if isinstance(self.layer, nn.Sequential):
# Tensorly case
_, (f_cout, f_cin, f_h, f_w) = parafac(kruskal_to_tensor(
(None, self.weight)),
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd=None,
cvg_criterion='rec_error')
else:
# Tensorly case
_, (f_cout, f_cin, f_h, f_w) = parafac(self.weight,
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd=None,
cvg_criterion='rec_error')
# # Reshape factor matrices to 4D weight tensors
# f_cin: (cin, rank) -> (rank, cin, 1, 1)
# f_h: (h, rank) -> (rank, 1, h, 1)
# f_w: (w, rank) -> (rank, 1, 1, w)
# f_cout: (count, rank) -> (count, rank, 1, 1)
# Pytorh case
f_cin = f_cin.t().unsqueeze_(2).unsqueeze_(3).contiguous()
f_h = f_h.t().unsqueeze_(1).unsqueeze_(3).contiguous()
f_w = f_w.t().unsqueeze_(1).unsqueeze_(2).contiguous()
f_cout = f_cout.unsqueeze_(2).unsqueeze_(3).contiguous()
return [f_cin, f_h, f_w, f_cout], [None, None, None, bias]
def parafac(tensor,
rank,
n_iter_max=100,
tol=1e-8,
random_state=None,
verbose=False,
return_errors=False,
mode_three_val=[[0.5, 0.5, 0.0], [0.0, 0.5, 0.5]]):
"""CANDECOMP/PARAFAC decomposition via alternating least squares (ALS)
Computes a rank-`rank` decomposition of `tensor` [1]_ such that,
``tensor = [| factors[0], ..., factors[-1] |]``.
Parameters
----------
tensor : ndarray
rank : int
Number of components.
n_iter_max : int
Maximum number of iteration
tol : float, optional
(Default: 1e-6) Relative reconstruction error tolerance. The
algorithm is considered to have found the global minimum when the
reconstruction error is less than `tol`.
random_state : {None, int, np.random.RandomState}
verbose : int, optional
Level of verbosity
return_errors : bool, optional
Activate return of iteration errors
Returns
-------
factors : ndarray list
List of factors of the CP decomposition element `i` is of shape
(tensor.shape[i], rank)
errors : list
A list of reconstruction errors at each iteration of the algorithms.
References
----------
.. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications",
SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009.
"""
factors = initialize_factors(tensor, rank, random_state=random_state)
rec_errors = []
norm_tensor = tl.norm(tensor, 2)
# Mode-3 values that control the country factors are set using the
# mode_three_val argument.
fixed_ja = mode_three_val[0]
fixed_ko = mode_three_val[1]
for iteration in range(n_iter_max):
for mode in range(tl.ndim(tensor)):
pseudo_inverse = tl.tensor(np.ones((rank, rank)),
**tl.context(tensor))
factors[2][0] = fixed_ja # set mode-3 values
factors[2][1] = fixed_ko # set mode-3 values
for i, factor in enumerate(factors):
if i != mode:
pseudo_inverse = pseudo_inverse * tl.dot(
tl.transpose(factor), factor)
factor = tl.dot(unfold(tensor, mode),
khatri_rao(factors, skip_matrix=mode))
factor = tl.transpose(
tl.solve(tl.transpose(pseudo_inverse), tl.transpose(factor)))
factors[mode] = factor
if tol:
rec_error = tl.norm(tensor - kruskal_to_tensor(factors),
2) / norm_tensor
rec_errors.append(rec_error)
if iteration > 1:
if verbose:
print('reconstruction error={}, variation={}.'.format(
rec_errors[-1], rec_errors[-2] - rec_errors[-1]))
if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol:
if verbose:
print('converged in {} iterations.'.format(iteration))
break
if return_errors:
return factors, rec_errors
else:
return factors
import tensorly.random as tl_rand
import numpy as np
from timeit import default_timer as timer
from BLOCK_SPG_CPD import bras_CPD
from BLOCK_SPG_CPD import ada_CPD
# Set up
for i in range(10):
# Each entry of the factor matrix is uniformly sampled from (0,1) and kruskal_tensor is used to from X
rank = 100
F = tl_rand.random_kruskal((300, 300, 300),
rank,
full=False,
random_state=np.random.RandomState(seed=i))
X = tl_kruskal.kruskal_to_tensor(F)
# Hetero noise
# H**o noise
# Parameters
B = 18
eta = 1
b = 10**-6
eps = 0
num_iterations = 0
max_time = 600
# Run ada_cpd update
total_time, res_error, time = ada_CPD(F, X, rank, B, eta, b, eps,
def sketched_residual_error(X, norm_x, idx, A, m):
X_bar = tl_kruskal.kruskal_to_tensor(A)
return norm(unfold(X, mode=m)[:, idx] -
unfold(X_bar, mode=m)[:, idx]) / norm_x
def decomp_plot(edge_len=25,
iterations=[1, 2, 3, 4],
ranks=[1, 5, 25, 50, 125, 130, 150, 200],
decomp='CP'):
#Params
print(ranks)
#Generate random samples
rng = check_random_state(7)
X = T.tensor(rng.normal(size=(1000, edge_len, edge_len), loc=0, scale=1))
#For plotting
n_rows = len(iterations)
n_columns = len(ranks) + 1
fig = plt.figure()
for i, _ in enumerate(iterations):
#Generate tensor
weight_img = X[i * edge_len:(i + 1) * edge_len, :, :]
ax = fig.add_subplot(n_rows, n_columns, i * n_columns + 1)
#Plot image corresponding to 3-D Tensor
ax.imshow(T.to_numpy(np.sum(weight_img, axis=0)),
cmap=plt.cm.OrRd,
interpolation='nearest')
ax.set_axis_off()
if i == 0:
ax.set_title('Original')
for j, rank in enumerate(ranks):
#Tensor decomposition, image_edge x rank (25x1, 25x5, 25x25 ...)
if decomp == 'CP':
#CP decomposition
components = parafac(weight_img, rank=rank)
ax = fig.add_subplot(n_rows, n_columns, i * n_columns + j + 2)
# Aggregate the factors for visualization
simg = np.sum(components[k] for k in range(len(components)))
ax.imshow(T.to_numpy(simg),
cmap=plt.cm.OrRd,
interpolation='nearest')
ax.text(.5,
2.0,
'{:.2f}'.format(
tensor_distance(kruskal_to_tensor(components),
weight_img)),
color='r')
# ax.set_autoscaley_on(False)
ax.set_axis_off()
else:
#Tucker decomposition
components, f = tucker(weight_img, ranks=[3, 25, rank])
#print(components.shape)
ax = fig.add_subplot(n_rows, n_columns, i * n_columns + j + 2)
# Aggregate the factors for visualization
simg = np.sum(components[k] for k in range(len(components)))
ax.imshow(T.to_numpy(simg),
cmap=plt.cm.OrRd,
interpolation='nearest')
ax.text(.5,
2.0,
'{:.2f}'.format(
tensor_distance(kruskal_to_tensor(components),
weight_img)),
color='r')
# ax.set_autoscaley_on(False)
ax.set_axis_off()
if i == 0:
ax.set_title('\n{}'.format(rank))
plt.suptitle('Tensor Decompositions')
plt.show()
# Set up
congruence = 0.9
lamb = 0.001
shape = (300,300,300)
nu = 2
rank = 20
num_iterations = 100000
eps = 1/num_iterations
# Sketching rates
sketching_rates = list(np.linspace(10**(-3), 10**(-1), 4)) + [1]
# Generate random latent factors
F = np.array(tl_rand.random_kruskal(shape=shape, rank=rank, full=False, random_state=np.random.RandomState(seed=0)))
X = tl_kruskal.kruskal_to_tensor(F)
# Generate ill conditioned factors
F_ill = generate_collinear_factors(shape, rank, congruence)
X_ill = tl_kruskal.kruskal_to_tensor(F_ill)
# Run experiment for sketching with weight update
sketching_rates = list(np.linspace(10**(-3), 10**(-1), 4)) + [1]
start = timer()
A,B,C, error, res_time = CPD_MWU(X, F, sketching_rates, lamb, eps, nu, rank, num_iterations)
end = timer()
# Print out total time
print("Total time", end-start)
print("CPD time", end-start-res_time)
print("Residual error time", res_time)