def get_cp_factors(self): if self.pretrained is not None: mat_dict = scipy.io.loadmat(self.pretrained) if mat_dict['R'][0][0] != self.rank: print('WRONG FACTORS, do not correspond to desired rank') PU_z, PU_cout, PU_cin = [Ui[0] for Ui in mat_dict['P_bals_epc_U']] Plmbda = mat_dict['P_bals_epc_lambda'].ravel() f_cin = np.array(PU_cin) f_cout = np.array(PU_cout) f_z = (np.array(PU_z)*(Plmbda)) else: bias = self.bias if isinstance(self.layer, nn.Sequential): # Tensorly case _, (f_cout, f_cin, f_z) = parafac(kruskal_to_tensor((None, self.weight)), self.rank, n_iter_max=5000, init='random', tol=1e-8, svd = None, cvg_criterion = 'rec_error') else: # Tensorly case _, (f_cout, f_cin, f_z) = parafac(self.weight, self.rank, n_iter_max=5000, init='random', tol=1e-8, svd = None, cvg_criterion = 'rec_error') # # Reshape factor matrices to 4D weight tensors # f_cin: (cin, rank) -> (rank, cin, 1, 1) # f_z: (z, rank) -> (rank, 1, h, w) # f_cout: (count, rank) -> (count, rank, 1, 1) # Pytorh case f_cin = f_cin.t().unsqueeze_(2).unsqueeze_(3).contiguous() f_z = torch.einsum('hwr->rhw', f_z.resize_((*self.kernel_size, self.rank))\ ).unsqueeze_(1).contiguous() f_cout = f_cout.unsqueeze_(2).unsqueeze_(3).contiguous() return [f_cin, f_z, f_cout], [None, None, bias]
def get_cp_factors(self): tl.set_backend('pytorch') bias = self.bias if isinstance(self.layer, nn.Sequential): # Tensorly case _, (f_cout, f_cin, f_h, f_w) = parafac(kruskal_to_tensor( (None, self.weight)), self.rank, n_iter_max=5000, init='random', tol=1e-8, svd=None, cvg_criterion='rec_error') else: # Tensorly case _, (f_cout, f_cin, f_h, f_w) = parafac(self.weight, self.rank, n_iter_max=5000, init='random', tol=1e-8, svd=None, cvg_criterion='rec_error') # # Reshape factor matrices to 4D weight tensors # f_cin: (cin, rank) -> (rank, cin, 1, 1) # f_h: (h, rank) -> (rank, 1, h, 1) # f_w: (w, rank) -> (rank, 1, 1, w) # f_cout: (count, rank) -> (count, rank, 1, 1) # Pytorh case f_cin = f_cin.t().unsqueeze_(2).unsqueeze_(3).contiguous() f_h = f_h.t().unsqueeze_(1).unsqueeze_(3).contiguous() f_w = f_w.t().unsqueeze_(1).unsqueeze_(2).contiguous() f_cout = f_cout.unsqueeze_(2).unsqueeze_(3).contiguous() return [f_cin, f_h, f_w, f_cout], [None, None, None, bias]
def parafac(tensor, rank, n_iter_max=100, tol=1e-8, random_state=None, verbose=False, return_errors=False, mode_three_val=[[0.5, 0.5, 0.0], [0.0, 0.5, 0.5]]): """CANDECOMP/PARAFAC decomposition via alternating least squares (ALS) Computes a rank-`rank` decomposition of `tensor` [1]_ such that, ``tensor = [| factors[0], ..., factors[-1] |]``. Parameters ---------- tensor : ndarray rank : int Number of components. n_iter_max : int Maximum number of iteration tol : float, optional (Default: 1e-6) Relative reconstruction error tolerance. The algorithm is considered to have found the global minimum when the reconstruction error is less than `tol`. random_state : {None, int, np.random.RandomState} verbose : int, optional Level of verbosity return_errors : bool, optional Activate return of iteration errors Returns ------- factors : ndarray list List of factors of the CP decomposition element `i` is of shape (tensor.shape[i], rank) errors : list A list of reconstruction errors at each iteration of the algorithms. References ---------- .. [1] tl.G.Kolda and B.W.Bader, "Tensor Decompositions and Applications", SIAM REVIEW, vol. 51, n. 3, pp. 455-500, 2009. """ factors = initialize_factors(tensor, rank, random_state=random_state) rec_errors = [] norm_tensor = tl.norm(tensor, 2) # Mode-3 values that control the country factors are set using the # mode_three_val argument. fixed_ja = mode_three_val[0] fixed_ko = mode_three_val[1] for iteration in range(n_iter_max): for mode in range(tl.ndim(tensor)): pseudo_inverse = tl.tensor(np.ones((rank, rank)), **tl.context(tensor)) factors[2][0] = fixed_ja # set mode-3 values factors[2][1] = fixed_ko # set mode-3 values for i, factor in enumerate(factors): if i != mode: pseudo_inverse = pseudo_inverse * tl.dot( tl.transpose(factor), factor) factor = tl.dot(unfold(tensor, mode), khatri_rao(factors, skip_matrix=mode)) factor = tl.transpose( tl.solve(tl.transpose(pseudo_inverse), tl.transpose(factor))) factors[mode] = factor if tol: rec_error = tl.norm(tensor - kruskal_to_tensor(factors), 2) / norm_tensor rec_errors.append(rec_error) if iteration > 1: if verbose: print('reconstruction error={}, variation={}.'.format( rec_errors[-1], rec_errors[-2] - rec_errors[-1])) if tol and abs(rec_errors[-2] - rec_errors[-1]) < tol: if verbose: print('converged in {} iterations.'.format(iteration)) break if return_errors: return factors, rec_errors else: return factors
import tensorly.random as tl_rand import numpy as np from timeit import default_timer as timer from BLOCK_SPG_CPD import bras_CPD from BLOCK_SPG_CPD import ada_CPD # Set up for i in range(10): # Each entry of the factor matrix is uniformly sampled from (0,1) and kruskal_tensor is used to from X rank = 100 F = tl_rand.random_kruskal((300, 300, 300), rank, full=False, random_state=np.random.RandomState(seed=i)) X = tl_kruskal.kruskal_to_tensor(F) # Hetero noise # H**o noise # Parameters B = 18 eta = 1 b = 10**-6 eps = 0 num_iterations = 0 max_time = 600 # Run ada_cpd update total_time, res_error, time = ada_CPD(F, X, rank, B, eta, b, eps,
def sketched_residual_error(X, norm_x, idx, A, m): X_bar = tl_kruskal.kruskal_to_tensor(A) return norm(unfold(X, mode=m)[:, idx] - unfold(X_bar, mode=m)[:, idx]) / norm_x
def decomp_plot(edge_len=25, iterations=[1, 2, 3, 4], ranks=[1, 5, 25, 50, 125, 130, 150, 200], decomp='CP'): #Params print(ranks) #Generate random samples rng = check_random_state(7) X = T.tensor(rng.normal(size=(1000, edge_len, edge_len), loc=0, scale=1)) #For plotting n_rows = len(iterations) n_columns = len(ranks) + 1 fig = plt.figure() for i, _ in enumerate(iterations): #Generate tensor weight_img = X[i * edge_len:(i + 1) * edge_len, :, :] ax = fig.add_subplot(n_rows, n_columns, i * n_columns + 1) #Plot image corresponding to 3-D Tensor ax.imshow(T.to_numpy(np.sum(weight_img, axis=0)), cmap=plt.cm.OrRd, interpolation='nearest') ax.set_axis_off() if i == 0: ax.set_title('Original') for j, rank in enumerate(ranks): #Tensor decomposition, image_edge x rank (25x1, 25x5, 25x25 ...) if decomp == 'CP': #CP decomposition components = parafac(weight_img, rank=rank) ax = fig.add_subplot(n_rows, n_columns, i * n_columns + j + 2) # Aggregate the factors for visualization simg = np.sum(components[k] for k in range(len(components))) ax.imshow(T.to_numpy(simg), cmap=plt.cm.OrRd, interpolation='nearest') ax.text(.5, 2.0, '{:.2f}'.format( tensor_distance(kruskal_to_tensor(components), weight_img)), color='r') # ax.set_autoscaley_on(False) ax.set_axis_off() else: #Tucker decomposition components, f = tucker(weight_img, ranks=[3, 25, rank]) #print(components.shape) ax = fig.add_subplot(n_rows, n_columns, i * n_columns + j + 2) # Aggregate the factors for visualization simg = np.sum(components[k] for k in range(len(components))) ax.imshow(T.to_numpy(simg), cmap=plt.cm.OrRd, interpolation='nearest') ax.text(.5, 2.0, '{:.2f}'.format( tensor_distance(kruskal_to_tensor(components), weight_img)), color='r') # ax.set_autoscaley_on(False) ax.set_axis_off() if i == 0: ax.set_title('\n{}'.format(rank)) plt.suptitle('Tensor Decompositions') plt.show()
# Set up congruence = 0.9 lamb = 0.001 shape = (300,300,300) nu = 2 rank = 20 num_iterations = 100000 eps = 1/num_iterations # Sketching rates sketching_rates = list(np.linspace(10**(-3), 10**(-1), 4)) + [1] # Generate random latent factors F = np.array(tl_rand.random_kruskal(shape=shape, rank=rank, full=False, random_state=np.random.RandomState(seed=0))) X = tl_kruskal.kruskal_to_tensor(F) # Generate ill conditioned factors F_ill = generate_collinear_factors(shape, rank, congruence) X_ill = tl_kruskal.kruskal_to_tensor(F_ill) # Run experiment for sketching with weight update sketching_rates = list(np.linspace(10**(-3), 10**(-1), 4)) + [1] start = timer() A,B,C, error, res_time = CPD_MWU(X, F, sketching_rates, lamb, eps, nu, rank, num_iterations) end = timer() # Print out total time print("Total time", end-start) print("CPD time", end-start-res_time) print("Residual error time", res_time)