def compare_spaces(self): self._compute_graph_laplacians() sent_eigenvalues = torch.eig(self.sent_laplacian, eigenvectors=False)[0] sent_sorted_eigenvalues = torch.sort(sent_eigenvalues[:, 0], -1, True)[0] # .numpy() sent_eignvalue_target = 0.9 * torch.sum(sent_sorted_eigenvalues).item() sent_cumsum = torch.cumsum(sent_sorted_eigenvalues, dim=0) sent_min_k = torch.min(torch.where(sent_cumsum > sent_eignvalue_target)[0]).item() #sev = list(chain.from_iterable(sent_eigenvalues.numpy())) #sent_min_k = 0 #for k in range(len(sent_sorted_eigenvalues)): # val = sent_sorted_eigenvalues[0:k].sum() # check = [1 if val > x else 0 for x in sev] # if sum(check) < .9 * len(sent_sorted_eigenvalues): # sent_min_k = k print('Sent min k: {k}'.format(k=sent_min_k)) node_eigenvalues = torch.eig(self.node_laplacian, eigenvectors=False)[0] node_sorted_eigenvalues = torch.sort(node_eigenvalues[:, 0], -1, True)[0] # .numpy() node_eignvalue_target = 0.9 * torch.sum(node_sorted_eigenvalues).item() node_cumsum = torch.cumsum(node_sorted_eigenvalues, dim=0) node_min_k = torch.min(torch.where(node_cumsum > node_eignvalue_target)[0]).item() #nev = list(chain.from_iterable(node_eigenvalues.numpy())) #node_min_k = 0 #for k in range(len(node_eigenvalues)): # val = node_sorted_eigenvalues[0:k].sum() # check = [1 if val > x else 0 for x in nev] # if sum(check) < .9 * len(node_sorted_eigenvalues): # node_min_k = k print('Node min k: {k}'.format(k=node_min_k)) final_k = min(node_min_k, sent_min_k) node_k = node_sorted_eigenvalues[0: final_k] sent_k = sent_sorted_eigenvalues[0: final_k] delta = self.sum_of_squares(node_k, sent_k).item() return delta
def compute_fisher(self, action_log_probs): # compute logprob grad for each action grad_log_probs = [] self.F.zero_() for i in range(len(action_log_probs)): # For each log pi(a | s) get the gradient wrt theta self.zero_grad() action_log_probs[i].backward(retain_graph=True) grad = self.get_flattened_grad() grad_log_probs.append(grad) # Build matrix of gradients (nparams x nsamples) X = torch.cat(grad_log_probs, dim=1) self.F = torch.mm(X, torch.transpose(X, 0, 1)) self.F = self.F / float(len(action_log_probs)) # check if Fisher PSD, if not add eps * eye min_eig = torch.min(torch.eig(self.F)[0]) trys = 0 while min_eig < 0: trys += 1 print("Fisher eig < 0: ", float(min_eig), file=sys.stderr) self.F = self.F + 1e-5 * torch.eye(self.F.shape[0]) min_eig = torch.min(torch.eig(self.F)[0]) if trys > 100: print("Fisher numerically unstable, matrix is neg def.", file=sys.stderr) sys.exit()
def _update_inv(self, m): """Do eigen decomposition for computing inverse of the ~ fisher. :param m: The layer :return: no returns. """ if self.torch_symeig == 'true': eps = 1e-10 # for numerical stability self.d_a[m], self.Q_a[m] = torch.symeig(self.m_aa[m], eigenvectors=True) self.d_g[m], self.Q_g[m] = torch.symeig(self.m_gg[m], eigenvectors=True) self.d_a[m].mul_((self.d_a[m] > eps).float()) self.d_g[m].mul_((self.d_g[m] > eps).float()) # if self.steps != 0: self.S_l[m] = self.d_g[m].unsqueeze(1) @ self.d_a[m].unsqueeze(0) else: d_a, self.Q_a[m] = torch.eig(self.m_aa[m], eigenvectors=True) d_g, self.Q_g[m] = torch.eig(self.m_gg[m], eigenvectors=True) self.d_a[m] = d_a[:, 0] self.d_g[m] = d_g[:, 0] self.d_a[m].mul_((self.d_a[m] > eps).float()) self.d_g[m].mul_((self.d_g[m] > eps).float()) self.S_l[m] = self.d_g[m].unsqueeze(1) @ self.d_a[m].unsqueeze(0)
def hottaBasisVectors(X,sub_dim): d,n = X.shape if d < n: C = torch.mm(X, X.transpose(0,1)) matrank = torch.matrix_rank(C) tmp_val, tmp_vec = torch.eig(C, eigenvectors = True) value, index = torch.sort(tmp_val,descending = True) eig_vec = tmp_vec[:,index[:matrank]] eig_val = value[:matrank] eig_vec = eig_vec[:,:sub_dim] eig_val = value[:sub_dim] else: C = torch.mm(X.transpose(0,1), X) matrank = torch.matrix_rank(C) tmp_val, tmp_vec = torch.eig(C, eigenvectors = True) # second column is zero if the eig vals are real tmp_val = tmp_val[:,0] value, index = torch.sort(tmp_val,descending = True) # tmp_vec = tmp_vec[:,index[:matrank]] # eig_vec = torch.mm(X, tmp_vec) eig_vec = torch.zeros((X.shape[0],matrank)) for i in range(matrank): eig_vec[:,i] = (X.mv(tmp_vec[:,index[i]])).div((value[i]).sqrt()) eig_vec = normalize_dataset(eig_vec[:,:sub_dim]) eig_val = value[:sub_dim] return eig_vec, eig_val
def qt_cal(x, y): input_outer_product = torch.outer(x, x) output_outer_product = torch.outer(y, y) (input_evals, input_evecs) = torch.eig(input_outer_product, eigenvectors=True) (output_evals, output_evecs) = torch.eig(output_outer_product, eigenvectors=True)
def forward(self, x_train, y_train, x_test=None): # See the autograd section for explanation of what happens here. n = x_train.size(0) #dm = torch.Tensor([[1,0,0],[0,1,0],[0,1,0],[0,0,1]]) SIGMA = torch.diag(self.sigmaentr.pow(2).view( -1)) #dm.mm(self.sigmaentr.pow(2).view(3,1)).view(2,2) trOnes = torch.ones(1, n) xext = torch.cat((trOnes, x_train.t()), 0) xSx = xext.t().mm(SIGMA).mm(xext) xSxd = torch.diag(xSx).view(n, 1) kyy = (2.0 / math.pi) * torch.asin(2.0 * xSx / torch.sqrt( (1 + 2.0 * xSxd).mm(1 + 2.0 * xSxd.t()))) kyy = 0.5 * (kyy + kyy.t()) + self.sigma_n.pow(2) * torch.eye(n) add = torch.eig(kyy)[0][:, 0].min().detach() while add < 0: kyy = kyy - 2 * add * torch.eye(n) add = torch.eig(kyy)[0][:, 0].min().detach() c = torch.cholesky(kyy, upper=True) # v = torch.potrs(y_train, c, upper=True) v, _ = torch.gesv(y_train, kyy) if x_test is None: out = (c, v) if x_test is not None: nt = x_test.size(0) teOnes = torch.ones(1, nt) xexte = torch.cat((teOnes, x_test.t()), 0) xSx_m = xext.t().mm(SIGMA).mm(xexte) xSx_te_d = torch.sum(xexte.t() * (SIGMA.mm(xexte)).t(), dim=1).view(nt, 1) kyf = (2.0 / math.pi) * torch.asin(2.0 * xSx_m / torch.sqrt( (1 + 2.0 * xSxd).mm(1 + 2.0 * xSx_te_d.t()))) # solve f_test = kyf.t().mm(v) # tmp = torch.potrs(kfy.t(), c, upper=True) # tmp = torch.sum(kfy * tmp.t(), dim=1) # #cov_f = something - tmp out = f_test #, cov_f) return out
def forward(self, s_inputs, s_outputs, t_input): num_src_domains = len(s_inputs) train_losses, discs = [], [] Phi_t = torch.cat([t_input, torch.ones_like(t_input)], dim=1) M_t = torch.mm(Phi_t.transpose(0, 1), Phi_t) for i in range(num_src_domains): prediction = self.regress_net(s_inputs[i]) train_loss = torch.mean((prediction - s_outputs[i]) ** 2) train_losses.append(train_loss) Phi_s = torch.cat([s_inputs[i], torch.ones_like(s_inputs[i])], dim=1) M_s = torch.mm(Phi_s.transpose(0, 1), Phi_s) disc = torch.max(torch.norm(torch.eig(M_t - M_s)[0], dim=1)) discs.append(disc) train_losses = torch.stack(train_losses) discs = torch.stack(discs) g = self.gamma * (train_losses + self.mu * discs) alpha = self.proj(g) loss = torch.dot(g, alpha) + torch.norm(alpha) alpha = alpha.detach().cpu().numpy() return loss, alpha
def spectral_normalization(W): ''' return net/||net||_spectral ''' eigenvalues = torch.eig(W.mm(W)) spectral_norm = torch.max(eigenvalues)**0.5 return spectral_norm
def calculate_frechet_distance(mu1: torch.Tensor, sigma1: torch.Tensor, mu2: torch.Tensor, sigma2: torch.Tensor, eps: float = 1e-8) -> torch.Tensor: """Numpy implementation of the Frechet Distance. The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1) and X_2 ~ N(mu_2, C_2) is d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)). Stable version by Dougal J. Sutherland. Params: -- mu1 : Numpy array containing the activations of a layer of the inception net (like returned by the function 'get_predictions') for generated samples. -- mu2 : The sample mean over activations, precalculated on an representative data set. -- sigma1: The covariance matrix over activations for generated samples. -- sigma2: The covariance matrix over activations, precalculated on an representative data set. Returns: -- : The Frechet Distance. """ diff = mu1 - mu2 offset = eps * torch.eye(*sigma1.shape, device=sigma1.device) eigenvals, _ = torch.eig((sigma1 + offset) @ (sigma2 + offset), eigenvectors=False) tr_covmean = torch.sum(torch.sqrt(torch.abs(eigenvals[:, 0]))) return (diff.dot(diff) + torch.trace(sigma1) + torch.trace(sigma2) - 2 * tr_covmean)
def hessian_max_min_eig(grads, model): hess = hessian(grads, model) #.to(device) eigvalues = torch.eig(hess)[0] real_eigvalues = [ real for real, img in eigvalues if torch.abs(img) < 1e-6 ] return max(real_eigvalues), min(real_eigvalues)
def test_remote_tensor_multi_var_methods(self): hook = TorchHook(verbose=False) local = hook.local_worker remote = VirtualWorker(hook, 1) local.add_worker(remote) x = torch.FloatTensor([[1, 2], [4, 3], [5, 6]]) x.send(remote) y, z = torch.max(x, 1) assert torch.equal(y.get(), torch.FloatTensor([2, 4, 6])) assert torch.equal(z.get(), torch.LongTensor([1, 0, 1])) x = torch.FloatTensor([[0, 0], [1, 0]]).send(remote) y, z = torch.qr(x) assert (y.get() == torch.FloatTensor([[0, -1], [-1, 0]])).all() assert (z.get() == torch.FloatTensor([[-1, 0], [0, 0]])).all() x = torch.arange(1, 6).send(remote) y, z = torch.kthvalue(x, 4) assert (y.get() == torch.FloatTensor([4])).all() assert (z.get() == torch.LongTensor([3])).all() x = torch.FloatTensor([[0, 0], [1, 1]]).send(remote) y, z = torch.eig(x, True) assert (y.get() == torch.FloatTensor([[1, 0], [0, 0]])).all() assert ((z.get() == torch.FloatTensor([[0, 0], [1, 0]])) == torch.ByteTensor([[1, 0], [1, 0]])).all() x = torch.zeros(3, 3).send(remote) w, y, z = torch.svd(x) assert (w.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all() assert (y.get() == torch.FloatTensor([0, 0, 0])).all() assert (z.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def reset_parameters(self): weight_dict = self.state_dict() for key, value in weight_dict.items(): if key == 'weight_ih_l0': nn.init.uniform_(value, -1, 1) value *= self.w_ih_scale[1:] elif re.fullmatch('weight_ih_l[^0]*', key): nn.init.uniform_(value, -1, 1) elif re.fullmatch('bias_ih_l[0-9]*', key): nn.init.uniform_(value, -1, 1) value *= self.w_ih_scale[0] elif re.fullmatch('weight_hh_l[0-9]*', key): w_hh = torch.Tensor(self.hidden_size * self.hidden_size) w_hh.uniform_(-1, 1) if self.density < 1: zero_weights = torch.randperm( int(self.hidden_size * self.hidden_size)) zero_weights = zero_weights[:int(self.hidden_size * self.hidden_size * (1 - self.density))] w_hh[zero_weights] = 0 w_hh = w_hh.view(self.hidden_size, self.hidden_size) abs_eigs = (torch.eig(w_hh)[0]**2).sum(1).sqrt() weight_dict[key] = w_hh * (self.spectral_radius / torch.max(abs_eigs)) self.load_state_dict(weight_dict)
def test_remote_tensor_multi_var_methods(self): hook = TorchHook(verbose=False) local = hook.local_worker remote = VirtualWorker(hook, 1) local.add_worker(remote) x = torch.FloatTensor([[1, 2], [4, 3], [5, 6]]) x.send(remote) y, z = torch.max(x, 1) assert torch.equal(y.get(), torch.FloatTensor([2, 4, 6])) assert torch.equal(z.get(), torch.LongTensor([1, 0, 1])) x = torch.FloatTensor([[0, 0], [1, 0]]).send(remote) y, z = torch.qr(x) assert (y.get() == torch.FloatTensor([[0, -1], [-1, 0]])).all() assert (z.get() == torch.FloatTensor([[-1, 0], [0, 0]])).all() x = torch.arange(1, 6).send(remote) y, z = torch.kthvalue(x, 4) assert (y.get() == torch.FloatTensor([4])).all() assert (z.get() == torch.LongTensor([3])).all() x = torch.FloatTensor([[0, 0], [1, 1]]).send(remote) y, z = torch.eig(x, True) assert (y.get() == torch.FloatTensor([[1, 0], [0, 0]])).all() assert ((z.get() == torch.FloatTensor( [[0, 0], [1, 0]], )) == torch.ByteTensor([[1, 0], [1, 0]])).all() x = torch.zeros(3, 3).send(remote) w, y, z = torch.svd(x) assert (w.get() == torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all() assert (y.get() == torch.FloatTensor([0, 0, 0])).all() assert (z.get() == torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
def test_local_tensor_multi_var_methods(self): x = torch.FloatTensor([[1, 2], [2, 3], [5, 6]]) t, s = torch.max(x, 1) assert (t == torch.FloatTensor([2, 3, 6])).float().sum() == 3 assert (s == torch.LongTensor([1, 1, 1])).float().sum() == 3 x = torch.FloatTensor([[0, 0], [1, 1]]) y, z = torch.eig(x, True) assert (y == torch.FloatTensor([[1, 0], [0, 0]])).all() assert (torch.equal( z == torch.FloatTensor([[0, 0], [1, 0]], ), torch.ByteTensor([[1, 0], [1, 0]]), )) x = torch.FloatTensor([[0, 0], [1, 0]]) y, z = torch.qr(x) assert (y == torch.FloatTensor([[0, -1], [-1, 0]])).all() assert (z == torch.FloatTensor([[-1, 0], [0, 0]])).all() x = torch.arange(1, 6) y, z = torch.kthvalue(x, 4) assert (y == torch.FloatTensor([4])).all() assert (z == torch.LongTensor([3])).all() x = torch.zeros(3, 3) w, y, z = torch.svd(x) assert (w == torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all() assert (y == torch.FloatTensor([0, 0, 0])).all() assert (z == torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
def __force_spectral_radius(self): # Make the reservoir weight matrix - a unit spectral radius rad = torch.max(torch.abs(torch.eig(self.W_r)[0])) self.W_r = self.W_r / rad # Force spectral radius self.W_r = self.W_r * self.spectral_radius
def forward(ctx, A): # normalize the shape to be batched Ashape = A.shape if A.ndim == 2: A = A.unsqueeze(0) elif A.ndim > 3: A = A.view(*A.shape[:-2], A.shape[-2], A.shape[-1]) nbatch = A.shape[0] evecs = torch.empty_like(A).to(A.device) evals = torch.empty(A.shape[0], A.shape[-1]).to(A.dtype).to(A.device) for i in range(nbatch): evalue, evec = torch.eig(A[i], eigenvectors=True) # check if the eigenvalues contain complex numbers if not torch.allclose(evalue[:,1], torch.zeros_like(evalue[:,1])): raise ValueError("The eigenvalues contain complex numbers") # sort eigenvalues from lowest to highest evalue, idxsort = torch.sort(evalue[:,0], dim=-1) # idxsort (na,) evec = torch.index_select(evec, dim=-1, index=idxsort) evecs[i] = evec evals[i] = evalue # reshape the results evecs = evecs.view(*Ashape) evals = evals.view(*Ashape[:-1]) ctx.evecs = evecs ctx.evals = evals return evals, evecs
def special_sylvester(A, B, d=None): '''Solves the eqations `AX+XA=B` for positive definite `A`. This is a special case of Sylvester equation `AX+XB=C`. https://en.wikipedia.org/wiki/Sylvester_equation A unique solution exists when `A` and `-A` have no common eigenvalues. Sources: https://math.stackexchange.com/a/820313 Explicit solution of the operator equation A*X+X*A=B: https://core.ac.uk/download/pdf/82315631.pdf Args: A: The matrix `A`. B: The matrix `B`. d: The eigenvalues or the singular values of `A` if available. If `d` is provided, `A` must be the eigenvectors, instead. Returns: The matrix `X`. ''' if d is None: D, Q = torch.eig(A, eigenvectors=True) d = D[:, 0] else: Q = A C = Q.t().mm(B.mm(Q)) Y = C / (d.view(-1, 1) + d.view(1, -1)) return Q.mm(Y.mm(Q.t()))
def test_torch_function_with_multiple_output_on_remote_var(self): hook = TorchHook(verbose=False) me = hook.local_worker remote = VirtualWorker(id=2, hook=hook) me.add_worker(remote) x = Var(torch.FloatTensor([[1, 2], [4, 3], [5, 6]])) x.send(remote) y, z = torch.max(x, 1) y.get() assert torch.equal(y, Var(torch.FloatTensor([2, 4, 6]))) x = Var(torch.FloatTensor([[0, 0], [1, 0]])).send(remote) y, z = torch.qr(x) assert (y.get() == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all() assert (z.get() == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all() x = Var(torch.arange(1, 6)).send(remote) y, z = torch.kthvalue(x, 4) assert (y.get() == Var(torch.FloatTensor([4]))).all() assert (z.get() == Var(torch.LongTensor([3]))).all() x = Var(torch.FloatTensor([[0, 0], [0, 0]])) x.send(remote) y, z = torch.eig(x, True) assert (y.get() == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all() assert (z.get() == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all() x = Var(torch.zeros(3, 3)).send(remote) w, y, z = torch.svd(x) assert (w.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all() assert (y.get() == Var(torch.FloatTensor([0, 0, 0]))).all() assert (z.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def test_torch_function_with_multiple_output_on_local_var(self): x = Var(torch.FloatTensor([[1, 2], [2, 3], [5, 6]])) t, s = torch.max(x, 1) assert (t == Var(torch.FloatTensor([2, 3, 6]))).all() assert (s == Var(torch.LongTensor([1, 1, 1]))).all() x = Var(torch.FloatTensor([[0, 0], [0, 0]])) y, z = torch.eig(x, True) assert (y == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all() assert (z == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all() x = Var(torch.FloatTensor([[0, 0], [1, 0]])) y, z = torch.qr(x) assert (y == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all() assert (z == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all() x = Var(torch.arange(1, 6)) y, z = torch.kthvalue(x, 4) assert (y == Var(torch.FloatTensor([4]))).all() assert (z == Var(torch.LongTensor([3]))).all() x = Var(torch.zeros(3, 3)) w, y, z = torch.svd(x) assert (w == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all() assert (y == Var(torch.FloatTensor([0, 0, 0]))).all() assert (z == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def test_batch_symeig_top(): # input = torch.randn(8, 4, 4).float().cuda() num_q = 800 input_lrf = torch.randn(num_q, 8, 4).cuda() input_lrf = F.normalize(input_lrf, p=2, dim=-1) input_lrf = input_lrf.view(-1, 4) test = torch.bmm(input_lrf.unsqueeze(2), input_lrf.unsqueeze(1)) test = test.view(num_q, 8, 4, 4) input = torch.sum(test, 1) input3 = input.clone() input3.requires_grad = True input.requires_grad = True # batch wise back w, v = S.symeig(input) v_max = v[:, 3].clone() bool_vmax = v_max[:, 0] / torch.abs(v_max[:, 0]) bool_vmax = bool_vmax.contiguous().view(-1, 1) bool_vmax = bool_vmax.expand(v_max.size(0), 4) v_max = v_max * bool_vmax (v_max.mean()).backward() # pytorch version loop back averaged_Q4 = torch.rand(num_q, 4) for i in range(input3.size(0)): eigenValues, eigenVectors = torch.eig(input3[i], eigenvectors=True) e_values, e_indices = torch.max(eigenValues, 0) averaged_Q4[i] = eigenVectors[:, e_indices] if (averaged_Q4[i][0] < 0): averaged_Q4[i] = -averaged_Q4[i] (averaged_Q4.mean()).backward() torch.testing.assert_allclose(input.grad, input3.grad)
def get_stably_bounded_shapes(N, n, a, b, c, d, h, w): ''' This could be made into a class for the data loader. input : positive integer, non-negative integer, number, number with a <= b, number, number with c <= d, positive integer, positive integer output : [N, h, w] tensor ''' x, y = torch.meshgrid(torch.linspace(a, b, h), -torch.linspace(c, d, w)) X = torch.stack([x**(n // 2 - i) * y**i for i in range(n // 2 + 1)]) M = torch.tensor([]) for _ in range(N): M_i = torch.randn(n // 2 + 1, n // 2 + 1) M_i = torch.tril(M_i) + torch.tril(M_i, diagonal=-1).t() while not torch.all(torch.eig(M_i)[0][:, 0] > 0): M_i = torch.randn(n // 2 + 1, n // 2 + 1) M_i = torch.tril(M_i) + torch.tril(M_i, diagonal=-1).t() M = torch.cat((M, M_i.view(1, *M_i.shape))) image = torch.einsum('ikl,mij,jkl->mkl', X, M, X) C = torch.randn(N, int(nCk(n + 2, 2)) - n - 1) B = torch.stack([x**i * y**j for i in range(n) for j in range(n - i)]) return (image + torch.einsum('li,ijk->jk', C, B) < 0).float()
def generate_reservoir(size, radius, degree): sparsity = degree / float(size) A = torch.tensor(sparse.rand(size, size, density=sparsity).todense(), dtype=torch.float32) e = torch.max(torch.eig(A, eigenvectors=False)[0]) A = (A / e) * radius return A
def test_torch_function_with_multiple_output_on_local_var(self): x = Var(torch.FloatTensor([[1, 2], [2, 3], [5, 6]])) t, s = torch.max(x, 1) assert (t == Var(torch.FloatTensor([2, 3, 6]))).all() assert (s == Var(torch.LongTensor([1, 1, 1]))).all() x = Var(torch.FloatTensor([[0, 0], [0, 0]])) y, z = torch.eig(x, True) assert (y == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all() assert (z == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all() x = Var(torch.FloatTensor([[0, 0], [1, 0]])) y, z = torch.qr(x) assert (y == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all() assert (z == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all() x = Var(torch.arange(1, 6)) y, z = torch.kthvalue(x, 4) assert (y == Var(torch.FloatTensor([4]))).all() assert (z == Var(torch.LongTensor([3]))).all() x = Var(torch.zeros(3, 3)) w, y, z = torch.svd(x) assert (w == Var(torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all() assert (y == Var(torch.FloatTensor([0, 0, 0]))).all() assert (z == Var(torch.FloatTensor( [[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
def test_local_tensor_multi_var_methods(self): x = torch.FloatTensor([[1, 2], [2, 3], [5, 6]]) t, s = torch.max(x, 1) assert (t == torch.FloatTensor([2, 3, 6])).float().sum() == 3 assert (s == torch.LongTensor([1, 1, 1])).float().sum() == 3 x = torch.FloatTensor([[0, 0], [1, 1]]) y, z = torch.eig(x, True) assert (y == torch.FloatTensor([[1, 0], [0, 0]])).all() assert (torch.equal(z == torch.FloatTensor([[0, 0], [1, 0]]), torch.ByteTensor([[1, 0], [1, 0]]))) x = torch.FloatTensor([[0, 0], [1, 0]]) y, z = torch.qr(x) assert (y == torch.FloatTensor([[0, -1], [-1, 0]])).all() assert (z == torch.FloatTensor([[-1, 0], [0, 0]])).all() x = torch.arange(1, 6) y, z = torch.kthvalue(x, 4) assert (y == torch.FloatTensor([4])).all() assert (z == torch.LongTensor([3])).all() x = torch.zeros(3, 3) w, y, z = torch.svd(x) assert (w == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all() assert (y == torch.FloatTensor([0, 0, 0])).all() assert (z == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def priorTrainNN(self, x_train, y_train): """ Updates the prior of the Bayesian NN Args: X_train (th.DoubleTensor): [N x D_in] column matrix of training inputs Y_train (th.DoubleTensor): [N x D_out] column matrix of training outputs """ x_t = Variable(x_train) y_t = Variable(y_train, requires_grad=False) #Get the hessian of the error function and its eigen values hess = self.getHessian(x_train, y_train) e, v = th.eig(self.beta * hess.data, eigenvectors=False) #Compute (w^t)*(w) d = 0 for param in self.model.parameters(): target = Variable(th.zeros(param.size())).type(dtype) d += self.reg_fn(param, target).data #Iterate the alpha calculations to hopefully converge (if hessian is nice to us) for i in range(20): gamma = th.sum(e[:, 0] / (self.alpha + e[:, 0]), 0) #Eq. 5.179 self.alpha = (gamma / d)[0] #Eq. 5.178 print(self.alpha) print('Alpha Updated to: ' + str(self.alpha))
def test_torch_function_with_multiple_output_on_remote_var(self): hook = TorchHook(verbose=False) me = hook.local_worker remote = VirtualWorker(id=2, hook=hook) me.add_worker(remote) x = Var(torch.FloatTensor([[1, 2], [4, 3], [5, 6]])) x.send(remote) y, z = torch.max(x, 1) y.get() assert torch.equal(y, Var(torch.FloatTensor([2, 4, 6]))) x = Var(torch.FloatTensor([[0, 0], [1, 0]])).send(remote) y, z = torch.qr(x) assert (y.get() == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all() assert (z.get() == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all() x = Var(torch.arange(1, 6)).send(remote) y, z = torch.kthvalue(x, 4) assert (y.get() == Var(torch.FloatTensor([4]))).all() assert (z.get() == Var(torch.LongTensor([3]))).all() x = Var(torch.FloatTensor([[0, 0], [0, 0]])) x.send(remote) y, z = torch.eig(x, True) assert (y.get() == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all() assert (z.get() == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all() x = Var(torch.zeros(3, 3)).send(remote) w, y, z = torch.svd(x) assert (w.get() == Var( torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all() assert (y.get() == Var(torch.FloatTensor([0, 0, 0]))).all() assert (z.get() == Var( torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
def PCA(data, k=2): X = torch.from_numpy(data) X_mean = torch.mean(X, 0) X = X - X_mean.expand_as(X) v, w = torch.eig(torch.mm(X, torch.t(X)), eigenvectors=True) return torch.mm(w[:k, :], X)
def get_obs(self, H, SpinOp, solver_params): # [ESpec, U] = EigenSolver.apply(H) [ESpec, U] = torch.eig(H, eigenvectors=True) ESpec = ESpec[:, 0] Sztot = sum(SpinOp.SzP).double() Sxtot = sum(SpinOp.SxP).double() Mnn = torch.diag(U.transpose(1, 0).mm(Sztot).mm(U)) Mnn_x = torch.diag(U.transpose(1, 0).mm(Sxtot).mm(U)) MSqnn = torch.diag(U.transpose(1, 0).mm(Sztot).mm(Sztot).mm(U)) for i in range(self.N): self.Z[i] = (torch.exp(-self.beta[i] * ESpec)).sum() for i in range(self.N): self.Fe[i, 0] = self.beta[i] self.Fe[i, 1] = -1 / self.beta[i] * torch.log( (torch.exp(-self.beta[i] * ESpec)).sum()) self.M[i] = (Mnn * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] self.M_paral[i] = ( Mnn_x * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] self.E[i] = (ESpec * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] for i in range(self.N): self.C[i] = ((ESpec ** 2 * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] - self.E[i] ** 2) * \ self.beta[i] ** 2 self.Chi[i] = ( (MSqnn * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] - self.M[i]**2) * self.beta[i] self.C = self.C / solver_params['L'] self.Chi = self.Chi / solver_params['L'] self.M = self.M / solver_params['L'] self.M_paral = self.M_paral / solver_params['L']
def PCA_eig(self, X, k, center=True, scale=False): n, p = X.size() print(X) ones = torch.ones(n).view([n, 1]) h = ((1 / n) * torch.mm(ones, ones.t())) if center else torch.zeros( n * n).view([n, n]) H = torch.eye(n) - h X_center = torch.mm(H.double(), X.double()) print("X_center", X_center) covariance = 1 / (n - 1) * torch.mm(X_center.t(), X_center).view(p, p) print("convariance", covariance) scaling = torch.sqrt(1 / torch.diag(covariance)).double( ) if scale else torch.ones(p).double() print("scaling", scaling) A = torch.diag(scaling).view(p, p) print("A", A) print("A size", A.size(), "covariance size", covariance.size()) scaled_covariance = torch.empty(p, p) scaled_covariance = torch.mm(A, covariance) print(scaled_covariance) eigenvalues, eigenvectors = torch.eig(scaled_covariance, True) components = (eigenvectors[:, :k]).t() projection = torch.mm(X, components.t()) return projection
def nearest_pd(A, f=np.spacing): """Find the nearest positive-definite matrix to input A Python/Numpy port of John D'Errico's `nearestSPD` MATLAB code [1], which credits [2] [1] https://www.mathworks.com/matlabcentral/fileexchange/42885-nearestspd [2] https://doi.org/10.1016/0024-3795(88)90223-6 """ B = (A + A.T) / 2 _, s, V = torch.svd(B) # For a comparison with kanga, see the following: # https://github.com/pytorch/pytorch/issues/16076#issuecomment-477755364 H = torch.matmul(V, torch.matmul(torch.diag(s), V.T)) A2 = (B + H) / 2 A3 = (A2 + A2.T) / 2 if is_pos_def(A3): return A3 spacing = f(torch.norm(A).item()) I = torch.eye(A.shape[0]) k = 1 while not is_pos_def(A3): eigenvals = torch.eig(A3, eigenvectors=False)[0][:, 0] mineig = eigenvals.min().item() A3 += I * (-mineig * k**2 + spacing) k += 1 return A3
def lda(x1, x2, device="cpu"): with torch.no_grad(): x1 = torch.tensor(x1, device=device, dtype=torch.float) x2 = torch.tensor(x2, device=device, dtype=torch.float) m1 = torch.mean(x1, dim=0) m2 = torch.mean(x2, dim=0) m = (len(x1) * m1 + len(x2) * m2) / (len(x1) + len(x2)) d1 = x1 - m1[None, :] scatter1 = d1.t() @ d1 d2 = x2 - m2[None, :] scatter2 = d2.t() @ d2 within_class_scatter = scatter1 + scatter2 d1 = m1 - m[None, :] scatter1 = len(x1) * (d1.t() @ d1) d2 = m2 - m[None, :] scatter2 = len(x2) * (d2.t() @ d2) between_class_scatter = scatter1 + scatter2 p = torch.pinverse(within_class_scatter) @ between_class_scatter eigenvalues, eigenvectors = torch.eig(p, eigenvectors=True) idx = torch.argsort(eigenvalues[:, 0], descending=True) eigenvalues = eigenvalues[idx, 0] eigenvectors = eigenvectors[idx, :] return eigenvectors[0, :].cpu().numpy()
def expm_eig(A): eigen_values, eigen_vector = th.eig(A, eigenvectors=True) eigen_values.exp_() return th.mm(th.mm(eigen_vector, th.diag(eigen_values[:, 0])), eigen_vector.t_())
def ARCoeffecientGeneration(arCoeffMeans, arCoeffecientNoiseVar, seed=-1): if (seed > 0): torch.manual_seed(seed) arCoeffsMatrix = torch.eye(2) arCoeffsMatrix[1] = arCoeffsMatrix[0] goodCoeffs = False # Pre-Allocating the arCoeffNoise array arCoeffNoise = torch.empty(2, 1, dtype=torch.float) eigValsEvaluated = torch.empty(2, dtype=torch.float) while (not goodCoeffs): # Generate new AR Coefficients until you get a pair who's eigenvalues are # less than 1. # We do this because the system would explode to infinity if the eigenvalues # were greater than 1. arCoeffNoise[0] = torch.randn(1) * torch.sqrt( torch.tensor(arCoeffecientNoiseVar, dtype=torch.float)) arCoeffNoise[1] = torch.randn(1) * torch.sqrt( torch.tensor(arCoeffecientNoiseVar, dtype=torch.float)) arCoeffsMatrix[0, 0] = arCoeffMeans[0] + arCoeffNoise[0] arCoeffsMatrix[0, 1] = arCoeffMeans[1] + arCoeffNoise[1] # Compute EigenValues of F eigValsEvaluated = torch.abs( torch.eig(arCoeffsMatrix, eigenvectors=False)[0][0]) > 1 # Determine if any of them have a greater magnitude than 1 if (not eigValsEvaluated.any()): goodCoeffs = True return arCoeffsMatrix
def _spectral_embedding(self, X): """ Helper function to embed the dataset X into the eigenvectors of the graph Laplacian matrix Returns ------- ht.DNDarray, shape=(m_lanczos): Eigenvalues of the graph's Laplacian matrix. ht.DNDarray, shape=(n, m_lanczos): Eigenvectors of the graph's Laplacian matrix. """ L = self._laplacian.construct(X) # 3. Eigenvalue and -vector calculation via Lanczos Algorithm v0 = ht.full( (L.shape[0], ), fill_value=1.0 / math.sqrt(L.shape[0]), dtype=L.dtype, split=0, device=L.device, ) V, T = ht.lanczos(L, self.n_lanczos, v0) # 4. Calculate and Sort Eigenvalues and Eigenvectors of tridiagonal matrix T eval, evec = torch.eig(T._DNDarray__array, eigenvectors=True) # If x is an Eigenvector of T, then y = V@x is the corresponding Eigenvector of L eval, idx = torch.sort(eval[:, 0], dim=0) eigenvalues = ht.array(eval) eigenvectors = ht.matmul(V, ht.array(evec))[:, idx] return eigenvalues, eigenvectors
def spectral_radius(m): """ Compute spectral radius of a square 2-D tensor :param m: squared 2D tensor :return: """ return torch.max(torch.abs(torch.eig(m)[0])).item()
def svd_wrapper(matrix, mode, ncomp, debug, verbose, usv=False, random_state=None, to_numpy=True): """ Wrapper for different SVD libraries (CPU and GPU). Parameters ---------- matrix : array_like, 2d 2d input matrix. mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy', 'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional Switch for the SVD method/library to be used. ``lapack`` uses the LAPACK linear algebra library through Numpy and it is the most conventional way of computing the SVD (deterministic result computed on CPU). ``arpack`` uses the ARPACK Fortran libraries accessible through Scipy (computation on CPU). ``eigen`` computes the singular vectors through the eigendecomposition of the covariance M.M' (computation on CPU). ``randsvd`` uses the randomized_svd algorithm implemented in Sklearn (computation on CPU). ``cupy`` uses the Cupy library for GPU computation of the SVD as in the LAPACK version. ``eigencupy`` offers the same method as with the ``eigen`` option but on GPU (through Cupy). ``randcupy`` is an adaptation f the randomized_svd algorithm, where all the computations are done on a GPU (through Cupy). ``pytorch`` uses the Pytorch library for GPU computation of the SVD. ``eigenpytorch`` offers the same method as with the ``eigen`` option but on GPU (through Pytorch). ``randpytorch`` is an adaptation of the randomized_svd algorithm, where all the linear algebra computations are done on a GPU (through Pytorch). ncomp : int Number of singular vectors to be obtained. In the cases when the full SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular vectors is truncated. debug : bool If True the explained variance ratio is computed and displayed. verbose: bool If True intermediate information is printed out. usv : bool optional If True the 3 terms of the SVD factorization are returned. random_state : int, RandomState instance or None, optional If int, random_state is the seed used by the random number generator. If RandomState instance, random_state is the random number generator. If None, the random number generator is the RandomState instance used by np.random. Used for ``randsvd`` mode. to_numpy : bool, optional If True (by default) the arrays computed in GPU are transferred from VRAM and converted to numpy ndarrays. Returns ------- V : array_like The right singular vectors of the input matrix. If ``usv`` is True it returns the left and right singular vectors and the singular values of the input matrix. References ---------- * For ``lapack`` SVD mode see: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html http://www.netlib.org/lapack/ * For ``eigen`` mode see: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html * For ``arpack`` SVD mode see: https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html http://www.caam.rice.edu/software/ARPACK/ * For ``randsvd`` SVD mode see: https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py Finding structure with randomness: Stochastic algorithms for constructing approximate matrix decompositions Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061 * For ``cupy`` SVD mode see: https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html * For ``eigencupy`` mode see: https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html * For ``pytorch`` SVD mode see: http://pytorch.org/docs/master/torch.html#torch.svd * For ``eigenpytorch`` mode see: http://pytorch.org/docs/master/torch.html#torch.eig """ def reconstruction(ncomp, U, S, V, var=1): if mode == 'lapack': rec_matrix = np.dot(U[:, :ncomp], np.dot(np.diag(S[:ncomp]), V[:ncomp])) rec_matrix = rec_matrix.T print(' Matrix reconstruction with {} PCs:'.format(ncomp)) print(' Mean Absolute Error =', MAE(matrix, rec_matrix)) print(' Mean Squared Error =', MSE(matrix, rec_matrix)) # see https://github.com/scikit-learn/scikit-learn/blob/c3980bcbabd9d2527548820581725df2904e4a0d/sklearn/decomposition/pca.py exp_var = (S ** 2) / (S.shape[0] - 1) full_var = np.sum(exp_var) explained_variance_ratio = exp_var / full_var # % of variance explained by each PC ratio_cumsum = np.cumsum(explained_variance_ratio) elif mode == 'eigen': exp_var = (S ** 2) / (S.shape[0] - 1) full_var = np.sum(exp_var) explained_variance_ratio = exp_var / full_var # % of variance explained by each PC ratio_cumsum = np.cumsum(explained_variance_ratio) else: rec_matrix = np.dot(U, np.dot(np.diag(S), V)) print(' Matrix reconstruction MAE =', MAE(matrix, rec_matrix)) exp_var = (S ** 2) / (S.shape[0] - 1) full_var = np.var(matrix, axis=0).sum() explained_variance_ratio = exp_var / full_var # % of variance explained by each PC if var == 1: pass else: explained_variance_ratio = explained_variance_ratio[::-1] ratio_cumsum = np.cumsum(explained_variance_ratio) msg = ' This info makes sense when the matrix is mean centered ' msg += '(temp-mean scaling)' print(msg) lw = 2; alpha = 0.4 fig = plt.figure(figsize=vip_figsize) fig.subplots_adjust(wspace=0.4) ax1 = plt.subplot2grid((1, 3), (0, 0), colspan=2) ax1.step(range(explained_variance_ratio.shape[0]), explained_variance_ratio, alpha=alpha, where='mid', label='Individual EVR', lw=lw) ax1.plot(ratio_cumsum, '.-', alpha=alpha, label='Cumulative EVR', lw=lw) ax1.legend(loc='best', frameon=False, fontsize='medium') ax1.set_ylabel('Explained variance ratio (EVR)') ax1.set_xlabel('Principal components') ax1.grid(linestyle='solid', alpha=0.2) ax1.set_xlim(-10, explained_variance_ratio.shape[0] + 10) ax1.set_ylim(0, 1) trunc = 20 ax2 = plt.subplot2grid((1, 3), (0, 2), colspan=1) # plt.setp(ax2.get_yticklabels(), visible=False) ax2.step(range(trunc), explained_variance_ratio[:trunc], alpha=alpha, where='mid', lw=lw) ax2.plot(ratio_cumsum[:trunc], '.-', alpha=alpha, lw=lw) ax2.set_xlabel('Principal components') ax2.grid(linestyle='solid', alpha=0.2) ax2.set_xlim(-2, trunc + 2) ax2.set_ylim(0, 1) msg = ' Cumulative explained variance ratio for {} PCs = {:.5f}' # plt.savefig('figure.pdf', dpi=300, bbox_inches='tight') print(msg.format(ncomp, ratio_cumsum[ncomp - 1])) # -------------------------------------------------------------------------- if matrix.ndim != 2: raise TypeError('Input matrix is not a 2d array') if usv: if mode not in ('lapack', 'arpack', 'randsvd', 'cupy', 'randcupy', 'pytorch', 'randpytorch'): msg = "Returning USV is supported with modes lapack, arpack, " msg += "randsvd, cupy, randcupy, pytorch or randpytorch" raise ValueError(msg) if ncomp > min(matrix.shape[0], matrix.shape[1]): msg = '{} PCs cannot be obtained from a matrix with size [{},{}].' msg += ' Increase the size of the patches or request less PCs' raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1])) if mode == 'eigen': # building C as np.dot(matrix.T,matrix) is slower and takes more memory C = np.dot(matrix, matrix.T) # covariance matrix e, EV = linalg.eigh(C) # EVals and EVs pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM' V = pc[::-1] # reverse since we need the last EVs S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals) S = S[::-1] # reverse since EVals go in increasing order if debug: reconstruction(ncomp, None, S, None) for i in range(V.shape[1]): V[:, i] /= S # scaling EVs by the square root of EVals V = V[:ncomp] if verbose: print('Done PCA with numpy linalg eigh functions') elif mode == 'lapack': # n_frames is usually smaller than n_pixels. In this setting taking the SVD of M' # and keeping the left (transposed) SVs is faster than taking the SVD of M (right SVs) U, S, V = linalg.svd(matrix.T, full_matrices=False) if debug: reconstruction(ncomp, U, S, V) V = V[:ncomp] # we cut projection matrix according to the # of PCs U = U[:, :ncomp] S = S[:ncomp] if verbose: print('Done SVD/PCA with numpy SVD (LAPACK)') elif mode == 'arpack': U, S, V = svds(matrix, k=ncomp) if debug: reconstruction(ncomp, U, S, V, -1) if verbose: print('Done SVD/PCA with scipy sparse SVD (ARPACK)') elif mode == 'randsvd': U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2, transpose='auto', random_state=random_state) if debug: reconstruction(ncomp, U, S, V) if verbose: print('Done SVD/PCA with randomized SVD') elif mode == 'cupy': if no_cupy: raise RuntimeError('Cupy is not installed') a_gpu = cupy.array(matrix) a_gpu = cupy.asarray(a_gpu) # move the data to the current device u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True, compute_uv=True) V = vh_gpu[:ncomp] if to_numpy: V = cupy.asnumpy(V) if usv: S = s_gpu[:ncomp] if to_numpy: S = cupy.asnumpy(S) U = u_gpu[:, :ncomp] if to_numpy: U = cupy.asnumpy(U) if verbose: print('Done SVD/PCA with cupy (GPU)') elif mode == 'randcupy': if no_cupy: raise RuntimeError('Cupy is not installed') U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy') if to_numpy: V = cupy.asnumpy(V) S = cupy.asnumpy(S) U = cupy.asnumpy(U) if debug: reconstruction(ncomp, U, S, V) if verbose: print('Done randomized SVD/PCA with cupy (GPU)') elif mode == 'eigencupy': if no_cupy: raise RuntimeError('Cupy is not installed') a_gpu = cupy.array(matrix) a_gpu = cupy.asarray(a_gpu) # move the data to the current device C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors pc = cupy.dot(EV.T, a_gpu) # PCs using a compact trick when cov is MM' V = pc[::-1] # reverse since last eigenvectors are the ones we want S = cupy.sqrt(e)[::-1] # reverse since eigenvalues are in increasing order if debug: reconstruction(ncomp, None, S, None) for i in range(V.shape[1]): V[:, i] /= S # scaling by the square root of eigenvalues V = V[:ncomp] if to_numpy: V = cupy.asnumpy(V) if verbose: print('Done PCA with cupy eigh function (GPU)') elif mode == 'pytorch': if no_torch: raise RuntimeError('Pytorch is not installed') a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T)) u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu) V = vh_gpu[:ncomp] S = s_gpu[:ncomp] U = torch.transpose(u_gpu, 0, 1)[:ncomp] if to_numpy: V = np.array(V) S = np.array(S) U = np.array(U) if verbose: print('Done SVD/PCA with pytorch (GPU)') elif mode == 'eigenpytorch': if no_torch: raise RuntimeError('Pytorch is not installed') a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32'))) C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1)) e, EV = torch.eig(C, eigenvectors=True) V = torch.mm(torch.transpose(EV, 0, 1), a_gpu) S = torch.sqrt(e[:, 0]) if debug: reconstruction(ncomp, None, S, None) for i in range(V.shape[1]): V[:, i] /= S V = V[:ncomp] if to_numpy: V = np.array(V) if verbose: print('Done PCA with pytorch eig function') elif mode == 'randpytorch': if no_torch: raise RuntimeError('Pytorch is not installed') U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch') if to_numpy: V = np.array(V) S = np.array(S) U = np.array(U) if debug: reconstruction(ncomp, U, S, V) if verbose: print('Done randomized SVD/PCA with randomized pytorch (GPU)') else: raise ValueError('The SVD mode is not available') if usv: if mode == 'lapack': return V.T, S, U.T elif mode == 'pytorch': if to_numpy: return V.T, S, U.T else: return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1) else: return U, S, V else: if mode == 'lapack': return U.T elif mode == 'pytorch': return U else: return V
def svd_wrapper(matrix, mode, ncomp, verbose, full_output=False, random_state=None, to_numpy=True): """ Wrapper for different SVD libraries (CPU and GPU). Parameters ---------- matrix : numpy ndarray, 2d 2d input matrix. mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy', 'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional Switch for the SVD method/library to be used. ``lapack``: uses the LAPACK linear algebra library through Numpy and it is the most conventional way of computing the SVD (deterministic result computed on CPU). ``arpack``: uses the ARPACK Fortran libraries accessible through Scipy (computation on CPU). ``eigen``: computes the singular vectors through the eigendecomposition of the covariance M.M' (computation on CPU). ``randsvd``: uses the randomized_svd algorithm implemented in Sklearn (computation on CPU). ``cupy``: uses the Cupy library for GPU computation of the SVD as in the LAPACK version. ` `eigencupy``: offers the same method as with the ``eigen`` option but on GPU (through Cupy). ``randcupy``: is an adaptation of the randomized_svd algorithm, where all the computations are done on a GPU (through Cupy). ` `pytorch``: uses the Pytorch library for GPU computation of the SVD. ``eigenpytorch``: offers the same method as with the ``eigen`` option but on GPU (through Pytorch). ``randpytorch``: is an adaptation of the randomized_svd algorithm, where all the linear algebra computations are done on a GPU (through Pytorch). ncomp : int Number of singular vectors to be obtained. In the cases when the full SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular vectors is truncated. verbose: bool If True intermediate information is printed out. full_output : bool optional If True the 3 terms of the SVD factorization are returned. If ``mode`` is eigen then only S and V are returned. random_state : int, RandomState instance or None, optional If int, random_state is the seed used by the random number generator. If RandomState instance, random_state is the random number generator. If None, the random number generator is the RandomState instance used by np.random. Used for ``randsvd`` mode. to_numpy : bool, optional If True (by default) the arrays computed in GPU are transferred from VRAM and converted to numpy ndarrays. Returns ------- V : numpy ndarray The right singular vectors of the input matrix. If ``full_output`` is True it returns the left and right singular vectors and the singular values of the input matrix. If ``mode`` is set to eigen then only S and V are returned. References ---------- * For ``lapack`` SVD mode see: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html http://www.netlib.org/lapack/ * For ``eigen`` mode see: https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html * For ``arpack`` SVD mode see: https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html http://www.caam.rice.edu/software/ARPACK/ * For ``randsvd`` SVD mode see: https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py Finding structure with randomness: Stochastic algorithms for constructing approximate matrix decompositions Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061 * For ``cupy`` SVD mode see: https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html * For ``eigencupy`` mode see: https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html * For ``pytorch`` SVD mode see: http://pytorch.org/docs/master/torch.html#torch.svd * For ``eigenpytorch`` mode see: http://pytorch.org/docs/master/torch.html#torch.eig """ if matrix.ndim != 2: raise TypeError('Input matrix is not a 2d array') if ncomp > min(matrix.shape[0], matrix.shape[1]): msg = '{} PCs cannot be obtained from a matrix with size [{},{}].' msg += ' Increase the size of the patches or request less PCs' raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1])) if mode == 'eigen': # building C as np.dot(matrix.T,matrix) is slower and takes more memory C = np.dot(matrix, matrix.T) # covariance matrix e, EV = linalg.eigh(C) # EVals and EVs pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM' V = pc[::-1] # reverse since we need the last EVs S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals) S = S[::-1] # reverse since EVals go in increasing order for i in range(V.shape[1]): V[:, i] /= S # scaling EVs by the square root of EVals V = V[:ncomp] if verbose: print('Done PCA with numpy linalg eigh functions') elif mode == 'lapack': # n_frames is usually smaller than n_pixels. In this setting taking # the SVD of M' and keeping the left (transposed) SVs is faster than # taking the SVD of M (right SVs) U, S, V = linalg.svd(matrix.T, full_matrices=False) V = V[:ncomp] # we cut projection matrix according to the # of PCs U = U[:, :ncomp] S = S[:ncomp] if verbose: print('Done SVD/PCA with numpy SVD (LAPACK)') elif mode == 'arpack': U, S, V = svds(matrix, k=ncomp) if verbose: print('Done SVD/PCA with scipy sparse SVD (ARPACK)') elif mode == 'randsvd': U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2, transpose='auto', random_state=random_state) if verbose: print('Done SVD/PCA with randomized SVD') elif mode == 'cupy': if no_cupy: raise RuntimeError('Cupy is not installed') a_gpu = cupy.array(matrix) a_gpu = cupy.asarray(a_gpu) # move the data to the current device u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True, compute_uv=True) V = vh_gpu[:ncomp] if to_numpy: V = cupy.asnumpy(V) if full_output: S = s_gpu[:ncomp] if to_numpy: S = cupy.asnumpy(S) U = u_gpu[:, :ncomp] if to_numpy: U = cupy.asnumpy(U) if verbose: print('Done SVD/PCA with cupy (GPU)') elif mode == 'randcupy': if no_cupy: raise RuntimeError('Cupy is not installed') U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy') if to_numpy: V = cupy.asnumpy(V) S = cupy.asnumpy(S) U = cupy.asnumpy(U) if verbose: print('Done randomized SVD/PCA with cupy (GPU)') elif mode == 'eigencupy': if no_cupy: raise RuntimeError('Cupy is not installed') a_gpu = cupy.array(matrix) a_gpu = cupy.asarray(a_gpu) # move the data to the current device C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors pc = cupy.dot(EV.T, a_gpu) # using a compact trick when cov is MM' V = pc[::-1] # reverse to get last eigenvectors S = cupy.sqrt(e)[::-1] # reverse since EVals go in increasing order for i in range(V.shape[1]): V[:, i] /= S # scaling by the square root of eigvals V = V[:ncomp] if to_numpy: V = cupy.asnumpy(V) if verbose: print('Done PCA with cupy eigh function (GPU)') elif mode == 'pytorch': if no_torch: raise RuntimeError('Pytorch is not installed') a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T)) u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu) V = vh_gpu[:ncomp] S = s_gpu[:ncomp] U = torch.transpose(u_gpu, 0, 1)[:ncomp] if to_numpy: V = np.array(V) S = np.array(S) U = np.array(U) if verbose: print('Done SVD/PCA with pytorch (GPU)') elif mode == 'eigenpytorch': if no_torch: raise RuntimeError('Pytorch is not installed') a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32'))) C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1)) e, EV = torch.eig(C, eigenvectors=True) V = torch.mm(torch.transpose(EV, 0, 1), a_gpu) S = torch.sqrt(e[:, 0]) for i in range(V.shape[1]): V[:, i] /= S V = V[:ncomp] if to_numpy: V = np.array(V) if verbose: print('Done PCA with pytorch eig function') elif mode == 'randpytorch': if no_torch: raise RuntimeError('Pytorch is not installed') U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch') if to_numpy: V = np.array(V) S = np.array(S) U = np.array(U) if verbose: print('Done randomized SVD/PCA with randomized pytorch (GPU)') else: raise ValueError('The SVD `mode` is not recognized') if full_output: if mode == 'lapack': return V.T, S, U.T elif mode == 'pytorch': if to_numpy: return V.T, S, U.T else: return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1) elif mode in ('eigen', 'eigencupy', 'eigenpytorch'): return S, V else: return U, S, V else: if mode == 'lapack': return U.T elif mode == 'pytorch': return U else: return V